tuning

Author: jacobdenobel

.gitignore

@@ -116,7 +116,9 @@ scripts/distributions/*png
 scripts/distributions/*pkl
 scripts/distributions/figures
 
-scripts/tuning/*pdf
+scripts/tuning/figures
 scripts/tuning/configs_*.csv
+scripts/tuning/data*
+script/tuning/data_new/*
 
 data_old

scripts/repelling/repelling.py

@@ -54,9 +54,10 @@ def calculate_potential(centers, problem):
     return potential, n_duplicate_runs
 
 
-def plot_contour(X, Y, Z, colorbar=True):
+def plot_contour(X, Y, Z, colorbar=True, vmin=-2, vmax=2):
     plt.contourf(
-        X, Y, np.log10(Z), levels=200, cmap="Spectral", zorder=-1, vmin=-1, vmax=2.5
+        X, Y, np.log10(Z), levels=200, cmap="Spectral", 
+        zorder=-1, vmin=vmin, vmax=vmax
     )
     plt.xlabel(R"$x_1$")
     plt.ylabel(R"$x_2$")
@@ -498,12 +499,12 @@ def interactive(fid=21, instance=6, dim=2, rep=True, coverage=5, save_frames = F
     # c_cmaes.constants.tol_min_sigma = 0.01
 
     modules = c_cmaes.parameters.Modules()
-    modules.restart_strategy = c_cmaes.options.BIPOP
+    modules.restart_strategy = c_cmaes.options.RestartStrategy.RESTART
     # modules.bound_correction = c_cmaes.options.SATURATE
-    modules.elitist = True
-    modules.active = True
+    # modules.elitist = True
+    # modules.active = True
     modules.repelling_restart = rep
-    modules.center_placement = c_cmaes.options.UNIFORM
+    modules.center_placement = c_cmaes.options.CenterPlacement.UNIFORM
     settings = c_cmaes.parameters.Settings(
         dim,
         modules,

scripts/tuning/analysis.ipynb

@@ -3172,7 +3172,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 310,
    "id": "59d5a6c6",
    "metadata": {},
    "outputs": [
@@ -3273,7 +3273,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 332,
+   "execution_count": null,
    "id": "26fe56ff",
    "metadata": {},
    "outputs": [
@@ -3406,6 +3406,17 @@
      },
      "metadata": {},
      "output_type": "display_data"
+    },
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
+      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
+      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
     }
    ],
    "source": [
@@ -3426,25 +3437,6 @@
     "    plt.savefig(f\"{module}_shap_heat.pdf\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fce55604",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "np.float64(0.05)"
-      ]
-     },
-     "execution_count": 330,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": 317,
@@ -3487,7 +3479,7 @@
    ],
    "source": [
     "plt.plot(cat_cols, model.feature_importances_, linestyle='dashed', marker='o')\n",
-    "plt.xticks(rotation=90)"
+    "plt.xticks(rotation=90);"
    ]
   },
   {

scripts/tuning/run_smac.py

@@ -20,7 +20,7 @@
 from modcma import c_maes
 
 
-DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))
+DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data_new"))
 
 
 def calc_aoc(problem: ioh.ProblemType, logger: ioh.logger.Store, budget: int) -> float:
@@ -150,7 +150,7 @@ def get_configspace(dim, use_learning_rates, add_popsize, add_sigma):
 
 def run_smac(fid, dim, use_learning_rates, add_popsize, add_sigma, n_workers):
     print(f"Running SMAC with fid={fid}, lr={use_learning_rates} and d={dim}")
-    cs = get_configspace(dim ,use_learning_rates, add_popsize, add_sigma)
+    cs = get_configspace(dim, use_learning_rates, add_popsize, add_sigma)
     scenario = Scenario(
         cs,
         name=str(int(time.time())) + "-" + "CMA",
@@ -165,7 +165,7 @@ def run_smac(fid, dim, use_learning_rates, add_popsize, add_sigma, n_workers):
     eval_func = partial(get_bbob_performance, fid=fid, dim=dim)
     config_selector = ConfigSelector(
         scenario,
-        retrain_after=250,
+        retrain_after=100,
         min_trials=500,
         retries=16,
     )
@@ -183,7 +183,7 @@ def run_smac(fid, dim, use_learning_rates, add_popsize, add_sigma, n_workers):
             ratio_features=0.5,
             min_samples_split=10,
             min_samples_leaf=5,
-            max_depth=10,
+            max_depth=8,
             bootstrapping=True,
             pca_components=13
         ),

scripts/tuning/smac_info.py

@@ -58,14 +58,14 @@ def plot_convergence(fid: int, dim:int, config_rt: dict[str, list[float]]):
     parser = ArgumentParser()
     parser.add_argument("--fid", default=1, type=int)
     parser.add_argument("--dim", default=5, type=int)
-    parser.add_argument("--root", default="data", type=str) 
+    parser.add_argument("--root", default="data_new", type=str) 
     parser.add_argument("--show_all_feasible", action="store_true")
     parser.add_argument("--all", action="store_true")
     parser.add_argument("--plot", action="store_true")
     args = parser.parse_args()
 
     folders = os.listdir(args.root)
-    arg_folder = f"BBOB_F{args.fid}_{args.dim}D_LRFalse"
+    arg_folder = f"BBOB_F{args.fid}_{args.dim}D_LRFalseTrue"
 
     for folder in sorted(os.listdir(args.root)):
         if not args.all and not folder.endswith(arg_folder): continue

scripts/tuning/test_config.py

@@ -74,6 +74,7 @@ def get_ert(
 
         suc += problem.state.final_target_found
         rt  += problem.state.evaluations
+        print(problem.state)
         problem.reset()
 
     if suc == 0:
@@ -87,7 +88,21 @@ def get_ert(
     parser.add_argument("--reps", type=int, default=50)
     args = parser.parse_args() 
     config = {
-        'ssa': "PSR",
+        'active': True,
+    'elitist': True,
+    'lambda0': 4,
+    'matrix_adaptation': 'MATRIX',
+    'mirrored': 'PAIRWISE',
+    'mu0': 4,
+    'orthogonal': True,
+    'repelling_restart': True,
+    'restart_strategy': 'NONE',
+    'sample_transformation': 'LOGISTIC',
+    'sampler': 'HALTON',
+    'sequential_selection': True,
+    'ssa': 'MXNES',
+    'threshold_convergence': True,
+    'weights': 'DEFAULT'
     }
 
     settings = c_maes.settings_from_dict(5, **config)