Skip to content

Commit 94fa13f

Browse files
jacobdenobeljacobdenobel
committed
update readme
1 parent cd2316c commit 94fa13f

3 files changed

Lines changed: 115 additions & 31 deletions

File tree

‎scripts/tuning/run_smac.py‎

Lines changed: 70 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@
99
import ioh
1010
import numpy as np
1111

12-
from smac import Scenario,AlgorithmConfigurationFacade
12+
from smac import Scenario, AlgorithmConfigurationFacade
13+
from smac.acquisition.maximizer import (
14+
LocalAndSortedRandomSearch,
15+
)
1316
from smac.main.config_selector import ConfigSelector
1417
from ConfigSpace import Configuration, ConfigurationSpace
1518
from ConfigSpace.hyperparameters import CategoricalHyperparameter
@@ -20,9 +23,38 @@
2023
DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))
2124

2225

23-
def calc_aoc(logger: ioh.logger.Store, budget: int, fid: int, iid: int, dim: int) -> float:
26+
def calc_aoc(problem: ioh.ProblemType, logger: ioh.logger.Store, budget: int) -> float:
27+
"""
28+
Compute the Area Over the Curve (AOC) for an optimization run.
29+
30+
The AOC summarizes optimization performance over time by averaging
31+
the log-scaled best-so-far objective values across a fixed evaluation
32+
budget. Lower values indicate better and faster convergence.
33+
34+
Steps:
35+
- Extract best-so-far objective values ("raw_y_best") from the logger.
36+
- Replace NaNs with a large penalty value (1e8).
37+
- Pad the trajectory to the full budget using the best observed value.
38+
- Clip values to [1e-8, 1e2] and apply log10 scaling.
39+
- Shift values to [0, 10] and normalize to [0, 1].
40+
- Return the mean over the budget (the AOC score).
41+
42+
Parameters
43+
----------
44+
problem : ioh.ProblemType
45+
The evaluated problem
46+
logger : ioh.logger.Store
47+
IOH logger containing experiment data.
48+
budget : int
49+
Maximum number of function evaluations to consider.
50+
Returns
51+
-------
52+
float
53+
AOC score in [0, 1], where lower values indicate better performance.
54+
"""
55+
2456
data = logger.data()
25-
data1 = data['None'][fid][dim][iid][0]
57+
data1 = data['None'][problem.meta_data.problem_id][problem.meta_data.n_variables][problem.meta_data.instance][0]
2658
fvals = [x['raw_y_best'] for x in data1.values()]
2759
fvals = np.array(fvals)
2860
if np.isnan(fvals).any():
@@ -43,7 +75,7 @@ def get_bbob_performance(
4375

4476
problem = ioh.get_problem(fid, iid, dim)
4577
logger = ioh.logger.Store(
46-
triggers=[ioh.logger.trigger.ON_IMPROVEMENT],
78+
triggers=[ioh.logger.trigger.ALWAYS],
4779
properties=[ioh.logger.property.RAWYBEST]
4880
)
4981
problem.attach_logger(logger)
@@ -55,17 +87,30 @@ def get_bbob_performance(
5587
ub=problem.bounds.ub,
5688
lb=problem.bounds.lb
5789
)
90+
settings.modules.center_placement = c_maes.options.CenterPlacement.UNIFORM
5891
par = c_maes.Parameters(settings)
5992

6093
try:
6194
cma = c_maes.ModularCMAES(par)
6295
cma.run(problem)
96+
aoc = calc_aoc(problem, logger, BUDGET)
6397
except Exception as e:
6498
print(
6599
f"Found target {problem.state.current_best.y} target, but exception ({e}), so run failed"
66100
)
67-
return np.inf
68-
return calc_aoc(problem, logger, BUDGET)
101+
aoc = np.inf
102+
103+
extra = {
104+
"fid": fid,
105+
"iid": iid,
106+
"dim": dim,
107+
"target": float(problem.optimum.y + 9e-9),
108+
"final_y": float(problem.state.current_best.y),
109+
"evals": int(problem.state.evaluations),
110+
"hit_target": bool(problem.state.current_best.y <= problem.optimum.y + 9e-9),
111+
"precision": float(abs(problem.state.current_best.y - problem.optimum.y)),
112+
}
113+
return aoc, extra
69114

70115
def make_new(hp: CategoricalHyperparameter, filter: list[str]):
71116
new_choices = [c for c in hp.choices if c not in filter]
@@ -117,8 +162,8 @@ def run_smac(fid, dim, use_learning_rates, add_popsize, add_sigma, n_workers):
117162
eval_func = partial(get_bbob_performance, fid=fid, dim=dim)
118163
config_selector = ConfigSelector(
119164
scenario,
120-
retrain_after=500,
121-
min_trials=1000,
165+
retrain_after=250,
166+
min_trials=500,
122167
retries=16,
123168
)
124169

@@ -127,7 +172,23 @@ def run_smac(fid, dim, use_learning_rates, add_popsize, add_sigma, n_workers):
127172
intensifier=AlgorithmConfigurationFacade.get_intensifier(
128173
scenario, max_config_calls=25
129174
),
130-
config_selector=config_selector
175+
config_selector=config_selector,
176+
initial_design = AlgorithmConfigurationFacade.get_initial_design(scenario),
177+
model = AlgorithmConfigurationFacade.get_model(
178+
scenario,
179+
n_trees=5,
180+
ratio_features=0.5,
181+
min_samples_split=10,
182+
min_samples_leaf=5,
183+
max_depth=10,
184+
bootstrapping=True,
185+
pca_components=13
186+
),
187+
acquisition_maximizer=LocalAndSortedRandomSearch(
188+
scenario.configspace,
189+
seed=scenario.seed,
190+
challengers=500
191+
)
131192
)
132193
smac.optimize()
133194

‎scripts/tuning/smac_info.py‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@
99
import numpy as np
1010

1111

12+
def min_rt(config_rt):
13+
amin = float("inf")
14+
cid = None
15+
for k, values in config_rt.items():
16+
if len(values) < 10: continue
17+
if (kmin:= np.mean(values)) < amin:
18+
amin = kmin
19+
cid = k
20+
return cid, amin, config_rt[cid]
21+
22+
1223
if __name__ == "__main__":
1324
parser = ArgumentParser()
1425
parser.add_argument("--fid", default=1, type=int)
@@ -38,26 +49,39 @@
3849
print("cannot load data")
3950
continue
4051

41-
configs = defaultdict(list)
42-
52+
config_costs = defaultdict(list)
53+
config_rt = defaultdict(list)
54+
records = defaultdict(list)
55+
4356
for config in data['data']:
4457
cost, cid = config['cost'], str(config['config_id'])
58+
records[cid].append(config)
4559
if cost != 1_000_000 and np.isfinite(cost):
46-
configs[cid].append(float(cost))
60+
config_costs[cid].append(float(cost))
61+
rt = config['additional_info']['evals']
62+
solved = config['additional_info']['hit_target']
63+
config_rt[cid].append(rt if solved else 50_000)
4764

4865
amin = float('inf')
4966
cmin = None
50-
for cid, values in configs.items():
51-
mvalue = round(np.mean(values), 1)
67+
for cid, values in config_costs.items():
68+
mvalue = np.mean(values)
5269
if args.show_all_feasible:
5370
print(cid, mvalue, len(values), data['configs'][cid])
54-
if len(values) > 20 and mvalue < amin:
71+
if len(values) > 10 and mvalue < amin:
5572
amin = mvalue
5673
cmin = cid
74+
75+
5776
print(f"{len(data['data'])} configs evaluated")
5877
if cmin is None:
5978
print("No best solutions yet")
6079
else:
61-
print(f"lowest avg. cost ({cmin}):", amin)
80+
print(f"lowest avg. cost ({cmin}): {amin: .6f}", end = ' - ')
81+
print(f"avg. rt: {np.mean(config_rt[cmin]): .2f}")
6282
pprint(data['configs'][cmin])
83+
84+
# cid, m_rt, rts = min_rt(config_rt)
85+
# print(cid, m_rt, np.mean(config_costs[cid]))
86+
# pprint(records[cid])
6387
print()

‎scripts/tuning/test_config.py‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import ioh
22
import numpy as np
3-
from ConfigSpace import Configuration
43

54
from modcma import c_maes
65

@@ -84,22 +83,22 @@ def get_ert(
8483

8584
if __name__ == "__main__":
8685
config = {
87-
"active": True,
88-
"elitist": True,
89-
"matrix_adaptation": "MATRIX",
90-
"mirrored": "MIRRORED",
91-
"orthogonal": False,
92-
"repelling_restart": True,
93-
"restart_strategy": "RESTART",
94-
"sample_transformation": "GAUSSIAN",
95-
"sampler": "HALTON",
96-
"sequential_selection": True,
97-
"ssa": "SR",
98-
"threshold_convergence": False,
99-
"weights": "EQUAL"
86+
'active': True,
87+
'elitist': False,
88+
'matrix_adaptation': 'CMSA',
89+
'mirrored': 'MIRRORED',
90+
'orthogonal': False,
91+
'repelling_restart': False,
92+
'restart_strategy': 'IPOP',
93+
'sample_transformation': 'CAUCHY',
94+
'sampler': 'UNIFORM',
95+
'sequential_selection': True,
96+
'ssa': 'TPA',
97+
'threshold_convergence': True,
98+
'weights': 'DEFAULT'
10099
}
101100

102101
settings = c_maes.settings_from_dict(5, **config)
103102
print(settings)
104-
print(get_ert(settings, 1, 4))
103+
print(get_ert(settings, 1, 2))
105104

0 commit comments

Comments
 (0)