Selfplay clean

config/chess.ini

@@ -0,0 +1,101 @@
+[base]
+env_name = chess
+
+[selfplay]
+enabled = 1
+max_size = 500
+swap_winrate = 0.6
+min_games = 4096
+snapshot_interval = 1_000_000_000
+opp_timeout_steps = 4_000_000_000
+
+[vec]
+total_agents = 8192
+num_buffers = 1
+num_threads = 2
+num_frozen_banks = 1
+frozen_bank_pct = 0.1
+
+[env]
+max_moves = 5000
+reward_draw = 0
+reward_invalid_piece = 0
+reward_invalid_move = 0
+reward_repetition = 0
+enable_50_move_rule = 1
+enable_threefold_repetition = 1
+mode = 1
+random_fen = 0
+render_fps = 30
+fen_curric_pct = 0.9
+
+[policy]
+hidden_size = 512
+num_layers = 3
+expansion_factor = 1
+
+[train]
+gpus = 1
+seed = 42
+total_timesteps = 1_000_000_000_000
+learning_rate = 0.000572786
+anneal_lr = 1
+min_lr_ratio = 0.086914
+gamma = 0.994795
+gae_lambda = 0.754641
+replay_ratio = 0.25
+clip_coef = 0.557019
+vf_coef = 4.37465
+vf_clip_coef = 1.69524
+max_grad_norm = 3.54293
+ent_coef = 0.0984801
+anneal_ent_coef = 1
+min_ent_coef_ratio = 0.1
+beta1 = 0.972205
+beta2 = 0.9
+eps = 8.51435e-11
+minibatch_size = 32768
+horizon = 64
+vtrace_rho_clip = 1.99404
+vtrace_c_clip = 2.19484
+prio_alpha = 1
+prio_beta0 = 0.474524
+torch_deterministic = 1
+adam_beta1 = 0.963206
+adam_beta2 = 0.99999
+adam_eps = 5.09326e-08
+update_epochs = 1
+[sweep]
+# Score each trial by winrate in a 2-policy match against a fixed enemy rather
+# than the training-time self-play env/score. 'latest' resolves to the newest
+# .bin in checkpoint_dir/chess/**. Enemy arch must match the checkpoint's arch.
+match_enemy_model_path = 'resources/chess/10b_weights.bin'
+match_num_games = 4096
+match_enemy_hidden_size = 512
+match_enemy_num_layers = 3
+
+[sweep.train.total_timesteps]
+distribution = log_normal
+min = 7e9
+max = 15e9
+mean = 10e9
+scale = time
+
+[sweep.selfplay.swap_winrate]
+distribution = uniform
+min = 0.55
+max = 0.90
+scale = auto
+
+[sweep.vec.num_buffers]
+distribution = uniform_pow2
+min = 1
+max = 8
+scale = auto
+
+[sweep.vec.frozen_bank_pct]
+distribution = uniform
+min = 0.05
+max = 0.5
+scale = auto
+

config/default.ini

@@ -9,7 +9,7 @@ nccl_id = 'None'
 profile = False
 checkpoint_dir = checkpoints
 log_dir = logs
-checkpoint_interval = 200
+checkpoint_interval = 500
 eval_episodes = 10000
 
 # Epoch at which to capture CUDA graphs. -1 to disable.
@@ -25,6 +25,25 @@ total_agents = 4096
 num_buffers = 2
 num_threads = 16
 
+# Selfplay-pool training. When enabled, frozen_bank_pct of agent slots (see
+# [vec]) play against a snapshot of an older policy from a disk-backed pool.
+# Primary advances to the next pool entry once it beats the current opponent
+# at >= swap_winrate over at least min_games (counted only on historical envs).
+[selfplay]
+enabled = 0
+max_size = 16
+swap_winrate = 0.8
+min_games = 2048
+elo_init = 0.0
+elo_k = 16.0
+seed = 42
+# Add a snapshot to the pool every snapshot_interval global steps, independent
+# of swap. 0 disables interval snapshotting (pool stays at bootstrap).
+snapshot_interval = 1_000_000_000
+# Force an opponent swap if the current opponent has been active for this many
+# global steps without a winrate-driven swap. 0 disables the timeout.
+opp_timeout_steps = 500_000_000
+
 # Args used by your env's binding.c go here
 [env]
 
@@ -53,6 +72,12 @@ vf_coef = 2.0
 vf_clip_coef = 0.2
 max_grad_norm = 1.5
 ent_coef = 0.001
+# Cosine-anneal ent_coef like lr does. Off by default (matches old behavior).
+# When on: ent_coef decays from its base value to min_ent_coef_ratio * ent_coef
+# over total_timesteps. Useful late in training to let a learned policy commit
+# harder on its preferences once the entropy bonus is no longer load-bearing.
+anneal_ent_coef = 0
+min_ent_coef_ratio = 0.1
 beta1 = 0.95
 beta2 = 0.999
 eps = 1e-12
@@ -75,6 +100,13 @@ downsample = 5
 use_gpu = True
 prune_pareto = True
 early_stop_quantile = 0.3
+# When set, each sweep trial is scored by winrate in a match against a fixed
+# enemy checkpoint rather than by the training-time env/score. Score key emitted
+# as env/match_score; set match_enemy_model_path to '' to disable.
+match_enemy_model_path = ''
+match_num_games = 1024
+match_enemy_hidden_size = 0
+match_enemy_num_layers = 0
 
 [sweep.train.total_timesteps]
 distribution = log_normal
@@ -94,11 +126,11 @@ min = 1
 max = 8
 scale = auto
 
-[sweep.vec.total_agents]
-distribution = uniform_pow2
-min = 256
-max = 16384
-scale = auto
+#[sweep.vec.total_agents]
+#distribution = uniform_pow2
+#min = 256
+#max = 16384
+#scale = auto
 
 [sweep.vec.num_buffers]
 distribution = uniform
@@ -112,11 +144,11 @@ min = 8
 max = 1024
 scale = auto
 
-[sweep.train.minibatch_size]
-distribution = uniform_pow2
-min = 4096
-max = 65536
-scale = auto
+#[sweep.train.minibatch_size]
+#distribution = uniform_pow2
+#min = 4096
+#max = 65536
+#scale = auto
 
 [sweep.train.learning_rate]
 distribution = log_normal
@@ -139,7 +171,6 @@ scale = auto
 [sweep.train.gae_lambda]
 distribution = logit_normal
 min = 0.2
-#min = 0.6
 max = 0.995
 scale = auto