mirror of https://github.com/inclusionAI/AReaL
141 lines
3.1 KiB
YAML
141 lines
3.1 KiB
YAML
experiment_name: lite-boba-math
|
|
trial_name: run1
|
|
|
|
cluster:
|
|
n_nodes: 16
|
|
n_gpus_per_node: 8
|
|
cluster_name: na132
|
|
fileroot: /storage/openpsi/experiments
|
|
name_resolve:
|
|
type: nfs
|
|
nfs_record_root: /storage/openpsi/experiments/name_resolve/lite-boba-math
|
|
etcd3_addr: etcd-client.openpsi-etcd.svc.sigma-na130-lingbo.na130.wl-robby.local:2379
|
|
|
|
seed: 1
|
|
total_train_epochs: 10
|
|
total_train_steps: null
|
|
tokenizer_path: ${actor.path}
|
|
allocation_mode: sglang.d96p1t1+d32p1t1
|
|
async_training: true
|
|
|
|
rollout:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
max_concurrent_rollouts: 400
|
|
queue_size: null
|
|
consumer_batch_size: ${train_dataset.batch_size}
|
|
max_head_offpolicyness: 4
|
|
enable_rollout_tracing: true
|
|
|
|
gconfig:
|
|
n_samples: 16
|
|
min_new_tokens: 0
|
|
max_new_tokens: 30720
|
|
greedy: false
|
|
temperature: 1.0
|
|
|
|
actor:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
path: /storage/openpsi/models/deepseek-ai__DeepSeek-R1-Distill-Qwen-1.5B/
|
|
init_from_scratch: false
|
|
disable_dropout: true
|
|
gradient_checkpointing: true
|
|
dtype: bfloat16
|
|
mb_spec:
|
|
max_tokens_per_mb: 32768
|
|
optimizer:
|
|
type: adam
|
|
lr: 1e-5
|
|
weight_decay: 0.01
|
|
beta1: 0.9
|
|
beta2: 0.999
|
|
eps: 1e-8
|
|
lr_scheduler_type: constant
|
|
gradient_clipping: 1.0
|
|
warmup_steps_proportion: 0.001
|
|
backend: fsdp
|
|
|
|
group_size: ${gconfig.n_samples}
|
|
group_adv_norm: false
|
|
eps_clip: 0.4
|
|
temperature: ${gconfig.temperature}
|
|
reward_scaling: 10.0
|
|
reward_bias: -0.5
|
|
kl_ctl: 0.0
|
|
ppo_n_minibatches: 4
|
|
recompute_logprob: true
|
|
use_decoupled_loss: true
|
|
behav_imp_weight_cap: 5.0
|
|
|
|
ref:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
path: ${actor.path}
|
|
init_from_scratch: false
|
|
disable_dropout: true
|
|
dtype: ${actor.dtype}
|
|
mb_spec:
|
|
max_tokens_per_mb: 32768
|
|
optimizer: null
|
|
backend: fsdp
|
|
|
|
# SGLang
|
|
server_only: false
|
|
sglang:
|
|
model_path: ${actor.path}
|
|
random_seed: ${seed}
|
|
skip_tokenizer_init: true
|
|
dtype: ${actor.dtype}
|
|
max_running_requests: null
|
|
context_length: 32768
|
|
mem_fraction_static: 0.9
|
|
|
|
# datasets
|
|
train_dataset:
|
|
batch_size: 512
|
|
shuffle: true
|
|
pin_memory: true
|
|
|
|
# Utilities
|
|
saver:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
fileroot: ${cluster.fileroot}
|
|
freq_epochs: 1
|
|
freq_steps: null
|
|
freq_secs: null
|
|
|
|
checkpointer:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
fileroot: ${cluster.fileroot}
|
|
freq_epochs: 1
|
|
freq_steps: null
|
|
freq_secs: 3600
|
|
|
|
evaluator:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
fileroot: ${cluster.fileroot}
|
|
freq_epochs: null
|
|
freq_steps: null
|
|
freq_secs: null
|
|
|
|
stats_logger:
|
|
experiment_name: ${experiment_name}
|
|
trial_name: ${trial_name}
|
|
fileroot: ${cluster.fileroot}
|
|
wandb:
|
|
mode: online
|
|
|
|
# Launcher
|
|
launcher:
|
|
inference_server_cpus_per_gpu: 15
|
|
inference_server_mem_per_gpu: 153600
|
|
trainer_cpus_per_gpu: 15
|
|
trainer_mem_per_gpu: 153600
|
|
slurm:
|
|
mount: /storage:/storage
|
|
trainer_image: /storage/openpsi/images/arealite-20250712-update-hf-xet.sif
|
|
inference_server_image: /storage/openpsi/images/arealite-20250712-update-hf-xet.sif |