AReaL/training/configs/sft/sft-7b-gpu8.yaml

41 lines
870 B
YAML

experiment_name: sft-7b-gpu8
trial_name: my-trial
mode: ray
wandb:
mode: disabled
recover_mode: auto
recover_retries: 10
allocation_mode: d2p4m1
n_nodes: 1
n_gpus_per_node: 8
exp_ctrl:
total_train_epochs: 200
save_freq_epochs: 1
ckpt_freq_secs: 600
torch_cache_mysophobia: true
dataset:
train_path: /storage/datasets/boba-sft_200_0319.jsonl
valid_path: /storage/datasets/boba-sft_200_0319.jsonl
max_seqlen: 32768
train_bs_n_seqs: 16
valid_bs_n_seqs: 16
model:
type:
_class: qwen2
path: /storage/models/DeepSeek-R1-Distill-Qwen-7B
optimizer:
type: adam
lr_scheduler_type: constant
lr: 1e-5
warmup_steps_proportion: 0.03
initial_loss_scale: 262144.0
loss_scale_window: 10
hysteresis: 2
weight_decay: 0.1
eps: 1e-5
bf16: true
allocation:
mb_spec:
max_tokens_per_mb: 32768