cfg: {amp_enabled: false, attention_probs_dropout_prob: 0.1, embedding_dropout_prob: 0.1, head_size: 64, hidden_dropout_prob: 0.1, hidden_layers: 6, hidden_size: 768, initializer_range: 0.02, intermediate_size: 1536, layernorm_eps: 1.0e-05, model_type: t5, num_attention_heads: 12, num_tokentypes: 0, relative_attention_num_buckets: 32, vocab_size: 30522}
dataloader:
  train:
    _target_: libai.data.build.build_nlp_train_loader
    collate_fn: !!python/object:projects.T5.datasets.dataset.collate_fn {decoder_start_token_id: 1, eos_token_id: 2, expanded_inputs_length: 568, max_seq_length: 512, mean_noise_span_length: 3, noise_density: 0.15, pad_token_id: 0, targets_length: 114, vocab_size: 12902}
    dataset:
    - {_target_: projects.T5.datasets.dataset.UnsuperviseT5Dataset, data_path: projects/T5/data/training_data/part_0}
graph:
  auto_parallel: {enable_auto_parallel_ignore_user_sbp_config: false, enabled: false, mainstem_algo: true, sbp_collector: false}
  debug: 1
  enabled: true
  eval_graph: {_target_: libai.models.utils.GraphBase, is_train: false}
  train_graph: {_target_: libai.models.utils.GraphBase, is_train: true}
model:
  _target_: projects.T5.models.t5_model.T5ForPreTraining
  cfg: {amp_enabled: false, attention_probs_dropout_prob: 0.1, embedding_dropout_prob: 0.1, head_size: 64, hidden_dropout_prob: 0.1, hidden_layers: 12, hidden_size: 768, initializer_range: 0.02, intermediate_size: 3072, layernorm_eps: 1.0e-06, model_type: mt5, num_attention_heads: 12, num_tokentypes: 0, pretrained_model_path: null, relative_attention_num_buckets: 32, vocab_size: 12900}
optim:
  _target_: oneflow.nn.optimizer.adam.Adam
  betas: [0.9, 0.999]
  do_bias_correction: true
  eps: 1.0e-08
  lr: 1.0e-05
  params: {_target_: libai.optim.get_default_optimizer_params, clip_grad_max_norm: 1.0, clip_grad_norm_type: 2.0, weight_decay_bias: 0.0, weight_decay_norm: 0.0}
  weight_decay: 0.01
train:
  activation_checkpoint: {enabled: false}
  amp: {enabled: true}
  checkpointer: {max_to_keep: 10, period: 10000}
  consumed_train_samples: 0
  consumed_valid_samples: 0
  dist: {data_parallel_size: 4, num_gpus_per_node: 4, num_nodes: 2, pipeline_num_layers: 24, pipeline_parallel_size: 1, tensor_parallel_size: 2}
  evaluation:
    enabled: true
    eval_iter: 20
    eval_period: 5000
    evaluator: {_target_: libai.evaluation.PPLEvaluator}
  global_batch_size: 128
  input_placement_device: cuda
  load_weight: ''
  log_period: 1
  nccl_fusion_max_ops: 24
  nccl_fusion_threshold_mb: 16
  num_accumulation_steps: 8
  output_dir: test_logs/2n4g
  rdma_enabled: true
  resume: false
  samples: 28160
  scheduler: {_target_: libai.scheduler.WarmupExponentialLR, gamma: 1.0, warmup_factor: 0.001, warmup_iter: 0.0, warmup_method: linear}
  seed: 1234
  start_iter: 0
  test_micro_batch_size: 32
  train_epoch: 0
  train_iter: 220
  train_micro_batch_size: 4
  train_samples: null
  warmup_ratio: 0.01
  zero_optimization: {enabled: true, stage: 2}
