dataloader:
  train:
    _target_: libai.data.build_nlp_train_val_test_loader
    dataset:
    - _target_: libai.data.datasets.BertDataset
      binary_head: true
      data_prefix: ./data_test/bert_data/loss_compara_content_sentence
      indexed_dataset: {_target_: libai.data.data_utils.get_indexed_dataset, data_impl: mmap, data_prefix: ./data_test/bert_data/loss_compara_content_sentence, skip_warmup: false}
      mask_lm_prob: 0.15
      masking_style: bert-cn-wwm
      max_seq_length: 512
      name: bert
      seed: 1234
      short_seq_prob: 0.1
    num_workers: 4
    splits:
    - [949.0, 50.0, 1.0]
    train_val_test_num_samples: null
    weights: [1.0]
ds:
  _target_: libai.data.datasets.BertDataset
  binary_head: true
  data_prefix: ./data_test/bert_data/loss_compara_content_sentence
  indexed_dataset: {_target_: libai.data.data_utils.get_indexed_dataset, data_impl: mmap, data_prefix: ./data_test/bert_data/loss_compara_content_sentence, skip_warmup: false}
  mask_lm_prob: 0.15
  masking_style: bert-cn-wwm
  max_seq_length: 512
  name: bert
  seed: 1234
  short_seq_prob: 0.1
graph:
  auto_parallel: {enable_auto_parallel_ignore_user_sbp_config: false, enabled: false, sbp_collector: false, trunk_algo: true}
  debug: -1
  enabled: true
  eval_graph: {_target_: libai.models.utils.GraphBase, is_train: false}
  train_graph: {_target_: libai.models.utils.GraphBase, is_train: true}
model:
  _target_: libai.models.BertForPreTraining
  cfg: {add_binary_head: true, add_pooling_layer: true, amp_enabled: false, apply_query_key_layer_scaling: true, apply_residual_post_layernorm: false, attention_probs_dropout_prob: 0.1, bias_dropout_fusion: true, bias_gelu_fusion: true, ffn_hidden_size: 4096, head_size: 64, hidden_dropout_prob: 0.1, hidden_layers: 24, hidden_size: 1024, initializer_range: 0.02, intermediate_size: 4096, layernorm_eps: 1.0e-05, max_position_embeddings: 512, num_attention_heads: 16, num_tokentypes: 2, scale_mask_softmax_fusion: true, vocab_size: 30522}
optim:
  _target_: oneflow.nn.optimizer.adamw.AdamW
  betas: [0.9, 0.999]
  do_bias_correction: true
  eps: 1.0e-08
  lr: 0.0001
  params: {_target_: libai.optim.get_default_optimizer_params, clip_grad_max_norm: 1.0, clip_grad_norm_type: 2.0, weight_decay_bias: 0.0, weight_decay_norm: 0.0}
  weight_decay: 0.01
tokenization:
  append_eod: false
  make_vocab_size_divisible_by: 128
  tokenizer: {_target_: libai.tokenizer.BertTokenizer, do_chinese_wwm: true, do_lower_case: true, vocab_file: ./data_test/bert_data/bert-base-chinese-vocab.txt}
train:
  activation_checkpoint: {enabled: true}
  amp: {enabled: true}
  checkpointer: {max_to_keep: 100, period: 5000, save_model_after_n_epoch: null}
  consumed_train_samples: 0
  consumed_valid_samples: 0
  dist: {custom_pipeline_stage_id: null, data_parallel_size: 8, num_gpus_per_node: 8, num_nodes: 1, pipeline_num_layers: 24, pipeline_parallel_size: 1, tensor_parallel_size: 1}
  evaluation:
    enabled: false
    eval_after_n_epoch: null
    eval_iter: 100000.0
    eval_metric: Acc@1
    eval_mode: max
    eval_period: 5000
    evaluator: {_target_: libai.evaluation.PPLEvaluator}
  global_batch_size: 2048
  input_placement_device: cpu
  load_weight: ''
  log_period: 100
  nccl_fusion_max_ops: 24
  nccl_fusion_threshold_mb: 16
  num_accumulation_steps: 8
  output_dir: test_logs/oneflow-28/NVIDIA_GeForce_RTX_3080_Ti/1ea2bb7/LibAI_bert_large_pretrain_graph_nl24_nah16_hs1024_FP16_actrue_DP8_MP1_PP1_zerotrue_stage2_mbs32_gbs2048_acc8_1n8g
  rdma_enabled: true
  resume: false
  samples: 450560
  scheduler: {_target_: libai.scheduler.WarmupCosineLR, alpha: 0.01, warmup_factor: 0.001, warmup_method: linear}
  seed: 1234
  start_iter: 0
  test_micro_batch_size: 32
  train_epoch: 0
  train_iter: 220
  train_micro_batch_size: 32
  train_samples: null
  warmup_ratio: 0
  zero_optimization: {enabled: true, stage: 2}
