loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1

loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 65536
  batch_size_per_proc ............................. 8192
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. True
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. eager
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.5043032765388489, latency(ms): 81.8931093439459801 | 2022-04-26 10:12:26.628
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.751, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.756, Tesla V100-SXM2-32GB, 470.57.02, 87 %, 32 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.760, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.763, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.768, Tesla V100-SXM2-32GB, 470.57.02, 87 %, 32 %, 32510 MiB, 30138 MiB, 2372 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.769, Tesla V100-SXM2-32GB, 470.57.02, 85 %, 32 %, 32510 MiB, 30190 MiB, 2320 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.774, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.774, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.775, Tesla V100-SXM2-32GB, 470.57.02, 82 %, 30 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.775, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.779, Tesla V100-SXM2-32GB, 470.57.02, 85 %, 32 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.778, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.780, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.780, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.781, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.781, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.782, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.786, Tesla V100-SXM2-32GB, 470.57.02, 82 %, 30 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.786, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:12:26.787, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.787, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.789, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.790, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.790, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.794, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.794, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.794, Tesla V100-SXM2-32GB, 470.57.02, 42 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB
2022/04/26 10:12:26.795, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.796, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.797, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.798, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.798, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.802, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.802, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.803, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30138 MiB, 2372 MiB
2022/04/26 10:12:26.804, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.805, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.806, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.807, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.810, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.811, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.812, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 13 %, 32510 MiB, 30214 MiB, 2296 MiB
2022/04/26 10:12:26.812, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.813, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.815, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.815, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.819, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.820, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB
2022/04/26 10:12:26.821, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.822, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.823, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.824, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.827, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.828, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30198 MiB, 2312 MiB
2022/04/26 10:12:26.828, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.829, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.831, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.831, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.837, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.838, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30132 MiB, 2378 MiB
2022/04/26 10:12:26.841, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.841, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
2022/04/26 10:12:26.845, Tesla V100-SXM2-32GB, 470.57.02, 44 %, 17 %, 32510 MiB, 30074 MiB, 2436 MiB
2022/04/26 10:12:26.854, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 9 %, 32510 MiB, 30154 MiB, 2356 MiB
[rank:0] iter: 200/1100, loss: 0.4704145491123199, latency(ms): 23.4032363444566727 | 2022-04-26 10:12:28.969
[rank:0] iter: 300/1100, loss: 0.4637370705604553, latency(ms): 21.0766908526420593 | 2022-04-26 10:12:31.076
[rank:0] iter: 400/1100, loss: 0.4602520465850830, latency(ms): 20.9831067174673080 | 2022-04-26 10:12:33.175
[rank:0] iter: 500/1100, loss: 0.4579895734786987, latency(ms): 20.9237539768218994 | 2022-04-26 10:12:35.267
[rank:0] iter: 600/1100, loss: 0.4532188773155212, latency(ms): 20.6364433839917183 | 2022-04-26 10:12:37.331
[rank:0] iter: 700/1100, loss: 0.4463142454624176, latency(ms): 20.0956175103783607 | 2022-04-26 10:12:39.340
[rank:0] iter: 800/1100, loss: 0.4460747838020325, latency(ms): 21.5976446121931076 | 2022-04-26 10:12:41.500
[rank:0] iter: 900/1100, loss: 0.4466235637664795, latency(ms): 21.2276766449213028 | 2022-04-26 10:12:43.623
[rank:0] iter: 1000/1100, loss: 0.4444417655467987, latency(ms): 21.0801266878843307 | 2022-04-26 10:12:45.731
[rank:0] iter: 1100/1100, loss: 0.4474284648895264, latency(ms): 19.5550780743360519 | 2022-04-26 10:12:47.686