loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1
/usr/lib/x86_64-linux-gnu/libibverbs.so.1


loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 131072
  batch_size_per_proc ............................. 16384
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. False
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. graph
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.0618514902889729, latency(ms): 197.1980131044983864 | 2022-04-28 10:52:42.162
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.314, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.323, Tesla V100-SXM2-32GB, 470.57.02, 82 %, 34 %, 32510 MiB, 29665 MiB, 2845 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.328, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.330, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.334, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.335, Tesla V100-SXM2-32GB, 470.57.02, 82 %, 34 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.336, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.338, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.341, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.343, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.343, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.345, Tesla V100-SXM2-32GB, 470.57.02, 69 %, 28 %, 32510 MiB, 29849 MiB, 2661 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.348, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.352, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.353, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.353, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.355, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.358, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/28 10:52:42.358, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.361, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.362, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.362, Tesla V100-SXM2-32GB, 470.57.02, 69 %, 28 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.364, Tesla V100-SXM2-32GB, 470.57.02, 49 %, 20 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.367, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.367, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.368, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.368, Tesla V100-SXM2-32GB, 470.57.02, 51 %, 22 %, 32510 MiB, 29709 MiB, 2801 MiB
2022/04/28 10:52:42.370, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.371, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.371, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.373, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 25 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.377, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.377, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.378, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.379, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29665 MiB, 2845 MiB
2022/04/28 10:52:42.380, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.381, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.381, Tesla V100-SXM2-32GB, 470.57.02, 49 %, 20 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.387, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.387, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.388, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.388, Tesla V100-SXM2-32GB, 470.57.02, 22 %, 9 %, 32510 MiB, 29817 MiB, 2693 MiB
2022/04/28 10:52:42.390, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.391, Tesla V100-SXM2-32GB, 470.57.02, 49 %, 20 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.391, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.396, Tesla V100-SXM2-32GB, 470.57.02, 49 %, 20 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.397, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.397, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.398, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 6 %, 32510 MiB, 29785 MiB, 2725 MiB
2022/04/28 10:52:42.399, Tesla V100-SXM2-32GB, 470.57.02, 49 %, 20 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.400, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.405, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.406, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.406, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.407, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29849 MiB, 2661 MiB
2022/04/28 10:52:42.408, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.415, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.415, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.416, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29649 MiB, 2861 MiB
2022/04/28 10:52:42.429, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.429, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.430, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29537 MiB, 2973 MiB
2022/04/28 10:52:42.437, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
2022/04/28 10:52:42.437, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29697 MiB, 2813 MiB
[rank:0] iter: 200/1100, loss: 0.0581544786691666, latency(ms): 24.4768229871988297 | 2022-04-28 10:52:44.609
[rank:0] iter: 300/1100, loss: 0.0573139935731888, latency(ms): 21.6545754671096802 | 2022-04-28 10:52:46.775
[rank:0] iter: 400/1100, loss: 0.0562818609178066, latency(ms): 21.7862092703580856 | 2022-04-28 10:52:48.953
[rank:0] iter: 500/1100, loss: 0.0559847392141819, latency(ms): 21.9639616459608078 | 2022-04-28 10:52:51.150
[rank:0] iter: 600/1100, loss: 0.0555572547018528, latency(ms): 21.6177541390061378 | 2022-04-28 10:52:53.311
[rank:0] iter: 700/1100, loss: 0.0546472445130348, latency(ms): 21.6355301067233086 | 2022-04-28 10:52:55.475
[rank:0] iter: 800/1100, loss: 0.0547492317855358, latency(ms): 21.9103648513555527 | 2022-04-28 10:52:57.666
[rank:0] iter: 900/1100, loss: 0.0537523403763771, latency(ms): 22.1555766835808754 | 2022-04-28 10:52:59.882
[rank:0] iter: 1000/1100, loss: 0.0533347129821777, latency(ms): 22.0934312790632248 | 2022-04-28 10:53:02.091
[rank:0] iter: 1100/1100, loss: 0.0534068346023560, latency(ms): 21.6277541220188141 | 2022-04-28 10:53:04.254