loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
W20220512 02:42:34.071233  7637 rpc_client.cpp:190] LoadServer 10.7.151.28 Failed at 0 times error_code 14 error_message failed to connect to all addresses
------------------------ arguments ------------------------
  batch_size ...................................... 131072
  batch_size_per_proc ............................. 16384
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. True
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. eager
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.5020629763603210, latency(ms): 94.0197596698999405 | 2022-05-12 02:42:55.177
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.311, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.313, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.315, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.319, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.321, Tesla V100-SXM2-32GB, 470.57.02, 52 %, 29 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.325, Tesla V100-SXM2-32GB, 470.57.02, 52 %, 29 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.328, Tesla V100-SXM2-32GB, 470.57.02, 94 %, 25 %, 32510 MiB, 29960 MiB, 2550 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.332, Tesla V100-SXM2-32GB, 470.57.02, 94 %, 25 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.334, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.336, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.337, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.338, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.339, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.341, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.342, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.343, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.344, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.346, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.347, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.348, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.348, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.350, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.351, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.354, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.356, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.356, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.356, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.358, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:42:55.361, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.363, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.363, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.365, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.366, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.367, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 29888 MiB, 2622 MiB
2022/05/12 02:42:55.368, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.370, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.370, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.372, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.374, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.375, Tesla V100-SXM2-32GB, 470.57.02, 14 %, 4 %, 32510 MiB, 29908 MiB, 2602 MiB
2022/05/12 02:42:55.375, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.377, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.378, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.380, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.381, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.382, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29984 MiB, 2526 MiB
2022/05/12 02:42:55.383, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.384, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.385, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.389, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.391, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.392, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 29960 MiB, 2550 MiB
2022/05/12 02:42:55.392, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.398, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.400, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.401, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.403, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 10 %, 32510 MiB, 29968 MiB, 2542 MiB
2022/05/12 02:42:55.404, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.407, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.408, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29900 MiB, 2610 MiB
2022/05/12 02:42:55.412, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.412, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 2 %, 32510 MiB, 29844 MiB, 2666 MiB
2022/05/12 02:42:55.415, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
2022/05/12 02:42:55.416, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29924 MiB, 2586 MiB
[rank:0] iter: 200/1100, loss: 0.4704225957393646, latency(ms): 42.9932986199855804 | 2022-05-12 02:42:59.477
[rank:0] iter: 300/1100, loss: 0.4606869816780090, latency(ms): 40.4746338352560997 | 2022-05-12 02:43:03.524
[rank:0] iter: 400/1100, loss: 0.4521812200546265, latency(ms): 40.6693481281399727 | 2022-05-12 02:43:07.591
[rank:0] iter: 500/1100, loss: 0.4503655135631561, latency(ms): 41.2332376092672348 | 2022-05-12 02:43:11.714
[rank:0] iter: 600/1100, loss: 0.4464572072029114, latency(ms): 42.6739969477057457 | 2022-05-12 02:43:15.982
[rank:0] iter: 700/1100, loss: 0.4401575326919556, latency(ms): 42.0711933448910713 | 2022-05-12 02:43:20.189
[rank:0] iter: 800/1100, loss: 0.4423397183418274, latency(ms): 42.0948710665106773 | 2022-05-12 02:43:24.398
[rank:0] iter: 900/1100, loss: 0.4323423802852631, latency(ms): 42.1084734797477722 | 2022-05-12 02:43:28.609
[rank:0] iter: 1000/1100, loss: 0.4308993816375732, latency(ms): 42.1037431061267853 | 2022-05-12 02:43:32.820
[rank:0] iter: 1100/1100, loss: 0.4325624406337738, latency(ms): 43.2387197017669678 | 2022-05-12 02:43:37.144