loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1
/usr/lib/x86_64-linux-gnu/libibverbs.so.1


loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 65536
  batch_size_per_proc ............................. 8192
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. True
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. eager
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.5042718648910522, latency(ms): 90.9937280043959618 | 2022-04-12 16:58:53.326
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.459, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.467, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.467, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.473, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 26 %, 32510 MiB, 30252 MiB, 2258 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.474, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.474, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.478, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.480, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 26 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.480, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.480, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.481, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.486, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.488, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.489, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.489, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.489, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 26 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.495, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.497, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.497, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 26 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.497, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 26 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.498, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.501, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.502, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.503, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.503, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.502, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.504, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.504, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.509, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.511, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.511, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.511, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.512, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.512, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.512, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.519, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.519, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.520, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.520, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.521, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.521, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/12 16:58:53.527, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.528, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.528, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.528, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.529, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.532, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30156 MiB, 2354 MiB
2022/04/12 16:58:53.534, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.535, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.535, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.536, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.541, Tesla V100-SXM2-32GB, 470.57.02, 48 %, 23 %, 32510 MiB, 30176 MiB, 2334 MiB
2022/04/12 16:58:53.542, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.543, Tesla V100-SXM2-32GB, 470.57.02, 45 %, 17 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.550, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30252 MiB, 2258 MiB
2022/04/12 16:58:53.551, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.552, Tesla V100-SXM2-32GB, 470.57.02, 46 %, 18 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.560, Tesla V100-SXM2-32GB, 470.57.02, 21 %, 7 %, 32510 MiB, 30228 MiB, 2282 MiB
2022/04/12 16:58:53.562, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.563, Tesla V100-SXM2-32GB, 470.57.02, 38 %, 13 %, 32510 MiB, 30192 MiB, 2318 MiB
2022/04/12 16:58:53.566, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30236 MiB, 2274 MiB
2022/04/12 16:58:53.575, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30168 MiB, 2342 MiB
2022/04/12 16:58:53.584, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30112 MiB, 2398 MiB
2022/04/12 16:58:53.587, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30192 MiB, 2318 MiB
[rank:0] iter: 200/1100, loss: 0.4707847833633423, latency(ms): 24.3201851844787598 | 2022-04-12 16:58:55.758
[rank:0] iter: 300/1100, loss: 0.4636177718639374, latency(ms): 20.8787046000361443 | 2022-04-12 16:58:57.846
[rank:0] iter: 400/1100, loss: 0.4601595997810364, latency(ms): 21.2734775990247726 | 2022-04-12 16:58:59.973
[rank:0] iter: 500/1100, loss: 0.4573691487312317, latency(ms): 21.1068610474467278 | 2022-04-12 16:59:02.084
[rank:0] iter: 600/1100, loss: 0.4527715742588043, latency(ms): 21.4776729047298431 | 2022-04-12 16:59:04.231
[rank:0] iter: 700/1100, loss: 0.4457454979419708, latency(ms): 21.0622262209653854 | 2022-04-12 16:59:06.338
[rank:0] iter: 800/1100, loss: 0.4456437826156616, latency(ms): 20.5072374269366264 | 2022-04-12 16:59:08.388
[rank:0] iter: 900/1100, loss: 0.4459028244018555, latency(ms): 21.1244635283946991 | 2022-04-12 16:59:10.501
[rank:0] iter: 1000/1100, loss: 0.4440155923366547, latency(ms): 20.4876386374235153 | 2022-04-12 16:59:12.549
[rank:0] iter: 1100/1100, loss: 0.4470074176788330, latency(ms): 20.6111265718936920 | 2022-04-12 16:59:14.611