loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1

loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 512
  batch_size_per_proc ............................. 64
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. False
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. graph
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.0668884813785553, latency(ms): 174.9143005162477493 | 2022-04-26 10:09:51.965
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.125, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 4 %, 32510 MiB, 30427 MiB, 2083 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.133, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.140, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 4 %, 32510 MiB, 30427 MiB, 2083 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.141, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.145, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 4 %, 32510 MiB, 30427 MiB, 2083 MiB
2022/04/26 10:09:52.150, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.150, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.150, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 4 %, 32510 MiB, 30427 MiB, 2083 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:09:52.155, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.154, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30427 MiB, 2083 MiB
2022/04/26 10:09:52.157, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.158, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.158, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.159, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30427 MiB, 2083 MiB
2022/04/26 10:09:52.160, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30427 MiB, 2083 MiB
2022/04/26 10:09:52.162, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.162, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30427 MiB, 2083 MiB
2022/04/26 10:09:52.163, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.166, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.167, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.168, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.169, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.171, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.173, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.173, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30447 MiB, 2063 MiB
2022/04/26 10:09:52.173, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.177, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.177, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 2 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.178, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.180, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.181, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.183, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.183, Tesla V100-SXM2-32GB, 470.57.02, 25 %, 1 %, 32510 MiB, 30599 MiB, 1911 MiB
2022/04/26 10:09:52.184, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.187, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.188, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.188, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.190, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.192, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.194, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.194, Tesla V100-SXM2-32GB, 470.57.02, 10 %, 1 %, 32510 MiB, 30551 MiB, 1959 MiB
2022/04/26 10:09:52.194, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.198, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 2 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.199, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.201, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.202, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.204, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.205, Tesla V100-SXM2-32GB, 470.57.02, 6 %, 0 %, 32510 MiB, 30567 MiB, 1943 MiB
2022/04/26 10:09:52.205, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.208, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.209, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.211, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.212, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.214, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.215, Tesla V100-SXM2-32GB, 470.57.02, 20 %, 1 %, 32510 MiB, 30431 MiB, 2079 MiB
2022/04/26 10:09:52.215, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.219, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.221, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.222, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.224, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30319 MiB, 2191 MiB
2022/04/26 10:09:52.225, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.230, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.231, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
2022/04/26 10:09:52.233, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30479 MiB, 2031 MiB
[rank:0] iter: 200/1100, loss: 0.0602035224437714, latency(ms): 6.4326003193855286 | 2022-04-26 10:09:52.608
[rank:0] iter: 300/1100, loss: 0.0620070807635784, latency(ms): 2.8957373276352882 | 2022-04-26 10:09:52.897
[rank:0] iter: 400/1100, loss: 0.0617346651852131, latency(ms): 2.9068350046873093 | 2022-04-26 10:09:53.188
[rank:0] iter: 500/1100, loss: 0.0619304887950420, latency(ms): 3.0229406431317329 | 2022-04-26 10:09:53.490
[rank:0] iter: 600/1100, loss: 0.0604410134255886, latency(ms): 3.6973244696855545 | 2022-04-26 10:09:53.860
[rank:0] iter: 700/1100, loss: 0.0594948492944241, latency(ms): 3.3758119121193886 | 2022-04-26 10:09:54.198
[rank:0] iter: 800/1100, loss: 0.0603142790496349, latency(ms): 3.6080972105264664 | 2022-04-26 10:09:54.559
[rank:0] iter: 900/1100, loss: 0.0604432821273804, latency(ms): 3.6267378181219101 | 2022-04-26 10:09:54.921
[rank:0] iter: 1000/1100, loss: 0.0606918483972549, latency(ms): 3.4105899184942245 | 2022-04-26 10:09:55.262
[rank:0] iter: 1100/1100, loss: 0.0589934140443802, latency(ms): 3.5308005288243294 | 2022-04-26 10:09:55.615