loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 65536
  batch_size_per_proc ............................. 8192
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. False
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. graph
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.0621120743453503, latency(ms): 187.9322549328207970 | 2022-04-26 10:10:21.599
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:22.779, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:22.781, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.782, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.784, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:22.785, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.789, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:22.790, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:22.791, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:22.792, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:22.794, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.794, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.795, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.795, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.796, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.797, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:22.798, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.801, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.802, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:22.803, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.804, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:22.805, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.806, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:22.807, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.811, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.812, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.813, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.814, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.814, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.816, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.816, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.820, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.822, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.822, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.824, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.824, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.826, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:22.826, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.830, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.831, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.832, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.833, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.834, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.835, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:22.836, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.840, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.841, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.842, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.843, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.845, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:22.846, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.851, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.853, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.855, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:22.859, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.861, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/04/26 10:10:22.863, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/26 10:10:23.796, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/04/26 10:10:23.797, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30067 MiB, 2443 MiB
2022/04/26 10:10:23.798, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:23.798, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/04/26 10:10:23.799, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/04/26 10:10:23.800, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30075 MiB, 2435 MiB
2022/04/26 10:10:23.801, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/04/26 10:10:23.801, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30091 MiB, 2419 MiB
[rank:0] iter: 200/1100, loss: 0.0584693476557732, latency(ms): 34.2819569632411003 | 2022-04-26 10:10:25.027
[rank:0] iter: 300/1100, loss: 0.0577656887471676, latency(ms): 12.3602731153368950 | 2022-04-26 10:10:26.264
[rank:0] iter: 400/1100, loss: 0.0573597587645054, latency(ms): 12.2613574191927910 | 2022-04-26 10:10:27.490
[rank:0] iter: 500/1100, loss: 0.0570919960737228, latency(ms): 12.2811593487858772 | 2022-04-26 10:10:28.718
[rank:0] iter: 600/1100, loss: 0.0565990209579468, latency(ms): 12.1129871904850006 | 2022-04-26 10:10:29.929
[rank:0] iter: 700/1100, loss: 0.0558495670557022, latency(ms): 12.5039312243461609 | 2022-04-26 10:10:31.179
[rank:0] iter: 800/1100, loss: 0.0558091588318348, latency(ms): 12.1948463469743729 | 2022-04-26 10:10:32.399
[rank:0] iter: 900/1100, loss: 0.0557041913270950, latency(ms): 11.8603729456663132 | 2022-04-26 10:10:33.585
[rank:0] iter: 1000/1100, loss: 0.0554834455251694, latency(ms): 12.4776357412338257 | 2022-04-26 10:10:34.833
[rank:0] iter: 1100/1100, loss: 0.0557058714330196, latency(ms): 12.5436460971832275 | 2022-04-26 10:10:36.087