loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1

loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
W20220512 02:39:29.978504  2003 rpc_client.cpp:190] LoadServer 10.7.151.28 Failed at 0 times error_code 14 error_message failed to connect to all addresses
W20220512 02:39:29.979182  2004 rpc_client.cpp:190] LoadServer 10.7.151.28 Failed at 0 times error_code 14 error_message failed to connect to all addresses
W20220512 02:39:29.980103  2002 rpc_client.cpp:190] LoadServer 10.7.151.28 Failed at 0 times error_code 14 error_message failed to connect to all addresses
------------------------ arguments ------------------------
  batch_size ...................................... 65536
  batch_size_per_proc ............................. 8192
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. False
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. graph
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.0621328540146351, latency(ms): 184.7894232347607613 | 2022-05-12 02:40:00.064
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.206, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/12 02:40:00.211, Tesla V100-SXM2-32GB, 470.57.02, 81 %, 27 %, 32510 MiB, 30059 MiB, 2451 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.218, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.219, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.225, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.227, Tesla V100-SXM2-32GB, 470.57.02, 81 %, 27 %, 32510 MiB, 30059 MiB, 2451 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.229, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.230, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.230, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/12 02:40:00.232, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.232, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/12 02:40:00.233, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/12 02:40:00.235, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.235, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/12 02:40:00.236, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.237, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.237, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 10 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/12 02:40:00.239, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.240, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.241, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.243, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.244, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.244, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 23 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.245, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.246, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/12 02:40:00.247, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.248, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.249, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.251, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.252, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.252, Tesla V100-SXM2-32GB, 470.57.02, 59 %, 20 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.253, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.254, Tesla V100-SXM2-32GB, 470.57.02, 75 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.255, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.256, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.257, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.259, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.260, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.261, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.262, Tesla V100-SXM2-32GB, 470.57.02, 68 %, 22 %, 32510 MiB, 30171 MiB, 2339 MiB
2022/05/12 02:40:00.264, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.265, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.266, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.268, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.269, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.270, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.271, Tesla V100-SXM2-32GB, 470.57.02, 17 %, 6 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/12 02:40:00.272, Tesla V100-SXM2-32GB, 470.57.02, 59 %, 20 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.273, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.274, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.276, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.277, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.278, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.279, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/12 02:40:00.281, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.282, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.284, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.285, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.286, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.287, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29939 MiB, 2571 MiB
2022/05/12 02:40:00.289, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.290, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.296, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
2022/05/12 02:40:00.298, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB
[rank:0] iter: 200/1100, loss: 0.0584713108837605, latency(ms): 14.2303537949919701 | 2022-05-12 02:40:01.487
[rank:0] iter: 300/1100, loss: 0.0577599816024303, latency(ms): 11.8599948659539223 | 2022-05-12 02:40:02.673
[rank:0] iter: 400/1100, loss: 0.0573837235569954, latency(ms): 11.9792807102203369 | 2022-05-12 02:40:03.871
[rank:0] iter: 500/1100, loss: 0.0570589825510979, latency(ms): 12.2408868744969368 | 2022-05-12 02:40:05.095
[rank:0] iter: 600/1100, loss: 0.0565930120646954, latency(ms): 12.1736041083931923 | 2022-05-12 02:40:06.312
[rank:0] iter: 700/1100, loss: 0.0558603256940842, latency(ms): 12.2708814963698387 | 2022-05-12 02:40:07.540
[rank:0] iter: 800/1100, loss: 0.0557931438088417, latency(ms): 12.4811881780624390 | 2022-05-12 02:40:08.788
[rank:0] iter: 900/1100, loss: 0.0557068251073360, latency(ms): 12.4961658939719200 | 2022-05-12 02:40:10.037
[rank:0] iter: 1000/1100, loss: 0.0554809272289276, latency(ms): 12.5047399476170540 | 2022-05-12 02:40:11.288
[rank:0] iter: 1100/1100, loss: 0.0556668415665627, latency(ms): 12.3574947193264961 | 2022-05-12 02:40:12.523