loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1


loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
------------------------ arguments ------------------------
  batch_size ...................................... 65536
  batch_size_per_proc ............................. 8192
  data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord
  data_part_name_suffix_length .................... 5
  data_part_num ................................... 256
  dataset_format .................................. ofrecord
  ddp ............................................. False
  deep_dropout_rate ............................... 0.5
  deep_embedding_vec_size ......................... 16
  deep_vocab_size ................................. 2322444
  eval_after_training ............................. False
  eval_batchs ..................................... 20
  eval_interval ................................... 0
  execution_mode .................................. graph
  hidden_size ..................................... 1024
  hidden_units_num ................................ 2
  learning_rate ................................... 0.001
  loss_print_every_n_iter ......................... 100
  max_iter ........................................ 1100
  model_load_dir .................................. 
  model_save_dir .................................. ./checkpoint
  num_deep_sparse_fields .......................... 26
  num_dense_fields ................................ 13
  num_wide_sparse_fields .......................... 2
  save_initial_model .............................. False
  save_model_after_each_eval ...................... False
  test_name ....................................... noname_test
  wide_vocab_size ................................. 2322444
-------------------- end of arguments ---------------------
[rank:0] iter: 100/1100, loss: 0.0620612092316151, latency(ms): 186.3451025262475014 | 2022-05-13 10:59:11.990
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/13 10:59:12.126, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.129, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.133, Tesla V100-SXM2-32GB, 470.57.02, 86 %, 28 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.135, Tesla V100-SXM2-32GB, 470.57.02, 86 %, 28 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.138, Tesla V100-SXM2-32GB, 470.57.02, 85 %, 29 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.140, Tesla V100-SXM2-32GB, 470.57.02, 85 %, 29 %, 32510 MiB, 30211 MiB, 2299 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/13 10:59:12.144, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.147, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.148, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/13 10:59:12.150, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.153, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.156, Tesla V100-SXM2-32GB, 470.57.02, 86 %, 28 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.156, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.157, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/13 10:59:12.160, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/05/13 10:59:12.162, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.163, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.163, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.163, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.165, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.165, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.166, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.167, Tesla V100-SXM2-32GB, 470.57.02, 32 %, 11 %, 32510 MiB, 30039 MiB, 2471 MiB
2022/05/13 10:59:12.168, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.169, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.169, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.171, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.173, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.174, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.175, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.176, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB
2022/05/13 10:59:12.177, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.178, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.179, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.182, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.183, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.185, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB
2022/05/13 10:59:12.185, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.186, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.187, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.190, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.191, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.193, Tesla V100-SXM2-32GB, 470.57.02, 37 %, 14 %, 32510 MiB, 30163 MiB, 2347 MiB
2022/05/13 10:59:12.193, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.194, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.195, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.198, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.199, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.200, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 12 %, 32510 MiB, 30179 MiB, 2331 MiB
2022/05/13 10:59:12.200, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.201, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.203, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.205, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.206, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.211, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB
2022/05/13 10:59:12.213, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.214, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.220, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.221, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.222, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 1 %, 32510 MiB, 29963 MiB, 2547 MiB
2022/05/13 10:59:12.225, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.227, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.228, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
2022/05/13 10:59:12.229, Tesla V100-SXM2-32GB, 470.57.02, 12 %, 4 %, 32510 MiB, 30091 MiB, 2419 MiB
[rank:0] iter: 200/1100, loss: 0.0584711991250515, latency(ms): 14.7039233148097992 | 2022-05-13 10:59:13.460
[rank:0] iter: 300/1100, loss: 0.0577833764255047, latency(ms): 12.1106719970703125 | 2022-05-13 10:59:14.671
[rank:0] iter: 400/1100, loss: 0.0574144832789898, latency(ms): 11.9366354867815971 | 2022-05-13 10:59:15.865
[rank:0] iter: 500/1100, loss: 0.0570719912648201, latency(ms): 11.9187501817941666 | 2022-05-13 10:59:17.057
[rank:0] iter: 600/1100, loss: 0.0566050484776497, latency(ms): 12.1581739187240601 | 2022-05-13 10:59:18.272
[rank:0] iter: 700/1100, loss: 0.0558423548936844, latency(ms): 11.9182373583316803 | 2022-05-13 10:59:19.464
[rank:0] iter: 800/1100, loss: 0.0558202639222145, latency(ms): 12.1112355217337608 | 2022-05-13 10:59:20.675
[rank:0] iter: 900/1100, loss: 0.0557155385613441, latency(ms): 11.8995367735624313 | 2022-05-13 10:59:21.865
[rank:0] iter: 1000/1100, loss: 0.0555126145482063, latency(ms): 12.0267531275749207 | 2022-05-13 10:59:23.068
[rank:0] iter: 1100/1100, loss: 0.0557109080255032, latency(ms): 11.9308649003505707 | 2022-05-13 10:59:24.261