loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: loaded library: loaded library: loaded library: loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 ------------------------ arguments ------------------------ batch_size ...................................... 65536 batch_size_per_proc ............................. 8192 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. False deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. graph hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.0621010363101959, latency(ms): 185.0993230938911438 | 2022-05-10 02:37:40.339 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/10 02:37:40.512, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.513, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/10 02:37:40.519, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.521, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 21 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.522, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 21 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.522, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/10 02:37:40.527, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.528, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 21 %, 32510 MiB, 30059 MiB, 2451 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/10 02:37:40.530, Tesla V100-SXM2-32GB, 470.57.02, 71 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.531, Tesla V100-SXM2-32GB, 470.57.02, 71 %, 23 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.532, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 21 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.534, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.537, Tesla V100-SXM2-32GB, 470.57.02, 63 %, 21 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.535, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.537, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.537, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30047 MiB, 2463 MiB 2022/05/10 02:37:40.538, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.540, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.541, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.545, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.547, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.547, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.547, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.549, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30059 MiB, 2451 MiB 2022/05/10 02:37:40.549, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.551, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.552, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.556, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.558, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.558, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.558, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.560, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.560, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.562, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.563, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.567, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.568, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.569, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.569, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.571, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30163 MiB, 2347 MiB 2022/05/10 02:37:40.571, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.572, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.574, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.578, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.579, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.580, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.580, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.582, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 9 %, 32510 MiB, 30211 MiB, 2299 MiB 2022/05/10 02:37:40.582, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.583, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.585, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.589, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.590, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.591, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.591, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.593, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30043 MiB, 2467 MiB 2022/05/10 02:37:40.597, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.601, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.602, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.603, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.604, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29931 MiB, 2579 MiB 2022/05/10 02:37:40.610, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.612, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB 2022/05/10 02:37:40.614, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30123 MiB, 2387 MiB [rank:0] iter: 200/1100, loss: 0.0584475286304951, latency(ms): 15.4687339812517166 | 2022-05-10 02:37:41.886 [rank:0] iter: 300/1100, loss: 0.0577504634857178, latency(ms): 11.7855023965239525 | 2022-05-10 02:37:43.064 [rank:0] iter: 400/1100, loss: 0.0573721975088120, latency(ms): 12.2709645703434944 | 2022-05-10 02:37:44.291 [rank:0] iter: 500/1100, loss: 0.0570828579366207, latency(ms): 11.9795759394764900 | 2022-05-10 02:37:45.489 [rank:0] iter: 600/1100, loss: 0.0565493851900101, latency(ms): 11.9416857138276100 | 2022-05-10 02:37:46.683 [rank:0] iter: 700/1100, loss: 0.0558373369276524, latency(ms): 12.1625908091664314 | 2022-05-10 02:37:47.900 [rank:0] iter: 800/1100, loss: 0.0557901486754417, latency(ms): 12.4225749447941780 | 2022-05-10 02:37:49.142 [rank:0] iter: 900/1100, loss: 0.0557048805058002, latency(ms): 12.4124630913138390 | 2022-05-10 02:37:50.383 [rank:0] iter: 1000/1100, loss: 0.0554722212255001, latency(ms): 12.3315472900867462 | 2022-05-10 02:37:51.616 [rank:0] iter: 1100/1100, loss: 0.0556838326156139, latency(ms): 12.3283043131232262 | 2022-05-10 02:37:52.849