loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ------------------------ arguments ------------------------ batch_size ...................................... 65536 batch_size_per_proc ............................. 8192 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. True deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. eager hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.5040161013603210, latency(ms): 75.7120483368635178 | 2022-05-13 11:01:07.910 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.038, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.043, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.049, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.050, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.057, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.057, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.062, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.062, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.062, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:01:08.064, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.065, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.066, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.067, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.067, Tesla V100-SXM2-32GB, 470.57.02, 47 %, 27 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.067, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.068, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.068, Tesla V100-SXM2-32GB, 470.57.02, 70 %, 29 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/05/13 11:01:08.071, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.073, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.074, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.074, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.075, Tesla V100-SXM2-32GB, 470.57.02, 58 %, 22 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.075, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.077, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.077, Tesla V100-SXM2-32GB, 470.57.02, 29 %, 13 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/05/13 11:01:08.080, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.081, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.083, Tesla V100-SXM2-32GB, 470.57.02, 47 %, 27 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.083, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.083, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.084, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.085, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.086, Tesla V100-SXM2-32GB, 470.57.02, 54 %, 21 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/05/13 11:01:08.089, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.090, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.091, Tesla V100-SXM2-32GB, 470.57.02, 58 %, 22 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.092, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.093, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.094, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.095, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/05/13 11:01:08.097, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.098, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.100, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.100, Tesla V100-SXM2-32GB, 470.57.02, 47 %, 27 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.101, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.103, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.103, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 3 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/05/13 11:01:08.106, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.107, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.109, Tesla V100-SXM2-32GB, 470.57.02, 58 %, 22 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.110, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.112, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.113, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/05/13 11:01:08.116, Tesla V100-SXM2-32GB, 470.57.02, 58 %, 22 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.118, Tesla V100-SXM2-32GB, 470.57.02, 58 %, 22 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.120, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.121, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.123, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.123, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/05/13 11:01:08.126, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.127, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.133, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.134, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 20 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/05/13 11:01:08.134, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30154 MiB, 2356 MiB [rank:0] iter: 200/1100, loss: 0.4704783260822296, latency(ms): 23.2740388810634613 | 2022-05-13 11:01:10.237 [rank:0] iter: 300/1100, loss: 0.4637112319469452, latency(ms): 21.2728137150406837 | 2022-05-13 11:01:12.364 [rank:0] iter: 400/1100, loss: 0.4601495265960693, latency(ms): 21.3573182001709938 | 2022-05-13 11:01:14.500 [rank:0] iter: 500/1100, loss: 0.4576763212680817, latency(ms): 21.5586493164300919 | 2022-05-13 11:01:16.656 [rank:0] iter: 600/1100, loss: 0.4527861773967743, latency(ms): 21.2976907193660736 | 2022-05-13 11:01:18.786 [rank:0] iter: 700/1100, loss: 0.4462134838104248, latency(ms): 20.8508641645312309 | 2022-05-13 11:01:20.871 [rank:0] iter: 800/1100, loss: 0.4461249113082886, latency(ms): 21.5649201348423958 | 2022-05-13 11:01:23.027 [rank:0] iter: 900/1100, loss: 0.4462857544422150, latency(ms): 21.1260532215237617 | 2022-05-13 11:01:25.140 [rank:0] iter: 1000/1100, loss: 0.4443166255950928, latency(ms): 20.7071984559297562 | 2022-05-13 11:01:27.211 [rank:0] iter: 1100/1100, loss: 0.4472658038139343, latency(ms): 21.0171563550829887 | 2022-05-13 11:01:29.312