loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: loaded library: loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 W20220409 02:14:09.117520 6437 rpc_client.cpp:190] LoadServer 10.7.102.248 Failed at 0 times error_code 14 error_message failed to connect to all addresses ------------------------ arguments ------------------------ batch_size ...................................... 65536 batch_size_per_proc ............................. 8192 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. True deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. eager hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.5045508742332458, latency(ms): 98.1924055516719818 | 2022-04-09 02:14:30.692 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.823, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.831, Tesla V100-SXM2-32GB, 470.57.02, 80 %, 26 %, 32510 MiB, 30138 MiB, 2372 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.838, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.841, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.846, Tesla V100-SXM2-32GB, 470.57.02, 91 %, 29 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.846, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.851, Tesla V100-SXM2-32GB, 470.57.02, 80 %, 26 %, 32510 MiB, 30138 MiB, 2372 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.854, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.856, Tesla V100-SXM2-32GB, 470.57.02, 80 %, 26 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.856, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.858, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/09 02:14:30.860, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.861, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.862, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.863, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.863, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.864, Tesla V100-SXM2-32GB, 470.57.02, 80 %, 26 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.867, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.869, Tesla V100-SXM2-32GB, 470.57.02, 50 %, 17 %, 32510 MiB, 30118 MiB, 2392 MiB 2022/04/09 02:14:30.870, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.871, Tesla V100-SXM2-32GB, 470.57.02, 52 %, 18 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.872, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.873, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.873, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.874, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.877, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.880, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30138 MiB, 2372 MiB 2022/04/09 02:14:30.880, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.880, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.882, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.882, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.883, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.884, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.886, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.889, Tesla V100-SXM2-32GB, 470.57.02, 40 %, 16 %, 32510 MiB, 30214 MiB, 2296 MiB 2022/04/09 02:14:30.890, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.892, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.892, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.893, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.894, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.896, Tesla V100-SXM2-32GB, 470.57.02, 52 %, 18 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.899, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30190 MiB, 2320 MiB 2022/04/09 02:14:30.900, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.901, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.902, Tesla V100-SXM2-32GB, 470.57.02, 52 %, 18 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.903, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.903, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.906, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.909, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30198 MiB, 2312 MiB 2022/04/09 02:14:30.910, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.911, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.912, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.913, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.913, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.919, Tesla V100-SXM2-32GB, 470.57.02, 1 %, 1 %, 32510 MiB, 30130 MiB, 2380 MiB 2022/04/09 02:14:30.920, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.921, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.922, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.923, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.932, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30074 MiB, 2436 MiB 2022/04/09 02:14:30.933, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.934, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.935, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB 2022/04/09 02:14:30.942, Tesla V100-SXM2-32GB, 470.57.02, 7 %, 4 %, 32510 MiB, 30154 MiB, 2356 MiB [rank:0] iter: 200/1100, loss: 0.4704844653606415, latency(ms): 24.6078902110457420 | 2022-04-09 02:14:33.152 [rank:0] iter: 300/1100, loss: 0.4639819264411926, latency(ms): 21.7160859331488609 | 2022-04-09 02:14:35.324 [rank:0] iter: 400/1100, loss: 0.4604267776012421, latency(ms): 21.7783296853303909 | 2022-04-09 02:14:37.502 [rank:0] iter: 500/1100, loss: 0.4577780961990356, latency(ms): 21.6953754425048828 | 2022-04-09 02:14:39.671 [rank:0] iter: 600/1100, loss: 0.4533349275588989, latency(ms): 21.7467187345027924 | 2022-04-09 02:14:41.846 [rank:0] iter: 700/1100, loss: 0.4464609026908875, latency(ms): 21.3504859432578087 | 2022-04-09 02:14:43.981 [rank:0] iter: 800/1100, loss: 0.4461264908313751, latency(ms): 21.8216137215495110 | 2022-04-09 02:14:46.163 [rank:0] iter: 900/1100, loss: 0.4464380145072937, latency(ms): 21.3997002318501472 | 2022-04-09 02:14:48.303 [rank:0] iter: 1000/1100, loss: 0.4447224438190460, latency(ms): 21.4910069853067398 | 2022-04-09 02:14:50.452 [rank:0] iter: 1100/1100, loss: 0.4474626183509827, latency(ms): 21.5654853731393814 | 2022-04-09 02:14:52.609