loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 W20220513 10:59:30.477633 3604 rpc_client.cpp:190] LoadServer 10.7.98.26 Failed at 0 times error_code 14 error_message failed to connect to all addresses ------------------------ arguments ------------------------ batch_size ...................................... 131072 batch_size_per_proc ............................. 16384 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. False deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. graph hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.0618635751307011, latency(ms): 196.3541846349835396 | 2022-05-13 11:00:01.881 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:00:02.031, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:00:02.036, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.037, Tesla V100-SXM2-32GB, 470.57.02, 92 %, 38 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.038, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.045, Tesla V100-SXM2-32GB, 470.57.02, 92 %, 38 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.045, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.047, Tesla V100-SXM2-32GB, 470.57.02, 92 %, 38 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.053, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:00:02.053, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:00:02.054, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/05/13 11:00:02.057, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.058, Tesla V100-SXM2-32GB, 470.57.02, 65 %, 27 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.058, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.059, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.058, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.060, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.060, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.061, Tesla V100-SXM2-32GB, 470.57.02, 39 %, 17 %, 32510 MiB, 29645 MiB, 2865 MiB 2022/05/13 11:00:02.065, Tesla V100-SXM2-32GB, 470.57.02, 65 %, 27 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.065, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.066, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.067, Tesla V100-SXM2-32GB, 470.57.02, 65 %, 27 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.067, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.069, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.070, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.070, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29729 MiB, 2781 MiB 2022/05/13 11:00:02.073, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.074, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.075, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.076, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.076, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.078, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.079, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.079, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 10 %, 32510 MiB, 29881 MiB, 2629 MiB 2022/05/13 11:00:02.082, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.083, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.083, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.084, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.085, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.087, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.088, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.088, Tesla V100-SXM2-32GB, 470.57.02, 61 %, 26 %, 32510 MiB, 29769 MiB, 2741 MiB 2022/05/13 11:00:02.091, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.092, Tesla V100-SXM2-32GB, 470.57.02, 65 %, 27 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.093, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.094, Tesla V100-SXM2-32GB, 470.57.02, 65 %, 27 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.096, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.097, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.097, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29801 MiB, 2709 MiB 2022/05/13 11:00:02.101, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.103, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.104, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.106, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.106, Tesla V100-SXM2-32GB, 470.57.02, 31 %, 14 %, 32510 MiB, 29713 MiB, 2797 MiB 2022/05/13 11:00:02.110, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.111, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.113, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.114, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.114, Tesla V100-SXM2-32GB, 470.57.02, 16 %, 7 %, 32510 MiB, 29537 MiB, 2973 MiB 2022/05/13 11:00:02.118, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.119, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.121, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.122, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB 2022/05/13 11:00:02.122, Tesla V100-SXM2-32GB, 470.57.02, 15 %, 6 %, 32510 MiB, 29697 MiB, 2813 MiB [rank:0] iter: 200/1100, loss: 0.0581204518675804, latency(ms): 23.6051013320684433 | 2022-05-13 11:00:04.242 [rank:0] iter: 300/1100, loss: 0.0572878494858742, latency(ms): 21.1428834870457649 | 2022-05-13 11:00:06.356 [rank:0] iter: 400/1100, loss: 0.0562612004578114, latency(ms): 21.1492500454187393 | 2022-05-13 11:00:08.471 [rank:0] iter: 500/1100, loss: 0.0559651888906956, latency(ms): 21.4747426286339760 | 2022-05-13 11:00:10.618 [rank:0] iter: 600/1100, loss: 0.0555352196097374, latency(ms): 21.1685716360807419 | 2022-05-13 11:00:12.735 [rank:0] iter: 700/1100, loss: 0.0546161793172359, latency(ms): 20.6858397647738457 | 2022-05-13 11:00:14.804 [rank:0] iter: 800/1100, loss: 0.0547118671238422, latency(ms): 20.9763055667281151 | 2022-05-13 11:00:16.901 [rank:0] iter: 900/1100, loss: 0.0537142269313335, latency(ms): 21.0198327153921127 | 2022-05-13 11:00:19.003 [rank:0] iter: 1000/1100, loss: 0.0533116199076176, latency(ms): 20.8431471511721611 | 2022-05-13 11:00:21.088 [rank:0] iter: 1100/1100, loss: 0.0533719919621944, latency(ms): 21.1967292055487633 | 2022-05-13 11:00:23.207