loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 W20220412 16:59:20.296100 7634 rpc_client.cpp:190] LoadServer 10.7.18.33 Failed at 0 times error_code 14 error_message failed to connect to all addresses ------------------------ arguments ------------------------ batch_size ...................................... 131072 batch_size_per_proc ............................. 16384 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. True deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. eager hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.5033751130104065, latency(ms): 111.6304462775588036 | 2022-04-12 16:59:43.142 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.293, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.300, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.307, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.310, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.315, Tesla V100-SXM2-32GB, 470.57.02, 74 %, 21 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.316, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.320, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.321, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.324, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.325, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.327, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.328, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.329, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB 2022/04/12 16:59:43.331, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.332, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.335, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.334, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.337, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB 2022/04/12 16:59:43.338, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.339, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.342, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.344, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.344, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB 2022/04/12 16:59:43.347, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.347, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.349, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.351, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.353, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.354, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/12 16:59:43.356, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.356, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.360, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.360, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.362, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.362, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.363, Tesla V100-SXM2-32GB, 470.57.02, 23 %, 13 %, 32510 MiB, 29926 MiB, 2584 MiB 2022/04/12 16:59:43.364, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.365, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.368, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.370, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB 2022/04/12 16:59:43.371, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.371, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.373, Tesla V100-SXM2-32GB, 470.57.02, 9 %, 4 %, 32510 MiB, 29946 MiB, 2564 MiB 2022/04/12 16:59:43.374, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.374, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.378, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.379, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.381, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.383, Tesla V100-SXM2-32GB, 470.57.02, 27 %, 8 %, 32510 MiB, 30022 MiB, 2488 MiB 2022/04/12 16:59:43.383, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.384, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.389, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.390, Tesla V100-SXM2-32GB, 470.57.02, 26 %, 8 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.392, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29998 MiB, 2512 MiB 2022/04/12 16:59:43.392, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.400, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.402, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.404, Tesla V100-SXM2-32GB, 470.57.02, 2 %, 2 %, 32510 MiB, 30006 MiB, 2504 MiB 2022/04/12 16:59:43.408, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.411, Tesla V100-SXM2-32GB, 470.57.02, 5 %, 2 %, 32510 MiB, 29938 MiB, 2572 MiB 2022/04/12 16:59:43.419, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.421, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29882 MiB, 2628 MiB 2022/04/12 16:59:43.426, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB 2022/04/12 16:59:43.431, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29962 MiB, 2548 MiB [rank:0] iter: 200/1100, loss: 0.4684083163738251, latency(ms): 42.2503503784537315 | 2022-04-12 16:59:47.367 [rank:0] iter: 300/1100, loss: 0.4601974487304688, latency(ms): 40.2711191773414612 | 2022-04-12 16:59:51.395 [rank:0] iter: 400/1100, loss: 0.4517259895801544, latency(ms): 40.1843165606260300 | 2022-04-12 16:59:55.413 [rank:0] iter: 500/1100, loss: 0.4499328732490540, latency(ms): 40.6866158917546272 | 2022-04-12 16:59:59.482 [rank:0] iter: 600/1100, loss: 0.4460764825344086, latency(ms): 41.5729597210884094 | 2022-04-12 17:00:03.639 [rank:0] iter: 700/1100, loss: 0.4396787881851196, latency(ms): 40.4880973324179649 | 2022-04-12 17:00:07.688 [rank:0] iter: 800/1100, loss: 0.4419981241226196, latency(ms): 41.1582896485924721 | 2022-04-12 17:00:11.804 [rank:0] iter: 900/1100, loss: 0.4321220815181732, latency(ms): 40.7385318353772163 | 2022-04-12 17:00:15.877 [rank:0] iter: 1000/1100, loss: 0.4307324886322021, latency(ms): 42.2932581976056099 | 2022-04-12 17:00:20.107 [rank:0] iter: 1100/1100, loss: 0.4324755370616913, latency(ms): 42.7593124657869339 | 2022-04-12 17:00:24.383