loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 W20220418 02:21:02.865801 7626 rpc_client.cpp:190] LoadServer 10.7.199.68 Failed at 0 times error_code 14 error_message failed to connect to all addresses ------------------------ arguments ------------------------ batch_size ...................................... 131072 batch_size_per_proc ............................. 16384 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. True deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. eager hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.5022305846214294, latency(ms): 104.0410923957824707 | 2022-04-18 02:21:24.909 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:26.548, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB 2022/04/18 02:21:26.552, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:26.558, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:26.566, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:26.565, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:26.571, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:26.571, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:26.572, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:26.575, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB 2022/04/18 02:21:26.576, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:26.576, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:26.578, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:26.578, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB 2022/04/18 02:21:26.582, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:26.582, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:26.583, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:26.585, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:26.586, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:26.589, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:26.590, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:26.590, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:26.592, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:26.593, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:26.596, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:26.597, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:26.599, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:26.600, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:26.603, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:26.603, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:26.605, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:26.607, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:26.610, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:26.610, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:26.612, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:26.613, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:26.616, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:26.618, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:26.620, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:26.626, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:26.629, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:28.074, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:28.077, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:28.078, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB 2022/04/18 02:21:28.079, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/18 02:21:28.081, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:28.082, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:28.082, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29894 MiB, 2616 MiB 2022/04/18 02:21:28.084, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:28.085, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:28.086, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29914 MiB, 2596 MiB 2022/04/18 02:21:28.088, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:28.088, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:28.089, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29990 MiB, 2520 MiB 2022/04/18 02:21:28.091, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:28.092, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:28.093, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29966 MiB, 2544 MiB 2022/04/18 02:21:28.095, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:28.096, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:28.097, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29974 MiB, 2536 MiB 2022/04/18 02:21:28.099, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:28.100, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29906 MiB, 2604 MiB 2022/04/18 02:21:28.103, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB 2022/04/18 02:21:28.104, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29850 MiB, 2660 MiB 2022/04/18 02:21:28.107, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 29930 MiB, 2580 MiB [rank:0] iter: 200/1100, loss: 0.4710654318332672, latency(ms): 71.0763531178236008 | 2022-04-18 02:21:32.017 [rank:0] iter: 300/1100, loss: 0.4611218273639679, latency(ms): 40.2758428454399109 | 2022-04-18 02:21:36.045 [rank:0] iter: 400/1100, loss: 0.4524781703948975, latency(ms): 40.3074692562222481 | 2022-04-18 02:21:40.075 [rank:0] iter: 500/1100, loss: 0.4505708813667297, latency(ms): 40.4701227694749832 | 2022-04-18 02:21:44.122 [rank:0] iter: 600/1100, loss: 0.4464763998985291, latency(ms): 39.9934889748692513 | 2022-04-18 02:21:48.122 [rank:0] iter: 700/1100, loss: 0.4401213824748993, latency(ms): 41.1278539896011353 | 2022-04-18 02:21:52.234 [rank:0] iter: 800/1100, loss: 0.4426641166210175, latency(ms): 40.8851449191570282 | 2022-04-18 02:21:56.323 [rank:0] iter: 900/1100, loss: 0.4327013492584229, latency(ms): 40.2616763114929199 | 2022-04-18 02:22:00.349 [rank:0] iter: 1000/1100, loss: 0.4313701391220093, latency(ms): 41.1518376693129539 | 2022-04-18 02:22:04.464 [rank:0] iter: 1100/1100, loss: 0.4327896833419800, latency(ms): 42.1379320323467255 | 2022-04-18 02:22:08.678