loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1 ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 loaded library: loaded library: loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1 W20220421 16:43:13.350244 2005 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses W20220421 16:43:13.350255 2006 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses W20220421 16:43:13.350980 2003 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses W20220421 16:43:13.351254 2004 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses W20220421 16:43:13.365568 2008 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses W20220421 16:43:13.366528 2007 rpc_client.cpp:190] LoadServer 10.7.238.126 Failed at 0 times error_code 14 error_message failed to connect to all addresses ------------------------ arguments ------------------------ batch_size ...................................... 65536 batch_size_per_proc ............................. 8192 data_dir ........................................ /dataset/f9f659c5/wdl_ofrecord data_part_name_suffix_length .................... 5 data_part_num ................................... 256 dataset_format .................................. ofrecord ddp ............................................. False deep_dropout_rate ............................... 0.5 deep_embedding_vec_size ......................... 16 deep_vocab_size ................................. 2322444 eval_after_training ............................. False eval_batchs ..................................... 20 eval_interval ................................... 0 execution_mode .................................. graph hidden_size ..................................... 1024 hidden_units_num ................................ 2 learning_rate ................................... 0.001 loss_print_every_n_iter ......................... 100 max_iter ........................................ 1100 model_load_dir .................................. model_save_dir .................................. ./checkpoint num_deep_sparse_fields .......................... 26 num_dense_fields ................................ 13 num_wide_sparse_fields .......................... 2 save_initial_model .............................. False save_model_after_each_eval ...................... False test_name ....................................... noname_test wide_vocab_size ................................. 2322444 -------------------- end of arguments --------------------- [rank:0] iter: 100/1100, loss: 0.0620337054133415, latency(ms): 184.1236492246389389 | 2022-04-21 16:43:43.394 timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:45.720, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:45.727, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:45.732, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:45.739, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:45.739, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:45.746, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:45.747, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:45.753, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:45.753, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:45.755, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:45.758, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:45.759, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:45.760, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:45.760, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:45.762, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:45.765, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB 2022/04/21 16:43:45.765, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:45.767, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:45.767, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:45.768, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:45.771, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:45.773, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:45.773, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:45.774, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:45.777, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:45.778, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:45.778, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:45.780, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:45.782, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB 2022/04/21 16:43:45.783, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:45.784, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:45.785, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:45.788, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:45.788, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:45.789, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:45.797, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:45.797, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:45.798, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB 2022/04/21 16:43:45.802, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB 2022/04/21 16:43:45.803, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:46.035, Tesla V100-SXM2-32GB, 470.57.02, 36 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:46.037, Tesla V100-SXM2-32GB, 470.57.02, 89 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:46.039, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:46.040, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:46.042, Tesla V100-SXM2-32GB, 470.57.02, 28 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:46.044, Tesla V100-SXM2-32GB, 470.57.02, 78 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:46.045, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:46.046, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB] 2022/04/21 16:43:46.941, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:46.942, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30041 MiB, 2469 MiB 2022/04/21 16:43:46.943, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:46.944, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30061 MiB, 2449 MiB 2022/04/21 16:43:46.945, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:46.946, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30213 MiB, 2297 MiB 2022/04/21 16:43:46.947, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:46.948, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 30197 MiB, 2313 MiB 2022/04/21 16:43:46.948, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:46.949, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30181 MiB, 2329 MiB 2022/04/21 16:43:46.950, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:46.951, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30077 MiB, 2433 MiB 2022/04/21 16:43:46.951, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:46.952, Tesla V100-SXM2-32GB, 470.57.02, 0 %, 0 %, 32510 MiB, 29941 MiB, 2569 MiB 2022/04/21 16:43:46.953, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB 2022/04/21 16:43:46.954, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 0 %, 32510 MiB, 30093 MiB, 2417 MiB [rank:0] iter: 200/1100, loss: 0.0584756433963776, latency(ms): 47.6533529907464981 | 2022-04-21 16:43:48.159 [rank:0] iter: 300/1100, loss: 0.0577821210026741, latency(ms): 12.0924302190542221 | 2022-04-21 16:43:49.368 [rank:0] iter: 400/1100, loss: 0.0573836602270603, latency(ms): 11.8844956904649734 | 2022-04-21 16:43:50.557 [rank:0] iter: 500/1100, loss: 0.0570962615311146, latency(ms): 12.0218899473547935 | 2022-04-21 16:43:51.759 [rank:0] iter: 600/1100, loss: 0.0565864183008671, latency(ms): 11.8747371807694435 | 2022-04-21 16:43:52.947 [rank:0] iter: 700/1100, loss: 0.0558477677404881, latency(ms): 12.2505083307623863 | 2022-04-21 16:43:54.172 [rank:0] iter: 800/1100, loss: 0.0557967238128185, latency(ms): 11.9705031439661980 | 2022-04-21 16:43:55.369 [rank:0] iter: 900/1100, loss: 0.0557250864803791, latency(ms): 12.2828216850757599 | 2022-04-21 16:43:56.597 [rank:0] iter: 1000/1100, loss: 0.0555165559053421, latency(ms): 11.8712975457310677 | 2022-04-21 16:43:57.784 [rank:0] iter: 1100/1100, loss: 0.0556953921914101, latency(ms): 12.0114176720380783 | 2022-04-21 16:43:58.985