loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1
loaded library: loaded library: /usr/lib/x86_64-linux-gnu/libibverbs.so.1/usr/lib/x86_64-linux-gnu/libibverbs.so.1

W20220414 02:06:48.499596  3669 rpc_client.cpp:190] LoadServer 10.7.129.151 Failed at 0 times error_code 14 error_message failed to connect to all addresses
W20220414 02:06:48.499310  3665 rpc_client.cpp:190] LoadServer 10.7.129.151 Failed at 0 times error_code 14 error_message failed to connect to all addresses
W20220414 02:06:48.499614  3663 rpc_client.cpp:190] LoadServer 10.7.129.151 Failed at 0 times error_code 14 error_message failed to connect to all addresses
W20220414 02:06:48.499601  3664 rpc_client.cpp:190] LoadServer 10.7.129.151 Failed at 0 times error_code 14 error_message failed to connect to all addresses
------------------------ arguments ------------------------
  batches_per_epoch ............................... 312
  channel_last .................................... True
  ddp ............................................. False
  exit_num ........................................ 300
  fuse_bn_add_relu ................................ True
  fuse_bn_relu .................................... True
  gpu_stat_file ................................... None
  grad_clipping ................................... 0.0
  graph ........................................... True
  label_smoothing ................................. 0.1
  learning_rate ................................... 4.096
  legacy_init ..................................... False
  load_path ....................................... None
  lr_decay_type ................................... cosine
  metric_local .................................... True
  metric_train_acc ................................ True
  momentum ........................................ 0.875
  nccl_fusion_max_ops ............................. 24
  nccl_fusion_threshold_mb ........................ 16
  num_classes ..................................... 1000
  num_devices_per_node ............................ 8
  num_epochs ...................................... 1
  num_nodes ....................................... 1
  ofrecord_part_num ............................... 256
  ofrecord_path ................................... /dataset/79846248
  print_interval .................................. 100
  print_timestamp ................................. False
  samples_per_epoch ............................... 1281167
  save_init ....................................... False
  save_path ....................................... None
  scale_grad ...................................... True
  skip_eval ....................................... True
  synthetic_data .................................. False
  total_batches ................................... -1
  train_batch_size ................................ 512
  train_global_batch_size ......................... 4096
  use_fp16 ........................................ True
  use_gpu_decode .................................. False
  val_batch_size .................................. 50
  val_batches_per_epoch ........................... 125
  val_global_batch_size ........................... 400
  val_samples_per_epoch ........................... 50000
  warmup_epochs ................................... 5
  weight_decay .................................... 3.0517578125e-05
  zero_init_residual .............................. True
-------------------- end of arguments ---------------------
***** Model Init *****
***** Model Init Finish, time escapled: 2.84698 s *****
[rank:2] [train], epoch: 0/1, iter: 100/312, loss: 0.86809, top1: 0.00096, throughput: 445.20 | 2022-04-14 02:09:04.776
[rank:3] [train], epoch: 0/1, iter: 100/312, loss: 0.86788, top1: 0.00082, throughput: 445.19 | 2022-04-14 02:09:04.777
[rank:4] [train], epoch: 0/1, iter: 100/312, loss: 0.86788, top1: 0.00100, throughput: 445.20 | 2022-04-14 02:09:04.776
[rank:5] [train], epoch: 0/1, iter: 100/312, loss: 0.86796, top1: 0.00084, throughput: 445.19 | 2022-04-14 02:09:04.776
[rank:6] [train], epoch: 0/1, iter: 100/312, loss: 0.86800, top1: 0.00084, throughput: 445.20 | 2022-04-14 02:09:04.776
[rank:7] [train], epoch: 0/1, iter: 100/312, loss: 0.86789, top1: 0.00088, throughput: 445.19 | 2022-04-14 02:09:04.777
[rank:0] [train], epoch: 0/1, iter: 100/312, loss: 0.86803, top1: 0.00098, throughput: 445.20 | 2022-04-14 02:09:04.780
[rank:1] [train], epoch: 0/1, iter: 100/312, loss: 0.86811, top1: 0.00092, throughput: 445.17 | 2022-04-14 02:09:04.781
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.100, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.107, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 74 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.113, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 44 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.117, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8702 MiB, 23808 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.124, Tesla V100-SXM2-32GB, 470.57.02, 98 %, 93 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.124, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.131, Tesla V100-SXM2-32GB, 470.57.02, 98 %, 86 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.130, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.131, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 74 %, 32510 MiB, 8598 MiB, 23912 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.134, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.137, Tesla V100-SXM2-32GB, 470.57.02, 98 %, 89 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.138, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 74 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.138, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 90 %, 32510 MiB, 8768 MiB, 23742 MiB
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
timestamp, name, driver_version, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]
2022/04/14 02:09:05.140, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.141, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 74 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.143, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 91 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.142, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.143, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 90 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.144, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.144, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 63 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.145, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 82 %, 32510 MiB, 8629 MiB, 23881 MiB
2022/04/14 02:09:05.148, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 69 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.148, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 89 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.151, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 69 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.152, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.152, Tesla V100-SXM2-32GB, 470.57.02, 98 %, 90 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.153, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 69 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.205, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 69 %, 32510 MiB, 8598 MiB, 23912 MiB
2022/04/14 02:09:05.214, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 89 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.215, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.229, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 89 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.229, Tesla V100-SXM2-32GB, 470.57.02, 98 %, 90 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.230, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.236, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 89 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.238, Tesla V100-SXM2-32GB, 470.57.02, 97 %, 89 %, 32510 MiB, 8768 MiB, 23742 MiB
2022/04/14 02:09:05.252, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.252, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.270, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.272, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.273, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.299, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.305, Tesla V100-SXM2-32GB, 470.57.02, 99 %, 81 %, 32510 MiB, 8702 MiB, 23808 MiB
2022/04/14 02:09:05.323, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.323, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.328, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.329, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.329, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.330, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.331, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8778 MiB, 23732 MiB
2022/04/14 02:09:05.333, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.334, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.354, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.356, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.358, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.360, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 67 %, 32510 MiB, 8602 MiB, 23908 MiB
2022/04/14 02:09:05.362, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.362, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.366, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.368, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.373, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 62 %, 32510 MiB, 8470 MiB, 24040 MiB
2022/04/14 02:09:05.375, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.377, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.378, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
2022/04/14 02:09:05.378, Tesla V100-SXM2-32GB, 470.57.02, 100 %, 64 %, 32510 MiB, 8650 MiB, 23860 MiB
[rank:5] [train], epoch: 0/1, iter: 200/312, loss: 0.86785, top1: 0.00102, throughput: 1333.50 | 2022-04-14 02:09:43.171
[rank:2] [train], epoch: 0/1, iter: 200/312, loss: 0.86790, top1: 0.00109, throughput: 1333.43 | 2022-04-14 02:09:43.173
[rank:4] [train], epoch: 0/1, iter: 200/312, loss: 0.86784, top1: 0.00096, throughput: 1333.40 | 2022-04-14 02:09:43.174
[rank:0] [train], epoch: 0/1, iter: 200/312, loss: 0.86782, top1: 0.00104, throughput: 1333.50 | 2022-04-14 02:09:43.176
[rank:6] [train], epoch: 0/1, iter: 200/312, loss: 0.86790, top1: 0.00090, throughput: 1333.39 | 2022-04-14 02:09:43.174
[rank:1] [train], epoch: 0/1, iter: 200/312, loss: 0.86801, top1: 0.00084, throughput: 1333.55 | 2022-04-14 02:09:43.174
[rank:3] [train], epoch: 0/1, iter: 200/312, loss: 0.86819, top1: 0.00090, throughput: 1333.36 | 2022-04-14 02:09:43.176
[rank:7] [train], epoch: 0/1, iter: 200/312, loss: 0.86778, top1: 0.00133, throughput: 1333.38 | 2022-04-14 02:09:43.176
[rank:0] [train], epoch: 0/1, iter: 300/312, loss: 0.86798, top1: 0.00107, throughput: 1347.48 | 2022-04-14 02:10:21.173
[rank:2] [train], epoch: 0/1, iter: 300/312, loss: 0.86771, top1: 0.00092, throughput: 1347.35 | 2022-04-14 02:10:21.174
[rank:5] [train], epoch: 0/1, iter: 300/312, loss: 0.86813, top1: 0.00102, throughput: 1347.31 | 2022-04-14 02:10:21.173
[rank:4] [train], epoch: 0/1, iter: 300/312, loss: 0.86778, top1: 0.00102, throughput: 1347.38 | 2022-04-14 02:10:21.173
[rank:3] [train], epoch: 0/1, iter: 300/312, loss: 0.86797, top1: 0.00090, throughput: 1347.46 | 2022-04-14 02:10:21.173
[rank:6] [train], epoch: 0/1, iter: 300/312, loss: 0.86767, top1: 0.00117, throughput: 1347.40 | 2022-04-14 02:10:21.174
[rank:1] [train], epoch: 0/1, iter: 300/312, loss: 0.86798, top1: 0.00117, throughput: 1347.29[rank:7] [train], epoch: 0/1, iter: 300/312, loss: 0.86803, top1: 0.00096, throughput: 1347.34 | 2022-04-14 02:10:21.177
 | 2022-04-14 02:10:21.177