Does the test accuracy need to be synchronized in distributed.py? about pytorch-distributed HOT 7 CLOSED

acc1, acc5 = accuracy(output, target, topk=(1, 5))
...
dist.all_reduce(acc1, op=dist.reduce_op.SUM)
...
top1.update(acc1[0] / 4 , images.size(0))

import time
import random

import torch
import torch.distributed as dist
import torch.multiprocessing as mp


def evaluate(rank):
    torch.cuda.manual_seed(rank)
    local_acc = torch.randn(1)[0].cuda(rank)

    print("local_acc:", local_acc)

    return local_acc

def main_worker(gpu, ngpus_per_node, args):
    dist.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:23456', world_size=4, rank=gpu)

    local_acc = evaluate(gpu)

    global_acc, global_count = args['global_acc'], args['global_count']

    global_acc += local_acc.cpu()
    global_count += 1

    print("global_acc:", global_acc / global_count)

if __name__ == '__main__':
    global_acc = torch.tensor(.0)
    global_count = torch.tensor(.0)
    
    global_acc.share_memory_()
    global_count.share_memory_()

    args = {
        'global_acc': global_acc,
        'global_count': global_count
    }
    
    mp.spawn(main_worker, nprocs=4, args=(4, args))

import time
import random

import torch
import torch.distributed as dist
import torch.multiprocessing as mp


def evaluate(rank):
    torch.cuda.manual_seed(rank)
    local_acc = torch.randn(1)[0].cuda(rank)

    print("local_acc:", local_acc)

    return local_acc

def main_worker(gpu, ngpus_per_node, args):
    dist.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:23456', world_size=4, rank=gpu)

    local_acc = evaluate(gpu)

    dist.all_reduce(local_acc, op=dist.reduce_op.SUM)
    global_acc = local_acc / ngpus_per_node

    print("global:", global_acc)

if __name__ == '__main__':
    args = {}
    mp.spawn(main_worker, nprocs=4, args=(4, args))