self.reward_tracker['reranker_rewards'].append(reranker_rewards.detach().cpu().numpy().tolist())
self.reward_tracker['metric_rewards'].append(metric_rewards.detach().cpu().numpy().tolist())
if self.args.normalize_rewards:  #
    rererank_rewards_std = torch.std(reranker_rewards, dim=1, keepdim = True)
    metric_rewards_std = torch.std(metric_rewards, dim=1, keepdim = True)
    reranker_rewards = reranker_rewards / (rererank_rewards_std + eps)
    metric_rewards = metric_rewards / (metric_rewards_std + eps)
rewards = self.args.reranker_reward_scaler * reranker_rewards + self.args.metric_reward_scaler * metric_rewards
rewards = rewards.view(-1) #(B* C)
rewards = rewards.unsqueeze(1).expand_as(generated_probs)  #(B*C,L)