[Bug] 关于hf模型stream_chat()输出乱码”�“的问题 about internlm HOT 2 CLOSED

class ChatStreamer(BaseStreamer):
     def __init__(self, tokenizer) -> None:
        super().__init__()
        self.tokenizer = tokenizer
        self.queue = response_queue
        self.query = query
        self.history = history
        self.response = ""
        self.cache = []
        self.received_inputs = False
        self.queue.put((self.response, history + [(self.query, self.response)]))

        def put(self, value):
            if len(value.shape) > 1 and value.shape[0] > 1:
                raise ValueError("ChatStreamer only supports batch size 1")
            elif len(value.shape) > 1:
                value = value[0]

            if not self.received_inputs:
                # The first received value is input_ids, ignore here
                self.received_inputs = True
                return
                
            self.cache.extend(value.tolist())
            token = self.tokenizer.decode(self.cache, skip_special_tokens=True)
            if "�" in token and len(token) <= 5:
                return
            self.cache = []
            if token.strip() != "<eoa>":
                self.response = self.response + token
                history = self.history + [(self.query, self.response)]
                self.queue.put((self.response, history))
            else:
                self.end()