andreburgaud / robotspy Goto Github PK

def gen_uri(self, uri: str):
        """Instantiate a generator from a URI (either http:// or https:// or file:///"""
        try:
            with urllib.request.urlopen(uri) as f:
                self._time = time.time()
                encoding = f.headers.get_content_charset() or "utf-8"
                for line in f:
                    try:
                        yield line.decode(encoding)
                    except UnicodeDecodeError:
                        yield line.decode("utf-8", "replace")
        except urllib.error.HTTPError as err:
            if err.code in (401, 403):
                self.disallow_all = True
                self._errors.append((str(err.code), str(err)))
            elif 400 <= err.code < 500:
                self.allow_all = True
                self._warnings.append((str(err.code), str(err)))
            self._time = 0
        except urllib.error.URLError as err:
            self._errors.append(("", str(err)))