Distributed webpage crawler making use of Husky

import sys import re import json import bs4

from Crawler import crawler_dist as cd from Crawler import crawler_config import bindings.frontend.env as env

def parse_name(url, html): try: soup = bs4.BeautifulSoup(html, "lxml") content = soup.find("div", {"class" : "healthaz-header"}).text.strip() return json.dumps({"name": content.split('\n')[0], "url": url}) except: sys.stderr.write('parse failed: ' + url + '\n') return json.dumps({"url": url})

config = crawler_config.CrawlerConfig()

config.urls_init=['http://www.nhs.uk/Conditions/Pages/BodyMap.aspx?Index=A']

config.set_rules_include(['http://www.nhs.uk/conditions/[a-zA-Z- ]+'])

config.add_parse_handler('http://www.nhs.uk/conditions/[a-zA-Z\- ]+', parse_name)

config.hdfspath_output = '/datasets/crawl/nhs-new-A2/'

config.group_url = 5

config.max_retry_time = 10

config.conn_timeout = 100

config.read_timeout = 100

config.tor_port_list = range(18112, 18161)

config.port_switch_threshold = 5.0

config.num_iters = 2

env.pyhusky_start() cd.crawler_run(config)