Hi Terra,

Thank you for sharing your great code! I adjusted your code but suffer captcha problem in both scraping urls and scraping transcripts. Your design about "unique" user agent names do not work in my code. The following are my adjusted code. Do you have any idea about how to deal with the captcha issue. Thank you so much!

Import Libraries

Standard Libraries

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
#%matplotlib inline

Scraping Libraries

from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from time import sleep
import re

NLP Libraries

from textblob import TextBlob
import textstat

from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
import datetime

def Sentamentize(text):
return TextBlob(str(text)).sentiment.polarity

def open_browser(alt_user_name = 'Thank you for your website'):
opts = Options()
opts.add_argument("user-agent=" + str(alt_user_name))
path = '../Garage/chromedriver' # Path to Chromedriver
#return webdriver.Chrome(executable_path = path, options=opts)
return webdriver.Chrome(ChromeDriverManager().install(), options=opts)

Data Collection

Scraping urls of each transcript before the next step.

I suffer from captchas after only tens of urls

b_url='https://seekingalpha.com/earnings/earnings-call-transcripts/'
url_list=[]
for page_num in range(1,1001):
browser = open_browser("Y'all are great " + str(page_num))
# I put in a "unique" user agent name to help throw
# off the captcha. This worked for only so long.
current_ts_list = b_url + str(page_num)
browser.get(current_ts_list)
elements_list = browser.find_elements_by_class_name('dashboard-article-link')
urls = [el.get_attribute('href') + '?part=single' for el in elements_list]
headers = [el.text for el in elements_list]
print('scraping', page_num)
#sleep(20)
for transcript_num in range(len(urls)):
current_transcript = {
'header' : headers[transcript_num],
'url' : urls[transcript_num],
'page_list': page_num
}
url_list.append(current_transcript)
browser.close()

Scraping every individual url in the df to return the full

url_df1 = pd.DataFrame(url_list)
pd.DataFrame(url_df1).to_csv("url_df_1000.csv")

Need to create a variabe "transcript" before moving to the following steps

url_df = pd.read_csv('url_df_20.csv')

I suffer from captchas after only severa urls

for i in range(20): # This allows for timeout errors w/oo crashing
try:
for row_num in range(len(url_df['transcript'])):
if url_df['transcript'][row_num] == 0:

            # Go to url and obtain the full transcript
            browser = open_browser("Y'all are excellent " + str(row_num))
            # I put in a "unique" user agent name to help throw
            # off the captcha.  This worked for only so long.                
            browser.get(url_df['url'][row_num])
            print('1')                
            soup    = BeautifulSoup(browser.page_source)
            article = soup.find('article')
            url_df['transcript'][row_num] = [item.text for 
                item in article.find_all('p')]
            browser.close()
            sleep(20)
        else:
            pass
except:
    print('--------')
    print('ERROR', i)
    print('--------')
    sleep(100)
    browser.close()
    #### Suggestions for the future scraper.
    # if browser:
    #    browser.close()
    # else:
    #    print("""Browser was already closed.  
    #    This wasn't a timeout error""")
    # For info on catching the errors that we want try:
    # https://stackoverflow.com/questions/33239308/how-to-
    # get-exception-message-in-python-properly/33239954

url_df.head(3)
pd.DataFrame(url_df).to_csv("url_df_20_transcript.csv")