import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np

from wordcloud import WordCloud

import matplotlib.pyplot as plt
% matplotlib inline

# 1. Collect lists of songs from different years.

# https://www.billboard.com/charts/hot-100/1958-08-01

prepare_df = []

start = "https://www.billboard.com/charts/hot-100/"
year = 1958
end = "-08-01"

while year <= 2018:
    
    url = start + str(year) + end
    response = requests.get(url)

    soup = BeautifulSoup(response.text, 'html.parser')
    songs = soup.find_all("div", class_="chart-list-item")

    index = 0

    while index < len(songs):
        for song in songs:
            artist = songs[index]['data-artist']
            title = songs[index]['data-title']
            
            prepare_df.append({'year': year,
                               'artist': artist,
                               'title': title,
                               'lyric_url': "",
                               'lyric': ""})
            
            print("\r" + str(year) + " / " + title + " "*100, end='')
            index += 1
    
    year += 10
        
df = pd.DataFrame(prepare_df, columns = ['year', 'artist', 'title', 'lyric_url', 'lyric'])
    
df

# 2. Generate lyric URLs for these songs.

# https://genius.com/ricky-nelson-poor-little-fool-lyrics
start = "https://genius.com/"
end = "-lyrics"

for index, row in df.iterrows():
    
    df.at[index, 'lyric_url'] = (start 
                                 + df.at[index, 'artist'].lower().replace(" ", "-")
                                 + "-"
                                 + df.at[index, 'title'].lower().replace(" ", "-")
                                 + end)
    
df

# 3. Try to get the lyrics from web.

for index, row in df.iterrows():
    
    url = df.at[index, 'lyric_url']
    response = requests.get(url)

    try:
        soup = BeautifulSoup(response.text, 'html.parser')
        lyric_soup = soup.find("div", class_="lyrics")

        df.at[index, 'lyric'] = lyric_soup.text.replace("\n", " ")
    except:
        df.at[index, 'lyric'] = np.NaN
        
df

df.to_excel("_SONGS.xlsx")
df.to_csv("_SONGS.csv")

df = df.dropna()
df.info()

df.loc[(df["year"] == 1958)]

# 4. Combine all lyrics from one year to a single cell.

final = df.groupby('year')['lyric'].apply(' '.join).reset_index()

final.to_excel("_FINAL.xlsx")
final.to_csv("_FINAL.csv")

final

# 5. Data cleansing

final['lyric'] = final['lyric'].str.replace('\[.*?\]', '', regex=True)

final.to_excel("_FINAL.xlsx")
final.to_csv("_FINAL.csv")

final

# 6. Make a word cloud for each year and save it as .png.

for index, row in final.iterrows():

    year = final.at[index, 'year']
    text = final.at[index, 'lyric']

    wordcloud = WordCloud(collocations=False, background_color="white", max_words=100, width=1200, height=800).generate(text)

    title = str(year) + "\n"

    plt.figure(figsize=(12,8))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.title(title, fontsize=16)
    plt.savefig(str(year) + '.png', bbox_inches='tight')