Its was informative notebook about topic modeling and spacy.
texts = metadata['cleandata']
bigram = gensim.models.Phrases(texts)
def display_topics(model, feature_names, no_top_words):
for topic_idx, topic in enumerate(model.components_):
print("Topic:", (topic_idx))
print(" ".join([feature_names[i]
for i in topic.argsort()[:-no_top_words - 1:-1]]))
def tfidf_vectorizer(documents,total_features):
# TFIDF Vectorizer
tfidf_vectorizer = TfidfVectorizer(max_features=total_features, stop_words='english',ngram_range=[2,2])
tfidf = tfidf_vectorizer.fit_transform(documents)
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
return tfidf_vectorizer,tfidf,tfidf_feature_names
def count_vectorizer(documents,total_features):
# Count Vectorizer
tf_vectorizer = CountVectorizer(max_features=total_features, stop_words='english')
tf = tf_vectorizer.fit_transform(documents)
tf_feature_names = tf_vectorizer.get_feature_names()
return tf_vectorizer,tf,tf_feature_names