In Jupyter Notebook:

Open Anaconda Prompt and conda activate your environment. Otherwise, create a new environment and activate it.

Run jupyter notebook.

Import statements:

import pandas as pd
import nltk 
from nltk.corpus import wordnet 
import numpy as np
import nltk
nltk.download('wordnet')
from nltk.stem.wordnet import WordNetLemmatizer
from nltk import word_tokenize
from mxnet import np, npx
from mxnet.contrib import text
npx.set_np()
text.embedding.get_pretrained_file_names().keys()
import string
import re
from pymongo import MongoClient

Connect to MongoDB database collection. Fill in any info between <>. In this example, the database collection has two columns Title and Description.

#create a MongoClient to the running mongoDB instance (make sure that MongoDB instance is currently running)

#specify the host and port to connect on
MONGO_ADDRESS = "mongodb://<IP Address>"
from bson import json_util
from collections import Counter

def connection():
    # Connect to the database
    client = MongoClient(MONGO_ADDRESS)
    db = client.<DBName>
    # Handle to the connection
    collection = db.<CollectionName>

    data = list(collection.find({},{'Title','Description'}))
        
    return data

data = connection()

Convert the collection into a dataframe. Pickle the file to be stored on your local machine. You only have to run this code once, or as often as the database updates.

import pandas as pd

df = pd.DataFrame.from_records(data)

#Converts dataframe to a file
df.to_pickle('filename.pkl')

Read the pickled file back into a pandas dataframe. Each time you run Jupyter Notebook, you only have to start from the code in this step (unless the MongoDB collection is updated).

import pandas as pd

df = pd.read_pickle('filename.pkl')

Import the keywords that you would like to extract from each article if present. Save to variable signal_words as a set. Here I am importing keywords from a text file ‘signal_keywords.txt’.

path = 'signal_keywords.txt'
with open(path, 'r') as f:
    signal_words = set(f.read().splitlines())

Clean the signal_words. Return signal_words without extraneous whitespace at end of each element of set if present:

signal_words = [x.strip(' ') for x in signal_words]

Since our synonym detecting method can only work on single words, drop all but single word keywords (by returning only elements which do not contain whitespace and which do not contain -):

signals_one_word = [ x for x in signal_words if " " not in x ]

signals_cleaned = [ x for x in signals_one_word if "-" not in x ]

Lemmatize cleaned signal words:

lmtzr = WordNetLemmatizer()

signals_lemmatized = [lmtzr.lemmatize(x) for x in signals_cleaned]

Get synonyms for the lemmatized signals using knn algorithm and glove vectorizer. Get the glove vectorizer here.

glove_6b50d = text.embedding.create(
    'glove', pretrained_file_name='glove.6B.50d.txt')

def knn(W, x, k):
    # The added 1e-9 is for numerical stability
    cos = np.dot(W, x.reshape(-1,)) / (
        np.sqrt(np.sum(W * W, axis=1) + 1e-9) * np.sqrt((x * x).sum()))
    topk = npx.topk(cos, k=k, ret_typ='indices')
    return topk, [cos[int(i)] for i in topk]

glove_result = []

def get_similar_tokens(query_token, k, embed):
    topk, cos = knn(embed.idx_to_vec,
                    embed.get_vecs_by_tokens([query_token]), k+1)
    for i, c in zip(topk[1:], cos[1:]):  # Remove input words
        glove_result.append('cosine sim=%.1f: %s' % (c, (embed.idx_to_token[int(i)])))

[get_similar_tokens(item, 3, glove_6b50d) for item in signals_lemmatized]

Now to clean up the synonyms, return only the elements which do not contain nan.

glove_result = [ x for x in glove_result if "nan" not in x ]

Also, drop any synonyms which have a cosine similarity below .9.

glove_result = [ x for x in glove_result if "6" not in x ]

glove_result = [ x for x in glove_result if "7" not in x ]

glove_result = [ x for x in glove_result if "8" not in x ]

Create a list called test which will hold the cleaned synonyms alone without their cosine similarities:

test = []

for line in glove_result:
    test.append(line.split(': ', 1)[-1])

Set the glove_result equal to test

glove_result = test

Combine the original keywords without whitespace at end, cleaned keywords, lemmatized keywords and synonyms in a list. Drop any duplicates.

keywords_and_synonyms = signal_words + signals_cleaned + signals_lemmatized + glove_result

# drop duplicates
keywords_and_synonyms = list(dict.fromkeys(keywords_and_synonyms))
keywords_and_synonyms

Create a function which applies keywords_and_synonyms to new Signals column in pandas dataframe.

def extract_unique_words(description):
    # remove punctuation
    description_no_punct = description.translate(str.maketrans('','',string.punctuation))
    #lower case
    description_lower_case = description_no_punct.lower()
    # split on whitespace
    description_split = description_lower_case.split()
    # convert list to set
    return set(description_split)

Convert keywords_and_synonyms to set

keywords_and_synonyms_set = set(keywords_and_synonyms)

Apply extract_unique_words function to each article and apply to Signals column of each row of dataframe.

df["Signals"] = df.apply(lambda row: extract_unique_words(row['Description']).intersection(keywords_and_synonyms_set), axis=1)

View updated dataframe. The keywords, if present, in Description per article will now show up under the Signals column.

df

In PyCharm:

Get the glove vectorizer (needed for getting synonyms) here.

main.py:

#Import statements
import signals
import re
from pymongo import MongoClient

if __name__ == "__main__":
    # Read from database collection
    #create a MongoClient to the running mongoDB instance (make sure that 
    MongoDB instance is currently running)

    #specify the host and port to connect on
    MONGO_ADDRESS = "mongodb://<IP Address>"
    from bson import json_util
    from collections import Counter

    def connection():
        # Connect to the database
        client = MongoClient(MONGO_ADDRESS)
        db = client.<DBName>
        # Handle to the connection
        collection = db.<CollectionName>

        data = list(collection.find({},{'Title','Description'}))
        
        return data

    data = connection() 

    # Convert collection to dataframe
    df = pd.DataFrame.from_records(data)

    # Clean and preprocess data here

    # Signal keywords and synonyms
    # Import signals
    signal_words = signals.import_signals()
    # Convert signals to list
    signal_words = list(signals.import_signals())
    # Clean signals
    signals_cleaned = signals.clean_signals(signal_words)
    # Lemmatize signals
    signals_lemmatized = signals.lemmatize_signals()
    # Glove synonyms
    glove_results = signals.glove_synonyms(signals_lemmatized)
    # Spacy synonyms
    #spacy_results = signals.spacy_synonyms()
    # Get list of keywords and their synonyms
    keywords_and_synonyms = signals.get_keywords_and_synonyms()
    # Convert list of keywords and synonyms to set
    keywords_and_synonyms_set = signals.convert_to_set()
    # Apply to database in Signals column
    df["Signals"] = df.apply(
        lambda row: signals.apply_to_news(row['Description']).intersection(keywords_and_synonyms_set), axis=1)


    # Print updated df dataframe
    print(df)

signals.py

# Import statements
import pandas as pd
import nltk
from nltk.corpus import wordnet
import numpy as np
import nltk
nltk.download('wordnet')
from nltk.stem.wordnet import WordNetLemmatizer
from nltk import word_tokenize
from mxnet import np, npx
from mxnet.contrib import text
npx.set_np()
text.embedding.get_pretrained_file_names().keys()
import string


# Import signals
def import_signals():
    path = 'signal_keywords.txt'
    with open(path, 'r') as f:
        return set(f.read().splitlines())


# Clean signals
# Pass argument 'signal_words' to function since defined in function
def clean_signals(signal_words):
    # Remove whitespace at end of string if present
    signal_words = [x.strip(' ') for x in signal_words]
    # Remove any elements with whitespace
    signals_one_word = [x for x in signal_words if " " not in x]
    # Remove any elements with -
    return [x for x in signals_one_word if "-" not in x]


# Lemmatize signals
def lemmatize_signals():
    lmtzr = WordNetLemmatizer()
    return [lmtzr.lemmatize(x) for x in clean_signals()]


# Get glove synonyms
# Pass argument 'signal_lemmatized' to function since defined in function
def glove_synonyms(signals_lemmatized):
    npx.set_np()
    text.embedding.get_pretrained_file_names().keys()
    glove_6b50d = text.embedding.create(
        'glove', pretrained_file_name='glove.6B.50d.txt')

    def knn(W, x, k):
        # The added 1e-9 is for numerical stability
        cos = np.dot(W, x.reshape(-1, )) / (
                np.sqrt(np.sum(W * W, axis=1) + 1e-9) * np.sqrt((x * x).sum()))
        topk = npx.topk(cos, k=k, ret_typ='indices')
        return topk, [cos[int(i)] for i in topk]
    glove_result = []

    def get_similar_tokens(query_token, k, embed):
        topk, cos = knn(embed.idx_to_vec,
                        embed.get_vecs_by_tokens([query_token]), k + 1)
        for i, c in zip(topk[1:], cos[1:]):  # Remove input words
            glove_result.append('cosine sim=%.1f: %s' % (c, (embed.idx_to_token[int(i)])))

    [get_similar_tokens(item, 3, glove_6b50d) for item in signals_lemmatized]

    # Remove synonyms with nan and cosine sim below .9
    glove_result = [x for x in glove_result if "nan" not in x]
    glove_result = [x for x in glove_result if "6" not in x]
    glove_result = [x for x in glove_result if "7" not in x]
    glove_result = [x for x in glove_result if "8" not in x]

    test = []

    for line in glove_result:
        test.append(line.split(': ', 1)[-1])

    glove_result = test

    return glove_result


def get_keywords_and_synonyms():
    # Give variable names to some function results
    signals_cleaned = clean_signals()
    signals_lemmatized = lemmatize_signals()
    glove_results = glove_synonyms()
    #spacy_results = spacy_synonyms()

    # Combine original signals list, cleaned signals list, lemmatized list, and synonyms results
    keywords_and_synonyms = signal_words + signals_cleaned + signals_lemmatized + glove_results

    # Drop any duplicates
    return list(dict.fromkeys(keywords_and_synonyms))


def convert_to_set():
    # Give variable names to some function results
    signals_cleaned = clean_signals()
    signals_lemmatized = lemmatize_signals()
    glove_results = glove_synonyms()
    #spacy_results = spacy_synonyms()

    # Combine original signals list, cleaned signals list, lemmatized list, and synonyms results
    keywords_and_synonyms = signal_words + signals_cleaned + signals_lemmatized + glove_results

    # Convert keywords_and_synonyms to set
    keywords_and_synonyms_set = set(keywords_and_synonyms)
    return keywords_and_synonyms_set


def apply_to_news(description):

    # Remove punctuation
    description_no_punct = description.translate(str.maketrans('', '', string.punctuation))

    # Lower case
    description_lower_case = description_no_punct.lower()

    # Split on whitespace
    description_split = description_lower_case.split()

    # Convert list to set
    return set(description_split)