Say you have successfully extracted the full HTML of multiple articles using a web crawler and stored it in a database. You can easily extract the title and text from this article.

In this tutorial, the data is being stored in a MongoDB collection. There is a URL column containing article URLs and an HTML column containing the full HTML of each article.

Using Jupyter Notebook:

Step 1: Import the scraped data from the MongoDB collection:

import re

#create a MongoClient to the running mongod instance (make sure that MongoDB instance is currently running)
from pymongo import MongoClient

#specify the host and port to connect on
MONGO_ADDRESS = "mongodb://<IP ADDRESS HERE>"
from bson import json_util
from collections import Counter


def connection():
    # Connect to the database
    client = MongoClient(MONGO_ADDRESS)
    db = client.GoogleResult
    # Handle to the connection
    collection = db.Articles

    news = list(collection.find({},{'URL', 'HTML'}))
        
    return news

news = connection()

Step 2: Convert news to pandas dataframe:

import pandas as pd

df = pd.DataFrame.from_records(news)

Step 3: Get the title from each article:

import re

def get_article_title(article):
    if '<title>' in article:
        m=re.search('<title>([\d\D]+)</title>', article)
        return m.group(1)

Step 4: Apply function to each article:

df["Title"] = df.apply(lambda article: get_article_title(article['HTML']), axis=1)

Step 5: Remove \n from title:

df['Title'] = df['Title'].str.replace('\n', '')

Step 6: Use Beautiful Soup to remove script from article:

def clean_me(article):
    soup = BeautifulSoup(article)
    for s in soup(['script', 'style']):
        s.decompose()
    return ' '.join(soup.stripped_strings)

Step 7: Apply to all articles:

df['Text'] = df.apply(lambda article: clean_me(article['HTML']), axis=1)

Step 8: Drop any rows which are empty or NaN in the ‘Page’ column:

df = df[df['Text'] != '']
df = df[df['Text'].notna()]

Step 9: Drop any rows which contain ‘403 Forbidden’ in the ‘Page’ column:

df = df[~df['Text'].str.contains('403 Forbidden')]

Using an IDE such as PyCharm:

main.py:

from extract_text import get_title, clean
import pandas as pd
import re
from pymongo import MongoClient
from bson import json_util
from collections import Counter


# Read Db
MONGO_ADDRESS = "mongodb://<IP ADDRESS HERE>"

def connection():
    # Connect to the database
    client = MongoClient(MONGO_ADDRESS)
    db = client.GoogleResult
    # Handle to the connection
    collection = db.Articles

    news = list(collection.find({},{'<LIST COLUMN NAMES HERE>'}))
        
    return news

news = connection()

raw_data = pd.DataFrame.from_records(news)


# Get and clean article title
raw_data['Title'] = raw_data.apply(lambda article: get_title(article['HTML']), axis=1)
raw_data['Title'] = raw_data['Title'].str.replace('\n', '')


# Clean article body
raw_data['Text'] = raw_data.apply(lambda article: clean(article['HTML']), axis=1)
raw_data = raw_data[raw_data['Text'] != '']
raw_data = raw_data[raw_data['Text'].notna()]
raw_data = raw_data[~raw_data['Text'].str.contains('403 Forbidden')]

extract_text.py:

import pandas as pd
from bs4 import BeautifulSoup
import re
import string


# Get article title
def get_title(article):
    """
    Get title from article and clean it
    """
    if '<title>' in article:
        m = re.search('<title>([\d\D]+)</title>', article)
        return m.group(1)


# Clean article
def clean(article):
    """
    Clean the body of article
    """
    soup = BeautifulSoup(article)
    for s in soup(['script', 'style']):
        s.decompose()
    return ' '.join(soup.stripped_strings)