Finding Occurrences of Chemicals and Drugs in PubMed Abstracts

Searches the ElasticSearch index created during CaseOLAP pipeline run for curated list of Drugs and Chemicals related to oxidative stress

Output:

• Chemical_PMID_occurances.csv: CSV table where each row is the occurance of a chemical in PubMed
• Drug_PMID_occurances.csv: CSV table where each row is the occurance of a drug in PubMed

from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search, Q
import pandas as pd
from itertools import product
import seaborn as sns
import numpy as np
import time
import matplotlib.pyplot as plt
import json
import progressbar

Load Drug and Chemical lists, initialize Elastic Search

• Requires elastic search engine to be running on cluster. Must have PMID index

chemical_list_df = pd.read_csv('input/oxidative_stress_chemicals_SA_10222019.csv')
chemical_list_df['Molecule/Enzyme/Protein'] = chemical_list_df['Molecule/Enzyme/Protein'].str.lower().str.strip()

chemical_list_df.head()

drug_list_df = pd.read_csv('input/drug_list_SA_10222019.csv')
drug_list_df['Name'] = drug_list_df['Name'].str.lower().str.strip()
drug_list_df.head()

es = Elasticsearch(timeout=300)

Find PMIDs associated with every combination of drugs and chemicals via elastic search

• output file saved to output/chem_drug_pair_matches.csv

# All combinations of drugs and chemicals
drug_chemicals = product(
    drug_list_df['Name'].dropna().unique(),
    chemical_list_df['Molecule/Enzyme/Protein'].dropna().unique()
)

matches = pd.DataFrame()
for (drug, chemical) in progressbar.progressbar(drug_chemicals):

    mol_matches = {
            'PMID': [],
            'title': [],
            'Year': [],
            'Month': []
        }    
    # Match drug and chemical
    q = Q("match_phrase", abstract=drug.lower().strip()) & Q("match_phrase", abstract=chemical.lower().strip())

    # Search
    hits = Search(
        using=es,
        index="pubmed"
    ).params(
        request_timeout=300
    ).query(q)
    for h in hits.scan():
        date_dict = json.loads(h.date.replace("'", '"'))        
        mol_matches['PMID'].append(h.pmid)
        mol_matches['title'].append(h.title)
        mol_matches['Year'].append(date_dict['Year'])
        mol_matches['Month'].append(date_dict['Month'])

    match_df = pd.DataFrame.from_dict(mol_matches)
    match_df['drug'] = drug.lower().strip()
    match_df['chemical'] = chemical.lower().strip()
    matches = matches.append(match_df)

matches.head()

| |                   #                           | 21059 Elapsed Time: 0:13:02

matches.to_csv('output/chem_drug_pair_matches.csv', index=False)

matches = pd.read_csv('output/chem_drug_pair_matches.csv')

Plotting Drug, chemical co-occurence heatmap

• colors used to show category of drug, and oxidative stress category association of chemicals

chem_name_cats = chemical_list_df[['Molecule/Enzyme/Protein', 'Biological Events of Oxidative Stress']]\
    .drop_duplicates().rename(columns={
        'Molecule/Enzyme/Protein': 'chemical',
        'Biological Events of Oxidative Stress':'chem_cat'
    }).dropna()
drug_name_cats = drug_list_df[['Name', 'Drug Category']]\
    .drop_duplicates().rename(columns={
        'Name': 'drug',
        'Drug Category': 'drug_cat'
    }).dropna()
with_cats_matches = matches.merge(
    chem_name_cats,
    how='left',
    validate='m:m'
)
with_cats_matches = with_cats_matches.merge(
    drug_name_cats,
    how='left',
    validate='m:m'
)
print(matches.dropna().shape, with_cats_matches.dropna().shape)
with_cats_matches.head()

(199479, 6) (204265, 8)

with_cats_matches.chem_cat.unique()

array(['Initiation of Oxidative', 'Outcomes of Oxidative Stress',
       'Regulation of Oxidative Stress'], dtype=object)

# Creating color palettes to label drug and chemical categories
chems = with_cats_matches.chem_cat.unique()
chem_pal = sns.color_palette("hls", n_colors=with_cats_matches.chem_cat.nunique())
chem_pal_dict = dict(zip(chems, chem_pal))

drugs = with_cats_matches.drug_cat.unique()
drug_pal = sns.color_palette("tab20c", n_colors=with_cats_matches.drug_cat.nunique())
drug_pal[-5:] = sns.color_palette("tab20b", n_colors=5)
drug_pal_dict = dict(zip(drugs, drug_pal))

with_cats_matches['chem_color'] = with_cats_matches.chem_cat.map(chem_pal_dict)
with_cats_matches['drug_color'] = with_cats_matches.drug_cat.map(drug_pal_dict)

sns.palplot(chem_pal)
plt.gca().set_xticklabels(chems)
plt.xticks(rotation=60)
sns.palplot(drug_pal)
plt.gca().set_xticklabels(drugs)
plt.xticks(rotation=90);

## Save Color Palettes
with open('drug_cat_palette.json', 'w') as fp:
    json.dump(drug_pal_dict, fp)

with open('chem_cat_palette.json', 'w') as fp:
    json.dump(chem_pal_dict, fp)

# Set NaN category color to white
with_cats_matches.loc[with_cats_matches.drug_color.isna(), 'drug_color'] = "white"

# Count articles per drug-chemical co-occurrence
article_count = pd.DataFrame(
    with_cats_matches.groupby(['drug', 'chemical', 'drug_cat', 'chem_cat']).PMID.nunique()
).reset_index().rename(columns={'PMID': 'Article Count'})
article_count['log_count'] = np.log10(article_count['Article Count'])

chem_colors_df = with_cats_matches[['chemical', 'chem_color']].drop_duplicates()
chem_colors = [chem_colors_df[chem_colors_df.chemical == chem].chem_color.unique()[0] for chem in piv_count.index]

drug_colors_df = with_cats_matches[['drug', 'drug_color']].drop_duplicates()
drug_colors = [drug_colors_df[drug_colors_df.drug == drug].drug_color.unique()[0] for drug in piv_count.columns]

piv_count = article_count.pivot_table(
    index='chemical',
    columns='drug',
    values='log_count',
    fill_value=0
)
sns.clustermap(
    piv_count,
    figsize=(22,13),
    cmap='viridis',
    row_colors=chem_colors,
    col_colors=drug_colors
)

<seaborn.matrix.ClusterGrid at 0x7ff7afcb4898>

Find PMIDS assocaited with drugs via elastic search

• Searches abstracts for drug names or synonyms of drug names
• Finds number of occurances of drug name or synonyms in abstract

Saves to output/Drug_PMID_occurances.csv

drug_matches = pd.DataFrame()
tot = drug_list_df.Name.nunique()

for (drug, synonyms, category), m_df in progressbar.progressbar(drug_list_df.groupby(['Name', 'Synonyms', 'Drug Category'])):
    drug_match = {
            'PMID': [],
            'title': [],
            'MeSH': [],
            'count': [],
            'Year': [],
            'Month': []
        }
    synonyms = synonyms.split(', ')
    drug = drug.lower()
    q = Q('match_phrase', abstract=drug)

    if synonyms:
        synonyms = [s.lower() for s in synonyms]
        for s in synonyms:
            q = q | Q('match_phrase', abstract=s)

    hits = Search(
        using=es,
        index="pubmed"
    ).query(q)

    for h in hits.scan():
        date_dict = json.loads(h.date.replace("'", '"'))        
        drug_match['PMID'].append(h.pmid)
        drug_match['title'].append(h.title)
        drug_match['MeSH'].append(h.MeSH)
        drug_match['Year'].append(date_dict['Year'])
        drug_match['Month'].append(date_dict['Month'])

        entity_count = 0
        for phrase in [drug] + synonyms:
            entity_lower = phrase.lower().replace("-", " ")
            entity_count += abs_lower.count(entity_lower)

        drug_match['count'].append(entity_count)

    drug_match_df = pd.DataFrame.from_dict(drug_match)
    drug_match_df['drug'] = drug
    drug_match_df['category'] = category
    drug_matches = drug_matches.append(drug_match_df)


drug_matches.head()

100% (161 of 161) |######################| Elapsed Time: 0:37:59 Time:  0:37:59

drug_matches.to_csv('output/Drug_PMID_occurances.csv', index=False)
drug_matches.shape

(2702853, 8)

drug_category_PMID_count = pd.DataFrame(drug_matches.groupby('category').PMID.nunique()).reset_index()
drug_category_PMID_count

Searching for PMIDs associated with each chemical

• If there is a MeSH id, searches pubmed index for abstract containing drug name OR MeSH terms containin mesh term
• If there is no MeSH id, only searches for drug name in abstract

saves to output/Chemical_PMID_occurances.csv

has_data_df = chemical_list_df[
    (~chemical_list_df['Molecule/Enzyme/Protein'].isnull()) |
    (~chemical_list_df['MeSH Heading'].isnull())
]
chem_matches_df = pd.DataFrame()

for (name, mesh, category), m_df in progressbar.progressbar(has_data_df.groupby(['Molecule/Enzyme/Protein', 'MeSH Heading', 'Biological Events of Oxidative Stress'])):
    hit_dict = {
        'PMID': [],
        'Article MeSH': [],
        'Year': [],
        'Month': [],
    }
    if mesh.lower() == 'none listed':
        q = Q('match_phrase', abstract=name.lower())
    else:
        q = Q('match_phrase', abstract=name.lower()) | Q('match_phrase', MeSH=mesh)

    hits = Search(
        using=es,
        index="pubmed"
    ).params(
        request_timeout=300
    ).query(q)
    for h in hits.scan():
        date_dict = json.loads(h.date.replace("'", '"'))
        hit_dict['PMID'].append(h.pmid)
        hit_dict['Article MeSH'].append(h.MeSH)
        hit_dict['Year'].append(date_dict['Year'])
        hit_dict['Month'].append(date_dict['Month'])


    hit_df = pd.DataFrame.from_dict(hit_dict)
    hit_df['category'] = category
    hit_df['chemical'] = name
    hit_df['MeSH'] = mesh
    chem_matches_df = chem_matches_df.append(hit_df)

chem_matches_df.head()

100% (157 of 157) |######################| Elapsed Time: 0:44:52 Time:  0:44:52

chem_matches_df.to_csv('output/Chemical_PMID_occurances.csv', index=False)
chem_matches_df.shape

(3291433, 7)