HS_weighted_adjancey_matrix.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hierarchy-Based Semantic Encoding \n",
    "Input Required\n",
    "    1. A csv file containing hierarchy in the form of two columns: parent node and child node or (subject object)\n",
    "    2. Root node of the hierarchy\n",
    "<br>\n",
    "Output:\n",
    "Dataframe, each row represnts embeddings for a uniqe value. \n",
    "<br>\n",
    "\n",
    "Steps for calculating semantic embeddings\n",
    "1. Load the hierarchy from CSV into the networkx Graph\n",
    "2. Convert the graph into tree by specifying the root node of the hierarchy\n",
    "3. Create an attribute \"node-level\" for each node to store the level of the node\n",
    "4. Create unique paris for all nodes\n",
    "5. For each pair(i,j), find it's lowest common ancestor and replace it with it's level\n",
    "6. Create adjancey matrix where row and columns represents all nodes and each element(i,j) represents level of lowest common ancestor\n",
    "7. Use any similarity function in range (0,1) to calculate similarity (here we calculate similarity using our proposed measure: hierarchy-based semantic similarity)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import timeit\n",
    "import csv\n",
    "from sklearn import tree, linear_model\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import itertools\n",
    "from multiprocessing import Pool, cpu_count\n",
    "from anytree import Node, RenderTree\n",
    "import os\n",
    "import networkx as nx\n",
    "import glob\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filename : contains input filename with full path \n",
    "# file should have at least two columns named 'subject' and 'object'\n",
    "# rootnode: specify the url for root node of the hierarchy\n",
    "# target_filename: save embeddings with key name\n",
    "# lambda_factor : tuneable factor to create embeddings, read http://ceur-ws.org/Vol-2600/paper16.pdf\n",
    "def create_embeddings(filename, rootnode,target_filename, lambda_factor=0.6):\n",
    "    df = pd.read_csv(filename)\n",
    "    # Create the Directed Graph \n",
    "    G = nx.from_pandas_edgelist(df,\n",
    "                            source='object',\n",
    "                            target='subject',\n",
    "                            create_using=nx.DiGraph())\n",
    "    # create tree by specifying root node\n",
    "    tree = nx.bfs_tree(G, rootnode) #\n",
    "    # find level of node(shortest path from root to current node)\n",
    "    optional_attrs = nx.shortest_path_length(tree ,rootnode)\n",
    "    nx.set_node_attributes(tree ,  optional_attrs, 'node_level' )\n",
    "    \n",
    "    ls_leafnodes = [node for node in tree.nodes()]\n",
    "    pairs = list(itertools.product(ls_leafnodes, repeat=2)) # create pair of all nodes \n",
    "    all_ancestors = nx.algorithms.all_pairs_lowest_common_ancestor(tree, pairs=pairs) # get lowest common ancestors of alll pairs of nodes\n",
    "\n",
    "\n",
    "    # replace ancestor node with its level in the hierarchy\n",
    "    ls_ancestors_levels = {}\n",
    "    for i in all_ancestors:\n",
    "        ls_ancestors_levels[i[0]] = tree.node[i[1]]['node_level'] \n",
    "        \n",
    "    chunked_data = [[k[0],k[1], v] for k, v in ls_ancestors_levels.items()]\n",
    "    df_nodes = pd.DataFrame(chunked_data)\n",
    "    df_nodes = df_nodes.rename(columns= {0:'node1', 1:'node2', 2:'weight'})\n",
    "    depth = df_nodes.weight.max() # find the maximum levels in the hierarchy\n",
    "\n",
    "    # create adjancey matrix\n",
    "    vals = np.unique(df_nodes[['node1', 'node2']])\n",
    "    df_nodes = df_nodes.pivot(index='node1', columns='node2', values='weight'\n",
    "                      ).reindex(columns=vals, index=vals, fill_value=0)\n",
    "\n",
    "    df_adjacency = df_nodes.apply( lambda x:  np.power(  lambda_factor, depth - x))\n",
    "\n",
    "    # set diagnoal to 1\n",
    "    pd.DataFrame.set_diag = set_diag\n",
    "    df_adjacency.set_diag(1)\n",
    "    df_adjacency.fillna(0, inplace=True)\n",
    "\n",
    "\n",
    "    df_adjacency.to_csv(filepath+'/embeddings/all_nodes'+target_filename)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_diag(self, values): \n",
    "    n = min(len(self.index), len(self.columns))\n",
    "    self.values[tuple([np.arange(n)] * 2)] = values\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Store root nodes for all hierarchies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dict = { 'Inorganic_': 'http://id.nlm.nih.gov/mesh/D007287',\n",
    "                'Organic_chemicals' : 'http://id.nlm.nih.gov/mesh/D009930',\n",
    "               'Hetrocyclic_compounds' :'http://id.nlm.nih.gov/mesh/D006571',\n",
    "                'Polycyclic_compounds':        'http://id.nlm.nih.gov/mesh/D011083',\n",
    "                'Macromolecular_substances':    'http://id.nlm.nih.gov/mesh/D046911',\n",
    "                'Hormones_Hormone_Substitutes_Hormone_Antagonists':   'http://id.nlm.nih.gov/mesh/D006730',\n",
    "                'Enzymes_and_Coenzymes' :        'http://id.nlm.nih.gov/mesh/D045762',\n",
    "                 'Carbohydrates':      'http://id.nlm.nih.gov/mesh/D002241',\n",
    "                  'Lipids':    'http://id.nlm.nih.gov/mesh/D008055',\n",
    "                 'Amino_Acids_Peptides_Proteins' :   'http://id.nlm.nih.gov/mesh/D000602',\n",
    "                   'Nucleic_Acids_Nucleotides_Nucleosides':   'http://id.nlm.nih.gov/mesh/D009706',\n",
    "                  'Complex_Mixtures':  'http://id.nlm.nih.gov/mesh/D045424',\n",
    "                   'Biological_Factors' : 'http://id.nlm.nih.gov/mesh/D001685',\n",
    "                    'biomedical_Dental_Materials':'http://id.nlm.nih.gov/mesh/D001697',\n",
    "                   'Pharmaceutical_Preparations': 'http://id.nlm.nih.gov/mesh/D004364',\n",
    "                    'Chemical_Actions_Uses' : 'http://id.nlm.nih.gov/mesh/D020164',\n",
    "                    'taxonomy': 'https://www.ncbi.nlm.nih.gov/taxonomy/taxon/1'}\n",
    "\n",
    "\n",
    "# fetch all csv files that contains hierarchy pair (subject object pair)\n",
    "filepath = os.getcwd()\n",
    "file_list = glob.glob(filepath +'/data/*.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create embeddings for all hierarchies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file .. Inorganic_chemicals_hierarchy.csv\n",
      "Processing file .. Organic_chemicals_hierarchy.csv\n",
      "Processing file .. Hetrocyclic_compounds_hierarchy.csv\n",
      "Processing file .. Polycyclic_compounds_hierarchy.csv\n",
      "Processing file .. Macromolecular_substances_hierarchy.csv\n",
      "Processing file .. Hormones_Hormone_Substitutes_Hormone_Antagonists_hierarchy.csv\n",
      "Processing file .. Carbohydrates_hierarchy.csv\n",
      "Processing file .. Lipids_hierarchy.csv\n",
      "Processing file .. Amino_Acids_Peptides_Proteins_hierarchy.csv\n",
      "Processing file .. Biological_Factors_hierarchy.csv\n",
      "Processing file .. biomedical_Dental_Materials_hierarchy.csv\n",
      "Processing file .. Chemical_Actions_Uses_hierarchy.csv\n",
      "Processing file .. taxonomy_hierarchy_only0.csv\n"
     ]
    }
   ],
   "source": [
    "# 0 < lambda_factor < 1\n",
    "#\n",
    "lambda_factor = 0.7\n",
    "for key, root_node in root_dict.items():\n",
    "    if key in str(file_list ):\n",
    "        filename = [i for i in file_list if key in i]\n",
    "        for file in filename:\n",
    "            print ('Processing file ..',file.rpartition(\"/\")[2] )\n",
    "            create_embeddings(file,  root_node , file.rpartition(\"/\")[2], lambda_factor)\n",
    "\n",
    "        \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}