benchmark_tasks_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook for analyzing benchmark tasks results\n",
    "\n",
    "Contains the following functionalities:\n",
    "\n",
    "1. Gets identifying attributes [1] per task (Benchmark tasks located in benchmark_tasks.gzip --> Please unzip before using). \n",
    "2. Produces results table\n",
    "3. Calculates profiling dimensions per task\n",
    "\n",
    "[1] Primpeli, Anna, and Christian Bizer. \"Profiling entity matching benchmark tasks.\" Proceedings of the 29th ACM International Conference on Information & Knowledge Management. 2020."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns;\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score, cross_validate,cross_val_predict, StratifiedShuffleSplit, GridSearchCV, PredefinedSplit\n",
    "from sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score, f1_score, precision_recall_fscore_support\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from os.path import isfile, join\n",
    "from os import listdir\n",
    "import re\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "from copy import copy\n",
    "import statistics\n",
    "import networkx as nx\n",
    "import similaritymeasures\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Get ID Features/ Attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get identifying features\n",
    "selected_task = \"lspc_computers\"\n",
    "fv_vector_train=pd.read_csv('../benchmark_tasks/'+selected_task+'/train_pairs_fv.csv')\n",
    "fv_vector_test=pd.read_csv('../benchmark_tasks/'+selected_task+'/test_pairs_fv.csv')\n",
    "fv_vector = pd.concat([fv_vector_train,fv_vector_test])\n",
    "\n",
    "def getIDFeatures(fv_vector):\n",
    "\n",
    "        X = fv_vector.drop(['source_id', 'target_id', 'pair_id', 'label','source','target','agg_score','unsupervised_label'], axis=1)\n",
    "        y =  fv_vector['label'].values\n",
    "        clf = RandomForestClassifier(random_state=1)\n",
    "        model = clf.fit(X,y)     \n",
    "        features_in_order, feature_weights = showFeatureImportances(X.columns.values,model,'rf',display=False) \n",
    "        \n",
    "        # all features that are relevant for the matching\n",
    "        matching_relevant_features = []\n",
    "        xval_scoring = {'precision' : make_scorer(precision_score),'recall' : make_scorer(recall_score),'f1_score' : make_scorer(f1_score)}     \n",
    "\n",
    "        max_result = cross_validate(clf, X, y, cv=StratifiedShuffleSplit(n_splits=4,random_state =1),\n",
    "                                    scoring=xval_scoring, n_jobs=-1)\n",
    "\n",
    "        max_f1_score = round(np.mean(max_result['test_f1_score']),2)\n",
    "        #gather features that are relevant for 95% of the max f1 score\n",
    "        sub_result = 0.0\n",
    "        for i in range(1,len(features_in_order)+1):\n",
    "            results_subvector = cross_validate(clf, X[features_in_order[:i]], y, cv=StratifiedShuffleSplit(n_splits=4,random_state =1),  scoring=xval_scoring, n_jobs=-1)\n",
    "            sub_result = round(np.mean(results_subvector ['test_f1_score']),2)\n",
    "            if (sub_result>0.95*max_f1_score): break;\n",
    "           \n",
    "        \n",
    "        important_features = features_in_order[:i]\n",
    "        \n",
    "        print(important_features)\n",
    "        \n",
    "def showFeatureImportances(column_names, model, classifierName,display=True):\n",
    "      \n",
    "    importances = get_model_importances(model, classifierName)\n",
    "       \n",
    "    column_names = [c.replace('<http://schema.org/Product/', '').replace('>','') for c in column_names]\n",
    "    sorted_zipped = sorted(list(zip(column_names, importances)), key = lambda x: x[1], reverse=True)[:50]\n",
    "   \n",
    "    features_in_order = [val[0] for val in sorted_zipped]\n",
    "    feature_weights_in_order = [round(val[1],2) for val in sorted_zipped]\n",
    "    if (display):\n",
    "        plt.figure(figsize=(18,3))\n",
    "        plt.title('Feature importances for classifier %s (max. top 50 features)' % classifierName)\n",
    "        plt.bar(range(len(sorted_zipped)), [val[1] for val in sorted_zipped], align='center', width = 0.8)\n",
    "        plt.xticks(range(len(sorted_zipped)), [val[0] for val in sorted_zipped])\n",
    "        plt.xticks(rotation=90)\n",
    "        plt.show() \n",
    "\n",
    "    return features_in_order,feature_weights_in_order\n",
    "\n",
    "def get_model_importances(model,classifierName=None):\n",
    " \n",
    "    if classifierName == 'logr':\n",
    "        importances = model.coef_.ravel()\n",
    "    elif classifierName == 'svm':\n",
    "        if model.kernel != 'linear':\n",
    "            display(\"Cannot print feature importances without a linear kernel\")\n",
    "            return\n",
    "        else: importances = model.coef_.ravel()\n",
    "    else:\n",
    "        importances = model.feature_importances_\n",
    "    \n",
    "    return importances\n",
    "\n",
    "getIDFeatures(fv_vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Produce results table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BENCHMARK TASKS PROFILE AND COMPARISON\n",
    "main_path = \"../benchmark_tasks/\"\n",
    "\n",
    "\n",
    "#ignore first for auc calculation\n",
    "ignore_first=50\n",
    "\n",
    "def getAUC(f1_scores_all, ignore_first=20):\n",
    "    f1_scores = f1_scores_all[ignore_first:]\n",
    "    x_ax_data= np.zeros((len(f1_scores),2))\n",
    "    x_ax_data[:, 0] = [0]*len(f1_scores)\n",
    "    x_ax_data[:, 1] = np.arange(0,len(f1_scores))\n",
    "\n",
    "    f1_scores_data= np.zeros((len(f1_scores),2))\n",
    "    f1_scores_data[:, 0] = f1_scores\n",
    "    f1_scores_data[:, 1] = np.arange(0,len(f1_scores))\n",
    "\n",
    "    area, d = similaritymeasures.dtw(f1_scores_data, x_ax_data)\n",
    "    return area\n",
    "\n",
    "#get all results, calculate AUC and plot scatter plots based on a certain dimension\n",
    "all_results =pd.DataFrame(columns=['method','task','AUC', \n",
    "                                   'F1_85','F1_150', 'F1_last_it', 'STD_last_it'])\n",
    "\n",
    "result_type = 'micro' \n",
    "methods = ['HeALER','ALMSER', 'ALMSERgroup'] \n",
    "\n",
    "finished_settings =0\n",
    "\n",
    "for setting_path in listdir(main_path):\n",
    "   \n",
    "    for res_file in listdir(main_path+'/'+setting_path+\"/results/\"):\n",
    "        if res_file == \"ALMSERgroup.csv\":\n",
    "            file_almser_group=main_path+'/'+setting_path+\"/results/\"+res_file\n",
    "        elif res_file == \"ALMSER.csv\":\n",
    "            file_almser=main_path+'/'+setting_path+\"/results/\"+res_file\n",
    "        elif res_file == \"HeALER.csv\":\n",
    "            file_healer=main_path+'/'+setting_path+\"/results/\"+res_file\n",
    "       \n",
    "    \n",
    "    results_almser_group = pd.read_csv(file_almser_group)\n",
    "    results_almser= pd.read_csv(file_almser)\n",
    "    results_healer= pd.read_csv(file_healer)\n",
    "\n",
    "    \n",
    "    auc_results = dict()\n",
    "    f1_last_it = dict()\n",
    "    f1_at_85 = dict()\n",
    "    f1_at_150 = dict()\n",
    "    std_last_it = dict()\n",
    "    \n",
    "    #HeALER\n",
    "    f1_healer = results_healer['F1_'+result_type]\n",
    "    auc_results['HeALER'] = getAUC(f1_healer,ignore_first)  \n",
    "    f1_last_it['HeALER'] = f1_healer.tail(1).values[0]\n",
    "    f1_at_85['HeALER'] = f1_healer[85]\n",
    "    f1_at_150['HeALER'] = f1_healer[150]\n",
    "    std_last_it['HeALER'] = results_healer['F1_'+result_type+'_std'].tail(1).values[0]\n",
    " \n",
    "    \n",
    "    #ALMSER\n",
    "    f1_almser = results_almser['F1_'+result_type]\n",
    "    auc_results['ALMSER'] = getAUC(f1_almser,ignore_first)  \n",
    "    f1_last_it['ALMSER'] = f1_almser.tail(1).values[0]\n",
    "    f1_at_85['ALMSER'] = f1_almser[85]\n",
    "    f1_at_150['ALMSER'] = f1_almser[150]\n",
    "    std_last_it['ALMSER'] = results_almser['F1_'+result_type+'_std'].tail(1).values[0]\n",
    " \n",
    "     #ALMSERgroup\n",
    "    f1_almsergroup = results_almser_group['F1_'+result_type]\n",
    "    auc_results['ALMSERgroup'] = getAUC(f1_almsergroup,ignore_first)  \n",
    "    f1_last_it['ALMSERgroup'] = f1_almsergroup.tail(1).values[0]\n",
    "    f1_at_85['ALMSERgroup'] = f1_almsergroup[85]\n",
    "    f1_at_150['ALMSERgroup'] = f1_almsergroup[150]\n",
    "    std_last_it['ALMSERgroup'] = results_almser_group['F1_'+result_type+'_std'].tail(1).values[0]\n",
    " \n",
    "    \n",
    "    for method in methods:\n",
    "        all_results = all_results.append({'task':setting_path,'method':method, 'AUC':auc_results[method],\n",
    "                                          'F1_85':f1_at_85[method],'F1_150':f1_at_150[method], \n",
    "                                          'F1_last_it': f1_last_it[method], 'STD_last_it':std_last_it[method]}, ignore_index=True)\n",
    "\n",
    "display(all_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Calculate VH and EO profiling dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#todo calculate value heterogeneity for each data source\n",
    "for setting_path in listdir(main_path):\n",
    "    #read train test\n",
    "    train_ = pd.read_csv(main_path+'/'+setting_path+'/train_pairs_fv.csv')\n",
    "    test_ = pd.read_csv(main_path+'/'+setting_path+'/test_pairs_fv.csv')\n",
    "    pairs_fv = pd.concat([train_,test_])    \n",
    "    matching_pairs = pairs_fv[pairs_fv.label]\n",
    "\n",
    "    print(\"Matching Pairs: \", matching_pairs.shape[0])\n",
    "\n",
    "    #read all sources\n",
    "    data_sources = dict()\n",
    "    for source_path in listdir(main_path+'/'+setting_path+'/sources'):\n",
    "        print(\"Read source \",source_path)\n",
    "        source_name = source_path.replace('.csv','')\n",
    "        if 'computers' in setting_path: sep=';'\n",
    "        else: sep=','\n",
    "        source_pd = pd.read_csv(main_path+'/'+setting_path+'/sources/'+source_path, sep=sep)\n",
    "        data_sources[source_name] = source_pd\n",
    "    \n",
    "    Graphtype = nx.Graph()\n",
    "    G = nx.from_pandas_edgelist(matching_pairs, source= 'source', target='target', create_using=Graphtype)\n",
    "\n",
    "    con_components = list(nx.connected_components(G))\n",
    "    print(\"Entities: \", len(con_components))   \n",
    "    \n",
    "    #identifying attributes\n",
    "    id_attr = []\n",
    "    \n",
    "    if 'lspc_computers_mutated' in setting_path : id_attr=['Part Number','title']\n",
    "    elif 'lspc_computers' in setting_path : id_attr=['Part Number']\n",
    "    elif 'restaurants' in setting_path : id_attr=['address','name']\n",
    "    elif 'musicbrainz20K_mutated' in setting_path: id_attr=['title','length','artist','number']\n",
    "    else : id_attr=['title','length','artist','album']\n",
    "\n",
    "    hetereogeneous_entities = 0\n",
    "    con_components_lengths = []\n",
    "    print(\"ID attributes:\", id_attr)\n",
    "\n",
    "    for c in nx.connected_components(G):\n",
    "        con_components_lengths.append(len(c))\n",
    "        entity_nodes=pd.DataFrame()\n",
    "        for node_ in G.subgraph(c).nodes:\n",
    "           \n",
    "            node_id= node_.split('_')[-1]\n",
    "            node_source= node_.replace('_'+node_id,'')\n",
    "            if 'restaurant' in setting_path:\n",
    "                node_desc = copy(data_sources[node_source][data_sources[node_source].id==node_][id_attr])\n",
    "            else: \n",
    "                node_desc = copy(data_sources[node_source][data_sources[node_source].id==int(node_id)][id_attr])\n",
    "            entity_nodes =pd.concat([entity_nodes,node_desc])\n",
    "        is_heter=True\n",
    "        for id_attr_ in id_attr:\n",
    "            if len(set(list(entity_nodes[id_attr_].values)))<entity_nodes.shape[0]:\n",
    "                is_heter=False\n",
    "                \n",
    "                continue;\n",
    "                        \n",
    "        if is_heter: hetereogeneous_entities+=1\n",
    "\n",
    "    print(Counter(con_components_lengths))\n",
    "\n",
    "    print(\"Heteregoneous entities:%i (%.2f)\" % (hetereogeneous_entities,hetereogeneous_entities/len(con_components)))\n",
    "display(all_results)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}