meetup_groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time as time\n",
    "import pickle\n",
    "from sklearn.utils import shuffle\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = \"data/meetups\"\n",
    "min_groupsize = 2\n",
    "max_groupsize = 500\n",
    "num_categories = 33\n",
    "group_blacklist = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "members = pd.read_csv(datadir+'/members.csv',delimiter = ',' , encoding = 'unicode_escape')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "membersSF = members[members.city == \"San Francisco\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(len(membersSF))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_sizes = {}\n",
    "for index, row in membersSF.iterrows():\n",
    "    group_id = str(row[\"group_id\"])\n",
    "    if group_id in group_sizes:\n",
    "        group_sizes[group_id] += 1\n",
    "    else:\n",
    "        group_sizes[group_id] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for group in group_sizes:\n",
    "    if group_sizes[group]>max_groupsize or group_sizes[group]<min_groupsize:\n",
    "        group_blacklist.add(group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = pd.read_csv(datadir+'/groups.csv')\n",
    "grouplookup = {}\n",
    "for index, row in groups.iterrows():\n",
    "    if str(row[\"group_id\"]) not in group_blacklist:\n",
    "        m = str(row[\"category_id\"])\n",
    "        if m == \"36\":\n",
    "            m = \"19\"\n",
    "        if m == \"34\":\n",
    "            m = \"7\"\n",
    "        grouplookup[str(row[\"group_id\"])] = m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperedges = {}\n",
    "for index, row in membersSF.iterrows():\n",
    "    if str(row[\"group_id\"]) not in group_blacklist:\n",
    "        group_id = str(row[\"group_id\"])\n",
    "        if group_id in hyperedges:\n",
    "            hyperedges[group_id][\"members\"].append(str(row[\"member_id\"]))\n",
    "        else:\n",
    "            group = []\n",
    "            group.append(str(row[\"member_id\"]))\n",
    "            hyperedges[group_id] = {}\n",
    "            hyperedges[group_id][\"members\"] = group\n",
    "            hyperedges[group_id][\"category\"] = grouplookup[group_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vertexMemberships = {}\n",
    "for index, row in membersSF.iterrows():\n",
    "    if str(row[\"group_id\"]) not in group_blacklist:\n",
    "        member_id = str(row[\"member_id\"])\n",
    "        if member_id in vertexMemberships:\n",
    "            vertexMemberships[member_id].append(str(row[\"group_id\"]))\n",
    "        else:\n",
    "            memberGroupList = []\n",
    "            memberGroupList.append(str(row[\"group_id\"]))\n",
    "            vertexMemberships[member_id] = memberGroupList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(hyperedges))\n",
    "print(len(vertexMemberships))\n",
    "with open(datadir+'/hyperedges.p', 'wb') as fp:\n",
    "    pickle.dump(hyperedges, fp)\n",
    "with open(datadir+'/vertexMemberships.p', 'wb') as fp:\n",
    "    pickle.dump(vertexMemberships, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_amounts = {}\n",
    "cat_names = {}\n",
    "cat_vals = []\n",
    "cat_csv = pd.read_csv(datadir+'/categories.csv')\n",
    "for index, row in cat_csv.iterrows():\n",
    "    cat_names[str(row[\"category_id\"])] = row[\"category_name\"]\n",
    "    \n",
    "print(cat_names)\n",
    "\n",
    "\n",
    "for h in hyperedges:\n",
    "    if cat_names[str(hyperedges[h][\"category\"])] not in cat_vals:\n",
    "        cat_vals.append(cat_names[str(hyperedges[h][\"category\"])])\n",
    "    if cat_names[str(hyperedges[h][\"category\"])] not in cat_amounts:\n",
    "        cat_amounts[cat_names[str(hyperedges[h][\"category\"])]]=1\n",
    "    else:\n",
    "        cat_amounts[cat_names[str(hyperedges[h][\"category\"])]]+=1\n",
    "\n",
    "pd_df = pd.DataFrame(list(cat_amounts.items()))\n",
    "pd_df.columns =[\"Dim\",\"Count\"]\n",
    "# sort df by Count column\n",
    "pd_df = pd_df.sort_values(['Count']).reset_index(drop=True)\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "# plot barh chart with index as x values\n",
    "ax = sns.barplot(pd_df.index, pd_df.Count)\n",
    "ax.get_yaxis().set_major_formatter(plt.FuncFormatter(lambda x, loc: \"{:,}\".format(int(x))))\n",
    "ax.set(xlabel=\"Dim\", ylabel='Count')\n",
    "# add proper Dim values as x labels\n",
    "ax.set_xticklabels(pd_df.Dim)\n",
    "for item in ax.get_xticklabels(): item.set_rotation(90)\n",
    "for i, v in enumerate(pd_df[\"Count\"].iteritems()):        \n",
    "    ax.text(i ,v[1], \"{:,}\".format(v[1]), color='m', va ='bottom', rotation=45)\n",
    "print(cat_vals)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cat_names = {}\n",
    "# cat_csv = pd.read_csv(datadir+'/categories.csv')\n",
    "# for index, row in cat_csv.iterrows():\n",
    "#     cat_names[str(row[\"category_id\"])] = row[\"category_name\"]\n",
    "# cat_vals = [cat_names[key] for key in cat_names]\n",
    "# print(cat_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for h_index in hyperedges:\n",
    "#     hyperedge = hyperedges[h_index]\n",
    "#     if hyperedge[\"category\"]==\"2\" or hyperedge[\"category\"]==\"34\":\n",
    "#         hyperedge[\"category\"] = \"0\"\n",
    "#     else:\n",
    "#         hyperedge[\"category\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HypergraphRandomWalks import SubsampleAndTraverse\n",
    "\n",
    "walksSAT = SubsampleAndTraverse(length=50, \n",
    "                                   num_walks=50, \n",
    "                                   hyperedges=hyperedges, \n",
    "                                   vertexMemberships=vertexMemberships,\n",
    "                                   alpha=1.,\n",
    "                                   beta=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HypergraphRandomWalks import TraverseAndSelect\n",
    "\n",
    "walksTAS = TraverseAndSelect(length=50, \n",
    "                               num_walks=50, \n",
    "                               hyperedges=hyperedges, \n",
    "                               vertexMemberships=vertexMemberships,\n",
    "                               alpha=1.,\n",
    "                               beta=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(datadir+'/walksSAT.p', 'wb') as fp:\n",
    "    pickle.dump(walksSAT, fp)\n",
    "\n",
    "with open(datadir+'/walksTAS.p', 'wb') as fp:\n",
    "    pickle.dump(walksTAS, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Embeddings import EmbedWord2Vec\n",
    "\n",
    "vertex_embedding_dimension = 16\n",
    "hyperedge_embedding_dimension = 128\n",
    "\n",
    "vertex_ids, vertex_embeddings = EmbedWord2Vec(walks=walksSAT,dimension=vertex_embedding_dimension)\n",
    "print(\"Vertex embeddings finished.\")\n",
    "hyperedge_ids, hyperedge_embeddings = EmbedWord2Vec(walks=walksTAS,dimension=hyperedge_embedding_dimension)\n",
    "print(\"Hyperedge embeddings finished.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(vertex_embeddings))\n",
    "with open(datadir+'/vertex_embeddings_'+str(max_groupsize)+'.p', 'wb') as fp:\n",
    "    pickle.dump(vertex_embeddings, fp)\n",
    "print(len(hyperedge_embeddings))\n",
    "with open(datadir+'/hyperedge_embeddings_'+str(max_groupsize)+'.p', 'wb') as fp:\n",
    "    pickle.dump(hyperedge_embeddings, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(datadir+'/hyperedges.p', 'rb') as rb:\n",
    "#      hyperedges = pickle.load(rb)\n",
    "# with open(datadir+'/vertexMemberships.p', 'rb') as rb:\n",
    "#     vertexMemberships = pickle.load(rb)\n",
    "# with open(datadir+'/walks_hyperedge.p', 'rb') as rb:\n",
    "#     walks_hyperedge = pickle.load(rb)\n",
    "# with open(datadir+'/walk_labels_hyperedge.p', 'rb') as rb:\n",
    "#     walk_labels_hyperedge = pickle.load(rb)\n",
    "# with open(datadir+'/walks_node.p', 'rb') as rb:\n",
    "#     walks_node = pickle.load(rb)\n",
    "# with open (datadir+'/node_embeddings.p', 'rb') as rb:\n",
    "#     node_embeddings = pickle.load(rb)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "def visualizePCA(embeddings,obj,with_labels=None,ids=None):\n",
    "    transform = PCA\n",
    "\n",
    "    trans = transform(n_components=2)\n",
    "    embeddings_2d = trans.fit_transform(embeddings)\n",
    "\n",
    "    alpha = 0.7\n",
    "    plt.figure(figsize=(7,7))\n",
    "    plt.axes().set(aspect=\"equal\")\n",
    "    \n",
    "    if with_labels:\n",
    "        targets = [int(with_labels[identifier]['category'])-1 for identifier in ids]\n",
    "        cmap = cm.rainbow(np.linspace(0.0, 1.0, num_categories))\n",
    "        colors = cmap[targets]\n",
    "        \n",
    "        plt.scatter(embeddings_2d[:,0], \n",
    "                    embeddings_2d[:,1], \n",
    "                    cmap=\"jet\", c=colors, alpha=alpha)\n",
    "    else:\n",
    "        plt.scatter(embeddings_2d[:,0], \n",
    "                    embeddings_2d[:,1], \n",
    "                    cmap=\"jet\", alpha=alpha)\n",
    "    \n",
    "    plt.title('PCA visualization of '+obj+' embeddings in the hypergraph.'.format(transform.__name__))\n",
    "    plt.show()\n",
    "    \n",
    "def visualizeTSNE(embeddings,obj,with_labels=None,ids=None):\n",
    "    transform = TSNE\n",
    "\n",
    "    trans = transform(n_components=2)\n",
    "    embeddings_2d = trans.fit_transform(embeddings)\n",
    "\n",
    "    alpha = 0.7\n",
    "    plt.figure(figsize=(7,7))\n",
    "    plt.axes().set(aspect=\"equal\")\n",
    "    \n",
    "    if with_labels:\n",
    "        targets = [int(with_labels[identifier]['category'])-1 for identifier in ids]\n",
    "        cmap = cm.rainbow(np.linspace(0.0, 1.0, num_categories))\n",
    "        colors = cmap[targets]\n",
    "        \n",
    "        plt.scatter(embeddings_2d[:,0], \n",
    "                    embeddings_2d[:,1], \n",
    "                    cmap=\"jet\", c=colors, alpha=alpha)\n",
    "    else:\n",
    "        plt.scatter(embeddings_2d[:,0], \n",
    "                    embeddings_2d[:,1], \n",
    "                    cmap=\"jet\", alpha=alpha)\n",
    "    \n",
    "    plt.title('TSNE visualization of '+obj+' embeddings in the hypergraph.'.format(transform.__name__))\n",
    "    plt.show()\n",
    "    \n",
    "visualizePCA(vertex_embeddings,\"vertex\")\n",
    "visualizeTSNE(hyperedge_embeddings,\"hyperedge\",hyperedges,hyperedge_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTrainingData():\n",
    "    i = 0\n",
    "    lists = []\n",
    "    labels = []\n",
    "    maxi = 0\n",
    "    for h in hyperedges:\n",
    "        \n",
    "        vertex_embedding_list = []\n",
    "        hyperedge = hyperedges[h]\n",
    "        for vertex in hyperedge[\"members\"]:\n",
    "            i+=1\n",
    "            if i%100000==0:\n",
    "                print(i)\n",
    "            try: # Good ol' nondeterminism\n",
    "                vertex_embedding_list.append(vertex_embeddings[vertex_ids.index(vertex)].tolist())\n",
    "            except:\n",
    "                print(\"Missed one: \",vertex)\n",
    "        lists.append({\"v\":vertex_embedding_list,\"h\":hyperedge_embeddings[hyperedge_ids.index(h)].tolist()})\n",
    "        label = np.zeros((num_categories,))\n",
    "        label[int(hyperedge[\"category\"])-1] = 1\n",
    "        labels.append(label)\n",
    "    X_unshuffled = []\n",
    "    \n",
    "    for hlist in lists:\n",
    "        np_vertex_embeddings = np.asarray(hlist[\"v\"])\n",
    "        x = np.zeros((hyperedge_embedding_dimension + vertex_embedding_dimension*max_groupsize,))\n",
    "        i = 0\n",
    "        x[:hyperedge_embedding_dimension] = hlist[\"h\"]\n",
    "        \n",
    "        for embedding in np_vertex_embeddings:\n",
    "            x[hyperedge_embedding_dimension + i*embedding.shape[0]:hyperedge_embedding_dimension + (i+1)*embedding.shape[0]] = embedding\n",
    "            i+=1\n",
    "        X_unshuffled.append(x)\n",
    "    labels = np.asarray(labels)\n",
    "    X_arr, Y_arr = shuffle(X_unshuffled, labels)\n",
    "    X = np.asarray(X_arr)\n",
    "    Y = np.asarray(Y_arr)\n",
    "    return X, Y\n",
    "\n",
    "X, Y = getTrainingData()\n",
    "\n",
    "print(X.shape)\n",
    "print(Y.shape)\n",
    "\n",
    "# np.save(datadir+'/X_hyperedge_'+str(max_groupsize)+'.npy', X)\n",
    "# np.save(datadir+'/Y_hyperedge_'+str(max_groupsize)+'.npy', Y)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.9, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from Models import DeepHyperedges\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras.utils import plot_model\n",
    "\n",
    "dataset_name = \"meetup\"\n",
    "batch_size = 16\n",
    "num_epochs = 50\n",
    "\n",
    "deephyperedges_model = DeepHyperedges(vertex_embedding_dimension=vertex_embedding_dimension,\n",
    "                                     hyperedge_embedding_dimension=hyperedge_embedding_dimension,\n",
    "                                     max_hyperedge_size=max_groupsize,\n",
    "                                     num_outputs=num_categories,\n",
    "                                     dataset_name=dataset_name)\n",
    "plot_model(deephyperedges_model, to_file='images/'+dataset_name+'_deephyperedges_model.png')\n",
    "checkpointer = ModelCheckpoint(filepath='weights/'+dataset_name+'/deephyperedges_weights.hdf5', verbose=0, save_best_only=True)\n",
    "tbCallBack = TensorBoard(log_dir='logs/'+dataset_name+'/deephyperedges_logs_multi', histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=True, write_images=True, update_freq='batch')\n",
    "\n",
    "history = deephyperedges_model.fit(X_train, Y_train, epochs=num_epochs, batch_size=batch_size,\n",
    "        shuffle=True, validation_split=0.1,\n",
    "        callbacks=[checkpointer,tbCallBack], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "target_names = cat_vals\n",
    "print(target_names)\n",
    "y_pred = deephyperedges_model.predict(X_test, batch_size=16, verbose=1)\n",
    "finals_pred = []\n",
    "finals_test = []\n",
    "for p in y_pred:\n",
    "    m = 0\n",
    "    ind = 0\n",
    "    final = 0\n",
    "    for i in p:\n",
    "        if i>m:\n",
    "            m=i\n",
    "            final=ind\n",
    "        ind+=1\n",
    "    finals_pred.append(final)\n",
    "\n",
    "for i in Y_test:\n",
    "    ind=0\n",
    "    for j in i:\n",
    "        if j==1:\n",
    "            finals_test.append(ind)\n",
    "        ind+=1\n",
    "           \n",
    "print(classification_report(finals_test, finals_pred, target_names=target_names[:-1]))\n",
    "print(accuracy_score(finals_test, finals_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMLPTrainingData():\n",
    "    i = 0\n",
    "    lists = []\n",
    "    labels = []\n",
    "    maxi = 0\n",
    "    for h in hyperedges:\n",
    "        vertex_embedding_list = []\n",
    "        hyperedge = hyperedges[h]\n",
    "        lists.append({\"h\":hyperedge_embeddings[hyperedge_ids.index(h)].tolist()})\n",
    "        label = np.zeros((num_categories,))\n",
    "        label[int(hyperedge[\"category\"])-1] = 1\n",
    "        labels.append(label)\n",
    "    X_unshuffled = []\n",
    "    for hlist in lists:\n",
    "        x = np.zeros((hyperedge_embedding_dimension,))\n",
    "        x[:hyperedge_embedding_dimension] = hlist[\"h\"]\n",
    "        X_unshuffled.append(x)\n",
    "    labels = np.asarray(labels)\n",
    "    X_arr, Y_arr = shuffle(X_unshuffled, labels)\n",
    "    X_MLP = np.asarray(X_arr)\n",
    "    Y_MLP = np.asarray(Y_arr)\n",
    "    return X_MLP, Y_MLP\n",
    "\n",
    "X_MLP, Y_MLP = getMLPTrainingData()\n",
    "# np.save(datadir+'/X_MLP_'+str(max_groupsize)+'.npy', X_MLP)\n",
    "# np.save(datadir+'/Y_MLP_'+str(max_groupsize)+'.npy', Y_MLP)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_MLP_train, X_MLP_test, Y_MLP_train, Y_MLP_test = train_test_split(X_MLP, Y_MLP, train_size=0.9, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%autoreload 2\n",
    "from Models import MLP\n",
    "dataset_name = \"meetup\"\n",
    "batch_size = 16\n",
    "num_epochs = 50\n",
    "\n",
    "MLP_model = MLP(input_dimension=hyperedge_embedding_dimension,\n",
    "                 num_outputs=num_categories,\n",
    "                 dataset_name=dataset_name)\n",
    "plot_model(MLP_model, to_file='images/'+dataset_name+'_MLP_model.png')\n",
    "checkpointer = ModelCheckpoint(filepath='weights/'+dataset_name+'/MLP_weights.hdf5', verbose=0, save_best_only=True)\n",
    "tbCallBack = TensorBoard(log_dir='logs/'+dataset_name+'/MLP_logs_multi', histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=True, write_images=True, update_freq='batch')\n",
    "\n",
    "history = MLP_model.fit(X_MLP_train, Y_MLP_train, epochs=num_epochs, batch_size=batch_size,\n",
    "        shuffle=True, validation_split=0.1,\n",
    "        callbacks=[checkpointer,tbCallBack], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_MLP = MLP_model.predict(X_MLP_test, batch_size=16, verbose=1)\n",
    "finals_pred_MLP = []\n",
    "finals_test_MLP = []\n",
    "for p in Y_pred_MLP:\n",
    "    m = 0\n",
    "    ind = 0\n",
    "    final = 0\n",
    "    for i in p:\n",
    "        if i>m:\n",
    "            m=i\n",
    "            final=ind\n",
    "        ind+=1\n",
    "    finals_pred_MLP.append(final)\n",
    "\n",
    "for i in Y_MLP_test:\n",
    "    ind=0\n",
    "    for j in i:\n",
    "        if j==1:\n",
    "            finals_test_MLP.append(ind)\n",
    "        ind+=1\n",
    "            \n",
    "print(classification_report(finals_test_MLP, finals_pred_MLP, target_names=target_names[:-1]))\n",
    "print(accuracy_score(finals_test_MLP, finals_pred_MLP))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTrainingData():\n",
    "    i = 0\n",
    "    lists = []\n",
    "    labels = []\n",
    "    maxi = 0\n",
    "    for h in hyperedges:\n",
    "        vertex_embedding_list = []\n",
    "        hyperedge = hyperedges[h]\n",
    "        for vertex in hyperedge[\"members\"]:\n",
    "            i+=1\n",
    "            if i%100000==0:\n",
    "                print(i)\n",
    "            try: # Good ol' nondeterminism\n",
    "                vertex_embedding_list.append(vertex_embeddings[vertex_ids.index(vertex)].tolist())\n",
    "            except:\n",
    "                print(\"Missed one: \",vertex)\n",
    "        lists.append({\"v\":vertex_embedding_list})\n",
    "        label = np.zeros((num_categories,))\n",
    "        label[int(hyperedge[\"category\"])-1] = 1\n",
    "        labels.append(label)\n",
    "    X_unshuffled = []\n",
    "    \n",
    "    for hlist in lists:\n",
    "        np_vertex_embeddings = np.asarray(hlist[\"v\"])\n",
    "        x = np.zeros((vertex_embedding_dimension*max_groupsize,))\n",
    "        i = 0        \n",
    "        for embedding in np_vertex_embeddings:\n",
    "            x[i*embedding.shape[0]:(i+1)*embedding.shape[0]] = embedding\n",
    "            i+=1\n",
    "        X_unshuffled.append(x)\n",
    "    labels = np.asarray(labels)\n",
    "    X_arr, Y_arr = shuffle(X_unshuffled, labels)\n",
    "    X = np.asarray(X_arr)\n",
    "    Y = np.asarray(Y_arr)\n",
    "    return X, Y\n",
    "\n",
    "X_deepset, Y_deepset = getTrainingData()\n",
    "\n",
    "print(X_deepset.shape)\n",
    "print(Y_deepset.shape)\n",
    "\n",
    "# np.save(datadir+'/X_deepset_'+str(max_groupsize)+'.npy', X_deepset)\n",
    "# np.save(datadir+'/Y_deepset_'+str(max_groupsize)+'.npy', Y_deepset)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_deepset_train, X_deepset_test, Y_deepset_train, Y_deepset_test = train_test_split(X_deepset, Y_deepset, train_size=0.9, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from Models import DeepSets\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras.utils import plot_model\n",
    "\n",
    "dataset_name = \"meetup\"\n",
    "batch_size = 16\n",
    "num_epochs = 50\n",
    "\n",
    "deepsets_model = DeepSets(vertex_embedding_dimension=vertex_embedding_dimension,\n",
    "                             max_hyperedge_size=max_groupsize,\n",
    "                             num_outputs=num_categories,\n",
    "                             dataset_name=dataset_name)\n",
    "plot_model(deepsets_model, to_file='images/'+dataset_name+'_deepsets_model.png')\n",
    "checkpointer = ModelCheckpoint(filepath='weights/'+dataset_name+'/deepsets_weights.hdf5', verbose=0, save_best_only=True)\n",
    "tbCallBack = TensorBoard(log_dir='logs/'+dataset_name+'/deepsets_logs_multi', histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=True, write_images=True, update_freq='batch')\n",
    "\n",
    "history = deepsets_model.fit(X_deepset_train, Y_deepset_train, epochs=num_epochs, batch_size=batch_size,\n",
    "        shuffle=True, validation_split=0.1,\n",
    "        callbacks=[checkpointer,tbCallBack], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "Y_pred_deepsets = deepsets_model.predict(X_deepset_test, batch_size=16, verbose=1)\n",
    "target_names = cat_vals\n",
    "finals_pred_deepsets = []\n",
    "finals_test_deepsets = []\n",
    "for p in Y_pred_deepsets:\n",
    "    m = 0\n",
    "    ind = 0\n",
    "    final = 0\n",
    "    for i in p:\n",
    "        if i>m:\n",
    "            m=i\n",
    "            final=ind\n",
    "        ind+=1\n",
    "    finals_pred_deepsets.append(final)\n",
    "\n",
    "for i in Y_deepset_test:\n",
    "    ind=0\n",
    "    for j in i:\n",
    "        if j==1:\n",
    "            finals_test_deepsets.append(ind)\n",
    "        ind+=1\n",
    "            \n",
    "print(classification_report(finals_test_deepsets, finals_pred_deepsets, target_names=target_names[:-1]))\n",
    "print(accuracy_score(finals_test_deepsets, finals_pred_deepsets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}