Data evaluation and visualization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Vizualizacija podatkov\n",
        "\n",
        "## Okolje\n",
        "\n"
      ],
      "metadata": {
        "id": "Az_ogsaWk-7U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install bertopic\n",
        "!pip install --upgrade joblib==1.1.0 \n",
        "!pip install matplotlib --upgrade\n",
        "!pip install classla\n",
        "!pip install tweet-preprocessor\n",
        "!pip install flair"
      ],
      "metadata": {
        "id": "b72dEmDMmK87"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ByJVLWq3k7ro"
      },
      "outputs": [],
      "source": [
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import string\n",
        "import copy\n",
        "import json\n",
        "import time\n",
        "import random\n",
        "import nltk\n",
        "import classla\n",
        "import preprocessor as tpre\n",
        "nltk.download('stopwords')\n",
        "#classla.download('sl')\n",
        "import re\n",
        "from collections import Counter\n",
        "from nltk.corpus import stopwords\n",
        "from wordcloud import WordCloud\n",
        "\n",
        "import matplotlib.patheffects as pe\n",
        "\n",
        "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "from transformers import AutoTokenizer, AutoModelForMaskedLM, AutoModelForSequenceClassification, pipeline\n",
        "\n",
        "from bertopic import BERTopic\n",
        "from flair.embeddings import TransformerDocumentEmbeddings\n",
        "from transformers.pipelines import pipeline\n",
        "from umap import UMAP\n",
        "from hdbscan import HDBSCAN\n",
        "from bertopic.vectorizers import ClassTfidfTransformer\n",
        "\n",
        "import plotly.express as px\n",
        "import plotly.graph_objects as go\n",
        "\n",
        "from gensim.models.coherencemodel import CoherenceModel\n",
        "from gensim.corpora.dictionary import Dictionary\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')\n",
        "\n",
        "from google.colab import output\n",
        "output.disable_custom_widget_manager()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setting constants\n",
        "\n",
        "LOCAL = False\n",
        "\n",
        "google_data_dir = \"/content/drive/MyDrive/Diploma/Data\"\n",
        "local_data_dir = \"/data\"\n",
        "\n",
        "root_dir = \"\"\n",
        "if LOCAL:\n",
        "    root_dir = local_data_dir\n",
        "else:\n",
        "    root_dir = google_data_dir"
      ],
      "metadata": {
        "id": "tLmJ_Buel9_0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Matplotlib settings\n",
        "\n",
        "font = {'weight' : 'normal', 'size'   : 14}\n",
        "\n",
        "matplotlib.rc('font', **font)"
      ],
      "metadata": {
        "id": "slgL5-WKQVIW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Funkcije in razredi"
      ],
      "metadata": {
        "id": "w_n7Nhd1l-8o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def load_data(file_name):\n",
        "  \n",
        "  # Load data\n",
        "  data = []\n",
        "\n",
        "  with open(file_name, 'r', encoding='utf8') as sample_data:\n",
        "    data = json.load(sample_data)\n",
        "\n",
        "  return data\n",
        "\n",
        "def load_labelled_tweets(dir, topic_names, shuffle_arrays=True, random_state=77):\n",
        "  topics = []\n",
        "  for t in topic_names:\n",
        "    with open(f'{dir}/labelled_topics/topic_{t}.json', 'r', encoding='utf8') as topic_data:\n",
        "      data = json.load(topic_data)\n",
        "      topics.extend(data)\n",
        "  \n",
        "  topic_lemmas = []\n",
        "  topic_labels = []\n",
        "\n",
        "  topic_labels = [ topic_names.index(x) for x in topic_labels]\n",
        "\n",
        "  if shuffle_arrays:\n",
        "    shuffle(topic_lemmas, topic_labels, random_state=random_state)\n",
        "  return topic_lemmas, topic_labels\n",
        "\n",
        "def preprocess_tweets(preprocess_pipeline, tweets, tweet_stop_words=[], tweet_upos=[], min_words=4, verbose=False, debug=False):\n",
        "\n",
        "  stop_words = stopwords.words('slovene')\n",
        "  stop_words.extend(tweet_stop_words)\n",
        "\n",
        "  # Remove stopwords\n",
        "  stop_words = list(set(stop_words))\n",
        "\n",
        "  data = []\n",
        "  start_time = 0\n",
        "\n",
        "  for index, tweet in enumerate(tweets):\n",
        "\n",
        "    if index%1000 == 0 and verbose:\n",
        "      # Import time\n",
        "      print(f'-- Progress: {index}/{len(tweets)}')\n",
        "      if index >= 1000:\n",
        "        print(f'-- Time elapsed: {time.time() - start_time}s')\n",
        "        print(f'-- Tweets preprocessed: {len(data)}')\n",
        "      start_time = time.time()\n",
        "\n",
        "    # Take attributes\n",
        "    tweet_full_text = tweet['full_text']\n",
        "\n",
        "    # Skip if retweet\n",
        "    if tweet_full_text.startswith(\"RT\"):\n",
        "      continue\n",
        "\n",
        "    tweet_id = tweet['id']\n",
        "    #tweet_hashtags = tweet['entities']['hashtags']\n",
        "    #tweet_mentions = tweet['entities']['mentions']\n",
        "    tweet_hashtags = tweet['hashtags']\n",
        "    tweet_mentions = tweet['mentions']\n",
        "    tweet_created_at = tweet['created_at']\n",
        "\n",
        "    tweet_user_name = tweet['user']['name']\n",
        "    tweet_user_screen_name = tweet['user']['screen_name']\n",
        "    tweet_user_description = clean_tweet_text(tweet['user']['description']).lower()\n",
        "    \n",
        "    # Remove hashtags, mentions, links, emojis and others\n",
        "    tweet_full_text = clean_tweet_text(tweet_full_text)\n",
        "\n",
        "    if debug:\n",
        "      print(f'Raw text:\\n{tweet_full_text}')\n",
        "\n",
        "    # Preprocess with preprocessing pipeline\n",
        "    tweet_lemma_text = preprocess_pipeline(tweet_full_text)\n",
        "\n",
        "    tweet_processed_lemmas = []\n",
        "    for sentence in tweet_lemma_text.sentences:\n",
        "\n",
        "      sentence_words = []\n",
        "\n",
        "      for i, word in enumerate(sentence.words):\n",
        " \n",
        "        # If the tweet is a retweet\n",
        "        if word.lemma.lower() == 'rt' and i == 0:\n",
        "          break\n",
        "\n",
        "        if debug:\n",
        "          print(f'Lemma: {word.lemma.lower()} -------------- Upos: {word.upos}')\n",
        "\n",
        "        # Not punctuation and not number + clean stopwords\n",
        "        if word.upos not in tweet_upos and word.lemma.lower() not in stop_words:\n",
        "          sentence_words.append(word.lemma.lower())\n",
        "        elif debug:\n",
        "          print(f'Discarded word: {word.lemma}')\n",
        "\n",
        "      tweet_processed_lemmas.extend(sentence_words)\n",
        "    \n",
        "    if debug:\n",
        "      print(f'Preprocessed lemmas:\\n{tweet_processed_lemmas}')\n",
        "\n",
        "    # If less than n words\n",
        "    if len(tweet_processed_lemmas) > min_words:\n",
        "      tweet_data = {\n",
        "          \"id\": tweet_id,\n",
        "          \"created_at\": tweet_created_at,\n",
        "          \"raw_text\": tweet_full_text,\n",
        "          \"lemma_text\": ' '.join(tweet_processed_lemmas),\n",
        "          \"hashtags\": tweet_hashtags,\n",
        "          \"mentions\": tweet_mentions,\n",
        "          \"user\": {\n",
        "              \"name\": tweet_user_name,\n",
        "              \"screen_name\": tweet_user_screen_name,\n",
        "              \"description\": tweet_user_description\n",
        "          }\n",
        "      }\n",
        "\n",
        "      # Append all the data\n",
        "      data.append(tweet_data)\n",
        "    \n",
        "  return data\n",
        "\n",
        "def clean_tweet_text(tweet_text):\n",
        "  tweet_text = tpre.clean(tweet_text)\n",
        "  tweet_text = re.sub(\"&gt;|&lt;|&amp;\", \"\", tweet_text)\n",
        "  tweet_text = remove_emojis(tweet_text)\n",
        "  return tweet_text\n",
        "\n",
        "def remove_emojis(data):\n",
        "    emoj = re.compile(\"[\"\n",
        "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "        u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
        "        u\"\\U00002702-\\U000027B0\"\n",
        "        u\"\\U00002702-\\U000027B0\"\n",
        "        u\"\\U000024C2-\\U0001F251\"\n",
        "        u\"\\U0001f926-\\U0001f937\"\n",
        "        u\"\\U00010000-\\U0010ffff\"\n",
        "        u\"\\u2640-\\u2642\" \n",
        "        u\"\\u2600-\\u2B55\"\n",
        "        u\"\\u200d\"\n",
        "        u\"\\u23cf\"\n",
        "        u\"\\u23e9\"\n",
        "        u\"\\u231a\"\n",
        "        u\"\\ufe0f\"  # dingbats\n",
        "        u\"\\u3030\"\n",
        "                      \"]+\", re.UNICODE)\n",
        "    return re.sub(emoj, '', data)\n",
        "\n",
        "def overwrite_labelled_topics(file_topic, topic_tweets, data_dir):\n",
        "    data=[]\n",
        "    with open(f'{data_dir}/labelled_topics/topic_{file_topic}.json', 'r', encoding='utf8') as topic_data:\n",
        "      data = json.load(topic_data)\n",
        "      data.extend(topic_tweets)\n",
        "    with open(f'{data_dir}/labelled_topics/topic_{file_topic}.json', 'w', encoding='utf8') as topic_data_n:\n",
        "      json.dump(data, topic_data_n, ensure_ascii=False)\n",
        "\n",
        "def overwrite_labelled_tweets(topic, tweets, data_dir):\n",
        "  # Overwrite\n",
        "  data=[]\n",
        "  with open(f'{data_dir}/process/tweets_{topic}.json', 'r', encoding='utf8') as topic_data:\n",
        "    data = json.load(topic_data)\n",
        "    data.extend(tweets)\n",
        "  with open(f'{data_dir}/process/tweets_{topic}.json', 'w', encoding='utf8') as topic_data_n:\n",
        "    json.dump(data, topic_data_n, ensure_ascii=False)\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "Bertopic model for modeling topics\n",
        "\n",
        "\"\"\"\n",
        "class BertopicModel:\n",
        "\n",
        "  def __init__(self, model_name, embed_model, config):\n",
        "    self.model_name = model_name\n",
        "    self.embed_model = embed_model\n",
        "    self.config = config\n",
        "\n",
        "    # Create the Bertopic model with config\n",
        "    self.make_model()\n",
        "\n",
        "  def make_model(self):\n",
        "    self.umap_model = UMAP(**self.config[\"umap_conf\"])\n",
        "    self.hdbscan_model = HDBSCAN(**self.config[\"hdbscan_conf\"])\n",
        "    self.bertopic = BERTopic(embedding_model=self.embed_model, umap_model=self.umap_model, hdbscan_model=self.hdbscan_model, **self.config[\"bertopic_conf\"])\n",
        "\n",
        "  def load_tweet_data(self, tweet_data):\n",
        "    doc_tweet_lemmas = [ t['lemma_text'] for t in tweet_data ]\n",
        "    \n",
        "    self.data = {}\n",
        "    self.data[\"tweets\"] = tweet_data\n",
        "    self.data[\"docs\"] = doc_tweet_lemmas\n",
        "\n",
        "  def load_topic_data(self, topic_docs, topic_labels):\n",
        "    if not hasattr(self, 'data'):\n",
        "      self.data = {}\n",
        "      \n",
        "    self.data[\"docs\"] = topic_docs\n",
        "    self.data[\"labels\"] = topic_labels\n",
        "\n",
        "  def train_model(self, only_fit):\n",
        "    data_keys = self.data.keys()\n",
        "\n",
        "    if \"docs\" in data_keys and not only_fit:\n",
        "      topics, probs = self.bertopic.fit_transform(self.data[\"docs\"])\n",
        "      self.result = {}\n",
        "      self.result[\"topic_ids\"] = topics\n",
        "      self.result[\"topic_probs\"] = probs\n",
        "    elif \"docs\" in data_keys and \"labels\" in data_keys and only_fit:\n",
        "      self.bertopic = self.bertopic.fit(self.data[\"docs\"], y=self.data[\"labels\"])\n",
        "    else:\n",
        "      print(\"Error: Missing data!\")\n",
        "  \n",
        "  def predict(self):\n",
        "    data_keys = self.data.keys()\n",
        "\n",
        "    if \"docs\" in data_keys:\n",
        "      topics, probs = self.bertopic.transform(self.data[\"docs\"])\n",
        "      self.result = {}\n",
        "      self.result[\"topic_ids\"] = topics\n",
        "      self.result[\"topic_probs\"] = probs\n",
        "\n",
        "  def reduce(self, nr):\n",
        "    if hasattr(self, 'data') and hasattr(self, 'result'):\n",
        "      topics, probs = self.bertopic.reduce_topics(self.data[\"docs\"], self.data[\"labels\"], nr_topics=nr)\n",
        "      self.result[\"topic_ids\"] = topics\n",
        "      self.result[\"topic_probs\"] = probs\n",
        "\n",
        "  def merge_topics(self, indexes):\n",
        "    if hasattr(self, 'data'):\n",
        "      self.bertopic.merge_topics(self.data[\"docs\"], self.data[\"labels\"], indexes)\n",
        "\n",
        "  def tweets_from_topic(self, ntopic):\n",
        "    if self.result:\n",
        "      tw = []\n",
        "\n",
        "      for i, x in enumerate(self.result.topic_ids):\n",
        "        if x == ntopic:\n",
        "          tw.append(self.data[\"docs\"][i])\n",
        "\n",
        "      return tw\n",
        "    else:\n",
        "      print(\"Error: Missing data!\")\n",
        "      return []\n",
        "\n",
        "  def collect_topic_indices(self, ntopic, tweet_prob=0.5):\n",
        "    if hasattr(self, 'result'):\n",
        "\n",
        "      tweet_ids = []\n",
        "      for i, x in enumerate(self.result[\"topic_ids\"]):\n",
        "\n",
        "        # Check if topic id and probability higher\n",
        "        if ntopic == x and self.result[\"topic_probs\"][i] > tweet_prob:\n",
        "          tweet_ids.append(i)\n",
        "\n",
        "      return tweet_ids\n",
        "    else:\n",
        "      print(\"Error: Missing data!\")\n",
        "      return []\n",
        "\n",
        "  def find_politic_topics(self, keywords, topn=3, sim_threshold=0.5, tweet_prob=0.5, include_prob=False):\n",
        "    if hasattr(self, 'bertopic'):\n",
        "      indices = set()\n",
        "\n",
        "      # Find relating topics\n",
        "      for keyword in keywords:\n",
        "        sim_ids, sim_probs = self.bertopic.find_topics(keyword, top_n=topn)\n",
        "\n",
        "        # Filter based on similarity\n",
        "        sim_topics = [ sim_ids[i] for i, x in enumerate(sim_probs) if x > sim_threshold ]\n",
        "\n",
        "        if len(sim_topics) > 0:\n",
        "          for topic in sim_topics:\n",
        "            indices.update(self.collect_topic_indices(topic, tweet_prob=tweet_prob))\n",
        "      \n",
        "      tweet_docs = []\n",
        "      for i in indices:\n",
        "        tdoc = self.data[\"tweets\"][i]\n",
        "        if include_prob:\n",
        "          tdoc[\"topic_probability\"] = self.result[\"topic_probs\"][i]\n",
        "\n",
        "        tweet_docs.append(tdoc)\n",
        "\n",
        "      return tweet_docs\n",
        "    else:\n",
        "      print(\"Error: Missing data!\")\n",
        "      return []\n",
        "\n",
        "  def visualize(self, t='distance_map'):\n",
        "    if hasattr(self, 'bertopic'):\n",
        "      #return self.bertopic.visualize_topics()\n",
        "      if t == 'barchart':\n",
        "        return self.bertopic.visualize_barchart()\n",
        "      elif t == 'hierarchy':\n",
        "        return self.bertopic.visualize_hierarchy()\n",
        "      elif t == 'heatmap':\n",
        "        return self.bertopic.visualize_heatmap()\n",
        "      elif t == 'term_rank':\n",
        "        return self.bertopic.visualize_term_rank()\n",
        "      else:\n",
        "        return self.bertopic.visualize_topics()\n",
        "      #elif t == 'documents':\n",
        "      #  self.bertopic.visualize_documents()\n",
        "    else:\n",
        "      print(\"Error: Model not yet initiated!\")\n",
        "\n",
        "  def save_model(self, model_dir):\n",
        "    self.bertopic.save(str(model_dir + self.model_name))\n",
        "  \n",
        "  def load_model(self, model_dir):\n",
        "    self.bertopic.load(str(model_dir + self.model_name), embedding_model=self.embed_model)\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "Tweetiment Model\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "class TweetimentModel:\n",
        "  def __init__(self, name, model, tokenizer, topic_bias, party_bias, device=0):\n",
        "    self.model_name = name\n",
        "\n",
        "    self.topic_bias = topic_bias\n",
        "    self.party_bias = party_bias\n",
        "\n",
        "    self.labels = [\"levo\", \"desno\", \"nevtralno\"]\n",
        "    \n",
        "    self.tokenizer = tokenizer\n",
        "    self.model = model\n",
        "\n",
        "    # Create the pipeline\n",
        "    self.make_model(device)\n",
        "  \n",
        "  def make_model(self, device):\n",
        "    self.tweetiment = pipeline(\"sentiment-analysis\", model=self.model, tokenizer=self.tokenizer, device=device)\n",
        "\n",
        "  def predict_text(self, txt):\n",
        "    if hasattr(self, 'tweetiment'):\n",
        "      return self.tweetiment(txt)\n",
        "\n",
        "  def classify(self, bias_party, bias_topic):\n",
        "\n",
        "    if bias_party is None and bias_topic is None:\n",
        "      return self.labels[2]\n",
        "    elif bias_party is None:\n",
        "      return bias_topic\n",
        "    elif bias_topic is None:\n",
        "      return bias_party\n",
        "    \n",
        "    return bias_topic\n",
        "\n",
        "  def calculate_biases(self, tweet, explain=False):\n",
        "    if tweet['raw_text']:\n",
        "      prediction = self.predict_text(tweet['raw_text'])[0]\n",
        "\n",
        "      bias_party, party = self.bias_sentiment_party(prediction, tweet)\n",
        "      #bias_user = self.bias_user(prediction)\n",
        "      bias_topic, topic = self.bias_sentiment_topic(prediction, tweet)\n",
        "\n",
        "      if explain:\n",
        "        explanation = self.make_explanation(prediction['label'], bias_party, party, bias_topic, topic)\n",
        "      \n",
        "      label = self.classify(bias_party, bias_topic)\n",
        "\n",
        "      return {\n",
        "          'label': label,\n",
        "          'sentiment': prediction['label'].lower(),\n",
        "          'sentiment_score': prediction['score'],\n",
        "          'topic_bias': bias_topic,\n",
        "          'topic_mentioned': topic,\n",
        "          'topic_score': tweet['topic_probability'],\n",
        "          'party_bias': bias_party,\n",
        "          'party_mentioned': party,\n",
        "      }\n",
        "    return None\n",
        "\n",
        "  # Bias based on negativity/positivity towards a party mentioned in a tweet\n",
        "  def bias_sentiment_party(self, prediction, tweet, single=True):\n",
        "    \"\"\"\n",
        "    Args:\n",
        "      single (bool): Detect only a single party in tweet\n",
        "      \n",
        "    Returns:\n",
        "      bias\n",
        "    \"\"\"\n",
        "    bias = None\n",
        "    party_detected = None\n",
        "    parties_mentioned = 0\n",
        "\n",
        "    for party in self.party_bias:\n",
        "      \n",
        "      for mention in tweet['mentions']:\n",
        "        # Check for mentions or in lemma text\n",
        "        if mention in party['clani'] or party['kratica_stranke'].lower() in tweet['lemma_text'].split(\" \"):\n",
        "          parties_mentioned = parties_mentioned+1\n",
        "          if parties_mentioned == 1:\n",
        "            party_detected = party\n",
        "          break\n",
        "\n",
        "    if single and parties_mentioned == 1 and party_detected is not None:\n",
        "\n",
        "      # If text is neutral\n",
        "      if prediction['label'] == \"Neutral\":\n",
        "        bias = self.labels[2]\n",
        "      # Supports the party\n",
        "      elif prediction['label'] == \"Positive\":\n",
        "        bias = self.labels[party_detected['usmerjenost']]\n",
        "      # Opposes the party\n",
        "      elif prediction['label'] == \"Negative\":\n",
        "        bias = self.labels[int(not party_detected['usmerjenost'])]\n",
        "\n",
        "      return bias, party_detected['kratica_stranke']\n",
        "\n",
        "    # If no parties are mentioned in a tweet\n",
        "    return None, None\n",
        "  \n",
        "  # Bias based on negativity/positivity towards a certain topic of the tweet\n",
        "  def bias_sentiment_topic(self, prediction, tweet):\n",
        "    \"\"\"\n",
        "    Args:\n",
        "\n",
        "    Returns:\n",
        "      bias\n",
        "    \"\"\"\n",
        "    bias = None\n",
        "    topic_detected = None\n",
        "\n",
        "    for topic in self.topic_bias:\n",
        "      if tweet['topic'] == topic and prediction['label'] != 'Neutral':\n",
        "        \n",
        "        bias = self.labels[self.topic_bias[topic][prediction['label'].lower()]]\n",
        "        topic_detected = topic\n",
        "        break\n",
        "\n",
        "    return bias, topic_detected\n",
        "  \n",
        "  # User a known member of a party?\n",
        "  def is_user_in_party(self):\n",
        "    \"\"\"\n",
        "    Args:\n",
        "\n",
        "    Returns:\n",
        "      bias\n",
        "    \"\"\"\n",
        "    return\n",
        "  \n",
        "  # Bias based on the user profile\n",
        "  def bias_user(self, prediction, tweet):\n",
        "    \"\"\"\n",
        "    Args:\n",
        "      only_desc (bool): Analyze description on user profile only\n",
        "\n",
        "    Returns:\n",
        "      bias\n",
        "    \"\"\"\n",
        "    return\n",
        "\n",
        "  def make_explanation(self, sentiment, bias_party, party, bias_topic, topic):\n",
        "    # TODO\n",
        "    return\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "Politic bias model\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "class PoliticBiasModel:\n",
        "  def __init__(self,\n",
        "                name,\n",
        "                working_dir,\n",
        "                preprocess_pipeline,\n",
        "                topic_model,\n",
        "                sentiment_model,\n",
        "                config\n",
        "               ):\n",
        "\n",
        "    self.name = name\n",
        "    self.working_dir = working_dir\n",
        "    self.preprocess_pipeline = preprocess_pipeline\n",
        "    self.config = config\n",
        "\n",
        "    # Make models\n",
        "    self.make_models(topic_model, sentiment_model)\n",
        "\n",
        "  def make_models(self, topic_model, sentiment_model):\n",
        "\n",
        "    # Create Bertopic SL\n",
        "    self.bertopic_SL = BertopicModel(\"Bertopic_SL\", embed_model=topic_model, config=self.config['bertopic_SL_config'])\n",
        "\n",
        "    # Create Tweetiment\n",
        "    self.tweetiment = TweetimentModel(\"Tweetiment\", model=sentiment_model['model'], tokenizer=sentiment_model['tokenizer'], topic_bias=self.config['tweetiment_config']['topic_bias'], party_bias=self.config['tweetiment_config']['party_bias'], device=self.config['tweetiment_config']['device'])\n",
        "\n",
        "  def train_models(self, X_train, y_train, optimization=False):\n",
        "\n",
        "    # Load training data\n",
        "    self.bertopic_SL.load_topic_data(X_train, y_train)\n",
        "\n",
        "    # Train the model\n",
        "    self.bertopic_SL.train_model(only_fit=True)\n",
        "\n",
        "    # Optimize\n",
        "    if optimization:\n",
        "      self.optimize_models(X_train)\n",
        "\n",
        "  def optimize_topics(self, topn=3, n_sim_subtopics=3):\n",
        "    sim_topics = similar_topics(self.bertopic_SL, self.config['topic_info'], topn=topn, n_sim_subtopics=n_sim_subtopics)\n",
        "\n",
        "    all_labels = [ x for x in self.bertopic_SL.bertopic.get_topics()]\n",
        "    to_elim = []\n",
        "    merging = False\n",
        "\n",
        "    for st in sim_topics:\n",
        "      lbl = sim_topics[st]\n",
        "      lbl = [ t for t,p in lbl if p > 0.9]\n",
        "\n",
        "      if len(lbl) > 1:\n",
        "        to_elim.append(lbl)\n",
        "        all_labels = [ x for x in all_labels if x not in lbl]\n",
        "        merging = True\n",
        "\n",
        "    #for e in to_elim:\n",
        "      #all_labels.append(e)\n",
        "    print(to_elim)\n",
        "    if merging:\n",
        "      self.bertopic_SL.merge_topics(to_elim)\n",
        "\n",
        "  def optimize_models(self, docs):\n",
        "    vectorizer_model = CountVectorizer(ngram_range=(1, 1))\n",
        "    self.bertopic_SL.bertopic.update_topics(docs, self.bertopic_SL.result['topic_ids'], vectorizer_model=vectorizer_model)\n",
        "\n",
        "  def bias_pipeline(self, tweets, topic_mapper, topn=3, n_sim_subtopics=4, do_preprocess=True, do_topic_predict=True):\n",
        "    \"\"\"\n",
        "    ## Automated Slovenian Political bias pipeline\n",
        "    \"\"\"\n",
        "\n",
        "    # Preprocess instances\n",
        "    if do_preprocess or do_topic_predict:\n",
        "\n",
        "      if do_preprocess:\n",
        "        tweets = preprocess_tweets(self.preprocess_pipeline, tweets, self.config['preprocess_config']['tweet_stop_words'], self.config['preprocess_config']['tweet_stop_words'], verbose=self.config['verbose'], debug=self.config['debug'])\n",
        "      \n",
        "      if do_topic_predict:\n",
        "        # Load the tweet data\n",
        "        self.bertopic_SL.load_tweet_data(tweets)\n",
        "\n",
        "        # Predict the instances\n",
        "        self.bertopic_SL.predict()\n",
        "\n",
        "        # Topic unlabelled tweets\n",
        "        #unlabelled_tweets = [ tweets[i] for i, tx in enumerate(self.bertopic_SL.result['topic_ids']) if tx == -1]\n",
        "\n",
        "        # Topic labelled tweets\n",
        "        #labelled_tweets = label_politic_tweets(self.bertopic_SL, self.config['topic_info'], self.working_dir, topn=topn, n_sim_subtopics=n_sim_subtopics, save_tweets=False, verbose=self.config['verbose'])\n",
        "\n",
        "        labelled_tweets = []\n",
        "        # Label tweets\n",
        "        for ix, pred_t in enumerate(self.bertopic_SL.result['topic_ids']):\n",
        "          if pred_t in topic_mapper:\n",
        "            predicted_topic = topic_mapper[pred_t]\n",
        "          else:\n",
        "            predicted_topic = -1\n",
        "          \n",
        "          tweet = tweets[ix]\n",
        "          tweet['topic'] = predicted_topic\n",
        "          tweet['topic_probability'] = self.bertopic_SL.result['topic_probs'][ix]\n",
        "          labelled_tweets.append(tweet)\n",
        "          \n",
        "    else:\n",
        "      labelled_tweets = tweets\n",
        "\n",
        "    bias_predictions = []\n",
        "    for t in labelled_tweets:\n",
        "      bias_predictions.append(self.tweetiment.calculate_biases(t))\n",
        "\n",
        "    predictions = []\n",
        "    for twt in tweets:\n",
        "      id = twt['id']\n",
        "      found = False\n",
        "\n",
        "      for ix, lbt in enumerate(labelled_tweets):\n",
        "        if lbt['id'] == id:\n",
        "          predictions.append(bias_predictions[ix])\n",
        "          found = True\n",
        "          break\n",
        "      \n",
        "      if not found:\n",
        "        predictions.append(None)\n",
        "\n",
        "    return labelled_tweets, predictions\n",
        "\n",
        "def label_politic_tweets(model : BertopicModel, topic_info, data_dir, topn=3, n_sim_subtopics=4, save_tweets=False, verbose=True):\n",
        "\n",
        "  if verbose:\n",
        "    print(f'-- Collected batch topic distribution summary:')\n",
        "\n",
        "  sim_topics = similar_topics(model, topic_info, topn=topn, n_sim_subtopics=n_sim_subtopics)\n",
        "\n",
        "  print(sim_topics)\n",
        "\n",
        "  tweets = model.data['tweets']\n",
        "  labels = model.result['topic_ids']\n",
        "  probs = model.result['topic_probs']\n",
        "\n",
        "  ids = set()\n",
        "\n",
        "  for i in range(len(tweets)):\n",
        "    tweet = tweets[i]\n",
        "    label = labels[i]\n",
        "    prob = probs[i]\n",
        "\n",
        "    if tweet['id'] in ids:\n",
        "      continue\n",
        "    else:\n",
        "      ids.add(tweet['id'])\n",
        "\n",
        "    most_likely_topic = None\n",
        "    most_likely_prob = 0\n",
        "\n",
        "    for st in sim_topics:\n",
        "      for sbt, prob in sim_topics[st]:\n",
        "        if sbt == label and prob > most_likely_prob:\n",
        "          most_likely_topic = st\n",
        "          most_likely_prob = prob\n",
        "\n",
        "\n",
        "    if most_likely_topic is not None and topic_info[most_likely_topic]['strict']:\n",
        "      kw = []\n",
        "      kw.extend(topic_info[most_likely_topic]['keywords'])\n",
        "      kw.extend(topic_info[most_likely_topic]['search_term'])\n",
        "\n",
        "      founds = False\n",
        "      for key in kw:\n",
        "        if key in tweet['lemma_text']:\n",
        "          tweet['topic'] = most_likely_topic\n",
        "          tweet['topic_probability'] = prob\n",
        "          founds = True\n",
        "          break\n",
        "      \n",
        "      if not founds:\n",
        "        tweet['topic'] = None\n",
        "        tweet['topic_probability'] = 0\n",
        "    else:\n",
        "      tweet['topic'] = most_likely_topic\n",
        "      tweet['topic_probability'] = prob\n",
        "  \n",
        "  if save_tweets:\n",
        "    for tp in topic_info:\n",
        "      t = list(filter(lambda x: x['topic'] == tp, tweets))\n",
        "      if verbose:\n",
        "        print(f'-- {tp} : {len(t)}')\n",
        "      overwrite_labelled_topics(tp, t, data_dir)\n",
        "\n",
        "  return model.data['tweets']\n",
        "\n",
        "def similar_topics(model : BertopicModel, topic_info, topn=3, n_sim_subtopics=4) -> dict:\n",
        "    \n",
        "  sim_topics = {}\n",
        "\n",
        "  for topic in topic_info:\n",
        "    tt = topic_info[topic]\n",
        "\n",
        "    tpcs1 = {}\n",
        "    for keyword in tt['search_term']:\n",
        "      sims = model.bertopic.find_topics(keyword, top_n=topn)\n",
        "\n",
        "      sims = tuple(zip(sims[0], sims[1]))\n",
        "\n",
        "      tpcs2 = dict((x, y) for x, y in sims)\n",
        "\n",
        "      tpcs1 = {\n",
        "        key: tpcs1.get(key, 0) + tpcs2.get(key, 0) for key in set(tpcs1) | set(tpcs2)\n",
        "      }\n",
        "    \n",
        "    # Normalize\n",
        "    mv = max(tpcs1.values())\n",
        "    for kj in tpcs1:\n",
        "      tpcs1[kj] = float(tpcs1[kj] / mv)\n",
        "      \n",
        "    subtopics = []\n",
        "    for i in range(n_sim_subtopics):\n",
        "      if tpcs1:\n",
        "        k1 = max(tpcs1, key=tpcs1.get)\n",
        "        if k1 != -1:\n",
        "          subtopics.append((k1, tpcs1[k1]))\n",
        "        tpcs1.pop(k1)\n",
        "    \n",
        "    sim_topics[topic] = subtopics\n",
        "  \n",
        "  return sim_topics"
      ],
      "metadata": {
        "id": "qzKGQ43zl-zD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Vizualizacija podatkovne množice"
      ],
      "metadata": {
        "id": "lLI__3yNmcQe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Configurations\n",
        "YEAR = 2021\n",
        "\n",
        "BATCHES = (1,10)\n",
        "\n",
        "EPOCHES = [5, 18, 26, 32, 41]"
      ],
      "metadata": {
        "id": "SzGEdcxqtP-D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect data\n",
        "\n",
        "viz_data = []\n",
        "\n",
        "for e in EPOCHES:\n",
        "\n",
        "  e_data = {}\n",
        "  e_data['e_len'] = 0\n",
        "  e_data['e_words_len'] = 0\n",
        "\n",
        "  words = set()\n",
        "\n",
        "  raw_stpt_data = load_data(f'{root_dir}/stpt/{YEAR}-{e}/{YEAR}_{e}_ALL.json')\n",
        "  e_data['e_len'] = e_data['e_len'] + len(raw_stpt_data)\n",
        "\n",
        "  for t in raw_stpt_data:\n",
        "    for w in t['lemma_text'].split(\" \"):\n",
        "      words.add(w)\n",
        "\n",
        "  e_data['e_words_len'] = len(words)\n",
        "\n",
        "\n",
        "  viz_data.append(e_data)\n"
      ],
      "metadata": {
        "id": "Zv1WeT7psnYp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Visualize data\n",
        "\n",
        "colors = ['red', 'lime']\n",
        "labels = ['Število tvitov', 'Število unikatnih besed']\n",
        "\n",
        "print(viz_data)"
      ],
      "metadata": {
        "id": "FHScoPw_EX01"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Vizualizacija rezultatov"
      ],
      "metadata": {
        "id": "jhM_qZT8skKo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "names = ['splav', 'lgbtq', 'begunci', 'religija', 'levo', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'desno']\n",
        "\n",
        "viz_data = []\n",
        "\n",
        "for i, n in enumerate(names):\n",
        "  raw_t_data = load_data(f'{root_dir}/final/tweets_{n}.json')\n",
        "  viz_data.append(len(raw_t_data))\n",
        "\n",
        "print(viz_data)"
      ],
      "metadata": {
        "id": "xqAbYoApMtt0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(figsize=(30,15))\n",
        "\n",
        "nms = ['Splav', 'LGBTQ+', 'Migracije', 'Tuje religije', 'Levi govor', 'Krščanstvo', 'Militarizem', 'Nacionalna varnost', 'Denacionalizacija', 'Desni govor']\n",
        "clrs = ['indigo', 'hotpink', 'green', 'orange', 'firebrick', 'seagreen', 'darkgray', 'chocolate', 'darkkhaki', 'royalblue']\n",
        "\n",
        "c = ax.bar(nms, viz_data, width=0.8, edgecolor=\"white\", color=clrs)\n",
        "ax.set_xlabel('Tema')\n",
        "ax.set_ylabel('Število tvitov')\n",
        "ax.set_title('Razporeditev tvitov po temah')\n",
        "ax.bar_label(c)\n",
        "fig.show()\n"
      ],
      "metadata": {
        "id": "gPL6HjLoPxBr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "names = ['splav', 'lgbtq','begunci','religija', 'levo', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'desno',]\n",
        "\n",
        "viz_data = []\n",
        "\n",
        "for i, n in enumerate(names):\n",
        "  raw_t_data = load_data(f'{root_dir}/final/tweets_{n}.json')\n",
        "  lev = []\n",
        "  desn = []\n",
        "  nevtraln = []\n",
        "  for t in raw_t_data:\n",
        "    if t['prediction']['label'] == 'desno':\n",
        "      desn.append(t)\n",
        "    elif t['prediction']['label'] == 'levo':\n",
        "      lev.append(t)\n",
        "    elif t['prediction']['label'] == 'nevtralno':\n",
        "      nevtraln.append(t)\n",
        "  \n",
        "  viz_data.append((len(lev), len(nevtraln), len(desn)))\n",
        "\n",
        "print(viz_data)"
      ],
      "metadata": {
        "id": "_qHI9srUR-Fo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(figsize=(30,15))\n",
        "\n",
        "nms = ['Splav', 'LGBTQ+', 'Migracije', 'Tuje religije', 'Levi govor', 'Krščanstvo', 'Militarizem', 'Nacionalna varnost', 'Denacionalizacija', 'Desni govor']\n",
        "#clrs = ['green', 'indigo', 'hotpink', 'orange', 'royalblue', 'firebrick']\n",
        "\n",
        "x = np.arange(len(nms))\n",
        "width = 0.35  # the width of the bars\n",
        "\n",
        "\n",
        "rc1 = ax.bar(x - width/2, [l for l,n,d in viz_data], width, color='indianred', label=\"Levo\")\n",
        "rc2 = ax.bar(x + width/2, [d for l,n,d in viz_data], width, color='cornflowerblue', label=\"Desno\")\n",
        "ax.set_xlabel('Tema')\n",
        "ax.set_ylabel('Število tvitov')\n",
        "ax.set_xticks(x, nms)\n",
        "ax.set_title('Razporeditev politične usmerjenosti tvitov po temah')\n",
        "\n",
        "ax.legend()\n",
        "\n",
        "ax.bar_label(rc1, padding=3)\n",
        "ax.bar_label(rc2, padding=3)\n",
        "\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "xe3KK6ngVxrh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "names = ['splav', 'lgbtq','begunci','religija', 'desno', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'levo']\n",
        "\n",
        "viz_data = {}\n",
        "\n",
        "for i, n in enumerate(names):\n",
        "  raw_t_data = load_data(f'{root_dir}/final/tweets_{n}.json')\n",
        "\n",
        "  for t in raw_t_data:\n",
        "    if t['prediction']['party_mentioned'] is not None:\n",
        "      if t['prediction']['party_mentioned'] in viz_data:\n",
        "        viz_data[t['prediction']['party_mentioned']].append(t)\n",
        "      else:\n",
        "        viz_data[t['prediction']['party_mentioned']] = []\n",
        "        viz_data[t['prediction']['party_mentioned']].append(t)\n",
        "\n",
        "  \n",
        "viz_data['PS'] = [ t for t in viz_data['PS'] if t['mentions'] in ['piratskastranka'] ]\n",
        "#viz_data['GS'] = [ t for t in viz_data['GS'] if t['mentions'] in ['Gibanje_Svoboda'] ]\n",
        "\n",
        "stranka_data = []\n",
        "\n",
        "for v in viz_data:\n",
        "  tviti = viz_data[v]\n",
        "  if len(tviti) == 0:\n",
        "    continue\n",
        "  poz = 0\n",
        "  neg = 0\n",
        "  nevt = 0\n",
        "  for t in tviti:\n",
        "    if t['prediction']['sentiment'] == 'positive':\n",
        "      poz = poz+1\n",
        "    elif t['prediction']['sentiment'] == 'negative':\n",
        "      neg = neg+1\n",
        "    elif t['prediction']['sentiment'] == 'neutral':\n",
        "      nevt = nevt+1\n",
        "  \n",
        "  stranka_data.append((poz, nevt, neg))"
      ],
      "metadata": {
        "id": "Mow9uornM5Q-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(stranka_data)"
      ],
      "metadata": {
        "id": "xd8ZwaUAhPY9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(figsize=(10,10))\n",
        "\n",
        "nms = [ z for z in viz_data if len(viz_data[z]) > 0]\n",
        "\n",
        "#clrs = ['green', 'indigo', 'hotpink', 'orange', 'royalblue', 'firebrick']\n",
        "\n",
        "x = np.arange(len(nms))\n",
        "width = 0.25  # the width of the bars\n",
        "\n",
        "\n",
        "rc1 = ax.bar(x - width, [p for p,n,neg in stranka_data], width, color='springgreen', label=\"Pozitivno\")\n",
        "rc2 = ax.bar(x, [n for p,n,neg in stranka_data], width, color='slategrey', label=\"Nevtralno\")\n",
        "rc3 = ax.bar(x + width, [neg for p,n,neg in stranka_data], width, color='orangered', label=\"Negativno\")\n",
        "ax.set_xlabel('Stranka')\n",
        "ax.set_ylabel('Število tvitov')\n",
        "ax.set_xticks(x, nms)\n",
        "ax.set_title('Razporeditev sentimentov tvitov po strankah')\n",
        "\n",
        "ax.legend()\n",
        "\n",
        "#ax.bar_label(rc1, padding=3)\n",
        "#ax.bar_label(rc2, padding=3)\n",
        "#ax.bar_label(rc3, padding=3)\n",
        "\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "bHFk1oeXO3H-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "punct = str.maketrans('', '', string.punctuation)\n",
        "\n",
        "def clean_str(test_str):\n",
        "  return \" \".join(test_str.translate(punct).split())\n",
        "\n",
        "def make_wordcloud(wrds):\n",
        "  fig, ax = plt.subplots(figsize = (7, 7), facecolor = None)\n",
        "  wc = WordCloud(width = 800, height = 800, max_font_size=120, min_font_size = 10, max_words=80, \n",
        "                 stopwords = ['2x', '3x', 'stopnja', '4x', '5x', '6x', 'omeniti', 'obsedenost', 'imeti', 'lahko'],\n",
        "                 background_color=\"white\").generate(wrds)\n",
        "  ax.imshow(wc)\n",
        "  ax.axis(\"off\")\n",
        "  fig.tight_layout(pad = 0)\n",
        "  fig.show()"
      ],
      "metadata": {
        "id": "UY3vK5uJJmSs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Wordclouds for themes\n",
        "names = ['denacionalizacija']# 'splav', 'lgbt', 'begunci', 'religija', 'desno', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'levo']\n",
        "\n",
        "viz_data = []\n",
        "\n",
        "for i, n in enumerate(names):\n",
        "  raw_t_data = load_data(f'{root_dir}/final/tweets_{n}.json')\n",
        "\n",
        "  words = \"\"\n",
        "  for t in raw_t_data:\n",
        "    words = words + \" \" + t['lemma_text']\n",
        "\n",
        "  make_wordcloud(words)\n",
        "  break"
      ],
      "metadata": {
        "id": "NoitffGDe7Yn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "names = ['splav', 'lgbtq', 'begunci', 'religija', 'levo', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'desno']\n",
        "\n",
        "viz_data = []\n",
        "\n",
        "party_left = []\n",
        "party_right = []\n",
        "\n",
        "for i, n in enumerate(names):\n",
        "  raw_t_data = load_data(f'{root_dir}/final/tweets_{n}.json')\n",
        "  for t in raw_t_data:\n",
        "    if t['prediction']['party_mentioned'] != None and t['prediction']['party_bias'] != None:\n",
        "      if t['prediction']['party_bias'] == 'levo':\n",
        "        party_left.append(t)\n",
        "      elif t['prediction']['party_bias'] == 'desno':\n",
        "        party_right.append(t)\n",
        "\n",
        "print(party_right)"
      ],
      "metadata": {
        "id": "sy3s9YagUL0M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "num_samples = 10\n",
        "\n",
        "party_left_sample = random.sample(party_left, num_samples)\n",
        "party_right_sample = random.sample(party_right, num_samples)"
      ],
      "metadata": {
        "id": "1zv8IemWUmUt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(num_samples):\n",
        "  print(\"LEFT:\" + party_left_sample[i]['raw_text'])\n",
        "  print(\"RIGHT:\" + party_right_sample[i]['raw_text'])"
      ],
      "metadata": {
        "id": "Hj12DczkWNWG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evalvacija modela tematik"
      ],
      "metadata": {
        "id": "AQppghp7q8kM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Configurations for automated bias pipeline\n",
        "\n",
        "VERBOSE = True\n",
        "SAVING = False\n",
        "DEBUG = False\n",
        "\n",
        "## Tweetiment configuration\n",
        "topic_bias_path = root_dir + '/configs/topic-bias.json'\n",
        "party_bias_path = root_dir + '/configs/party-bias.json'\n",
        "\n",
        "tweetiment_config = {\n",
        "    'topic_bias': load_data(topic_bias_path),\n",
        "    'party_bias': load_data(party_bias_path),\n",
        "    'device': \"cpu\"\n",
        "}\n",
        "\n",
        "## Bertopic second layer configuration\n",
        "tweetiment_bertopic_SL_config = {\n",
        "    'bertopic_conf': {\n",
        "        \"top_n_words\": 10,\n",
        "        \"min_topic_size\": 20,\n",
        "        \"n_gram_range\": (1,2),\n",
        "        \"nr_topics\": 10,\n",
        "        \"diversity\": 0.1,\n",
        "        \"verbose\": VERBOSE\n",
        "    },\n",
        "    'umap_conf': {\n",
        "        \"n_neighbors\": 20,\n",
        "        \"n_components\": 15,\n",
        "        \"metric\": 'cosine'\n",
        "    },\n",
        "    'hdbscan_conf': {\n",
        "        \"min_cluster_size\": 15,\n",
        "        \"metric\": 'euclidean',\n",
        "        \"prediction_data\": True\n",
        "    },\n",
        "    'topn': 3,\n",
        "    'n_sim_subtopics': 3,\n",
        "}\n",
        "\n",
        "## Topic information\n",
        "tweetiment_topics_info = {\n",
        "    'begunci': {\n",
        "        'keywords': [\"migrant\", \"migriranje\", \"beg\", \"begunec\", \"meja\", \"begunci\", \"migracija\"],\n",
        "        'regexes': [r'\\bmigr\\w+', r'\\bbeg\\w+']\n",
        "    },\n",
        "    'lgbt': {\n",
        "        'keywords': [\"lgbtq\", \"lgbt\", \"lgbtqia\", \"istospolen\", \"spol\", \"gej\", \"lezbijka\", \"lezbijski\", \"trans\", \"seksualnost\"],\n",
        "        'regexes': [r'\\btrans\\w+', r'\\bseks\\w+', r'\\blgbt\\w+', r'\\bistospol\\w+', r'\\bgej\\w+', r'\\blezb\\w+', r'\\bspol\\w+']\n",
        "    },\n",
        "    'religija': {\n",
        "        'keywords': [\"musliman\", \"islam\", \"radikalen\", \"islamski\", \"muslimanski\", \"jud\", \"izrael\", \"izraelski\", \"vera\"],\n",
        "        'regexes': [r'\\bver\\w+', r'\\bislam\\w+', r'\\bžid\\w+', r'\\bzid\\w+', r'\\bislam\\w+', r'\\bjud\\w+', r'\\bmusli\\w+']\n",
        "    },\n",
        "    'splav': {\n",
        "        'keywords': [\"splav\", \"zarodek\", \"kontracepcija\", \"vazektomija\", \"sterilizacija\", \"diafragma\", \"kondom\", \"maternica\", \"fetus\"],\n",
        "        'regexes': [r'\\bkontracep\\w+', r'\\bsplav\\w+', r'\\bnoseč\\w+', r'\\bnosec\\w+', r'\\bsteril\\w+', r'\\bkastri\\w+']\n",
        "    },\n",
        "    'levo': {\n",
        "        'keywords': [\"levica\", \"levicar\", \"lev\", \"levičar\", \"mesec\", \"levi\"],\n",
        "        'regexes': [r'\\blev\\w+']\n",
        "    },\n",
        "    'krscanstvo': {\n",
        "        'keywords': [\"cerkev\", \"župnik\", \"verouk\", \"vera\", \"bog\", \"mučenik\", \"vernik\", \"verniki\", \"otrok\", \"papež\"],\n",
        "        'regexes': [r'\\bkrscan\\w+', r'\\bkrščan\\w+', r'\\bcerkv\\w+', r'\\bkatoli\\w+', r'\\bdruzi\\w+', r'\\bdruži\\w+', r'\\bteolo\\w+']\n",
        "    },\n",
        "    'militarizem': {\n",
        "      'keywords': [\"nato\", \"vojska\", \"vojak\", \"meja\", \"obramba\", \"zasčita\", \"sila\", \"varnost\", \"orožje\", \"orozje\"],\n",
        "      'regexes': [r'\\bmilitar\\w+', r'\\bvoj\\w+', r'\\bnaborni\\w+', r'\\bpovelj\\w+', r'\\bzavez\\w+', r'\\bpatri\\w+']\n",
        "    },\n",
        "    'varnost': {\n",
        "      'keywords': [\"represiven\", \"policija\", \"protest\", \"varda\", \"nadzor\", \"varovanje\", \"varnost\", \"varen\", \"red\", \"mir\"],\n",
        "      'regexes': [r'\\bpolici\\w+', r'\\bprotest\\w+', r'\\bshod\\w+', r'\\bteror\\w+']\n",
        "    },\n",
        "    'denacionalizacija': {\n",
        "      'keywords': [\"denacionalizacija\", \"privat\", \"last\", \"premoženje\", \"kapital\"],\n",
        "      'regexes': [r'\\bprivat\\w+', r'\\bzaseb\\w+', r'\\blast\\w+', r'\\bpremož\\w+', r'\\bpodrža\\w+', r'\\bdenacional\\w+', r'\\bkapital\\w+']\n",
        "    },\n",
        "    'desno': {\n",
        "        'keywords': [\"desnica\", \"desno\", \"jj\", \"sds\", \"desničar\", \"janša\", \"jansa\", \"nsi\", \"janez\"],\n",
        "        'regexes': [r'\\bdesni\\w+', r'\\bjan\\w+']\n",
        "    }\n",
        "}\n",
        "\n",
        "pbm_config = {}\n",
        "\n",
        "pbm_config['bertopic_SL_config'] = tweetiment_bertopic_SL_config\n",
        "\n",
        "ctfidf_model = ClassTfidfTransformer(bm25_weighting=True, reduce_frequent_words=True)\n",
        "pbm_config['bertopic_SL_config']['bertopic_conf']['ctfidf_model'] = ctfidf_model\n",
        "\n",
        "pbm_config['preprocess_config'] = None\n",
        "pbm_config['tweetiment_config'] = tweetiment_config\n",
        "pbm_config['topic_info'] = tweetiment_topics_info\n",
        "pbm_config['verbose'] = VERBOSE\n",
        "pbm_config['debug'] = DEBUG"
      ],
      "metadata": {
        "id": "nE9QAJ8br2dW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading models\n",
        "\n",
        "# Embedding model\n",
        "topic_model = pipeline('feature-extraction', model='EMBEDDIA/sloberta', device=0)#\n",
        "\n",
        "sent_model = {}\n",
        "sent_model['tokenizer'] = AutoTokenizer.from_pretrained(\"EMBEDDIA/sloberta-tweetsentiment\", device=0)\n",
        "sent_model['model'] = AutoModelForSequenceClassification.from_pretrained(\"EMBEDDIA/sloberta-tweetsentiment\")"
      ],
      "metadata": {
        "id": "YW58CDvqQCKd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect and split test data\n",
        "\n",
        "RANDOM_STATE = 992\n",
        "\n",
        "topic_names = ['splav', 'lgbtq', 'begunci','religija', 'desno', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'levo']\n",
        "\n",
        "data = []\n",
        "\n",
        "X_train = []\n",
        "y_train = []\n",
        "\n",
        "X_test = []\n",
        "y_test = []\n",
        "\n",
        "for t in topic_names:\n",
        "  raw_result_data = load_data(f'{root_dir}/labelled_topics/topic_{t}.json')\n",
        "  raw_result_data.extend(load_data(f'{root_dir}/final/tweets_{t}.json'))\n",
        "\n",
        "  docs = []\n",
        "  labels = []\n",
        "\n",
        "  for d in raw_result_data:\n",
        "    if not 'id' in d:\n",
        "      continue\n",
        "    if 'topic' in d:\n",
        "      labels.append(topic_names.index(d['topic']))\n",
        "    elif 'prediction' in d and d['prediction']['topic_mentioned'] is not None:\n",
        "      labels.append(topic_names.index(d['prediction']['topic_mentioned']))\n",
        "    d['topic'] = None\n",
        "    docs.append(d)\n",
        "  \n",
        "  iX_train, iX_test, iy_train, iy_test = train_test_split(docs, labels, test_size=0.33, random_state=RANDOM_STATE)\n",
        "  \n",
        "  X_train.extend(iX_train)\n",
        "  y_train.extend(iy_train)\n",
        "  X_test.extend(iX_test)\n",
        "  y_test.extend(iy_test)\n",
        "\n",
        "\n",
        "# Shuffle for good measure\n",
        "\n",
        "X_train, y_train = shuffle(X_train, y_train, random_state=(RANDOM_STATE+13))"
      ],
      "metadata": {
        "id": "sn_ut_ANsqIy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pt_model = PoliticBiasModel(\"TopicEvaluation\",\n",
        "                            root_dir,\n",
        "                            None,\n",
        "                            topic_model,\n",
        "                            sent_model,\n",
        "                            pbm_config)\n",
        "\n",
        "# Manually train the topic model\n",
        "pt_model.bertopic_SL.load_topic_data([d['lemma_text'] for d in X_train], y_train)\n",
        "\n",
        "# Train model\n",
        "print('Training the model with training data...')\n",
        "pt_model.bertopic_SL.train_model(only_fit=True)\n",
        "\n",
        "# Load data\n",
        "pt_model.bertopic_SL.load_tweet_data(X_test)\n",
        "\n",
        "# Predict on test data\n",
        "print('Predicting on test data...')\n",
        "pt_model.bertopic_SL.predict()\n",
        "\n",
        "# Label\n",
        "pt_model.bertopic_SL.visualize()"
      ],
      "metadata": {
        "id": "55EI87gKD6WU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pt_model.bertopic_SL.bertopic.get_topics()"
      ],
      "metadata": {
        "id": "YV33ZBz7D0ad"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mapper = {\n",
        "    -1: -1,\n",
        "    0: topic_names.index('varnost'),\n",
        "    1: topic_names.index('begunci'),\n",
        "    2: topic_names.index('levo'),\n",
        "    3: topic_names.index('splav'),\n",
        "    4: topic_names.index('denacionalizacija'),\n",
        "    5: topic_names.index('desno'),\n",
        "    6: topic_names.index('lgbtq'),\n",
        "    7: topic_names.index('militarizem'),\n",
        "    8: topic_names.index('krscanstvo'),\n",
        "    9: topic_names.index('religija')\n",
        "}"
      ],
      "metadata": {
        "id": "lYVWzgLXF3Mj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filter_non_topics = True\n",
        "\n",
        "y_test_f = []\n",
        "y_pred = []\n",
        "\n",
        "for i, d in enumerate(pt_model.bertopic_SL.result['topic_ids']):\n",
        "  if filter_non_topics and d != -1 and d < 10:\n",
        "    y_pred.append(mapper[d])\n",
        "    y_test_f.append(y_test[i])\n",
        "  elif not filter_non_topics:\n",
        "    if d > 9:\n",
        "      d = -1\n",
        "    y_pred.append(mapper[d])\n",
        "    y_test_f.append(y_test[i])\n",
        "\n",
        "print(y_pred)\n",
        "print(y_test_f)"
      ],
      "metadata": {
        "id": "SODjgMPUQJQo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking distribution of classified topics\n",
        "\n",
        "Counter(y_pred)"
      ],
      "metadata": {
        "id": "UcsdjcDqHqqb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Classification scores\n",
        "\n",
        "accscore = accuracy_score(y_test_f, y_pred)\n",
        "f1score = f1_score(y_test_f, y_pred, average='weighted')\n",
        "precisionscore = precision_score(y_test_f, y_pred, average='weighted')\n",
        "recallscore = recall_score(y_test_f, y_pred, average='weighted')\n",
        "\n",
        "print(f'Accuracy score: {accscore}')\n",
        "print(f'F1 score: {f1score}')\n",
        "print(f'Precision score: {precisionscore}')\n",
        "print(f'Recall score: {recallscore}')"
      ],
      "metadata": {
        "id": "_t-WuajuRvTY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Heat map for topics\n",
        "\n",
        "nms = ['Splav', 'LGBTQ+', 'Migracije', 'Tuje religije', 'Desni govor', 'Krščanstvo', 'Militarizem', 'Nacionalna varnost', 'Denacionalizacija', 'Levi govor']\n",
        "\n",
        "cf_m = confusion_matrix(y_test_f, y_pred)\n",
        "\n",
        "data_masked = np.ma.masked_where(cf_m == 0, cf_m)\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(15,15))\n",
        "im = ax.imshow(data_masked, cmap='viridis_r', interpolation = 'none', vmin = 0, aspect='auto')\n",
        "\n",
        "\n",
        "plt.xticks(np.arange(len(nms)), labels=nms, rotation=45)\n",
        "plt.yticks(np.arange(len(nms)), labels=nms)\n",
        "ax.xaxis.tick_top()\n",
        "\n",
        "#plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n",
        "\n",
        "# Loop over data dimensions and create text annotations.\n",
        "for i in range(len(nms)):\n",
        "    for j in range(len(nms)):\n",
        "        text = ax.text(j, i, cf_m[i, j],\n",
        "                       ha=\"center\", va=\"center\", color=\"white\", fontsize=\"x-large\", path_effects=[pe.withStroke(linewidth=3, foreground=\"black\")])\n",
        "\n",
        "ax.set_xlabel('Klasificirana tema')\n",
        "ax.set_ylabel('Prava tema')\n",
        "ax.set_title('Matrika zamenjav za teme', pad=20)\n",
        "fig.tight_layout()\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "yzZslfMqS6io"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evalvacija modela za določanje usmerjenosti"
      ],
      "metadata": {
        "id": "YiMJ9A4RodM6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect and split test data\n",
        "\n",
        "RANDOM_STATE = 466\n",
        "\n",
        "topic_names = ['splav', 'lgbtq', 'begunci','religija', 'desno', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'levo']\n",
        "\n",
        "bias_labels = ['levo', 'nevtralno', 'desno']\n",
        "\n",
        "data = []\n",
        "\n",
        "X_train = []\n",
        "y_train = []\n",
        "\n",
        "X_test = []\n",
        "y_test = []\n",
        "\n",
        "# Collect labelled topics as training data\n",
        "\n",
        "for t in topic_names:\n",
        "  raw_result_data = load_data(f'{root_dir}/labelled_topics/topic_{t}.json')\n",
        "\n",
        "  docs = []\n",
        "  labels = []\n",
        "\n",
        "  for d in raw_result_data:\n",
        "    if not 'id' in d:\n",
        "      continue\n",
        "    if 'topic' in d:\n",
        "      labels.append((topic_names.index(d['topic']), -1))\n",
        "    #elif 'prediction' in d and d['prediction']['topic_mentioned'] is not None:\n",
        "    #  labels.append(topic_names.index(d['prediction']['topic_mentioned']))\n",
        "    d['topic'] = None\n",
        "    docs.append((d['lemma_text'], d['raw_text']))\n",
        "  \n",
        "  X_train.extend(docs)\n",
        "  y_train.extend(labels)\n",
        "\n",
        "# Collect final tweets as training + test data\n",
        "\n",
        "for t in topic_names:\n",
        "  raw_result_data = load_data(f'{root_dir}/final/tweets_{t}.json')\n",
        "\n",
        "  docs = []\n",
        "  labels = []\n",
        "\n",
        "  for d in raw_result_data:\n",
        "    if not 'id' in d:\n",
        "      continue\n",
        "    elif 'prediction' in d and d['prediction']['topic_mentioned'] is not None:\n",
        "      labels.append((topic_names.index(d['prediction']['topic_mentioned']), bias_labels.index(d['prediction']['label'])))\n",
        "    docs.append(d)\n",
        "\n",
        "  iX_train, iX_test, iy_train, iy_test = train_test_split(docs, labels, test_size=0.5, random_state=RANDOM_STATE)\n",
        "  \n",
        "  X_train.extend([(z['lemma_text'], z['raw_text']) for z in iX_train])\n",
        "  y_train.extend(iy_train)\n",
        "  X_test.extend(iX_test)\n",
        "  y_test.extend(iy_test)\n",
        "\n",
        "# Shuffle for good measure\n",
        "X_train, y_train = shuffle(X_train, y_train, random_state=(RANDOM_STATE+13))"
      ],
      "metadata": {
        "id": "Q6TlhXXM0we-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pt_model = PoliticBiasModel(\"PoliticalBiasEvaluation\",\n",
        "                            root_dir,\n",
        "                            None,\n",
        "                            topic_model,\n",
        "                            sent_model,\n",
        "                            pbm_config)\n",
        "\n",
        "# Manually train the topic model\n",
        "pt_model.bertopic_SL.load_topic_data([d[0] for d in X_train], [l[0] for l in y_train])\n",
        "\n",
        "# Train model\n",
        "print('Training the model with training data...')\n",
        "pt_model.bertopic_SL.train_model(only_fit=True)\n",
        "\n",
        "# Load data\n",
        "pt_model.bertopic_SL.load_tweet_data(X_test)\n",
        "\n",
        "# Predict on test data\n",
        "print('Predicting on test data...')\n",
        "pt_model.bertopic_SL.predict()\n",
        "\n",
        "# Label\n",
        "pt_model.bertopic_SL.visualize()"
      ],
      "metadata": {
        "id": "u4bLKaz13H6Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pt_model.bertopic_SL.bertopic.get_topics()"
      ],
      "metadata": {
        "id": "IZ5hQ2rF5E6O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mapper = {\n",
        "    -1: -1,\n",
        "    0: topic_names.index('desno'),\n",
        "    1: topic_names.index('varnost'),\n",
        "    2: topic_names.index('begunci'),\n",
        "    3: topic_names.index('denacionalizacija'),\n",
        "    4: topic_names.index('levo'),\n",
        "    5: topic_names.index('splav'),\n",
        "    6: topic_names.index('militarizem'),\n",
        "    7: topic_names.index('lgbtq'),\n",
        "    8: topic_names.index('religija'),\n",
        "    9: topic_names.index('krscanstvo')\n",
        "}"
      ],
      "metadata": {
        "id": "2WrZ9t--5Frj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filter_non_topics = False\n",
        "\n",
        "y_test_f = []\n",
        "y_pred_topic = []\n",
        "\n",
        "for i, d in enumerate(pt_model.bertopic_SL.result['topic_ids']):\n",
        "  if filter_non_topics and d != -1 and d < 10:\n",
        "    y_pred_topic.append(mapper[d])\n",
        "    y_test_f.append(y_test[i][0])\n",
        "  elif not filter_non_topics:\n",
        "    if d > 9:\n",
        "      d = -1\n",
        "    y_pred_topic.append(mapper[d])\n",
        "    y_test_f.append(y_test[i][0])\n",
        "\n",
        "print(y_pred_topic)\n",
        "print(y_test_f)"
      ],
      "metadata": {
        "id": "RLCHnkI45I-u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "accscore = accuracy_score(y_test_f, y_pred_topic)\n",
        "\n",
        "print(accscore)"
      ],
      "metadata": {
        "id": "_v8yWGCklfVh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "correct_indices = []\n",
        "\n",
        "for i, d in enumerate(pt_model.bertopic_SL.result['topic_ids']):\n",
        "  if d != -1 and mapper[d] == y_test[i][0]:\n",
        "    correct_indices.append(i)\n",
        "\n",
        "X_test_bias = [ X_test[c] for c in correct_indices ]\n",
        "y_test_bias = [ y_test[c][1] for c in correct_indices ]"
      ],
      "metadata": {
        "id": "-5V6i7rTUt6T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "xy = random.sample(list(zip(X_test_bias, y_test_bias)), 1295)\n",
        "\n",
        "X_test_bias = [ y[0] for y in xy ]\n",
        "y_test_bias = [ y[1] for y in xy ]"
      ],
      "metadata": {
        "id": "GcK2aQ-nZe11"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred_bias = []\n",
        "\n",
        "for i, t in enumerate(X_test_bias):\n",
        "  nt = copy.deepcopy(t)\n",
        "  nt.pop('prediction')\n",
        "  nt['topic'] = t['prediction']['topic_mentioned']\n",
        "  #if y_test_bias[i] != -1:\n",
        "  #  nt['topic'] = topic_names[y_test_bias[i]]\n",
        "  #else:\n",
        "  #  nt['topic'] = None\n",
        "  nt['topic_probability'] = None\n",
        "\n",
        "  pr = pt_model.tweetiment.calculate_biases(nt)['label']\n",
        "\n",
        "  y_pred_bias.append(bias_labels.index(pr))"
      ],
      "metadata": {
        "id": "nBrfz_Z_rURg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Classification scores for political bias\n",
        "\n",
        "accscore = accuracy_score(y_test_bias, y_pred_bias)\n",
        "f1score = f1_score(y_test_bias, y_pred_bias, average='weighted')\n",
        "precisionscore = precision_score(y_test_bias, y_pred_bias, average='weighted')\n",
        "recallscore = recall_score(y_test_bias, y_pred_bias, average='weighted')\n",
        "\n",
        "print(f'Accuracy score: {accscore}')\n",
        "print(f'F1 score: {f1score}')\n",
        "print(f'Precision score: {precisionscore}')\n",
        "print(f'Recall score: {recallscore}')"
      ],
      "metadata": {
        "id": "06L8uaoptiwU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Heat map for political bias\n",
        "\n",
        "nms = ['Levo', 'Nevtralno', 'Desno']\n",
        "\n",
        "#cf_m = confusion_matrix(y_test_bias, y_pred_bias)#[1:]\n",
        "\n",
        "cf_m = np.array([[\n",
        "    393, 92, 10\n",
        "],\n",
        "[\n",
        "    0, 226, 0\n",
        "],\n",
        "[\n",
        "    3, 68, 475\n",
        "]])\n",
        "\n",
        "data_masked = np.ma.masked_where(cf_m == 0, cf_m)\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(8,8))\n",
        "im = ax.imshow(data_masked, cmap='viridis_r', interpolation = 'none', vmin = 0, aspect='auto')\n",
        "\n",
        "plt.xticks(np.arange(len(nms)), labels=nms, rotation=45)\n",
        "plt.yticks(np.arange(len(nms)), labels=nms)\n",
        "ax.xaxis.tick_top()\n",
        "\n",
        "#plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n",
        "\n",
        "# Loop over data dimensions and create text annotations.\n",
        "for i in range(len(nms)):\n",
        "    for j in range(len(nms)):\n",
        "        text = ax.text(j, i, cf_m[i, j],\n",
        "                       ha=\"center\", va=\"center\", color=\"white\", fontsize=\"xx-large\", path_effects=[pe.withStroke(linewidth=5, foreground=\"black\")])\n",
        "\n",
        "ax.set_xlabel('Klasificirana usmerjenost')\n",
        "ax.set_ylabel('Prava usmerjenost')\n",
        "#ax.set_title('Matrika zamenjav za usmerjenosti', pad=20)\n",
        "fig.tight_layout()\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "enn8_8Hf2_GL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## More visualization"
      ],
      "metadata": {
        "id": "7xht40iQKDeN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect and split test data\n",
        "\n",
        "# 0 - initial, 1 - final, 2 - both\n",
        "n_dataset = 2\n",
        "\n",
        "RANDOM_STATE = 200\n",
        "\n",
        "topic_names = ['splav', 'lgbtq', 'begunci','religija', 'desno', 'krscanstvo', 'militarizem', 'varnost', 'denacionalizacija', 'levo']\n",
        "\n",
        "X = []\n",
        "y = []\n",
        "\n",
        "for t in topic_names:\n",
        "  if n_dataset == 2:\n",
        "    raw_result_data = load_data(f'{root_dir}/labelled_topics/topic_{t}.json')\n",
        "    raw_result_data.extend(load_data(f'{root_dir}/final/tweets_{t}.json'))\n",
        "  elif n_dataset == 1:\n",
        "    raw_result_data = load_data(f'{root_dir}/final/tweets_{t}.json')\n",
        "  else:\n",
        "    raw_result_data = load_data(f'{root_dir}/labelled_topics/topic_{t}.json')\n",
        "\n",
        "  for d in raw_result_data:\n",
        "    if not 'id' in d:\n",
        "      continue\n",
        "    if 'topic' in d:\n",
        "      y.append(topic_names.index(d['topic']))\n",
        "    elif 'prediction' in d and d['prediction']['topic_mentioned'] is not None:\n",
        "      y.append(topic_names.index(d['prediction']['topic_mentioned']))\n",
        "    d['topic'] = None\n",
        "    X.append(d['lemma_text'])\n",
        "\n",
        "# Shuffle for good measure\n",
        "X, y = shuffle(X, y, random_state=(RANDOM_STATE+13))"
      ],
      "metadata": {
        "id": "-dZcBPx-NAGn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Prepare Politic bias model\n",
        "pt_model = PoliticBiasModel(\"PoliticBiasVisualization\",\n",
        "                            root_dir,\n",
        "                            None,\n",
        "                            topic_model,\n",
        "                            sent_model,\n",
        "                            pbm_config)\n",
        "\n",
        "# Train the models\n",
        "pt_model.train_models(X, y)"
      ],
      "metadata": {
        "id": "PQQdv_ceabWo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = pt_model.bertopic_SL.visualize()\n",
        "fig"
      ],
      "metadata": {
        "id": "QTHEkgDfavQj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pt_model.bertopic_SL.bertopic.get_topics()"
      ],
      "metadata": {
        "id": "I8m3jy0ojK47"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "teme = {0: \"Levi govor\",\n",
        "1: \"Desni govor\",\n",
        "2: \"Nacionalna varnost\",\n",
        "3: \"Migracije\",\n",
        "4: \"Tuje religije\",\n",
        "5: \"Krščanstvo\",\n",
        "6: \"LGBTQ+\",\n",
        "7: \"Militarizem\",\n",
        "8: \"Splav\",\n",
        "9: \"Denacionalizacija\",\n",
        "10: \"Neznano\"}\n",
        "\n",
        "\n",
        "pt_model.bertopic_SL.bertopic.set_topic_labels(teme)\n",
        "print(pt_model.bertopic_SL.bertopic.custom_labels_)"
      ],
      "metadata": {
        "id": "lUxGMynEi7vl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figd = pt_model.bertopic_SL.bertopic.visualize_documents(docs=X, sample=0.6, hide_annotations=False, hide_document_hover=True, custom_labels=False, width=1200, height=800)\n",
        "figd.show()"
      ],
      "metadata": {
        "id": "ZibRdLjiq0Mm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figd['data'][1]['name'] = teme[0]\n",
        "figd['data'][1]['marker']['size'] = 15\n",
        "figd['data'][1]['marker']['color'] = 'firebrick'\n",
        "figd['data'][1]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][1]['text']\n",
        "figd['data'][1]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][2]['name'] = teme[1]\n",
        "figd['data'][2]['marker']['size'] = 15\n",
        "figd['data'][2]['marker']['color'] = 'royalblue'\n",
        "figd['data'][2]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][2]['text']\n",
        "figd['data'][2]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][3]['name'] = teme[2]\n",
        "figd['data'][3]['marker']['size'] = 15\n",
        "figd['data'][3]['marker']['color'] = 'chocolate'\n",
        "figd['data'][3]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][3]['text']\n",
        "figd['data'][3]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][4]['name'] = teme[3]\n",
        "figd['data'][4]['marker']['size'] = 15\n",
        "figd['data'][4]['marker']['color'] = 'green'\n",
        "figd['data'][4]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][4]['text']\n",
        "figd['data'][4]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][5]['name'] = teme[4]\n",
        "figd['data'][5]['marker']['size'] = 15\n",
        "figd['data'][5]['marker']['color'] = 'palegreen'\n",
        "figd['data'][5]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][5]['text']\n",
        "figd['data'][5]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][6]['name'] = teme[5]\n",
        "figd['data'][6]['marker']['size'] = 15\n",
        "figd['data'][6]['marker']['color'] = 'orange'\n",
        "figd['data'][6]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][6]['text']\n",
        "figd['data'][6]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][7]['name'] = teme[6]\n",
        "figd['data'][7]['marker']['size'] = 15\n",
        "figd['data'][7]['marker']['color'] = 'hotpink'\n",
        "figd['data'][7]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][7]['text']\n",
        "figd['data'][7]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][8]['name'] = teme[7]\n",
        "figd['data'][8]['marker']['size'] = 15\n",
        "figd['data'][8]['marker']['color'] = 'darkgray'\n",
        "figd['data'][8]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][8]['text']\n",
        "figd['data'][8]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][9]['name'] = teme[8]\n",
        "figd['data'][9]['marker']['size'] = 15\n",
        "figd['data'][9]['marker']['color'] = 'indigo'\n",
        "figd['data'][9]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][9]['text']\n",
        "figd['data'][9]['text'][len(ars)-1] = ''\n",
        "\n",
        "figd['data'][10]['name'] = teme[9]\n",
        "figd['data'][10]['marker']['size'] = 15\n",
        "figd['data'][10]['marker']['color'] = 'darkkhaki'\n",
        "figd['data'][10]['marker']['opacity'] = 0.8\n",
        "ars = figd['data'][10]['text']\n",
        "figd['data'][10]['text'][len(ars)-1] = ''\n",
        "\n",
        "print(figd['data'][1])"
      ],
      "metadata": {
        "id": "wTfCx1vonZXn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figd.show()"
      ],
      "metadata": {
        "id": "PuDtR2d6rQwQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Change to fit colors\n",
        "\n",
        "clrs = ['indigo', 'hotpink', 'green', 'orange', 'royalblue', 'firebrick']\n",
        "\n",
        "fig['data'][0]['marker']['color'] = np.array(['royalblue', 'firebrick', 'orange', 'green', 'darkkhaki', 'seagreen', 'chocolate', 'darkgray', 'indigo', 'hotpink'])"
      ],
      "metadata": {
        "id": "rZR6yPoJT0Ue"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#nms = ['Splav', 'LGBTQ+', 'Migracije', 'Tuje religije', 'Levi govor', 'Krščanstvo', 'Militarizem', 'Nacionalna varnost', 'Denacionalizacija', 'Desni govor']\n",
        "#clrs = ['indigo', 'hotpink', 'green', 'orange', 'firebrick', 'seagreen', 'darkgray', 'chocolate', 'darkkhaki', 'royalblue']\n",
        "\n",
        "fig['data'][0]['text'] = np.array([\"Desni govor\", \"Levi govor\", \"Tuje religije\", \"Migracije\", \"Denacionalizacija\", \" Krščanstvo\", \"Nacionalna varnost\", \"Militarizem\", \"Splav\", \"LGBTQ+ \"])\n",
        "fig['data'][0]['textposition'] = np.array(['middle right', 'middle left', 'middle right', 'top center', 'top center', 'bottom center', 'middle right','bottom left', 'middle left', 'bottom left'])\n",
        "fig['data'][0]['mode'] = \"markers+text\""
      ],
      "metadata": {
        "id": "4mq302ggZvcW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig.show()"
      ],
      "metadata": {
        "id": "BDYd5JZYrHse"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figm['data'][0]['hovertemplate'] = 'x: %{x}<br>y: %{y}<br>Legenda podobnosti: %{z}<extra></extra>'"
      ],
      "metadata": {
        "id": "PFcW31nEqEQN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figm['data'][0]['x'] = np.array(['Levi govor', 'Migracije', 'Religije', 'LGBTQ+', 'Desni govor', 'Splav'])\n",
        "figm['data'][0]['y'] = np.array(['Levi govor', 'Migracije', 'Religije', 'LGBTQ+', 'Desni govor', 'Splav'])"
      ],
      "metadata": {
        "id": "3i2PxRaYpfP0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figm['data'][0]['z'] = np.array([[1.        , 0.98312585, 0.97783003, 0.97631899, 0.97725498,\n",
        "                  0.97912109],\n",
        "                 [0.98312585, 1.        , 0.98378234, 0.98044662, 0.97803244,\n",
        "                  0.98592705],\n",
        "                 [0.97783003, 0.98378234, 1.        , 0.97651127, 0.97041834,\n",
        "                  0.98143742],\n",
        "                 [0.97631899, 0.98044662, 0.97651127, 1.        , 0.97815931,\n",
        "                  0.98451635],\n",
        "                 [0.97725498, 0.97803244, 0.97041834, 0.97815931, 1.        ,\n",
        "                  0.97417223],\n",
        "                 [0.97912109, 0.98592705, 0.98143742, 0.98451635, 0.97417223,\n",
        "                  1.        ]])"
      ],
      "metadata": {
        "id": "pmE6_fqlq9iB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figm.update_layout(\n",
        "    xaxis={'side': 'top'}, \n",
        "    yaxis={'side': 'left'}  \n",
        ")"
      ],
      "metadata": {
        "id": "-7pTHkdlsPMS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figm.show()"
      ],
      "metadata": {
        "id": "R83oGqgup7SO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = pt_model.bertopic_SL.bertopic.visualize_barchart([6,7,3,4,5,8,2,9,0,1], width=370, height=350, title=\"Distribucija besed v temah\")\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "HoGUCvc1ayoo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Splav\n",
        "#fig['data'][0]['xaxis'] = 'x'\n",
        "#fig['data'][0]['yaxis'] = 'y'\n",
        "fig['data'][0]['marker']['color'] = 'indigo'\n",
        "fig.layout.annotations[0].update(text=\"Splav\")\n",
        "\n",
        "# LGBTQ+\n",
        "#fig['data'][7]['xaxis'] = 'x2'\n",
        "#fig['data'][7]['yaxis'] = 'y2'\n",
        "fig['data'][1]['marker']['color'] = 'hotpink'\n",
        "fig.layout.annotations[1].update(text=\"LGBTQ+\")\n",
        "\n",
        "# Migracije\n",
        "#fig['data'][3]['xaxis'] = 'x3'\n",
        "#fig['data'][3]['yaxis'] = 'y3'\n",
        "fig['data'][2]['marker']['color'] = 'green'\n",
        "fig.layout.annotations[2].update(text=\"Migracije\")\n",
        "\n",
        "# Religije\n",
        "#fig['data'][4]['xaxis'] = 'x4'\n",
        "#fig['data'][4]['yaxis'] = 'y4'\n",
        "fig['data'][3]['marker']['color'] = 'orange'\n",
        "fig.layout.annotations[3].update(text=\"Religije\")\n",
        "\n",
        "# Krščanstvo\n",
        "#fig['data'][1]['xaxis'] = 'x5'\n",
        "#fig['data'][1]['yaxis'] = 'y5'\n",
        "fig['data'][4]['marker']['color'] = 'seagreen'\n",
        "fig.layout.annotations[4].update(text=\"Krščanstvo\")\n",
        "\n",
        "# Militarizem\n",
        "#fig['data'][6]['xaxis'] = 'x6'\n",
        "#fig['data'][6]['yaxis'] = 'y6'\n",
        "fig['data'][5]['marker']['color'] = 'darkgray'\n",
        "fig.layout.annotations[5].update(text=\"Militarizem\")\n",
        "\n",
        "# Nacionalna varnost\n",
        "#fig['data'][8]['xaxis'] = 'x7'\n",
        "#fig['data'][8]['yaxis'] = 'y7'\n",
        "fig['data'][6]['marker']['color'] = 'chocolate'\n",
        "fig.layout.annotations[6].update(text=\"Nacionalna varnost\")\n",
        "\n",
        "# Denacionalizacija\n",
        "#fig['data'][9]['xaxis'] = 'x8'\n",
        "#fig['data'][9]['yaxis'] = 'y8'\n",
        "fig['data'][7]['marker']['color'] = 'darkkhaki'\n",
        "fig.layout.annotations[7].update(text=\"Denacionalizacija\")\n",
        "\n",
        "# Levi govor\n",
        "#fig['data'][5]['xaxis'] = 'x9'\n",
        "#fig['data'][5]['yaxis'] = 'y9'\n",
        "fig['data'][8]['marker']['color'] = 'firebrick'\n",
        "fig.layout.annotations[8].update(text=\"Levi govor\")\n",
        "\n",
        "# Desni govor\n",
        "#fig['data'][2]['xaxis'] = 'x10'\n",
        "#fig['data'][2]['yaxis'] = 'y10'\n",
        "fig['data'][9]['marker']['color'] = 'royalblue'\n",
        "fig.layout.annotations[9].update(text=\"Desni govor\")"
      ],
      "metadata": {
        "id": "lzupRfPmcltL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig.show()"
      ],
      "metadata": {
        "id": "f0FoGlxTznYy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evalvacija koherence tematik"
      ],
      "metadata": {
        "id": "i0dV2a8GbpXe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "topics = [\n",
        "    ['splav', 'kontracepcija', 'zarodek', 'nosečnost'],\n",
        "    ['lgbt', 'homoseksualnost', 'spol', 'parada'],\n",
        "    ['prehod', 'begunec', 'meja', 'migrant'],\n",
        "    ['musliman', 'jud', 'islam', 'žid'],\n",
        "    ['cerkev', 'krščanstvo', 'kristjan', 'verouk'],\n",
        "    ['vojska', 'orožje', 'vojak', 'poveljnik'],\n",
        "    ['policija', 'zaščita', 'red', 'varnost'],\n",
        "    ['privaten', 'denacionalizacija', 'lastnina', 'premoženje'],\n",
        "    ['lev', 'levičar', 'levica', 'lmš', 'sd'],\n",
        "    ['desen', 'desničar', 'desnica', 'sds', 'nsi']\n",
        "]\n",
        "\n",
        "X_tokens = list(map(lambda x: x.split(\" \"), X))\n",
        "\n",
        "word2id = Dictionary(X_tokens)\n",
        "\n",
        "corpus = [ word2id.doc2bow(text) for text in X_tokens]\n",
        "\n",
        "cm = CoherenceModel(topics=topics, \n",
        "                    texts=X_tokens,\n",
        "                    coherence='c_v',  \n",
        "                    dictionary=word2id)\n",
        "\n",
        "coherence_per_topic = cm.get_coherence_per_topic()"
      ],
      "metadata": {
        "id": "5iuUCqVIbswD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "topics_str = [ '\\n '.join(t) for t in topics ]\n",
        "data_topic_score = pd.DataFrame( data=zip(topics_str, coherence_per_topic), columns=['Tematika', 'Koherenca'] )\n",
        "data_topic_score = data_topic_score.set_index('Tematika')\n",
        "\n",
        "fig, ax = plt.subplots( figsize=(20,20) )\n",
        "ax.set_title(\"Koherenca tematik\\n $C_v$\")\n",
        "sns.heatmap(data=data_topic_score, annot=True, square=True,\n",
        "            cmap='Reds', fmt='.2f',linecolor='black', ax=ax )\n",
        "plt.yticks( rotation=0 )\n",
        "ax.set_xlabel('')\n",
        "ax.set_ylabel('')\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "LNOcsT7_kljl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}