dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Importing Libraries"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Importing the dataset"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data = pd.read_json(\"datasets/02-data/data.json\")\n",
    "data.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## creating new dataframe for the model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model_df_x = pd.DataFrame(columns=[\n",
    "    \"full name\",\n",
    "    # Skills\n",
    "    \"programming_languages\",\n",
    "    \"backend_frameworks\",\n",
    "    \"devops_technologies\",\n",
    "    \"frontend_frameworks\",\n",
    "    \"system_design_skills\",\n",
    "    \"version_controls\",\n",
    "    \"ui-ux\",\n",
    "    \"data_related\",\n",
    "    \"databases\",\n",
    "    \"ai_models\",\n",
    "    \"ai_frameworks\",\n",
    "    \"network_skills\",\n",
    "    # \"message_brokers\",\n",
    "    \"hardware_skills\",\n",
    "    # \"os\",\n",
    "    # Educations\n",
    "    \"bachelor-computer engineering\",\n",
    "    \"master-computer engineering\",\n",
    "    \"phd-computer engineering\",\n",
    "    \"other-majors\",\n",
    "    # Experiences\n",
    "    # \"mobile developer\",\n",
    "    \"data scientist\",\n",
    "    \"ui-ux designer\",\n",
    "    \"network engineer\",\n",
    "    \"data engineer\",\n",
    "    \"software engineer\",\n",
    "    # \"full stack developer\",\n",
    "    \"frontend developer\",\n",
    "    \"hardware engineer\",\n",
    "    \"devops engineer\",\n",
    "    \"database administrator\",\n",
    "])\n",
    "model_df_x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Dataframe for the output"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model_df_y = pd.DataFrame(columns={\n",
    "    \"current experience\",\n",
    "})\n",
    "model_df_y"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Skill Buckets"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "SKILL_BUCKETS = {\n",
    "    \"programming_languages\": [\n",
    "        {\"java\", \"Core Java\", \"Java Enterprise Edition\", \"JavaSE\"},\n",
    "        {\"kotlin\", \"elixir\"},\n",
    "        {\"xml\", \"JSP\"},\n",
    "        {\"swift\", \"Swift (Programming Language)\"},\n",
    "        # {\"android\", \"android jetpack\", \"android data binding\", \"Android Development\", \"Android Studio\"},\n",
    "        # {\"ios\", \"ios development\"},\n",
    "        {\"r\", \"R Programming\"},\n",
    "        {\"python\", \"Python (Programming Language)\"},\n",
    "        {\"php\"},\n",
    "        {\"javascript\", \"js\", \"JavaScript eXtension (JSX)\", \"Three.js\", \"typescript\"},\n",
    "        {\"HTML\", \"HTML5\", \"HTML 5\"},\n",
    "        {\"CSS\", \"Cascading Style Sheets (CSS)\", \"less\", \"Tailwind CSS\", \"scss\", \"CSS Flexbox\"},\n",
    "        {\"golang\", \"Go (Programming Language)\"},\n",
    "        {\"c++\"},\n",
    "        {\"c#\"},\n",
    "        {\"c\", \"C (Programming Language)\" },\n",
    "        {\"ruby\", \"ruby or rails\"},#13\n",
    "        {\"perl\", \"Perl Script\"},\n",
    "        {\"Shell Scripting\", \"Bash\"},\n",
    "        {\"Scala\", \"Scala (Programming Language)\"},\n",
    "    ],\n",
    "    \"backend_frameworks\": [\n",
    "        {\"Back-End Web Development\"},\n",
    "        {\"ASP.NET\", \"ASP.NET Core\", \"ASP.NET MVC\", \".NET\", \".NET Framework\", \".NET Core\", \"Asp.net identity\", \"asp.net core Web API\", \"ASP.NET WEB API\", \"LINQ\", \"Entity Framework\", \"WCF\", \"blazor\"},\n",
    "        {\"django\", \"Django REST Framework\"},\n",
    "        {\"spring\", \"Spring Framework\", \"Spring MVC\", \"Spring Boot\"},\n",
    "        {\"laravel\"},\n",
    "        {\"Node.js\", \"Node\", \"Node js\"},\n",
    "        {\"Express.js\", \"Express\"},\n",
    "    ],\n",
    "    \"devops_technologies\": [\n",
    "        {\"kubernetes\", \"helm chart\", \"Helm Charts\", \"openshift\", \"prometheus\"},\n",
    "        {\"docker\", \"Docker Products\"},\n",
    "        {\"ansible\"},\n",
    "        {\"infrastructure\"},#3\n",
    "        {\"google cloud platform\", \"Google Cloud Platform (GCP)\"},\n",
    "        {\"azure\", \"Azure DevOps\", \"Microsoft Azure\", \"Microsoft Azure task management\"},\n",
    "        {\"containerization\", \"Virtualization\"},\n",
    "        {\"aws\", \"amazon web services certified solutions architect\", \"Amazon CloudFront\", \"AWS Lambda\",\n",
    "         \"Amazon Redshift\", \"Amazon Web Services (AWS)\", \"AWS CloudFormation\", \"Amazon ECS\"},\n",
    "        {\"jenkins\"},\n",
    "        {\"CI-CD\", \" Continuous Delivery\", \"Continuous Integration\", \"Continuous Improvement\"},\n",
    "        {\"cloud endure\", \"cloud migration\", \"cloud backup\", \"cloud computing\"},\n",
    "        {\"devops\"},\n",
    "    ],\n",
    "    \"frontend_frameworks\": [\n",
    "        {\"react\", \"React.js\", \"react hook\", \"react js\", \"React Native\", \"nextjs\"},\n",
    "        {\"vuejs\", \"vue.js\"},\n",
    "        {\"angular\", \"angularjs\", \"Angular Material\", \"Angular 6\"},\n",
    "        {\"redux\", \"Redux.js\", \"Redux Thunk\"},\n",
    "        {\"ajax\", \"jquery\"},\n",
    "        {\"bootstrap\"},\n",
    "    ],\n",
    "    \"system_design_skills\": [\n",
    "        {\"mobile application\", \"mobile applications\", \"iOS Development\", \"Android Development\"},\n",
    "        {\"oop\", \"Object-Oriented Programming (OOP)\", \"Object Oriented Design\"},\n",
    "        {\"design patterns\", \"software design patterns\", \"Software Design\"},\n",
    "        {\"Systems Engineering\", \"Distributed Systems\", \"software engineering\", \"sass\"},\n",
    "        {\"programming\"},\n",
    "        {\"algorithm\", \"algorithms\", \"Data Structures and algorithms\", \"data structures\", \"Graph Theory\"},\n",
    "        {\"web development\", \"grpc\"},\n",
    "        {\"web applications\", \"Web Application Firewall\", \"Web Services\", \"Web Application Security\", \"web application penetration testing\"},\n",
    "        {\"microservices\"},\n",
    "        {\"Software Testing\", \"Test Driven Development\", \"Test Automation\", \"unit testing\"},\n",
    "        {\"Unified Modeling Language (UML)\", \"uml\"},\n",
    "    ],\n",
    "    \"version_controls\": [\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "    ],\n",
    "    \"ui-ux\": [\n",
    "        {\"graphic design\", \"Computer Graphics\"},\n",
    "        {\"illustrator\"},\n",
    "        {\"figma\", \"adobe xd\", \"Google Material Design\"},\n",
    "        {\"photoshop\", \"adobe photoshop\"},\n",
    "        {\"ui-ux\", \"User Interface Design\", \"User Experience (UX)\", \"Ui Designer\", \"Ux design\",\n",
    "         \"User Interface Prototyping\", \"Material-UI\"},\n",
    "    ],\n",
    "    \"data_related\": [\n",
    "        {\"data integration\"},\n",
    "        {\"big data\", \"Big Data Analytics\", \"Data Engineering\"},\n",
    "        {\"data\"},\n",
    "        {\"spark\", \"Apache Spark\", \"PySpark\"},\n",
    "        {\"Information Retrieval\"},\n",
    "        {\"Data Visualization\"},\n",
    "        {\"Selenium\"},\n",
    "        {\"PySpark\"},\n",
    "        {\"mapreduce\"},\n",
    "        {\"Zeppelin\"},\n",
    "        {\"Hadoop\"},\n",
    "        {\"Apache Sqoop\", \"sqoop\"},\n",
    "        {\"Tableau\"},\n",
    "        {\"hdfs\"},\n",
    "        {\"Apache Oozie\", \"Oozie\"},\n",
    "        {\"Data Warehousing\"},\n",
    "        {\"Apache Ambari\"},\n",
    "        {\"Cloudera Impala\", \"Apache Impala\", \"Impala\"},\n",
    "        {\"Snowflake\"},\n",
    "    ],\n",
    "    \"databases\": [\n",
    "        {\"data migration\"},\n",
    "        {\"mongodb\"},\n",
    "        {\"database design\"},\n",
    "        {\"nosql\"},\n",
    "        {\"mysql\"},\n",
    "        {\"postgresql\"},\n",
    "        {\"graphql\"},\n",
    "        {\"elasticsearch\", \"Elastic Stack (ELK)\"},\n",
    "        {\"redis\"},\n",
    "        {\"data center\"},\n",
    "        {\"mariadb\"},\n",
    "        {\"oracle\"},\n",
    "        {\"sql\", \"Microsoft SQL Server\", \"SQL Server Management Studio\", \"PL/SQL\", \"T-SQL\", \"Transact-SQL (T-SQL)\", \"SSRS\", \"sqlite\"},\n",
    "    ],\n",
    "    \"ai_modeling\": [\n",
    "        {\"Artificial Intelligence (AI)\", \"AI\", \"Artificial Intelligence\"},\n",
    "        {\"regression testing\"},\n",
    "        {\"predictive modeling\"},\n",
    "        {\"Machine Learning\", \"Machine Learning Algorithms\"},\n",
    "        {\"image processing\"},\n",
    "        {\"data science\"},\n",
    "        {\"NLP\", \"Natural Language Processing (NLP)\"},\n",
    "        {\"data modeling\"},\n",
    "        {\"computer vision\"},\n",
    "        {\"deep learning\"},\n",
    "        {\"data mining\"},\n",
    "        {\"Data Analysis\"},\n",
    "        {\"MLOps\"},\n",
    "        {\"Supervised Learning\", \"Unsupervised Learning\"},\n",
    "        {\"Recommender Systems\"},\n",
    "        {\"Data Preprocessing\"},\n",
    "    ],\n",
    "    \"ai_frameworks\": [\n",
    "        {\"pandas\", \"Pandas (Software)\"},\n",
    "        {\"pytorch\"},\n",
    "        {\"keras\"},\n",
    "        {\"tensorflow\"},\n",
    "        {\"scikit-learn\"},\n",
    "        {\"seaborn\"},\n",
    "    ],\n",
    "    \"network_skills\": [\n",
    "        {\"Dynamic Host Configuration Protocol (DHCP)\"},\n",
    "        {\"Domain Name System (DNS)\"},\n",
    "        {\"routing protocols\", \"Routing\"},\n",
    "        {\"Routers\"},\n",
    "        {\"cisco\"},\n",
    "        {\"wireless\"},\n",
    "        {\"vpn\", \"SSLVPN\"},\n",
    "        {\"switching\"},\n",
    "        {\"dhcp server\"},\n",
    "        {\"lan-wan\"},\n",
    "        {\"penetration test\", \"Penetration Testing\", \"network security\", \"cybersecurity\", \"hacking\"},\n",
    "        {\"network administration\", \"Server Administration\", \"Networking\", \"Computer Networking\"},\n",
    "        {\"ip\"},\n",
    "        {\"TCP/IP\"},\n",
    "    ],\n",
    "    # \"message_brokers\": [\n",
    "    #     {\"apache kafka\"},\n",
    "    #     {\"rabbitmq\"},\n",
    "    # ],\n",
    "    \"hardware_skills\": [\n",
    "        {\"FPGA\", \"Field-Programmable Gate Arrays (FPGA)\"},\n",
    "        {\"PCB design\", \"Printed Circuit Board (PCB)\"},\n",
    "        {\"VLSI\", \"Very-Large-Scale Integration (VLSI)\"},\n",
    "        {\"IOT\"},\n",
    "        {\"circuit design\", \"Electronic Circuit Design\", \"Electronics Hardware Design\", \"Electrical Engineering\", \"Electronics\"},\n",
    "        {\"embedded\", \"Embedded C\", \"Embedded Systems\", \"Embedded Software\"},\n",
    "        {\"Computer Architecture\", \"unix\"},\n",
    "        {\"PSpice\", \"Hspice\"},\n",
    "        {\"microcontroller\"},\n",
    "        {\"Controller Area Network (CAN)\"},\n",
    "        {\"signal processing\"},\n",
    "        {\"raspberry pi\"},\n",
    "        {\"verilog\", \"System Verilog\"},\n",
    "        {\"stm32\"},\n",
    "        {\"ARM\", \"ARM Assembly\", \"Assembly Language\"},\n",
    "        {\"xilinx\"},\n",
    "        {\"arduino\"},\n",
    "        {\"simulink\", \"ModelSim\"},\n",
    "        {\"matlab\"},\n",
    "        {\"VHDL\"},\n",
    "        {\"proteus\"},\n",
    "    ],\n",
    "    # \"os\": [\n",
    "    #     {\"linux\", \"Red Hat Linux\", \"Linux Kernel\", \"Arch Linux\", \"Kali Linux\", \"CentOS\"},\n",
    "    #     {\"vmware\", \"VMware Infrastructure\"},\n",
    "    #     {\"Windows Server\", \"Windows\"},\n",
    "    # ]\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "counted_skills = {\n",
    "    \"programming_languages\": [False for i in range(len(SKILL_BUCKETS[\"programming_languages\"]))],\n",
    "    \"backend_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"backend_frameworks\"]))],\n",
    "    \"devops_technologies\": [False for i in range(len(SKILL_BUCKETS[\"devops_technologies\"]))],\n",
    "    \"frontend_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"frontend_frameworks\"]))],\n",
    "    \"system_design_skills\": [False for i in range(len(SKILL_BUCKETS[\"system_design_skills\"]))],\n",
    "    \"version_controls\": [False for i in range(len(SKILL_BUCKETS[\"version_controls\"]))],\n",
    "    \"ui-ux\": [False for i in range(len(SKILL_BUCKETS[\"ui-ux\"]))],\n",
    "    \"data_related\": [False for i in range(len(SKILL_BUCKETS[\"data_related\"]))],\n",
    "    \"databases\": [False for i in range(len(SKILL_BUCKETS[\"databases\"]))],\n",
    "    \"ai_modeling\": [False for i in range(len(SKILL_BUCKETS[\"ai_modeling\"]))],\n",
    "    \"ai_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"ai_frameworks\"]))],\n",
    "    \"network_skills\": [False for i in range(len(SKILL_BUCKETS[\"network_skills\"]))],\n",
    "    # \"message_brokers\": [False for i in range(len(SKILL_BUCKETS[\"message_brokers\"]))],\n",
    "    \"hardware_skills\": [False for i in range(len(SKILL_BUCKETS[\"hardware_skills\"]))],\n",
    "    # \"os\": [False for i in range(len(SKILL_BUCKETS[\"os\"]))],\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Convert skill to lowercase"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "SKILL_PROGRAMMING_LANGUAGE = []\n",
    "SKILL_BACKEND_FRAMEWORKS = []\n",
    "SKILL_DEVOPS_TECHNOLOGIES = []\n",
    "SKILL_FRONTEND_FRAMEWORKS = []\n",
    "SKILL_SYSTEM_DESIGN = []\n",
    "SKILL_VERSION_CONTROL = []\n",
    "SKILL_UI_UX = []\n",
    "SKILL_DATA_RELATED = []\n",
    "SKILL_DATABASES = []\n",
    "SKILL_AI_MODELING = []\n",
    "SKILL_AI_FRAMEWORKS = []\n",
    "SKILL_NETWORK = []\n",
    "SKILL_MESSAGE_BROKERS = []\n",
    "SKILL_HARDWARE= []\n",
    "SKILL_OS = []\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"programming_languages\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"programming_languages\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_PROGRAMMING_LANGUAGE.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"backend_frameworks\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"backend_frameworks\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_BACKEND_FRAMEWORKS.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"devops_technologies\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"devops_technologies\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DEVOPS_TECHNOLOGIES.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"frontend_frameworks\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"frontend_frameworks\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_FRONTEND_FRAMEWORKS.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"system_design_skills\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"system_design_skills\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_SYSTEM_DESIGN.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"version_controls\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"version_controls\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_VERSION_CONTROL.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"ui-ux\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"ui-ux\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_UI_UX.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"data_related\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"data_related\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DATA_RELATED.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"databases\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"databases\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DATABASES.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"ai_modeling\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"ai_modeling\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_AI_MODELING.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"ai_frameworks\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"ai_frameworks\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_AI_FRAMEWORKS.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"network_skills\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"network_skills\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_NETWORK.append(skill_set)\n",
    "\n",
    "# for i in range(len(SKILL_BUCKETS[\"message_brokers\"])):\n",
    "#     skill_set = set()\n",
    "#     for j in SKILL_BUCKETS[\"message_brokers\"][i]:\n",
    "#         skill_set.add(j.lower())\n",
    "#     SKILL_MESSAGE_BROKERS.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"hardware_skills\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"hardware_skills\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_HARDWARE.append(skill_set)\n",
    "\n",
    "# for i in range(len(SKILL_BUCKETS[\"os\"])):\n",
    "#     skill_set = set()\n",
    "#     for j in SKILL_BUCKETS[\"os\"][i]:\n",
    "#         skill_set.add(j.lower())\n",
    "#     SKILL_OS.append(skill_set)\n",
    "\n",
    "SKILL_BUCKETS_LOWERCASE = {\n",
    "    \"programming_languages\": SKILL_PROGRAMMING_LANGUAGE,\n",
    "    \"backend_frameworks\": SKILL_BACKEND_FRAMEWORKS,\n",
    "    \"devops_technologies\": SKILL_DEVOPS_TECHNOLOGIES,\n",
    "    \"frontend_frameworks\": SKILL_FRONTEND_FRAMEWORKS,\n",
    "    \"system_design_skills\": SKILL_SYSTEM_DESIGN,\n",
    "    \"version_controls\": SKILL_VERSION_CONTROL,\n",
    "    \"ui-ux\": SKILL_UI_UX,\n",
    "    \"data_related\": SKILL_DATA_RELATED,\n",
    "    \"databases\": SKILL_DATABASES,\n",
    "    \"ai_modeling\": SKILL_AI_MODELING,\n",
    "    \"ai_frameworks\": SKILL_AI_FRAMEWORKS,\n",
    "    \"network_skills\": SKILL_NETWORK,\n",
    "    # \"message_brokers\": SKILL_MESSAGE_BROKERS,\n",
    "    \"hardware_skills\": SKILL_HARDWARE,\n",
    "    # \"os\": SKILL_OS,\n",
    "}\n",
    "\n",
    "SKILL_BUCKETS_LOWERCASE"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Education Categories"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "EDUCATIONS = [\n",
    "    \"bachelor-computer engineering\",\n",
    "    \"master-computer engineering\",\n",
    "    \"phd-computer engineering\",\n",
    "    \"others\",\n",
    "]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Experience Categories"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "EXPERIENCES = [\n",
    "    \"mobile developer\",  #0\n",
    "    \"data scientist\",  #1\n",
    "    \"ui-ux designer\",  #2\n",
    "    \"network engineer\",  #3\n",
    "    \"data engineer\",  #4\n",
    "    \"software engineer\",  #5\n",
    "    \"full stack developer\",  #6\n",
    "    \"frontend developer\",  #7\n",
    "    \"hardware engineer\",  #8\n",
    "    \"devops engineer\",  #9\n",
    "    \"database administrator\",  #10\n",
    "]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create conditions and append it to the dataframe"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "for index, row in data.iterrows():\n",
    "    # counter for skills\n",
    "    counter_skills_programming = 0\n",
    "    counter_skills_backend = 0\n",
    "    counter_skills_devops = 0\n",
    "    counter_skills_frontend = 0\n",
    "    counter_skills_system_design = 0\n",
    "    counter_skills_version_control = 0\n",
    "    counter_skills_ui_ux = 0\n",
    "    counter_skills_data_related = 0\n",
    "    counter_skills_databases = 0\n",
    "    counter_skills_ai_modeling = 0\n",
    "    counter_skills_ai_frameworks = 0\n",
    "    counter_skills_network = 0\n",
    "    counter_skills_message_brokers = 0\n",
    "    counter_skills_hardware = 0\n",
    "    counter_skills_os = 0\n",
    "    # counter for educations\n",
    "    counter_educations_bachelor = 0\n",
    "    counter_educations_master = 0\n",
    "    counter_educations_phd = 0\n",
    "    counter_educations_other_majors = 0\n",
    "    # counter for experiences\n",
    "    counter_mobile_developer = 0\n",
    "    counter_data_scientist = 0\n",
    "    counter_ui_ux_designer = 0\n",
    "    counter_network_engineer = 0\n",
    "    counter_data_engineer = 0\n",
    "    counter_software_engineer = 0\n",
    "    counter_fullstack_developer = 0\n",
    "    counter_frontend_developer = 0\n",
    "    counter_hardware_engineer = 0\n",
    "    counter_devops_engineer = 0\n",
    "    counter_database_administrator = 0\n",
    "    counter_other_experiences = 0\n",
    "    current_job = \"\"\n",
    "    counted_skills = {\n",
    "        \"programming_languages\": [False for i in range(len(SKILL_BUCKETS[\"programming_languages\"]))],\n",
    "        \"backend_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"backend_frameworks\"]))],\n",
    "        \"devops_technologies\": [False for i in range(len(SKILL_BUCKETS[\"devops_technologies\"]))],\n",
    "        \"frontend_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"frontend_frameworks\"]))],\n",
    "        \"system_design_skills\": [False for i in range(len(SKILL_BUCKETS[\"system_design_skills\"]))],\n",
    "        \"version_controls\": [False for i in range(len(SKILL_BUCKETS[\"version_controls\"]))],\n",
    "        \"ui-ux\": [False for i in range(len(SKILL_BUCKETS[\"ui-ux\"]))],\n",
    "        \"data_related\": [False for i in range(len(SKILL_BUCKETS[\"data_related\"]))],\n",
    "        \"databases\": [False for i in range(len(SKILL_BUCKETS[\"databases\"]))],\n",
    "        \"ai_modeling\": [False for i in range(len(SKILL_BUCKETS[\"ai_modeling\"]))],\n",
    "        \"ai_frameworks\": [False for i in range(len(SKILL_BUCKETS[\"ai_frameworks\"]))],\n",
    "        \"network_skills\": [False for i in range(len(SKILL_BUCKETS[\"network_skills\"]))],\n",
    "        # \"message_brokers\": [False for i in range(len(SKILL_BUCKETS[\"message_brokers\"]))],\n",
    "        \"hardware_skills\": [False for i in range(len(SKILL_BUCKETS[\"hardware_skills\"]))],\n",
    "        # \"os\": [False for i in range(len(SKILL_BUCKETS[\"os\"]))],\n",
    "    }\n",
    "\n",
    "    # Full name\n",
    "    name = row[\"Full Name\"]\n",
    "\n",
    "    # Skills\n",
    "    for itemSkill in row[\"Skills\"]:\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"programming_languages\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"programming_languages\"][idx]:\n",
    "                if not counted_skills[\"programming_languages\"][idx]:\n",
    "                    counter_skills_programming += 1\n",
    "                    counted_skills[\"programming_languages\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"backend_frameworks\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"backend_frameworks\"][idx]:\n",
    "                if not counted_skills[\"backend_frameworks\"][idx]:\n",
    "                    counter_skills_backend += 1\n",
    "                    counted_skills[\"backend_frameworks\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"devops_technologies\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"devops_technologies\"][idx]:\n",
    "                if not counted_skills[\"devops_technologies\"][idx]:\n",
    "                    counter_skills_devops += 1\n",
    "                    counted_skills[\"devops_technologies\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"frontend_frameworks\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"frontend_frameworks\"][idx]:\n",
    "                if not counted_skills[\"frontend_frameworks\"][idx]:\n",
    "                    counter_skills_frontend += 1\n",
    "                    counted_skills[\"frontend_frameworks\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"system_design_skills\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"system_design_skills\"][idx]:\n",
    "                if not counted_skills[\"system_design_skills\"][idx]:\n",
    "                    counter_skills_system_design += 1\n",
    "                    counted_skills[\"system_design_skills\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"version_controls\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"version_controls\"][idx]:\n",
    "                if not counted_skills[\"version_controls\"][idx]:\n",
    "                    counter_skills_version_control += 1\n",
    "                    counted_skills[\"version_controls\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"ui-ux\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"ui-ux\"][idx]:\n",
    "                if not counted_skills[\"ui-ux\"][idx]:\n",
    "                    counter_skills_ui_ux += 1\n",
    "                    counted_skills[\"ui-ux\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"data_related\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"data_related\"][idx]:\n",
    "                if not counted_skills[\"data_related\"][idx]:\n",
    "                    counter_skills_data_related += 1\n",
    "                    counted_skills[\"data_related\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"databases\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"databases\"][idx]:\n",
    "                if not counted_skills[\"databases\"][idx]:\n",
    "                    counter_skills_databases += 1\n",
    "                    counted_skills[\"databases\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"ai_modeling\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"ai_modeling\"][idx]:\n",
    "                if not counted_skills[\"ai_modeling\"][idx]:\n",
    "                    counter_skills_ai_modeling += 1\n",
    "                    counted_skills[\"ai_modeling\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"ai_frameworks\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"ai_frameworks\"][idx]:\n",
    "                if not counted_skills[\"ai_frameworks\"][idx]:\n",
    "                    counter_skills_ai_frameworks += 1\n",
    "                    counted_skills[\"ai_frameworks\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"network_skills\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"network_skills\"][idx]:\n",
    "                if not counted_skills[\"network_skills\"][idx]:\n",
    "                    counter_skills_network += 1\n",
    "                    counted_skills[\"network_skills\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"hardware_skills\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"hardware_skills\"][idx]:\n",
    "                if not counted_skills[\"hardware_skills\"][idx]:\n",
    "                    counter_skills_hardware += 1\n",
    "                    counted_skills[\"hardware_skills\"][idx] = True\n",
    "\n",
    "\n",
    "        # for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"message_brokers\"]):\n",
    "        #     if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"message_brokers\"][idx]:\n",
    "        #         if not counted_skills[\"message_brokers\"][idx]:\n",
    "        #             counter_skills_message_brokers += 1\n",
    "        #             counted_skills[\"message_brokers\"][idx] = True\n",
    "\n",
    "        # for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"os\"]):\n",
    "        #     if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"os\"][idx]:\n",
    "        #         if not counted_skills[\"os\"][idx]:\n",
    "        #             counter_skills_os += 1\n",
    "        #             counted_skills[\"os\"][idx] = True\n",
    "\n",
    "    # Educations\n",
    "    for itemEducation in row[\"Educations\"]:\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[0]:\n",
    "            counter_educations_bachelor = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[1]:\n",
    "            counter_educations_master = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[2]:\n",
    "            counter_educations_phd = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[3]:\n",
    "            counter_educations_other_majors = 1\n",
    "\n",
    "    # Experiences\n",
    "    \"\"\"\n",
    "    Depending on the length of the experience, we build the model.\n",
    "    If the person has only 1 job, that will go to the output of the model.\n",
    "    If the person has more than one jobs, the last one is counted as model output and the rest are the input.\n",
    "    \"\"\"\n",
    "    for i in range(len(row[\"Experiences\"])):\n",
    "        if len(row[\"Experiences\"]) == 1:\n",
    "            current_job = (row[\"Experiences\"][i])[\"name\"]\n",
    "        else:\n",
    "            current_job = (row[\"Experiences\"][0])[\"name\"]\n",
    "            if i != 0:\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[0]:\n",
    "                    counter_mobile_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[1]:\n",
    "                    counter_data_scientist += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[2]:\n",
    "                    counter_ui_ux_designer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[3]:\n",
    "                    counter_network_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[4]:\n",
    "                    counter_data_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[5]:\n",
    "                    counter_software_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[6]:\n",
    "                    counter_frontend_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[7]:\n",
    "                    counter_frontend_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[8]:\n",
    "                    counter_hardware_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[9]:\n",
    "                    counter_devops_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[10]:\n",
    "                    counter_database_administrator += 1\n",
    "\n",
    "    # Appending to the dataframe\n",
    "    model_df_x = model_df_x.append({\n",
    "        # full name\n",
    "        \"full name\": name,\n",
    "        # skills\n",
    "        'programming_languages': counter_skills_programming / len(SKILL_BUCKETS[\"programming_languages\"]),\n",
    "        \"backend_frameworks\": counter_skills_backend / len(SKILL_BUCKETS[\"backend_frameworks\"]),\n",
    "        \"devops_technologies\": counter_skills_devops / len(SKILL_BUCKETS[\"devops_technologies\"]),\n",
    "        \"frontend_frameworks\": counter_skills_frontend / len(SKILL_BUCKETS[\"frontend_frameworks\"]),\n",
    "        \"system_design_skills\": counter_skills_system_design / len(SKILL_BUCKETS[\"system_design_skills\"]),\n",
    "        \"version_controls\": counter_skills_version_control / len(SKILL_BUCKETS[\"version_controls\"]),\n",
    "        \"ui-ux\": counter_skills_ui_ux / len(SKILL_BUCKETS[\"ui-ux\"]),\n",
    "        \"data_related\": counter_skills_data_related / len(SKILL_BUCKETS[\"data_related\"]),\n",
    "        \"databases\": counter_skills_databases / len(SKILL_BUCKETS[\"databases\"]),\n",
    "        \"ai_models\": counter_skills_ai_modeling / len(SKILL_BUCKETS[\"ai_modeling\"]),\n",
    "        \"ai_frameworks\": counter_skills_ai_frameworks / len(SKILL_BUCKETS[\"ai_frameworks\"]),\n",
    "        \"network_skills\": counter_skills_network / len(SKILL_BUCKETS[\"network_skills\"]),\n",
    "        \"hardware_skills\": counter_skills_hardware / len(SKILL_BUCKETS[\"hardware_skills\"]),\n",
    "        # \"message_brokers\": counter_skills_message_brokers / len(SKILL_BUCKETS[\"message_brokers\"]),\n",
    "        # \"os\": counter_skills_os / len(SKILL_BUCKETS[\"os\"]),\n",
    "        # education\n",
    "        'bachelor-computer engineering': counter_educations_bachelor,\n",
    "        \"master-computer engineering\": counter_educations_master,\n",
    "        \"phd-computer engineering\": counter_educations_phd,\n",
    "        \"other-majors\": counter_educations_other_majors,\n",
    "        # experience\n",
    "        # \"mobile developer\": counter_mobile_developer,\n",
    "        \"data scientist\": counter_data_scientist,\n",
    "        \"ui-ux designer\": counter_ui_ux_designer,\n",
    "        \"network engineer\": counter_network_engineer,\n",
    "        \"data engineer\": counter_data_engineer,\n",
    "        \"software engineer\": counter_software_engineer,\n",
    "        # \"full stack developer\": counter_fullstack_developer,\n",
    "        \"frontend developer\": counter_frontend_developer,\n",
    "        \"hardware engineer\": counter_hardware_engineer,\n",
    "        \"devops engineer\": counter_devops_engineer,\n",
    "        \"database administrator\": counter_database_administrator,\n",
    "    }, ignore_index=True)\n",
    "\n",
    "    model_df_y = model_df_y.append({\n",
    "        \"current experience\": current_job,\n",
    "    }, ignore_index=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Exporting dataframe to file"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "model_df_x.to_csv('dataframe/09/df_x.csv', index=False)\n",
    "model_df_y.to_csv('dataframe/09/df_y.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## counting jobs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "job_counts = [0 for i in range(len(EXPERIENCES))]\n",
    "for idx, val in enumerate(model_df_y.values):\n",
    "    if val == EXPERIENCES[0]:\n",
    "        job_counts[0] += 1\n",
    "    elif val == EXPERIENCES[1]:\n",
    "        job_counts[1] += 1\n",
    "    elif val == EXPERIENCES[2]:\n",
    "        job_counts[2] += 1\n",
    "    elif val == EXPERIENCES[3]:\n",
    "        job_counts[3] += 1\n",
    "    elif val == EXPERIENCES[4]:\n",
    "        job_counts[4] += 1\n",
    "    elif val == EXPERIENCES[5]:\n",
    "        job_counts[5] += 1\n",
    "    elif val == EXPERIENCES[6]:\n",
    "        job_counts[6] += 1\n",
    "    elif val == EXPERIENCES[7]:\n",
    "        job_counts[7] += 1\n",
    "    elif val == EXPERIENCES[8]:\n",
    "        job_counts[8] += 1\n",
    "    elif val == EXPERIENCES[9]:\n",
    "        job_counts[9] += 1\n",
    "    elif val == EXPERIENCES[10]:\n",
    "        job_counts[10] += 1\n",
    "job_counts = np.array(job_counts)\n",
    "\n",
    "counts_of_each_job = {\n",
    "    EXPERIENCES[0]: job_counts[0],\n",
    "    EXPERIENCES[1]: job_counts[1],\n",
    "    EXPERIENCES[2]: job_counts[2],\n",
    "    EXPERIENCES[3]: job_counts[3],\n",
    "    EXPERIENCES[4]: job_counts[4],\n",
    "    EXPERIENCES[5]: job_counts[5],\n",
    "    EXPERIENCES[6]: job_counts[6],\n",
    "    EXPERIENCES[7]: job_counts[7],\n",
    "    EXPERIENCES[8]: job_counts[8],\n",
    "    EXPERIENCES[9]: job_counts[9],\n",
    "    EXPERIENCES[10]: job_counts[10],\n",
    "}\n",
    "counts_of_each_job"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}