nearLabelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/AnnaValentinaHirsch/Web3CodeLLM/blob/main/NEARlabelling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VgmcUTdIp2mq"
   },
   "source": [
    "**This notebook is used to label the training data for finetuning the Starcoder2 model for the Near dApps domain. Openai API is used to generate the labels (user prompts) corresponding to the github repos, tree structures, and readme contents.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CynzDheZp2mu"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "%pip install pandas numpy openai python-dotenv datasets transformers torch matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xhxhHFcop2mx"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from datasets import Dataset, load_dataset # huggingface\n",
    "from transformers import pipeline # summarizer\n",
    "from openai import OpenAI # new\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "os.environ['OPENAI_API_KEY'] = 'secret'\n",
    "client = OpenAI(api_key=os.environ['OPENAI_API_KEY'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3SS33fXr-5CX"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yEBRMLjBuUt6"
   },
   "outputs": [],
   "source": [
    "# Step 2: Verify GPU availability\n",
    "device = 0 if torch.cuda.is_available() else -1\n",
    "print(f\"Using device: {'GPU' if device == 0 else 'CPU'}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "a0Lh5fUalxsS"
   },
   "outputs": [],
   "source": [
    "# Free up GPU memory\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gq2Gh6m3p2my"
   },
   "outputs": [],
   "source": [
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "\n",
    "# Load Dataset\n",
    "dataset = load_dataset('jcarbonnell/structTuningNEAR')\n",
    "\n",
    "# Convert the train split of the dataset to a pandas DataFrame\n",
    "train = dataset['train'].to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "v2SKeGLUp2mz"
   },
   "outputs": [],
   "source": [
    "# Order by size of readme file\n",
    "train = train.sort_values(by='readme', key=lambda x: x.str.len(), ascending=True)\n",
    "\n",
    "# Remove rows with empty readme files\n",
    "train = train[train['readme'].str.len() > 200]\n",
    "train = train[train['readme'].str.len() < 50000] # remove problematic files that cause crash\n",
    "\n",
    "# Reset index and drop the old index\n",
    "train = train.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XdSJnB9AAmos"
   },
   "outputs": [],
   "source": [
    "# Calculate the wordcount of readme files\n",
    "train['readme_word_count'] = train['readme'].apply(lambda x: len(str(x).split()))\n",
    "\n",
    "# Plot the histogram of README word counts\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(train['readme_word_count'], bins=30, edgecolor='black')\n",
    "plt.title('Histogram of README File Word Counts')\n",
    "plt.xlabel('Number of Words in README')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "# Display the top 20 longest README files\n",
    "print(train[['repoName', 'readme_word_count']].head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bOcxfpfzp2m0"
   },
   "outputs": [],
   "source": [
    "# Def params\n",
    "engine = \"gpt-4o\"\n",
    "max_output_tokens = 500\n",
    "example = \"\"\"Create a project structure for a NEAR DApp based on the following requirements:\n",
    "\n",
    "1. The project should be related to the Nearuko NFT, which can be converted into a character in the Etheruko game.\n",
    "2. Use necessary files typically required for a NEAR Protocol mainnet DApp.\n",
    "3. Include all relevant dependencies and packages for an NFT project on the NEAR Protocol.\n",
    "4. The main coding language should be TypeScript.\n",
    "5. Ensure the project includes configurations, tests, and NEAR-compatible contract files.\n",
    "6. Capture any common NEAR DApp structure conventions while setting up the project.\n",
    "\n",
    "Provide a well-organized directory structure and file list based on these requirements.\"\"\"\n",
    "\n",
    "# Function to generate labels (prompts)\n",
    "def generate_prompt(repoName, tree, readme, example):\n",
    "    # Create a user prompt for a coding assistant\n",
    "    prompt = (\n",
    "        f\"You are provided with a GitHub repository called \\n{repoName}\\n\\n. This repository has the following directory structure:\\n\"\n",
    "        f\"{tree}\\n\\n\"\n",
    "        f\"The README file contains the following information:\\n{readme}\\n\\n\"\n",
    "        f\"Step 1: Extract all the relevant information from the README file needed to predict the corresponding tree for a NEAR DApp, such as necessary files, dependencies, packages, and any particular coding languages or frameworks that should be used. \"\n",
    "        f\"Step 2: Write a perfect user prompt asking a coding assistant to create a project stucture based only on the extracted information from the README file. Only return the user prompt from Step 2. Do not return any information about the tree or file names. Here is an example: \\n{example}\\n\\n\"\n",
    "    )\n",
    "\n",
    "    response = client.chat.completions.create(\n",
    "        model=engine,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ],\n",
    "        max_tokens=max_output_tokens\n",
    "    )\n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XM4fpvGskLfQ"
   },
   "outputs": [],
   "source": [
    "# Print example prompt\n",
    "print(generate_prompt(train['repoName'][500], train['tree'][500], train['readme'][500], example))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WcqOyqwzqdwn"
   },
   "source": [
    "**Create labels (perfect user prompts) by extracting crucial information from readme files**\n",
    "\n",
    "\n",
    "*Please Note:*\n",
    "*   Summarizer is used to ensure large readme files don't exceed the api token limits.\n",
    "*   Time Delay introducted to ensure number of allowed requests per minute are not exceeded.\n",
    "*   Checkpoint introduced to save intermediate outputs in case of system crashing.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rpPeKPzB_EXW"
   },
   "outputs": [],
   "source": [
    "# Define the file path for checkpoint in Google Drive\n",
    "checkpoint_file = '/content/drive/My Drive/checkpoint.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qFuTCkwyonnL"
   },
   "outputs": [],
   "source": [
    "# Initialize user_prompts if not loaded from checkpoint\n",
    "if 'user_prompt' not in train.columns:\n",
    "    train['user_prompt'] = None\n",
    "    user_prompts = [None] * len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OGwCq2n0pMRE"
   },
   "outputs": [],
   "source": [
    "train = train.drop('readme_word_count', axis=1)\n",
    "train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UOmlTKq5p2m1"
   },
   "outputs": [],
   "source": [
    "# Define the summarizer\n",
    "summarizer = pipeline(\"summarization\", model=\"t5-base\", tokenizer=\"t5-base\", device=device)\n",
    "max_input_tokens = 2000\n",
    "delay_between_requests = 2 # in seconds\n",
    "\n",
    "# Function to summarize long readme files\n",
    "def summarize(text, max_length):\n",
    "    if len(text.split()) > max_length:\n",
    "        summary = summarizer(text, max_length=max_length, do_sample=False)\n",
    "        return summary[0]['summary_text']\n",
    "    return text\n",
    "\n",
    "# Function to count tokens\n",
    "def count_tokens(text, tokenizer):\n",
    "    return len(tokenizer.encode(text))\n",
    "\n",
    "# Save function\n",
    "def save_checkpoint(dataframe, prompts, filename):\n",
    "    dataframe['user_prompt'] = prompts\n",
    "    dataframe.to_csv(filename, index=False, escapechar='\\\\')\n",
    "\n",
    "# Load function\n",
    "def load_checkpoint(filename):\n",
    "    return pd.read_csv(filename, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yfQrNdxAaTkT"
   },
   "outputs": [],
   "source": [
    "# Processing loop\n",
    "for index, row in train.iterrows():\n",
    "    if user_prompts[index] is not None:\n",
    "        continue  # Skip already processed rows\n",
    "\n",
    "    repoName = row['repoName']\n",
    "    tree = row['tree']\n",
    "    readme = row['readme']\n",
    "    try:\n",
    "        # Check if readme is too long and summarize if needed\n",
    "        readme = summarize(readme, max_input_tokens)\n",
    "\n",
    "        # Generate the user prompt\n",
    "        user_prompt = generate_prompt(repoName, tree, readme, example)\n",
    "\n",
    "        # Count tokens in the user prompt\n",
    "        input_tokens = count_tokens(user_prompt, summarizer.tokenizer)\n",
    "\n",
    "        if input_tokens > max_input_tokens + max_output_tokens:\n",
    "            raise ValueError(f\"Prompt too long: {input_tokens} tokens (max allowed is {max_input_tokens + max_output_tokens})\")\n",
    "\n",
    "        user_prompts[index] = user_prompt  # None for errors\n",
    "\n",
    "        # Log the length of the input and prompt for debugging\n",
    "        print(f\"Row {index}: Input length {len(summarizer.tokenizer.encode(readme))} tokens, Prompt length {input_tokens} tokens\")\n",
    "\n",
    "        # Free up GPU memory\n",
    "        torch.cuda.empty_cache()\n",
    "\n",
    "        # Implement delay between requests\n",
    "        time.sleep(delay_between_requests)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error processing row {index}: {e}\")\n",
    "\n",
    "    # Save checkpoint after each row\n",
    "    save_checkpoint(train, user_prompts, checkpoint_file)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}