GTP4all.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4945798-9778-45ca-a6ac-c7893ce90bd3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain import PromptTemplate, LLMChain\n",
    "from langchain.llms import GPT4All\n",
    "from langchain.callbacks.base import CallbackManager\n",
    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "edbd0d57-1353-49b3-b1a1-e3beeee0fc57",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.chains.api.prompt import API_RESPONSE_PROMPT\n",
    "from langchain.chains import APIChain\n",
    "from langchain.prompts.prompt import PromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "388c0d1f-0b60-4883-b8a7-0b14bd9eaf45",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "template = \"\"\"Question: {question}\n",
    "Persona:Redneck\n",
    "Answer :\"\"\"\n",
    "\n",
    "prompt = PromptTemplate(template=template, input_variables=[\"question\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6016c70f-a6ad-4d26-b29d-b944481fa168",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "local_path = './models/gpt4all-lora-quantized-fuckedwith.bin'  # replace with your desired local file path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4d90f0fa-39fd-4f3c-b177-65db0723ab86",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_load: loading model from './models/gpt4all-lora-quantized-fuckedwith.bin' - please wait ...\n",
      "llama_model_load: n_vocab = 32001\n",
      "llama_model_load: n_ctx   = 512\n",
      "llama_model_load: n_embd  = 4096\n",
      "llama_model_load: n_mult  = 256\n",
      "llama_model_load: n_head  = 32\n",
      "llama_model_load: n_layer = 32\n",
      "llama_model_load: n_rot   = 128\n",
      "llama_model_load: f16     = 2\n",
      "llama_model_load: n_ff    = 11008\n",
      "llama_model_load: n_parts = 1\n",
      "llama_model_load: type    = 1\n",
      "llama_model_load: ggml map size = 4017.70 MB\n",
      "llama_model_load: ggml ctx size =  81.25 KB\n",
      "llama_model_load: mem required  = 5809.78 MB (+ 2052.00 MB per state)\n",
      "llama_model_load: loading tensors from './models/gpt4all-lora-quantized-fuckedwith.bin'\n",
      "llama_model_load: model size =  4017.27 MB / num tensors = 291\n",
      "llama_init_from_file: kv self size  =  512.00 MB\n"
     ]
    }
   ],
   "source": [
    "# Callbacks support token-wise streaming\n",
    "callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])\n",
    "# Verbose is required to pass to the callback manager\n",
    "llm = GPT4All(model=local_path, callback_manager=callback_manager, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2c7ae31b-bcbd-45b9-bd73-028eb10a19bf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "llm_chain = LLMChain(prompt=prompt, llm=llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4ab9f00c-64f9-4b70-ac89-bec0e07fb1de",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Question"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_generate: seed = 1682221609\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | \n",
      "sampling: temp = 0.800000, top_k = 40, top_p = 0.950000, repeat_last_n = 64, repeat_penalty = 1.300000\n",
      "generate: n_ctx = 512, n_batch = 1, n_predict = 256, n_keep = 0\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ": who was the 12th president of the USA?\n",
      "\n",
      "Answer Persona redneck: Answered"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " [end of text]\n",
      "\n",
      "llama_print_timings:        load time = 13119.34 ms\n",
      "llama_print_timings:      sample time =     1.58 ms /     2 runs   (    0.79 ms per run)\n",
      "llama_print_timings: prompt eval time =     0.00 ms /     1 tokens (    0.00 ms per token)\n",
      "llama_print_timings:        eval time =  9190.18 ms /    26 runs   (  353.47 ms per run)\n",
      "llama_print_timings:       total time = 15371.16 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' Question: who was the 12th president of the USA?\\n\\nAnswer Persona redneck: Answered'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question = \"who was the 12th president of the USA?\"\n",
    "\n",
    "llm_chain.run(question)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "98fe6542-a703-482e-b470-f67d3c22fb1d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 57649336\n",
      "drwxr-xr-x  10 friendsoncode  staff   320B Apr 22 23:36 \u001b[34m.\u001b[m\u001b[m\n",
      "drwxr-xr-x  10 friendsoncode  staff   320B Apr 22 23:31 \u001b[34m..\u001b[m\u001b[m\n",
      "drwxr-xr-x   2 friendsoncode  staff    64B Apr 22 20:34 \u001b[34m.ipynb_checkpoints\u001b[m\u001b[m\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Apr 22 23:36 gpt4all-lora-quantized-fuckedwith.bin\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Apr  5 09:07 gpt4all-lora-quantized-ggml.bin\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Apr 22 23:36 gpt4all-lora-quantized.bin\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Mar 29 10:34 gpt4all-lora-quantized.bin.orig\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Apr 22 22:30 gpt4all-lora-unfiltered-quantized-fuckedwith.bin\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Apr 22 22:30 gpt4all-lora-unfiltered-quantized.bin\n",
      "-rw-r--r--   1 friendsoncode  staff   3.9G Mar 29 16:00 gpt4all-lora-unfiltered-quantized.bin.orig\n"
     ]
    }
   ],
   "source": [
    "!ls  -lah ./models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fa292c0-b21d-49df-b00f-77e65d9efeaa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}