Sakura-32B-Galgame-Kaggle-llama.cpp.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["[![Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://kaggle.com/kernels/welcome?src=https://github.com/Isotr0py/SakuraLLM-Notebooks/blob/main/Sakura-32B-Galgame-Kaggle-llama.cpp.ipynb)"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-09T05:45:10.716734Z","iopub.status.busy":"2024-05-09T05:45:10.716355Z","iopub.status.idle":"2024-05-09T05:46:44.188293Z","shell.execute_reply":"2024-05-09T05:46:44.187029Z","shell.execute_reply.started":"2024-05-09T05:45:10.716703Z"},"trusted":true},"outputs":[],"source":["!git clone https://github.com/SakuraLLM/Sakura-13B-Galgame.git\n","%cd Sakura-13B-Galgame\n","\n","!pip install \"diskcache>=5.6.1\"\n","!pip install llama-cpp-python -i https://isotr0py.github.io/llama.cpp-python-index/wheels/cu124\n","!pip install -q -r requirements.llamacpp.txt\n","!pip install -q pyngrok\n","\n","# install localtunnel\n","!npm install -g localtunnel"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-09T05:50:57.287407Z","iopub.status.busy":"2024-05-09T05:50:57.286353Z","iopub.status.idle":"2024-05-09T05:52:29.778820Z","shell.execute_reply":"2024-05-09T05:52:29.777529Z","shell.execute_reply.started":"2024-05-09T05:50:57.287362Z"},"trusted":true},"outputs":[],"source":["# ngrokToken留空则使用localtunnel进行内网穿透\n","ngrokToken = \"\"\n","MODEL = \"sakura-32b-qwen2beta-v0.9.1-iq4xs\"\n","\n","\n","from huggingface_hub import hf_hub_download\n","from pathlib import Path\n","\n","if ngrokToken:\n","    from pyngrok import conf, ngrok\n","    conf.get_default().auth_token = ngrokToken\n","    conf.get_default().monitor_thread = False\n","    ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n","    if len(ssh_tunnels) == 0:\n","        ssh_tunnel = ngrok.connect(5000)\n","        print('address：'+ssh_tunnel.public_url)\n","    else:\n","        print('address：'+ssh_tunnels[0].public_url)\n","else:\n","    import subprocess\n","    import threading\n","    def start_localtunnel(port):\n","        p = subprocess.Popen([\"lt\", \"--port\", f\"{port}\"], stdout=subprocess.PIPE)\n","        for line in p.stdout:\n","            print(line.decode(), end='')\n","    threading.Thread(target=start_localtunnel, daemon=True, args=(5000,)).start()\n","\n","MODEL_PATH = f\"./models/{MODEL}.gguf\"\n","if not Path(MODEL_PATH).exists():\n","    hf_hub_download(repo_id=\"SakuraLLM/Sakura-32B-Qwen2beta-v0.9.1-GGUF\", filename=f\"{MODEL}.gguf\", local_dir=\"models/\")\n","\n","!python server.py \\\n","    --model_name_or_path $MODEL_PATH \\\n","    --llama_cpp \\\n","    --use_gpu \\\n","    --model_version 0.9 \\\n","    --trust_remote_code \\\n","    --no-auth"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":30683,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat":4,"nbformat_minor":4}