Merge pull request #77 from fractalego/llm-webserving

added the packaging module
fractalego · Dec 20, 2023 · 3c3f80c · 3c3f80c
2 parents 769aeab + 10836f0
commit 3c3f80c
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 6 deletions.
diff --git a/.github/workflows/development-tests1.yml b/.github/workflows/development-tests1.yml
@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install portaudio19-dev
+        sudo apt-get install portaudio19-dev python3-packaging
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

diff --git a/requirements.txt b/requirements.txt
@@ -5,7 +5,6 @@ accelerate==0.20.3
 bitsandbytes==0.41.3
 torch==2.0.0
 optimum==1.15.0
-flash-attn==2.3.6
 transformers==4.35.2
 nltk==3.6.2
 gensim==4.3.1

diff --git a/setup.py b/setup.py
@@ -54,7 +54,6 @@
         "bitsandbytes==0.41.3",
         "torch==2.0.0",
         "optimum==1.15.0",
-        "flash-attn==2.3.6",
         "transformers==4.35.2",
         "nltk==3.6.2",
         "gensim==4.3.1",

diff --git a/wafl/connectors/local/local_llm_connector.py b/wafl/connectors/local/local_llm_connector.py
@@ -1,7 +1,7 @@
 import logging
-from typing import List
-
 import torch
+
+from typing import List
 from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
 from transformers import StoppingCriteria
 
@@ -26,7 +26,6 @@ def __init__(self, config, last_strings=None):
             model = AutoModelForCausalLM.from_pretrained(
                 model_name,
                 config=AutoConfig.from_pretrained(model_name, trust_remote_code=True),
-                use_flash_attention_2=True,
                 torch_dtype=torch.half,
                 trust_remote_code=True,
                 device_map=device,