From c28ef80f30447cea98ab221e687fb30c8aad5de7 Mon Sep 17 00:00:00 2001 From: kaarthik108 Date: Wed, 23 Aug 2023 19:49:14 +1200 Subject: [PATCH] Fix ingest path --- pyproject.toml | 3 ++- snowdev/fillers/sproc/fill.toml | 5 +++++ snowdev/fillers/udf/fill.toml | 5 +++++ snowdev/functions/utils/ingest.py | 6 ++++-- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82c9c70..3879beb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snowdev" -version = "0.1.12" +version = "0.1.13" description = "snowdev: DevOps toolkit for Snowflake, facilitating seamless deployment of UDFs, stored procedures, and Streamlit apps using Snowpark's capabilities right from your local environment." authors = ["kaarthik "] readme = "README.md" @@ -22,6 +22,7 @@ pyyaml = "^6.0.1" toml = "^0.10.2" openai = "^0.27.8" langchain = "0.0.265" +chromadb = "0.4.6" tiktoken = "0.4.0" unstructured = "0.9.3" diff --git a/snowdev/fillers/sproc/fill.toml b/snowdev/fillers/sproc/fill.toml index a957e1e..74052e7 100644 --- a/snowdev/fillers/sproc/fill.toml +++ b/snowdev/fillers/sproc/fill.toml @@ -1,3 +1,8 @@ +[tool.connection] +database = "" +schema = "" +role = "" + [tool.poetry.dependencies] python = "3.10.0" snowflake-snowpark-python = "1.5.1" \ No newline at end of file diff --git a/snowdev/fillers/udf/fill.toml b/snowdev/fillers/udf/fill.toml index f3a61b6..50130c3 100644 --- a/snowdev/fillers/udf/fill.toml +++ b/snowdev/fillers/udf/fill.toml @@ -1,3 +1,8 @@ +[tool.connection] +database = "" +schema = "" +role = "" + [tool.poetry.dependencies] python = "3.10.0" snowflake-snowpark-python = "1.5.1" diff --git a/snowdev/functions/utils/ingest.py b/snowdev/functions/utils/ingest.py index b25e4d3..c71002b 100644 --- a/snowdev/functions/utils/ingest.py +++ b/snowdev/functions/utils/ingest.py @@ -1,6 +1,6 @@ -import os from typing import Any, Dict +import pkg_resources from langchain.document_loaders import DirectoryLoader from langchain.embeddings.openai import OpenAIEmbeddings from langchain.text_splitter import CharacterTextSplitter @@ -15,7 +15,9 @@ class Secrets(BaseModel): class Config(BaseModel): chunk_size: int = 1000 chunk_overlap: int = 0 - docs_dir: str = "snowdev/functions/utils/knowledge" + docs_dir: str = pkg_resources.resource_filename( + "snowdev.functions.utils", "knowledge" + ) docs_glob: str = "**/*"