Merge pull request #19 from flepied/tests

add integration tests to the run-tests Github action

.github/workflows/pr.yml

@@ -6,7 +6,7 @@ on:
     types: [opened, synchronize, reopened]
 jobs:
 
-  validate-tests:
+  run-tests:
     runs-on: ubuntu-latest
     steps:
 
@@ -23,8 +23,14 @@ jobs:
           set -ex
           curl -sSL https://install.python-poetry.org | python3 -
           poetry install --with test
+          poetry run pip3 install torch
           poetry run pre-commit run -a --show-diff-on-failure
 
+      - name: Run integration tests
+        run: |
+          set -ex
+          ./integration-test.sh
+
   check-all-dependencies-are-merged:
     runs-on: ubuntu-latest
     steps:

.pre-commit-config.yaml

@@ -16,9 +16,10 @@ repos:
       - id: pytest-check
         name: Pytest checks
         entry: poetry run pytest
-        language: system
+        language: python
         pass_filenames: false
         always_run: true
+        files: ".*\\.py$"
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0

Makefile

@@ -3,8 +3,14 @@ SHELL := /bin/bash
 all: compose.yaml
 
 compose.yaml: poetry.lock
-	VER=$$(poetry run pip freeze|grep chromadb==|sed "s/.*==//") \
-	&& poetry run pip freeze|grep chromadb \
-	&& test -n "$$VER" \
-	&& echo $$VER  _
-	&& sed -i -e "[email protected]/chroma-core/chroma:.*@ghcr.io/chroma-core/chroma:$$VER@" compose.yaml
+	set -x && \
+	. .venv/bin/activate && \
+	VER=$$(pip3 freeze|grep chromadb==|sed "s/.*==//") && \
+	pip3 freeze && \
+	test -n "$$VER" && \
+	echo "$$VER" && \
+	sed -i -e "[email protected]/chroma-core/chroma:.*@ghcr.io/chroma-core/chroma:$$VER@" compose.yaml
+
+poetry.lock: pyproject.toml
+	poetry lock --no-update
+	touch poetry.lock

README.md

@@ -50,7 +50,7 @@ The second brain agent is using the vector database to answer questions about yo
 
 ## Installation
 
-You need a Python interpreter. All this has been tested with Python 3.11 under Fedora Linux 38. Let me know if it work for you on your system.
+You need a Python 3 interpreter and the `inotify-tools` installed. All this has been tested with Python 3.11 under Fedora Linux 38. Let me know if it work for you on your system.
 
 Get the source code:
 

integration-test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -ex
+
+sudo apt-get install inotify-tools
+
+mkdir $HOME/.second-brain $HOME/Notes
+
+cat > .env <<EOF
+SRCDIR=$HOME/Notes
+DSTDIR=$HOME/.second-brain
+EOF
+
+bash -x ./install-systemd-services.sh
+
+cat > $HOME/Notes/langchain.md <<EOF
+## References
+
+- https://docs.langchain.com/docs/
+- https://github.com/kyrolabs/awesome-langchain
+
+Plan-and-Solve Prompting: Improving Zero-Shot
+Chain-of-Thought Reasoning by Large Language Models
+https://arxiv.org/pdf/2305.04091.pdf
+EOF
+
+TRY=0
+docker-compose ps
+while [ $TRY -lt 24 ]; do
+    TRY=$(( TRY + 1 ))
+    if docker-compose logs | grep -q "Application startup complete"; then
+        break
+    fi
+    docker-compose logs
+    sleep 5
+done
+
+sudo journalctl -u sba-md
+sudo journalctl -u sba-txt
+
+RES=$(poetry run ./similarity.py "What is langchain?")
+echo "$RES"
+test -n "$RES"
+
+# integration-test.sh ends here

pyproject.toml

@@ -31,6 +31,7 @@ unstructured = "*"
 youtube-transcript-api = "*"
 yt-dlp = "*"
 pydub = "*"
+markdown = "*"
 # workarounds for streamlit dependencies
 altair = "<5"
 protobuf = "~=3.19.0"