Update load_dataframe

.github/workflows/tests.yaml

@@ -1,43 +1,118 @@
 name: GerryDB client tests
-on: [push, pull_request]
+on: 
+  workflow_dispatch:
+  push:
+  pull_request:
+
+
 jobs:
   run:
-    runs-on: ${{ matrix.os }}
+    name: Run tests Linux
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgis/postgis:16-3.4
+        ports:
+          - 54320:5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: dev
+          POSTGRES_DB: gerrydb
+        options: >-
+          --health-cmd "pg_isready -U postgres" 
+          --health-interval 10s 
+          --health-timeout 20s 
+          --health-retries 10
     strategy:
       matrix:
-        python-version: ['3.10', '3.11']
-        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.10', '3.12']
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
+      GERRYDB_DATABASE_URI: "postgresql://postgres:dev@localhost:54320/gerrydb"
+      GERRYDB_TEST_SERVER: "localhost:8000"
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: dev
+      POSTGRES_DB: gerrydb
+
     steps:
-    - uses: actions/checkout@main
+    - name: Checkout repository into custom directory
+      uses: actions/checkout@v3
+
     - name: Set up Python
-      uses: actions/setup-python@master
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
+
+    - name: Install PostgreSQL client tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y postgresql-client
+
+    - name: Install GDAL
+      run: |
+        sudo apt-get install -y gdal-bin
+
+    - name: Clone backend repo
+      run: |
+        git clone --branch dev https://github.com/mggg/gerrydb-meta.git ../gerrydb-meta
+
     - name: Install dependencies
       run: |
+        cd ..
+        python -m venv .venv
+        source .venv/bin/activate
+        pip install --upgrade pip
         pip install poetry
-        pip install .
         pip install black isort
-    - name: Check formatting
+        pip install ./gerrydb-client-py/
+        pip install ./gerrydb-meta/
+
+    - name: Set up GerryDB and get API key
+      run: |
+        cd ..
+        source .venv/bin/activate
+        cd ./gerrydb-meta
+        echo "export GERRYDB_TEST_API_KEY=$(python init.py --name test --email test --reset)" > ../.env
+        echo "export GERRYDB_TEST_SERVER=localhost:8000" >> ../.env
+
+    - name: Wait for PostgreSQL to be ready
+      run: |
+        until pg_isready -h 127.0.0.1 -p 54320 -U postgres; do
+          echo "Waiting for PostgreSQL to be ready..."
+          sleep 2
+        done
+
+    - name: Check the postgres
       run: |
-        python -m black . --check
-        python -m isort . --diff
+        PGPASSWORD=dev psql -h 127.0.0.1 -p 54320 -U postgres -c '\l'
+        PGPASSWORD=dev psql -h 127.0.0.1 -p 54320 -U postgres -d gerrydb -c 'SELECT postgis_version();'
+
+    - name: Start the uvicorn server
+      run: |
+        source ../.venv/bin/activate
+        source ../.env
+        cd ../gerrydb-meta
+        nohup uvicorn gerrydb_meta.main:app --host 0.0.0.0 --port 8000 --log-level trace > uvicorn.log 2>&1 &
+
+    - name: Wait for Uvicorn to be ready
+      run: |
+        until curl -s http://localhost:8000/api/v1 > /dev/null; do
+          echo "Waiting for Uvicorn to be ready..."
+          sleep 2
+        done
+
     - name: Run tests and generate coverage report
       run: |
-        pip install pytest
-        pip install pytest-cov
-        pytest -v --cov=./ --cov-report=xml
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        directory: .
-        env_vars: OS,PYTHON
-        fail_ci_if_error: true
-        files: ./coverage.xml
-        flags: unittests
-        name: codecov-umbrella
-        verbose: true
+        source ../.venv/bin/activate
+        source ../.env
+        pip install pytest pytest-cov
+        pytest -v -s tests --cov=./ --cov-report=xml
+
+    - name: Print Uvicorn logs on Failure
+      if: failure()
+      run: |
+        echo "Displaying Uvicorn logs:"
+        cat ../gerrydb-meta/uvicorn.log
+

.gitignore

@@ -160,3 +160,7 @@ cython_debug/
 #.idea/
 
 .DS_Store
+
+
+# Ignore all of the backup files that might be floating around
+*.tar

TODO.md

@@ -0,0 +1,10 @@
+# For Docs
+
+[ ] Show how to make a column with the write context
+    [ ] Show how this all works with the the load_dataframe method
+
+
+# Ideas
+[ ] Maybe cache all of the columns in the database on the local machine so
+    validation can be done locally? Periodically checks for columns can be done
+    to ensure that the local cache is up to date.

gerrydb/cache.py

@@ -97,9 +97,6 @@ def upsert_view_gpkg(
             db_cursor.execute("SELECT SUM(file_size_kb) FROM view")
             total_db_size = db_cursor.fetchone()[0]
 
-            print(total_db_size)
-            print(f"max_size: {self.max_size_gb * 1024 * 1024}")
-
             while total_db_size > self.max_size_gb * 1024 * 1024:
                 db_cursor.execute("SELECT * FROM view ORDER BY cached_at ASC LIMIT 1")
                 oldest = db_cursor.fetchone()