Merge pull request #35 from CMCC-Foundation/feature_34

Feature 34 Hotfix
CMCC-Foundation · May 13, 2024 · 5eef173 · 5eef173
2 parents be914c5 + 9049c51
commit 5eef173
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 24 deletions.
diff --git a/.github/workflows/build_on_pull_request.yaml b/.github/workflows/build_on_pull_request.yaml
@@ -20,58 +20,68 @@ jobs:
         build
         --user
     - name: Build a binary wheel and a source for drivers
-      run: python3 -m build ./drivers    
+      run: python3 -m build ./drivers
     - name: Set Docker image tag name
       run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: TAG ECHO
+      run: echo ${{ env.TAG }}
     - name: Login to Scaleway Container Registry
       uses: docker/login-action@v2
       with:
         username: nologin
         password: ${{ secrets.DOCKER_PASSWORD }}
         registry: ${{ vars.DOCKER_REGISTRY }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2     
+      uses: docker/setup-buildx-action@v2
     - name: Build and push drivers
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./drivers
         file: ./drivers/Dockerfile
         push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         build-args: |
           REGISTRY=${{ vars.GEOKUBE_REGISTRY  }}
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest             
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest
     - name: Build and push datastore component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./datastore
         file: ./datastore/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest    
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest
     - name: Build and push api component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./api
         file: ./api/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest
     - name: Build and push executor component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./executor
         file: ./executor/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest                           
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest
diff --git a/.github/workflows/build_on_release.yaml b/.github/workflows/build_on_release.yaml
@@ -32,46 +32,54 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2     
     - name: Build and push drivers
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./drivers
         file: ./drivers/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOKUBE_REGISTRY  }}
           TAG=v0.2a6
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }}  
     - name: Build and push datastore component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./datastore
         file: ./datastore/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }}  
     - name: Build and push api component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./api
         file: ./api/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }}  
     - name: Build and push executor component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./executor
         file: ./executor/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }}
diff --git a/datastore/Dockerfile b/datastore/Dockerfile
@@ -1,10 +1,9 @@
 ARG REGISTRY=rg.fr-par.scw.cloud/geolake
 ARG TAG=latest
 FROM $REGISTRY/geolake-drivers:$TAG
-RUN conda install -c conda-forge --yes --freeze-installed psycopg2 \
-    && conda clean -afy
+
 COPY requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir -r /app/requirements.txt    
+RUN pip install --no-cache-dir -r /app/requirements.txt
 COPY ./datastore /app/datastore
 COPY ./workflow /app/workflow
 COPY ./dbmanager /app/dbmanager
@@ -13,3 +12,4 @@ COPY ./utils /app/utils
 COPY ./tests /app/tests
 COPY ./wait-for-it.sh /
 
+WORKDIR /
diff --git a/datastore/datastore/datastore.py b/datastore/datastore/datastore.py
@@ -50,7 +50,7 @@ def __init__(self) -> None:
 
     @log_execution_time(_LOG)
     def get_cached_product_or_read(
-        self, dataset_id: str, product_id: str
+        self, dataset_id: str, product_id: str, query: GeoQuery | None = None
     ) -> DataCube | Dataset:
         """Get product from the cache instead of loading files indicated in
         the catalog if `metadata_caching` set to `True`.
@@ -81,7 +81,7 @@ def get_cached_product_or_read(
             )
             return self.catalog(CACHE_DIR=self.cache_dir)[dataset_id][
                 product_id
-            ].read_chunked()
+            ].get(geoquery=query, compute=False).read_chunked()
         return self.cache[dataset_id][product_id]
 
     @log_execution_time(_LOG)
@@ -389,7 +389,7 @@ def estimate(
         # NOTE: we always use catalog directly and single product cache
         self._LOG.debug("loading product...")
         # NOTE: for estimation we use cached products
-        kube = self.get_cached_product_or_read(dataset_id, product_id)
+        kube = self.get_cached_product_or_read(dataset_id, product_id, query=query)
         self._LOG.debug("original kube len: %s", len(kube))
         return Datastore._process_query(kube, geoquery, False).nbytes
 

diff --git a/datastore/requirements.txt b/datastore/requirements.txt
@@ -1,2 +1,3 @@
 networkx
-pydantic<2.0.0
+pydantic<2.0.0
+psycopg2-binary
diff --git a/drivers/Dockerfile b/drivers/Dockerfile
@@ -1,8 +1,8 @@
 ARG REGISTRY=rg.fr-par.scw.cloud/geokube
+# ARG TAG=v0.2.6b1
 ARG TAG=latest
 FROM $REGISTRY/geokube:$TAG
-RUN conda install -c conda-forge --yes --freeze-installed intake=0.6.6
-RUN conda clean -afy
+
 COPY dist/intake_geokube-0.1a0-py3-none-any.whl /
 RUN pip install /intake_geokube-0.1a0-py3-none-any.whl
-RUN rm /intake_geokube-0.1a0-py3-none-any.whl
+RUN rm /intake_geokube-0.1a0-py3-none-any.whl