From 042f7a853ec0e22553ef4802678540d68a69ff9f Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Thu, 31 Jul 2025 12:56:12 -0500 Subject: [PATCH 1/3] feat(minio): Added MinIO support for calls storage --- .github/workflows/pylint.yml | 1 + .gitignore | 3 +- Dockerfile | 7 +- app/main.py | 6 ++ app/opensense.py | 1 - app/storage.py | 71 +++++++++++++++ k8s/cronjob.yml | 44 +++++++++ k8s/deployment.yml | 63 +++++++++++-- k8s/ingress.yml | 11 +++ requirements.in | 3 +- requirements.txt | 170 +++++++++++++++++++++++++++++++++-- tests/test_modules.py | 19 ++++ version.txt | 2 +- 13 files changed, 381 insertions(+), 20 deletions(-) create mode 100644 app/storage.py create mode 100644 k8s/cronjob.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 7af33ff..bff2404 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -32,6 +32,7 @@ jobs: pip install vcrpy pip install prometheus_client pip install redis + pip install minio - name: Analysing the code with pylint run: | # Set PYTHONPATH so pylint can find the app module diff --git a/.gitignore b/.gitignore index e50c92d..b03ec9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /env /__pycache__ /out.txt -/tests/__pycache__ \ No newline at end of file +/tests/__pycache__ +/.pytest_cache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b529ce0..48ee38d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,12 @@ RUN pip install --no-cache-dir -r /app/requirements.txt --require-hashes && \ ENV FLASK_APP=app.main.py:app \ PYTHONUNBUFFERED=1 \ - REDIS_HOST=localhost \ REDIS_PORT=6379 \ - CACHE_TTL=300 + REDIS_DB=0 \ + CACHE_TTL=300 \ + MINIO_PORT=9000 \ + MINIO_ACCESS_KEY=minioadmin \ + MINIO_SECRET_KEY=minioadmin USER appuser diff --git a/app/main.py b/app/main.py index df141fb..eac6782 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,7 @@ from flask import Flask, Response from prometheus_client import generate_latest, CONTENT_TYPE_LATEST from app import opensense +from app import storage app = Flask(__name__) @@ -26,5 +27,10 @@ def metrics(): '''Function to return Prometheus metrics.''' return Response(generate_latest(), mimetype=CONTENT_TYPE_LATEST) +@app.route('/store') +def store(): + '''Function to store results in MinIO.''' + return storage.store_temperature_data() + if __name__ == "__main__": app.run() diff --git a/app/opensense.py b/app/opensense.py index 3d3dd0a..e0728c8 100644 --- a/app/opensense.py +++ b/app/opensense.py @@ -22,7 +22,6 @@ socket_connect_timeout=240, socket_timeout=240 ) - redis_client.ping() REDIS_AVAILABLE = True print("Connected to Redis successfully!") diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..3b73f10 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,71 @@ +'''This script uploads the output to a MinIO bucket.''' +import os +import io +import datetime +from minio import Minio +from minio.error import S3Error, InvalidResponseError +from app import opensense + +MINIO_HOST = os.getenv('MINIO_HOST', 'localhost') +MINIO_PORT = int(os.environ.get('MINIO_PORT', 9000)) +MINIO_ACCESS_KEY = os.environ.get('MINIO_ACCESS_KEY', 'minioadmin') +MINIO_SECRET_KEY = os.environ.get('MINIO_SECRET_KEY', 'minioadmin') + +def store_temperature_data(): + '''Function to upload temperature data to MinIO.''' + try: + client = Minio(f"{MINIO_HOST}:{MINIO_PORT}", + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=False + ) + + # Check if the MinIO server is reachable + try: + client.list_buckets() + except Exception as conn_exc: + error_msg = f"Cannot connect to MinIO server: {conn_exc}" + print(error_msg) + return error_msg + + bucket_name = "temperature-data" + destination_file = f"temperature_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S%f')}.txt" + + # Get the original temperature data format + temperature_data = opensense.get_temperature() + + text_bytes = temperature_data.encode('utf-8') + text_stream = io.BytesIO(text_bytes) + + # Make the bucket if it doesn't exist. + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + print("Created bucket", bucket_name) + else: + print("Bucket", bucket_name, "already exists") + + # Upload the data + client.put_object( + bucket_name, + destination_file, + text_stream, + length=len(text_bytes), + content_type='text/plain' + ) + + return (f"Temperature data successfully uploaded as " + f"{destination_file} to bucket {bucket_name}") + + except (S3Error, InvalidResponseError) as exc: + error_msg = f"MinIO S3 error occurred: {exc}" + print(error_msg) + return error_msg + except Exception as exc: + error_msg = f"Unexpected error occurred: {exc}" + print(error_msg) + return error_msg + +if __name__ == "__main__": + result = store_temperature_data() + print(result) diff --git a/k8s/cronjob.yml b/k8s/cronjob.yml new file mode 100644 index 0000000..51e4b3c --- /dev/null +++ b/k8s/cronjob.yml @@ -0,0 +1,44 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: temperature-storage-cronjob + labels: + app: hivebox-cronjob +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody user + seccompProfile: + type: RuntimeDefault + containers: + - name: temperature-storage + image: curlimages/curl:8.15.0@sha256:4026b29997dc7c823b51c164b71e2b51e0fd95cce4601f78202c513d97da2922 + command: ["curl"] + args: + - "-f" # Fail on HTTP errors + - "-s" # Silent mode + - "-S" # Show errors + - "--max-time" + - "60" + - "http://hivebox-service/store" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + memory: "32Mi" + cpu: "50m" + requests: + memory: "16Mi" + cpu: "10m" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 diff --git a/k8s/deployment.yml b/k8s/deployment.yml index 3662a48..21876bc 100644 --- a/k8s/deployment.yml +++ b/k8s/deployment.yml @@ -26,12 +26,8 @@ spec: env: - name: REDIS_HOST value: "redis-service" - - name: REDIS_PORT - value: "6379" - - name: REDIS_DB - value: "0" - - name: CACHE_TTL - value: "300" + - name: MINIO_HOST + value: "minio-service" securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -60,9 +56,9 @@ spec: httpGet: path: /version port: 5000 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 60 + periodSeconds: 120 + timeoutSeconds: 10 failureThreshold: 3 volumeMounts: - name: tmp-volume @@ -119,3 +115,52 @@ spec: volumes: - name: valkey-data emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + securityContext: + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: minio + image: minio/minio:RELEASE.2025-07-23T15-54-02Z@sha256:d249d1fb6966de4d8ad26c04754b545205ff15a62e4fd19ebd0f26fa5baacbc0 + ports: + - containerPort: 9000 + command: ["minio"] + args: ["server", "/data"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + capabilities: + drop: + - ALL + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "128Mi" + cpu: "100m" + volumeMounts: + - name: minio-data + mountPath: /data + volumes: + - name: minio-data + emptyDir: {} diff --git a/k8s/ingress.yml b/k8s/ingress.yml index 972bfa9..c3accda 100644 --- a/k8s/ingress.yml +++ b/k8s/ingress.yml @@ -30,3 +30,14 @@ spec: ports: - port: 6379 targetPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + selector: + app: minio + ports: + - port: 9000 + targetPort: 9000 diff --git a/requirements.in b/requirements.in index 01b49b0..d3a1256 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ Flask==3.1.1 requests==2.32.4 prometheus-client==0.22.1 -redis==6.2.0 \ No newline at end of file +redis==6.2.0 +minio==7.2.16 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b74be37..e5e5bd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,14 +4,117 @@ # # pip-compile --generate-hashes --output-file=requirements.txt requirements.in # +argon2-cffi==25.1.0 \ + --hash=sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1 \ + --hash=sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741 + # via minio +argon2-cffi-bindings==25.1.0 \ + --hash=sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99 \ + --hash=sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6 \ + --hash=sha256:21378b40e1b8d1655dd5310c84a40fc19a9aa5e6366e835ceb8576bf0fea716d \ + --hash=sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44 \ + --hash=sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a \ + --hash=sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f \ + --hash=sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2 \ + --hash=sha256:5acb4e41090d53f17ca1110c3427f0a130f944b896fc8c83973219c97f57b690 \ + --hash=sha256:5d588dec224e2a83edbdc785a5e6f3c6cd736f46bfd4b441bbb5aa1f5085e584 \ + --hash=sha256:6dca33a9859abf613e22733131fc9194091c1fa7cb3e131c143056b4856aa47e \ + --hash=sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0 \ + --hash=sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f \ + --hash=sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623 \ + --hash=sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b \ + --hash=sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44 \ + --hash=sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98 \ + --hash=sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500 \ + --hash=sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94 \ + --hash=sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6 \ + --hash=sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d \ + --hash=sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85 \ + --hash=sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92 \ + --hash=sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d \ + --hash=sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a \ + --hash=sha256:da0c79c23a63723aa5d782250fbf51b768abca630285262fb5144ba5ae01e520 \ + --hash=sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb + # via argon2-cffi blinker==1.9.0 \ --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc # via flask -certifi==2025.7.9 \ - --hash=sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079 \ - --hash=sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39 - # via requests +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 + # via + # minio + # requests +cffi==1.17.1 \ + --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ + --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \ + --hash=sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1 \ + --hash=sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15 \ + --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \ + --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \ + --hash=sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8 \ + --hash=sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36 \ + --hash=sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17 \ + --hash=sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf \ + --hash=sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc \ + --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \ + --hash=sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed \ + --hash=sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702 \ + --hash=sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1 \ + --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \ + --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \ + --hash=sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6 \ + --hash=sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d \ + --hash=sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b \ + --hash=sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e \ + --hash=sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be \ + --hash=sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c \ + --hash=sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683 \ + --hash=sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9 \ + --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \ + --hash=sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8 \ + --hash=sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1 \ + --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \ + --hash=sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655 \ + --hash=sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67 \ + --hash=sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595 \ + --hash=sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0 \ + --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \ + --hash=sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41 \ + --hash=sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6 \ + --hash=sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401 \ + --hash=sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6 \ + --hash=sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3 \ + --hash=sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16 \ + --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \ + --hash=sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e \ + --hash=sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4 \ + --hash=sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964 \ + --hash=sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c \ + --hash=sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576 \ + --hash=sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0 \ + --hash=sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3 \ + --hash=sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662 \ + --hash=sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3 \ + --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \ + --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \ + --hash=sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd \ + --hash=sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f \ + --hash=sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5 \ + --hash=sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14 \ + --hash=sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d \ + --hash=sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9 \ + --hash=sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7 \ + --hash=sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382 \ + --hash=sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a \ + --hash=sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e \ + --hash=sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a \ + --hash=sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4 \ + --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 \ + --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \ + --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b + # via argon2-cffi-bindings charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 \ @@ -192,10 +295,61 @@ markupsafe==3.0.2 \ # flask # jinja2 # werkzeug +minio==7.2.16 \ + --hash=sha256:81e365c8494d591d8204a63ee7596bfdf8a7d06ad1b1507d6b9c1664a95f299a \ + --hash=sha256:9288ab988ca57c181eb59a4c96187b293131418e28c164392186c2b89026b223 + # via -r requirements.in prometheus-client==0.22.1 \ --hash=sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28 \ --hash=sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 # via -r requirements.in +pycparser==2.22 \ + --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ + --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc + # via cffi +pycryptodome==3.23.0 \ + --hash=sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4 \ + --hash=sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c \ + --hash=sha256:14e15c081e912c4b0d75632acd8382dfce45b258667aa3c67caf7a4d4c13f630 \ + --hash=sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f \ + --hash=sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27 \ + --hash=sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a \ + --hash=sha256:350ebc1eba1da729b35ab7627a833a1a355ee4e852d8ba0447fafe7b14504d56 \ + --hash=sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef \ + --hash=sha256:45c69ad715ca1a94f778215a11e66b7ff989d792a4d63b68dc586a1da1392ff5 \ + --hash=sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477 \ + --hash=sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886 \ + --hash=sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a \ + --hash=sha256:573a0b3017e06f2cffd27d92ef22e46aa3be87a2d317a5abf7cc0e84e321bd75 \ + --hash=sha256:63dad881b99ca653302b2c7191998dd677226222a3f2ea79999aa51ce695f720 \ + --hash=sha256:64093fc334c1eccfd3933c134c4457c34eaca235eeae49d69449dc4728079339 \ + --hash=sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625 \ + --hash=sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490 \ + --hash=sha256:6fe8258e2039eceb74dfec66b3672552b6b7d2c235b2dfecc05d16b8921649a8 \ + --hash=sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b \ + --hash=sha256:7ac1080a8da569bde76c0a104589c4f414b8ba296c0b3738cf39a466a9fb1818 \ + --hash=sha256:865d83c906b0fc6a59b510deceee656b6bc1c4fa0d82176e2b77e97a420a996a \ + --hash=sha256:89d4d56153efc4d81defe8b65fd0821ef8b2d5ddf8ed19df31ba2f00872b8002 \ + --hash=sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae \ + --hash=sha256:93837e379a3e5fd2bb00302a47aee9fdf7940d83595be3915752c74033d17ca7 \ + --hash=sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d \ + --hash=sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265 \ + --hash=sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39 \ + --hash=sha256:a176b79c49af27d7f6c12e4b178b0824626f40a7b9fed08f712291b6d54bf566 \ + --hash=sha256:a7fc76bf273353dc7e5207d172b83f569540fc9a28d63171061c42e361d22353 \ + --hash=sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b \ + --hash=sha256:b34e8e11d97889df57166eda1e1ddd7676da5fcd4d71a0062a760e75060514b4 \ + --hash=sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2 \ + --hash=sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575 \ + --hash=sha256:ce64e84a962b63a47a592690bdc16a7eaf709d2c2697ababf24a0def566899a6 \ + --hash=sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843 \ + --hash=sha256:d8e95564beb8782abfd9e431c974e14563a794a4944c29d6d3b7b5ea042110b4 \ + --hash=sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446 \ + --hash=sha256:ddb95b49df036ddd264a0ad246d1be5b672000f12d6961ea2c267083a5e19379 \ + --hash=sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa \ + --hash=sha256:e3f2d0aaf8080bda0587d58fc9fe4766e012441e2eed4269a77de6aea981c8be \ + --hash=sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7 + # via minio redis==6.2.0 \ --hash=sha256:c8ddf316ee0aab65f04a11229e94a64b2618451dab7a67cb2f77eb799d872d5e \ --hash=sha256:e821f129b75dde6cb99dd35e5c76e8c49512a5a0d8dfdc560b2fbd44b85ca977 @@ -204,10 +358,16 @@ requests==2.32.4 \ --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 # via -r requirements.in +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 + # via minio urllib3==2.5.0 \ --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc - # via requests + # via + # minio + # requests werkzeug==3.1.3 \ --hash=sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e \ --hash=sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746 diff --git a/tests/test_modules.py b/tests/test_modules.py index 9430757..960172b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -81,3 +81,22 @@ def test_opensense_cache_setex(): result = opensense.get_temperature() assert 'Average temperature' in result mock_setex.assert_called() + +def test_store_endpoint(): + """Test store endpoint with test client""" + client = app.test_client() + response = client.get('/store') + # Should return 200 or 500 depending on MinIO availability + assert response.status_code in [200, 500] + +def test_store_temperature_data(): + '''Test that store endpoint works''' + with mock.patch('app.storage.store_temperature_data') as mock_store: + mock_store.return_value = "Temperature data successfully uploaded" + + client = app.test_client() + response = client.get('/store') + + assert response.status_code == 200 + assert "successfully uploaded" in response.get_data(as_text=True) + mock_store.assert_called_once() diff --git a/version.txt b/version.txt index 79a2734..09a3acf 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0 \ No newline at end of file +0.6.0 \ No newline at end of file From 9618640a703df8581ef50a33e7b0e5c42458b536 Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Fri, 1 Aug 2025 12:52:10 -0500 Subject: [PATCH 2/3] fix(ci): Added tests for Sonarqube coverage --- app/storage.py | 6 +----- tests/test_modules.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/app/storage.py b/app/storage.py index 3b73f10..9fcc91d 100644 --- a/app/storage.py +++ b/app/storage.py @@ -23,7 +23,7 @@ def store_temperature_data(): # Check if the MinIO server is reachable try: client.list_buckets() - except Exception as conn_exc: + except ConnectionError as conn_exc: error_msg = f"Cannot connect to MinIO server: {conn_exc}" print(error_msg) return error_msg @@ -61,10 +61,6 @@ def store_temperature_data(): error_msg = f"MinIO S3 error occurred: {exc}" print(error_msg) return error_msg - except Exception as exc: - error_msg = f"Unexpected error occurred: {exc}" - print(error_msg) - return error_msg if __name__ == "__main__": result = store_temperature_data() diff --git a/tests/test_modules.py b/tests/test_modules.py index 960172b..26c5e68 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,6 +1,7 @@ '''This module contains tests for the Flask and OpenSense modules.''' import re import unittest.mock as mock +from app.storage import store_temperature_data from app.main import app from app import opensense @@ -100,3 +101,24 @@ def test_store_temperature_data(): assert response.status_code == 200 assert "successfully uploaded" in response.get_data(as_text=True) mock_store.assert_called_once() + +def test_store_temperature_data_integration(): + '''Test that storage function works with mocked MinIO''' + with mock.patch('app.storage.Minio') as mock_minio_class: + # Mock MinIO client + mock_client = mock.MagicMock() + mock_minio_class.return_value = mock_client + mock_client.bucket_exists.return_value = True + mock_client.put_object.return_value = None + mock_client.list_buckets.return_value = [] + + # Mock temperature data + with mock.patch('app.opensense.get_temperature') as mock_temp: + mock_temp.return_value = "Average temperature: 22.5 °C (Good)\nFrom: test\n" + + # Import and call the actual function + result = store_temperature_data() + + # Test the result + assert "successfully uploaded" in result + mock_client.put_object.assert_called_once() From b377345a573d71ff98c1813480eb8b926937cb02 Mon Sep 17 00:00:00 2001 From: GabrielPalmar Date: Fri, 1 Aug 2025 15:08:05 -0500 Subject: [PATCH 3/3] fix(tests) Added more functions to test MinIO --- tests/test_modules.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/test_modules.py b/tests/test_modules.py index 26c5e68..16b7cf8 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,6 +1,7 @@ '''This module contains tests for the Flask and OpenSense modules.''' import re import unittest.mock as mock +from minio.error import S3Error, InvalidResponseError from app.storage import store_temperature_data from app.main import app from app import opensense @@ -122,3 +123,74 @@ def test_store_temperature_data_integration(): # Test the result assert "successfully uploaded" in result mock_client.put_object.assert_called_once() + +def test_store_temperature_data_bucket_creation(): + '''Test bucket creation when bucket doesn't exist''' + with mock.patch('app.storage.Minio') as mock_minio_class: + mock_client = mock.MagicMock() + mock_minio_class.return_value = mock_client + + # Mock bucket doesn't exist initially + mock_client.bucket_exists.return_value = False + mock_client.make_bucket.return_value = None + mock_client.put_object.return_value = None + mock_client.list_buckets.return_value = [] + + with mock.patch('app.opensense.get_temperature') as mock_temp: + mock_temp.return_value = "Average temperature: 22.5 °C (Good)\nFrom: test\n" + + result = store_temperature_data() + + # Verify bucket creation was called + mock_client.bucket_exists.assert_called_once_with("temperature-data") + mock_client.make_bucket.assert_called_once_with("temperature-data") + mock_client.put_object.assert_called_once() + + assert "successfully uploaded" in result + assert "temperature-data" in result + +def test_store_temperature_data_s3_error(): + '''Test S3Error exception handling''' + with mock.patch('app.storage.Minio') as mock_minio_class: + mock_client = mock.MagicMock() + mock_minio_class.return_value = mock_client + + # Mock S3Error during bucket check + mock_client.list_buckets.return_value = [] + mock_client.bucket_exists.side_effect = S3Error( + code="AccessDenied", + message="Access denied", + resource="/temperature-data", + request_id="test-request-id", + host_id="test-host-id", + response=None + ) + + result = store_temperature_data() + + assert "MinIO S3 error occurred" in result + assert "Access denied" in result + + +def test_store_temperature_data_invalid_response_error(): + '''Test InvalidResponseError exception handling''' + with mock.patch('app.storage.Minio') as mock_minio_class: + mock_client = mock.MagicMock() + mock_minio_class.return_value = mock_client + + # Mock InvalidResponseError during put_object + mock_client.list_buckets.return_value = [] + mock_client.bucket_exists.return_value = True + mock_client.put_object.side_effect = InvalidResponseError( + "Invalid response", + content_type="application/json", + body=b"{}" + ) + + with mock.patch('app.opensense.get_temperature') as mock_temp: + mock_temp.return_value = "Test temperature data" + + result = store_temperature_data() + + assert "MinIO S3 error occurred" in result + assert "Invalid response" in result