[CVAT][Exchange Oracle] Refactor structure (#787)

* Refactor Exchange Oracle structure

.gitignore

@@ -45,7 +45,4 @@ build
 dist
 
 #cache
-cache
-
-#Postgres Data
-db
+cache

packages/apps/job-launcher/server/.gitignore

@@ -35,4 +35,7 @@ lerna-debug.log*
 !.vscode/extensions.json
 
 .env.development
-.env.production
+.env.production
+
+#Postgres Data
+db

packages/examples/cvat/exchange-oracle/alembic/env.py

@@ -5,7 +5,7 @@
 
 from alembic import context
 
-from src.config import Config
+from src.core.config import Config
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -22,9 +22,10 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from src.database import Base
-from src.modules.oracle_webhook.model import Webhook
-from src.modules.cvat.model import Task, Job
+
+from src.models.cvat import Task, Job
+from src.models.webhook import Webhook
+from src.db import Base
 
 target_metadata = Base.metadata
 

packages/examples/cvat/exchange-oracle/docker-compose.test.yml

@@ -17,7 +17,7 @@ services:
   blockchain-node:
     build: 
       context: ./
-      dockerfile: blockchain-node.Dockerfile
+      dockerfile: dockerfiles/blockchain-node.Dockerfile
     healthcheck:
       # Using a magic nubmer of 28 here because this is a block number when blockchain-node container is ready to use
       test: if [ $(( $(wget -q --post-data='{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' -O- http://blockchain-node:8545 | grep -o '"result":"[^"]*"' | awk -F'"' '{print $4}' ) )) -ge 28 ]; then exit 0; else exit 1; fi
@@ -70,7 +70,7 @@ services:
   test:
     build:
       context: ./
-      dockerfile: test.Dockerfile
+      dockerfile: dockerfiles/test.Dockerfile
     environment:
       SQLALCHEMY_SILENCE_UBER_WARNING: 1
       ENVIRONMENT: 'test'

packages/examples/cvat/exchange-oracle/run.py

@@ -1,5 +1,5 @@
 import uvicorn
-from src.config import Config
+from src.core.config import Config
 
 if __name__ == "__main__":
     is_dev = Config.environment == "development"

packages/examples/cvat/exchange-oracle/src/__init__.py

@@ -1,11 +1,10 @@
 import logging
 from fastapi import FastAPI
 
-import src.log
 from src.api import init_api
-from src.error_handlers import setup_error_handlers
-from src.cron import setup_cron_jobs
-from src.config import Config
+from src.handlers.error_handlers import setup_error_handlers
+from src.crons import setup_cron_jobs
+from src.core.config import Config
 
 
 app = FastAPI(

packages/examples/cvat/exchange-oracle/src/api.py → packages/examples/cvat/exchange-oracle/src/api/__init__.py

@@ -1,10 +1,11 @@
 """ API endpoints """
 from fastapi import APIRouter, FastAPI
 
-from src.api_schema import ValidationErrorResponse, ResponseError, MetaResponse
-from src.config import Config
+from src.schemas import ValidationErrorResponse, ResponseError, MetaResponse
+from src.core.config import Config
 
-from src.modules.api import router
+from src.api.cvat import router as cvat_router
+from src.api.webhook import router as webhook_router
 
 
 greet_router = APIRouter()
@@ -44,6 +45,7 @@ def init_api(app: FastAPI) -> FastAPI:
     }
 
     app.include_router(greet_router)
-    app.include_router(router, prefix="/webhook", responses=default_responses)
+    app.include_router(cvat_router, responses=default_responses)
+    app.include_router(webhook_router, responses=default_responses)
 
     return app

packages/examples/cvat/exchange-oracle/src/api/cvat.py

@@ -0,0 +1,19 @@
+from fastapi import APIRouter, Header, Request
+from src.handlers.webhook import cvat_webhook_handler
+from src.schemas.cvat import CvatWebhook
+from src.validators.signature import validate_cvat_signature
+
+router = APIRouter()
+
+
+@router.post(
+    "/cvat",
+    description="Consumes a webhook from a cvat",
+)
+async def cvat_webhook(
+    cvat_webhook: CvatWebhook,
+    request: Request,
+    x_signature_256: str = Header(),
+):
+    await validate_cvat_signature(request, x_signature_256)
+    cvat_webhook_handler(cvat_webhook)

packages/examples/cvat/exchange-oracle/src/modules/api.py → packages/examples/cvat/exchange-oracle/src/api/webhook.py

@@ -1,15 +1,12 @@
-from fastapi import APIRouter, HTTPException, Header, Request
 from typing import Union
 
-from src.modules.api_schema import OracleWebhook, OracleWebhookResponse, CvatWebhook
-from src.database import SessionLocal
-
-from src.modules.chain.escrow import validate_escrow
-from src.validators.signature import validate_webhook_signature, validate_cvat_signature
-
-from src.modules.cvat.handlers.webhook import cvat_webhook_handler
-from src.modules.oracle_webhook.service import create_webhook
-from src.modules.oracle_webhook.constants import OracleWebhookTypes
+from fastapi import APIRouter, Header, HTTPException, Request
+from src.chain.escrow import validate_escrow
+from src.core.constants import OracleWebhookTypes
+from src.db import SessionLocal
+from src.schemas.webhook import OracleWebhook, OracleWebhookResponse
+from src.services.webhook import create_webhook
+from src.validators.signature import validate_webhook_signature
 
 router = APIRouter()
 
@@ -39,16 +36,3 @@ async def job_launcher_webhook(
         return OracleWebhookResponse(id=webhook_id)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
-
-
-@router.post(
-    "/cvat",
-    description="Consumes a webhook from a cvat",
-)
-async def cvat_webhook(
-    cvat_webhook: CvatWebhook,
-    request: Request,
-    x_signature_256: str = Header(),
-):
-    await validate_cvat_signature(request, x_signature_256)
-    cvat_webhook_handler(cvat_webhook)

packages/examples/cvat/exchange-oracle/src/modules/chain/escrow.py → packages/examples/cvat/exchange-oracle/src/chain/escrow.py

@@ -1,12 +1,10 @@
 import json
 
-from web3 import Web3
-
 from human_protocol_sdk.constants import Status
 from human_protocol_sdk.escrow import EscrowClient
 from human_protocol_sdk.storage import StorageClient
 
-from src.modules.chain.web3 import get_web3
+from src.chain.web3 import get_web3
 
 
 def validate_escrow(chain_id: int, escrow_address: str) -> None:

packages/examples/cvat/exchange-oracle/src/modules/chain/kvstore.py → packages/examples/cvat/exchange-oracle/src/chain/kvstore.py

@@ -1,7 +1,7 @@
 from human_protocol_sdk.escrow import EscrowClient
 from human_protocol_sdk.kvstore import KVStoreClient
 
-from src.modules.chain.web3 import get_web3
+from src.chain.web3 import get_web3
 
 
 def get_recording_oracle_url(chain_id: int, escrow_address: str) -> str:

packages/examples/cvat/exchange-oracle/src/modules/chain/web3.py → packages/examples/cvat/exchange-oracle/src/chain/web3.py

@@ -4,8 +4,8 @@
 
 from eth_account.messages import encode_defunct
 
-from src.constants import Networks
-from src.config import Config
+from src.core.constants import Networks
+from src.core.config import Config
 
 
 def get_web3(chain_id: Networks):

packages/examples/cvat/exchange-oracle/src/modules/cvat/constants.py → packages/examples/cvat/exchange-oracle/src/core/constants.py

@@ -1,4 +1,11 @@
 from enum import Enum
+from src.core.config import Config
+
+
+class Networks(int, Enum):
+    polygon_mainnet = Config.polygon_mainnet.chain_id
+    polygon_mumbai = Config.polygon_mumbai.chain_id
+    localhost = Config.localhost.chain_id
 
 
 class EventTypes(str, Enum):
@@ -34,3 +41,14 @@ class CvatLabelTypes(str, Enum):
 class Providers(str, Enum):
     aws = "AWS_S3_BUCKET"
     gcs = "GOOGLE_CLOUD_STORAGE"
+
+
+class OracleWebhookTypes(str, Enum):
+    job_launcher = "job_launcher"
+    recording_oracle = "recoring_oracle"
+
+
+class OracleWebhookStatuses(str, Enum):
+    pending = "pending"
+    completed = "completed"
+    failed = "failed"

packages/examples/cvat/exchange-oracle/src/cron.py → packages/examples/cvat/exchange-oracle/src/crons/__init__.py

@@ -1,16 +1,16 @@
 from fastapi import FastAPI
 from apscheduler.schedulers.background import BackgroundScheduler
 
-from src.config import Config
-from src.modules.cvat.jobs.track_completed import (
+from src.core.config import Config
+from src.crons.track_completed import (
     track_completed_projects,
     track_completed_tasks,
 )
-from src.modules.cvat.jobs.retrieve_annotations import retrieve_annotations
-from src.modules.oracle_webhook.jobs.process_job_launcher_webhooks import (
+from src.crons.retrieve_annotations import retrieve_annotations
+from src.crons.process_job_launcher_webhooks import (
     process_job_launcher_webhooks,
 )
-from src.modules.oracle_webhook.jobs.process_recording_oracle_webhooks import (
+from src.crons.process_recording_oracle_webhooks import (
     process_recording_oracle_webhooks,
 )
 

packages/examples/cvat/exchange-oracle/src/modules/oracle_webhook/jobs/process_job_launcher_webhooks.py → packages/examples/cvat/exchange-oracle/src/crons/process_job_launcher_webhooks.py

@@ -1,13 +1,14 @@
 import logging
 
-from src.database import SessionLocal
-from src.config import CronConfig
+from src.db import SessionLocal
+from src.core.config import CronConfig
 
-from src.modules.cvat.job_flows import job_creation_process, revert_job_creation
-from src.modules.chain.escrow import get_escrow_manifest, validate_escrow
+from src.cvat.create_job import job_creation_process
+from src.cvat.revert_job import revert_job_creation
+from src.chain.escrow import get_escrow_manifest, validate_escrow
 
-from src.modules.oracle_webhook.constants import OracleWebhookTypes
-import src.modules.oracle_webhook.service as db_service
+from src.core.constants import OracleWebhookTypes
+import src.services.webhook as db_service
 
 
 LOG_MODULE = "[cron][webhook][process_job_launcher_webhooks]"

packages/examples/cvat/exchange-oracle/src/modules/oracle_webhook/jobs/process_recording_oracle_webhooks.py → packages/examples/cvat/exchange-oracle/src/crons/process_recording_oracle_webhooks.py

@@ -1,15 +1,15 @@
 import httpx
 import logging
 
-from src.database import SessionLocal
-from src.config import CronConfig
+from src.db import SessionLocal
+from src.core.config import CronConfig
 
-from src.modules.chain.kvstore import get_recording_oracle_url
+from src.chain.kvstore import get_recording_oracle_url
 
-from src.modules.oracle_webhook.constants import OracleWebhookTypes
-from src.modules.oracle_webhook.helpers import prepare_recording_oracle_webhook_body
+from src.core.constants import OracleWebhookTypes
+from src.utils.helpers import prepare_recording_oracle_webhook_body
 
-import src.modules.oracle_webhook.service as db_service
+import src.services.webhook as db_service
 
 
 LOG_MODULE = "[cron][webhook][process_recording_oracle_webhooks]"

packages/examples/cvat/exchange-oracle/src/modules/cvat/jobs/retrieve_annotations.py → packages/examples/cvat/exchange-oracle/src/crons/retrieve_annotations.py

@@ -1,20 +1,14 @@
 import logging
 
-from human_protocol_sdk.storage import StorageClient, Credentials
-
-from src.database import SessionLocal
-from src.config import CronConfig, StorageConfig
-
-from src.modules.cvat.constants import ProjectStatuses
-from src.modules.cvat.handlers.annotation import get_annotations_handler
-
-from src.modules.oracle_webhook.constants import OracleWebhookTypes
-from src.modules.oracle_webhook.helpers import prepare_signature
-
-import src.modules.cvat.api_calls as cvat_api
-import src.modules.cvat.service as cvat_db_service
-import src.modules.oracle_webhook.service as oracle_db_service
-
+import src.cvat.api_calls as cvat_api
+import src.services.cvat as cvat_db_service
+import src.services.webhook as oracle_db_service
+from human_protocol_sdk.storage import Credentials, StorageClient
+from src.core.config import CronConfig, StorageConfig
+from src.core.constants import OracleWebhookTypes, ProjectStatuses
+from src.db import SessionLocal
+from src.handlers.annotation import get_annotations_handler
+from src.utils.helpers import prepare_signature
 
 LOG_MODULE = "[cron][cvat][retrieve_annotations]"
 logger = logging.getLogger("app")

packages/examples/cvat/exchange-oracle/src/modules/cvat/jobs/track_completed.py → packages/examples/cvat/exchange-oracle/src/crons/track_completed.py

@@ -1,10 +1,10 @@
 import logging
 
-from src.database import SessionLocal
-from src.config import CronConfig
+from src.db import SessionLocal
+from src.core.config import CronConfig
 
-from src.modules.cvat.constants import ProjectStatuses, TaskStatuses, JobStatuses
-import src.modules.cvat.service as cvat_service
+from src.core.constants import ProjectStatuses, TaskStatuses, JobStatuses
+import src.services.cvat as cvat_service
 
 LOG_MODULE = "[cron][cvat]"
 logger = logging.getLogger("app")

packages/examples/cvat/exchange-oracle/src/modules/cvat/api_calls.py → packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py

@@ -5,7 +5,7 @@
 from http import HTTPStatus
 from typing import Dict, List
 
-from src.config import Config
+from src.core.config import Config
 from cvat_sdk.api_client import Configuration, ApiClient, models, exceptions
 
 configuration = Configuration(

packages/examples/cvat/exchange-oracle/src/modules/cvat/job_flows.py → packages/examples/cvat/exchange-oracle/src/cvat/create_job.py

@@ -1,9 +1,9 @@
-from src.database import SessionLocal
+from src.db import SessionLocal
 
-from src.modules.cvat.helpers import parse_manifest, compose_bucket_url
+from src.utils.helpers import parse_manifest, compose_bucket_url
 
-import src.modules.cvat.service as db_service
-import src.modules.cvat.api_calls as cvat_api
+import src.services.cvat as db_service
+import src.cvat.api_calls as cvat_api
 
 
 def job_creation_process(escrow_address: str, chain_id: int, manifest: dict) -> None:
@@ -31,15 +31,3 @@ def job_creation_process(escrow_address: str, chain_id: int, manifest: dict) ->
 
     # Actual job creation on CVAT. Async process (will be created in DB once 'update:task' or 'update:job' webhook is received)
     cvat_api.put_task_data(task.id, cloudstorage.id)
-
-
-def revert_job_creation(escrow_address: str) -> None:
-    with SessionLocal.begin() as session:
-        project = db_service.get_project_by_escrow_address(session, escrow_address)
-        if project is not None:
-            if project.cvat_cloudstorage_id:
-                cvat_api.delete_cloustorage(project.cvat_cloudstorage_id)
-            if project.cvat_id:
-                cvat_api.delete_project(project.cvat_id)
-            db_service.delete_project(session, project.id)
-            session.commit()