From 9052733d679c640fccbf4f6b8e2a9623a61310e5 Mon Sep 17 00:00:00 2001 From: peusterm Date: Mon, 3 Jun 2019 10:20:08 +0200 Subject: [PATCH] Added downloads of schemas at container build time. Container can now run w/o internet connection. Closes #174 Signed-off-by: peusterm --- Dockerfile | 30 +++++++++++++++++++++++++++--- doc/rest_api_model.json | 2 +- src/tngsdk/package/validator.py | 23 ++++++++++++++++------- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index dbc70c1..7754730 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,9 +47,34 @@ ENV LOGLEVEL INFO ENV LOGJSON True # Install basics -RUN apt-get update && apt-get install -y git # We net git to install other tng-* tools. -RUN pip install flake8 +RUN apt-get update && apt-get install -y git wget # We net git to install other tng-* tools. +RUN pip install flake8 pyaml +# Pre-fetch latest tng-schemas (so that container works w/o internet connection) +RUN mkdir /root/.tng-schema +RUN mkdir /root/.tng-schema/service-descriptor/ +WORKDIR /root/.tng-schema/service-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/service-descriptor/nsd-schema.yml +RUN mkdir /root/.tng-schema/function-descriptor/ +WORKDIR /root/.tng-schema/function-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/function-descriptor/vnfd-schema.yml +RUN mkdir /root/.tng-schema/test-descriptor/ +WORKDIR /root/.tng-schema/test-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/test-descriptor/test-descriptor-schema.yml -O test-schema.yml +RUN mkdir /root/.tng-schema/policy-descriptor/ +WORKDIR /root/.tng-schema/policy-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/policy-descriptor/policy-schema.yml +RUN mkdir /root/.tng-schema/sla-template-descriptor/ +WORKDIR /root/.tng-schema/sla-template-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/sla-template-descriptor/sla-template-schema.yml +RUN mkdir /root/.tng-schema/slice-descriptor/ +WORKDIR /root/.tng-schema/slice-descriptor/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/slice-descriptor/nst-schema.yml -O nstd-schema.yml +RUN mkdir /root/.tng-schema/package-specification/ +WORKDIR /root/.tng-schema/package-specification/ +RUN wget https://raw.githubusercontent.com/sonata-nfv/tng-schema/master/package-specification/napd-schema.yml + +WORKDIR / # Install other 5GTAGNO SDK components # - tng-sdk-project (required by validator) RUN pip install git+https://github.com/sonata-nfv/tng-sdk-project.git @@ -62,7 +87,6 @@ RUN tng-sdk-validate -h # # Installation (packager) # -WORKDIR / ADD . /tng-sdk-package WORKDIR /tng-sdk-package RUN python setup.py develop diff --git a/doc/rest_api_model.json b/doc/rest_api_model.json index 6a7e86d..bcd76a2 100644 --- a/doc/rest_api_model.json +++ b/doc/rest_api_model.json @@ -1 +1 @@ -{"swagger": "2.0", "basePath": "http://tng-package.5gtango.eu/api", "paths": {"/v1/packages": {"post": {"responses": {"400": {"description": "Bad package: Could not unpackage given package."}, "200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "operationId": "post_packages", "parameters": [{"name": "package", "in": "formData", "type": "file", "required": true, "description": "Uploaded package file"}, {"name": "callback_url", "in": "formData", "type": "string", "description": "URL called after unpackaging (optional)"}, {"name": "username", "in": "formData", "type": "string", "description": "Username of the uploader (optional)"}, {"name": "layer", "in": "formData", "type": "string", "description": "Layer tag to be unpackaged (optional)"}, {"name": "format", "in": "formData", "type": "string", "description": "Package format (optional)", "default": "eu.5gtango"}, {"name": "skip_store", "in": "formData", "type": "boolean", "description": "Skip catalog upload of contents (optional)", "default": false}, {"name": "skip_validation", "in": "formData", "type": "boolean", "description": "Skip service validation (optional)", "default": false}, {"name": "validation_level", "in": "formData", "type": "string", "description": "Set validation level.\n Possible values:\n 's' or 'syntax',\n 'i' or 'integrity',\n 't' or 'topology' ,\n 'skip'", "default": "t", "enum": ["s", "syntax", "i", "integrity", "t", "topology", "skip"], "collectionFormat": "multi"}, {"name": "workspace", "in": "formData", "type": "string", "description": "Workspace (ignored for now)"}, {"name": "output", "in": "formData", "type": "string", "description": "Output (ignored for now)"}, {"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "consumes": ["multipart/form-data"], "tags": ["v1"]}}, "/v1/packages/status": {"get": {"responses": {"200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusListGetReturn"}}}, "operationId": "get_packages_status_list", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/packages/status/{package_process_uuid}": {"parameters": [{"name": "package_process_uuid", "in": "path", "required": true, "type": "string"}], "get": {"responses": {"404": {"description": "Package process not found."}, "200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "operationId": "get_packages_status_item", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/pings": {"get": {"responses": {"200": {"description": "Success", "schema": {"$ref": "#/definitions/PingGetReturn"}}}, "operationId": "get_ping", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/projects": {"post": {"responses": {"200": {"description": "Success"}}, "operationId": "post_project", "parameters": [{"name": "project", "in": "formData", "type": "file", "required": true, "description": "Uploaded project archive"}], "consumes": ["multipart/form-data"], "tags": ["v1"]}}}, "info": {"title": "5GTANGO tng-package API", "version": "0.1", "description": "5GTANGO tng-package REST API to package/unpacke NFV packages."}, "produces": ["application/json"], "consumes": ["application/json"], "tags": [{"name": "default", "description": "Default namespace"}, {"name": "v1", "description": "tng-package API v1"}], "definitions": {"PackagesStatusItemGetReturn": {"required": ["package_process_uuid", "status"], "properties": {"package_process_uuid": {"type": "string", "description": "UUID of started unpackaging process."}, "status": {"type": "string", "description": "Status of the unpacking process: waiting|runnig|failed|done"}, "error_msg": {"type": "string", "description": "More detailed error message."}}, "type": "object"}, "PackagesStatusListGetReturn": {"properties": {"package_processes": {"type": "array", "items": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "type": "object"}, "PingGetReturn": {"required": ["alive_since"], "properties": {"alive_since": {"type": "string", "description": "system uptime"}}, "type": "object"}}, "responses": {"ParseError": {"description": "When a mask can't be parsed"}, "MaskError": {"description": "When any error occurs on mask"}}, "host": "tng-package.5gtango.eu"} \ No newline at end of file +{"swagger": "2.0", "basePath": "http://tng-package.5gtango.eu/api", "paths": {"/v1/packages": {"post": {"responses": {"400": {"description": "Bad package: Could not unpackage given package."}, "200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "operationId": "post_packages", "parameters": [{"name": "package", "in": "formData", "type": "file", "required": true, "description": "Uploaded package file"}, {"name": "callback_url", "in": "formData", "type": "string", "description": "URL called after unpackaging (optional)"}, {"name": "username", "in": "formData", "type": "string", "description": "Username of the uploader (optional)"}, {"name": "layer", "in": "formData", "type": "string", "description": "Layer tag to be unpackaged (optional)"}, {"name": "format", "in": "formData", "type": "string", "description": "Package format (optional)"}, {"name": "skip_store", "in": "formData", "type": "boolean", "description": "Skip catalog upload\n of contents (optional)"}, {"name": "skip_validation", "in": "formData", "type": "boolean", "description": "Skip service validation (optional)"}, {"name": "validation_level", "in": "formData", "type": "string", "description": "Set validation level.\n Possible values:\n 's' or 'syntax',\n 'i' or 'integrity',\n 't' or 'topology' ,\n 'skip'", "enum": ["s", "syntax", "i", "integrity", "t", "topology", "skip"], "collectionFormat": "multi"}, {"name": "workspace", "in": "formData", "type": "string", "description": "Workspace (ignored for now)"}, {"name": "output", "in": "formData", "type": "string", "description": "Output (ignored for now)"}, {"name": "offline", "in": "formData", "type": "string", "description": "Offline"}, {"name": "no_checksums", "in": "formData", "type": "string", "description": "Do not validate artifact checksums."}, {"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "consumes": ["multipart/form-data"], "tags": ["v1"]}}, "/v1/packages/status": {"get": {"responses": {"200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusListGetReturn"}}}, "operationId": "get_packages_status_list", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/packages/status/{package_process_uuid}": {"parameters": [{"name": "package_process_uuid", "in": "path", "required": true, "type": "string"}], "get": {"responses": {"404": {"description": "Package process not found."}, "200": {"description": "Success", "schema": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "operationId": "get_packages_status_item", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/pings": {"get": {"responses": {"200": {"description": "Success", "schema": {"$ref": "#/definitions/PingGetReturn"}}}, "operationId": "get_ping", "parameters": [{"name": "X-Fields", "in": "header", "type": "string", "format": "mask", "description": "An optional fields mask"}], "tags": ["v1"]}}, "/v1/projects": {"get": {"responses": {"200": {"description": "Success"}}, "summary": "Get a list created packages", "description": "Returns: List of dictionaries: [{'package_name: ,\n 'package_download_link': }, ..]", "operationId": "get_projects", "tags": ["v1"]}, "post": {"responses": {"400": {"description": "Bad project: Could not package given project."}, "200": {"description": "Successfully started packaging."}}, "operationId": "post_projects", "parameters": [{"name": "project", "in": "formData", "type": "file", "required": true, "description": "Uploaded project archive"}, {"name": "callback_url", "in": "formData", "type": "string", "description": "URL called after unpackaging (optional)"}, {"name": "username", "in": "formData", "type": "string", "description": "Username of the uploader (optional)"}, {"name": "format", "in": "formData", "type": "string", "description": "Package format (optional)"}, {"name": "skip_store", "in": "formData", "type": "boolean", "description": "Skip catalog upload\n of contents (ignored)"}, {"name": "skip_validation", "in": "formData", "type": "boolean", "description": "Skip service validation (optional)"}, {"name": "validation_level", "in": "formData", "type": "string", "description": "Set validation level.\n Possible values:\n 's' or 'syntax',\n 'i' or 'integrity',\n 't' or 'topology' ,\n 'skip'", "enum": ["s", "syntax", "i", "integrity", "t", "topology", "skip"], "collectionFormat": "multi"}, {"name": "output", "in": "formData", "type": "string", "description": "Output"}, {"name": "workspace", "in": "formData", "type": "string", "description": "Workspace (ignored for now)"}, {"name": "offline", "in": "formData", "type": "string", "description": "Offline"}, {"name": "no_checksums", "in": "formData", "type": "string", "description": "Do not validate artifact checksums."}], "consumes": ["multipart/form-data"], "tags": ["v1"]}}, "/v1/projects/{filename}": {"parameters": [{"name": "filename", "in": "path", "required": true, "type": "string"}], "get": {"responses": {"200": {"description": "Success"}}, "operationId": "get_project_download", "parameters": [{"name": "project", "in": "formData", "type": "file", "required": true, "description": "Uploaded project archive"}, {"name": "callback_url", "in": "formData", "type": "string", "description": "URL called after unpackaging (optional)"}, {"name": "username", "in": "formData", "type": "string", "description": "Username of the uploader (optional)"}, {"name": "format", "in": "formData", "type": "string", "description": "Package format (optional)"}, {"name": "skip_store", "in": "formData", "type": "boolean", "description": "Skip catalog upload\n of contents (ignored)"}, {"name": "skip_validation", "in": "formData", "type": "boolean", "description": "Skip service validation (optional)"}, {"name": "validation_level", "in": "formData", "type": "string", "description": "Set validation level.\n Possible values:\n 's' or 'syntax',\n 'i' or 'integrity',\n 't' or 'topology' ,\n 'skip'", "enum": ["s", "syntax", "i", "integrity", "t", "topology", "skip"], "collectionFormat": "multi"}, {"name": "output", "in": "formData", "type": "string", "description": "Output"}, {"name": "workspace", "in": "formData", "type": "string", "description": "Workspace (ignored for now)"}, {"name": "offline", "in": "formData", "type": "string", "description": "Offline"}, {"name": "no_checksums", "in": "formData", "type": "string", "description": "Do not validate artifact checksums."}], "consumes": ["multipart/form-data"], "tags": ["v1"]}}}, "info": {"title": "5GTANGO tng-package API", "version": "0.1", "description": "5GTANGO tng-package REST API to package/unpacke NFV packages."}, "produces": ["application/json"], "consumes": ["application/json"], "tags": [{"name": "default", "description": "Default namespace"}, {"name": "v1", "description": "tng-package API v1"}], "definitions": {"PackagesStatusItemGetReturn": {"required": ["package_process_uuid", "status"], "properties": {"package_process_uuid": {"type": "string", "description": "UUID of started unpackaging process."}, "status": {"type": "string", "description": "Status of the unpacking process: waiting|runnig|failed|done"}, "error_msg": {"type": "string", "description": "More detailed error message."}}, "type": "object"}, "PackagesStatusListGetReturn": {"properties": {"package_processes": {"type": "array", "items": {"$ref": "#/definitions/PackagesStatusItemGetReturn"}}}, "type": "object"}, "PingGetReturn": {"required": ["alive_since"], "properties": {"alive_since": {"type": "string", "description": "system uptime"}}, "type": "object"}}, "responses": {"ParseError": {"description": "When a mask can't be parsed"}, "MaskError": {"description": "When any error occurs on mask"}}, "host": "tng-package.5gtango.eu"} \ No newline at end of file diff --git a/src/tngsdk/package/validator.py b/src/tngsdk/package/validator.py index 41107dc..9f331b5 100644 --- a/src/tngsdk/package/validator.py +++ b/src/tngsdk/package/validator.py @@ -31,6 +31,7 @@ # partner consortium (www.5gtango.eu). import requests import yaml +import os from jsonschema import validate from tngsdk.package.logger import TangoLogger @@ -100,18 +101,26 @@ def validate_yaml_online(data, schema_uri=None): try: # try to download schema r = requests.get(schema_uri, timeout=3) - except BaseException as e: - LOG.error("Couldn't fetch schema from '{}': {}".format( - schema_uri, e)) - return False - try: # try to parse schema schema = yaml.load(r.text) except BaseException as e: - LOG.error("Couldn't parse schema from '{}': {}".format( + LOG.warning("Couldn't fetch schema from '{}': {}".format( schema_uri, e)) - return False + # ok, no internet? lets try to use a local NAPD schema + try: + path = os.path.join( + os.path.expanduser("~"), + ".tng-schema/package-specification/napd-schema.yml") + LOG.info("Using local schema: {}".format(path)) + with open(path, "r") as f: + schema = yaml.load(f) + except BaseException as e: + LOG.error("Get schema from '{}' or '{}': {}".format( + schema_uri, path, e)) + return False try: + if schema is None: + raise BaseException("No schema found online and offile") # validate data against schema validate(data, schema) except BaseException as e: