From 0faa0008e86628d09aadab070303a548b577c9cc Mon Sep 17 00:00:00 2001 From: Andrea Bonomi Date: Tue, 12 Jul 2022 09:34:08 +0000 Subject: [PATCH 1/7] Filesystem access refactory --- Makefile | 5 +- README.md | 47 ++-- airflow_code_editor/VERSION | 2 +- airflow_code_editor/app_builder_view.py | 1 - airflow_code_editor/code_editor_view.py | 36 ++- airflow_code_editor/commons.py | 11 +- airflow_code_editor/fs.py | 273 +++++++++++++++++++++ airflow_code_editor/git.py | 251 +++++++++++++++++++ airflow_code_editor/tree.py | 59 ++--- airflow_code_editor/utils.py | 307 ++++-------------------- changelog.txt | 12 + requirements.txt | 1 + scripts/config.sh | 6 +- setup.py | 20 +- sh.py | 104 ++++++++ tests/test_fs.py | 73 ++++++ tests/test_import.py | 13 + tests/test_tree.py | 64 +++-- tests/test_utils.py | 145 ++++++----- 19 files changed, 965 insertions(+), 465 deletions(-) create mode 100644 airflow_code_editor/fs.py create mode 100644 airflow_code_editor/git.py create mode 100644 sh.py create mode 100644 tests/test_fs.py diff --git a/Makefile b/Makefile index df56c6b..dd851eb 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,10 @@ help: @echo "- make npm-watch Run npm build when files change" lint: - python3 setup.py flake8 + flake8 airflow_code_editor tests + +black: + black -S airflow_code_editor tests tag: @grep -q "## $$(cat airflow_code_editor/VERSION)" changelog.txt || (echo "Missing changelog !!! Update changelog.txt"; exit 1) diff --git a/README.md b/README.md index 8eea2bb..d041689 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,15 @@ If git support is enabled, the DAGs are stored in a Git repository. You may use pip install airflow-code-editor ``` -2. (Optional) Install Black Python code formatter. +2. Install optional dependencies + +* black - Black Python code formatter +* fs-s3fs - S3FS Amazon S3 Filesystem +* fs-gcsfs - Google Cloud Storage Filesystem +* ... other filesystems supported by PyFilesystem - see https://www.pyfilesystem.org/page/index-of-filesystems/ ```bash - pip install black + pip install black fs-gcsfs ``` 3. Restart the Airflow Web Server @@ -60,12 +65,10 @@ All the settings are optional. * **git_author_email** email for the author/committer (default: logged user email) * **git_init_repo** initialize a git repo in DAGs folder (default: True) * **root_directory** root folder (default: Airflow DAGs folder) -* **mount_name**, **mount1_name**, ... configure additional file folder name (mount point) -* **mount_path**, **mount1_path**, ... configure additional file path * **line_length** Python code formatter - max line length (default: 88) * **string_normalization** Python code formatter - if true normalize string quotes and prefixes (default: False) +* **mount**, **mount1**, ... configure additional folder (mount point) - format: name=xxx,path=yyy -Example: ``` [code_editor] enabled = True @@ -76,12 +79,21 @@ Example: root_directory = /home/airflow/dags line_length = 88 string_normalization = False - mount_name = data - mount_path = /home/airflow/data - mount1_name = logs - mount1_path = /home/airflow/logs + mount = name=data,path=/home/airflow/data + mount1 = name=logs,path=/home/airflow/logs + mount2 = name=data,path=s3://example ``` +Mount Options: + +* **name**: mount name (destination) +* **path**: local path or PyFilesystem FS URLs - see https://docs.pyfilesystem.org/en/latest/openers.html + +Example: +* name=ftp_server,path=ftp://user:pass@ftp.example.org/private +* name=data,path=s3://example +* name=tmp,path=/tmp + You can also set options with the following environment variables: * AIRFLOW__CODE_EDITOR__ENABLED @@ -94,20 +106,14 @@ You can also set options with the following environment variables: * AIRFLOW__CODE_EDITOR__ROOT_DIRECTORY * AIRFLOW__CODE_EDITOR__LINE_LENGTH * AIRFLOW__CODE_EDITOR__STRING_NORMALIZATION -* AIRFLOW__CODE_EDITOR__MOUNT_NAME -* AIRFLOW__CODE_EDITOR__MOUNT_PATH -* AIRFLOW__CODE_EDITOR__MOUNT1_NAME, AIRFLOW__CODE_EDITOR__MOUNT2_NAME, ... -* AIRFLOW__CODE_EDITOR__MOUNT1_PATH, AIRFLOW__CODE_EDITOR__MOUNT2_PATH, ... +* AIRFLOW__CODE_EDITOR__MOUNT, AIRFLOW__CODE_EDITOR__MOUNT1, AIRFLOW__CODE_EDITOR__MOUNT2, ... Example: ``` export AIRFLOW__CODE_EDITOR__STRING_NORMALIZATION=True - export AIRFLOW__CODE_EDITOR__MOUNT_NAME='data' - export AIRFLOW__CODE_EDITOR__MOUNT_PATH='/home/airflow/data' - export AIRFLOW__CODE_EDITOR__MOUNT1_NAME='logs' - export AIRFLOW__CODE_EDITOR__MOUNT1_PATH='/home/airflow/logs' - export AIRFLOW__CODE_EDITOR__MOUNT2_NAME='tmp' - export AIRFLOW__CODE_EDITOR__MOUNT2_PATH='/tmp' + export AIRFLOW__CODE_EDITOR__MOUNT='name=data,path=/home/airflow/data' + export AIRFLOW__CODE_EDITOR__MOUNT1='name=logs,path=/home/airflow/logs' + export AIRFLOW__CODE_EDITOR__MOUNT2='name=tmp,path=/tmp' ``` ### Development Instructions @@ -167,3 +173,6 @@ Example: * Vue-tree, TreeView control for VueJS - https://github.com/grapoza/vue-tree * Splitpanes - https://github.com/antoniandre/splitpanes * Axios, Promise based HTTP client for the browser and node.js - https://github.com/axios/axios +* PyFilesystem2, Python's Filesystem abstraction layer - https://github.com/PyFilesystem/pyfilesystem2 +* Amazon S3 PyFilesystem - https://github.com/PyFilesystem/s3fs +* Google Cloud Storage PyFilesystem - https://github.com/Othoz/gcsfs diff --git a/airflow_code_editor/VERSION b/airflow_code_editor/VERSION index ce7f2b4..09b254e 100644 --- a/airflow_code_editor/VERSION +++ b/airflow_code_editor/VERSION @@ -1 +1 @@ -5.2.2 +6.0.0 diff --git a/airflow_code_editor/app_builder_view.py b/airflow_code_editor/app_builder_view.py index 796f9ce..e33c588 100644 --- a/airflow_code_editor/app_builder_view.py +++ b/airflow_code_editor/app_builder_view.py @@ -98,7 +98,6 @@ def _render(self, template, *args, **kargs): **kargs ) - except (ImportError, ModuleNotFoundError): from airflow_code_editor.auth import has_access from airflow.www_rbac.decorators import has_dag_access diff --git a/airflow_code_editor/code_editor_view.py b/airflow_code_editor/code_editor_view.py index 65f84aa..04f9e8d 100644 --- a/airflow_code_editor/code_editor_view.py +++ b/airflow_code_editor/code_editor_view.py @@ -15,11 +15,9 @@ # limitations under the Licens # -import os -import os.path import logging import mimetypes -from flask import abort, request, send_file +from flask import abort, request from flask_wtf.csrf import generate_csrf from airflow.version import version from airflow_code_editor.commons import HTTP_404_NOT_FOUND @@ -27,12 +25,14 @@ from airflow_code_editor.utils import ( get_plugin_boolean_config, get_plugin_int_config, - git_absolute_path, - execute_git_command, error_message, normalize_path, prepare_api_response, ) +from airflow_code_editor.git import ( + execute_git_command, +) +from airflow_code_editor.fs import RootFS __all__ = ["AbstractCodeEditorView"] @@ -48,22 +48,16 @@ def _index(self): def _save(self, path=None): try: - fullpath = git_absolute_path(path) mime_type = request.headers.get("Content-Type", "text/plain") is_text = mime_type.startswith("text/") if is_text: data = request.get_data(as_text=True) # Newline fix (remove cr) - data = data.replace("\r", "").rstrip() - os.makedirs(os.path.dirname(fullpath), exist_ok=True) - with open(fullpath, "w") as f: - f.write(data) - f.write("\n") + data = data.replace("\r", "").rstrip() + "\n" else: # Binary file data = request.get_data() - os.makedirs(os.path.dirname(fullpath), exist_ok=True) - with open(fullpath, "wb") as f: - f.write(data) + root_fs = RootFS() + root_fs.path(path).write_file(data=data, is_text=is_text) return prepare_api_response(path=normalize_path(path)) except Exception as ex: logging.error(ex) @@ -81,18 +75,19 @@ def _git_repo(self, path): return self._git_repo_get(path) def _git_repo_get(self, path): - " Get a file from GIT (invoked by the HTTP GET method) " + "Get a file from GIT (invoked by the HTTP GET method)" return execute_git_command(["cat-file", "-p", path]) def _git_repo_post(self, path): - " Execute a GIT command (invoked by the HTTP POST method) " + "Execute a GIT command (invoked by the HTTP POST method)" git_args = request.json.get('args', []) return execute_git_command(git_args) def _load(self, path): - " Send the contents of a file to the client " + "Send the contents of a file to the client" try: path = normalize_path(path) + root_fs = RootFS() if path.startswith("~git/"): # Download git blob - path = '~git//' _, path, filename = path.split("/", 3) @@ -109,14 +104,13 @@ def _load(self, path): return response else: # Download file - fullpath = git_absolute_path(path) - return send_file(fullpath, as_attachment=True) + return root_fs.path(path).send_file(as_attachment=True) except Exception as ex: logging.error(ex) abort(HTTP_404_NOT_FOUND) def _format(self): - " Format code " + "Format code" try: import black @@ -141,7 +135,7 @@ def _format(self): ) ) - def _tree(self, path, args = {}): + def _tree(self, path, args={}): return {'value': get_tree(path, args)} def _ping(self): diff --git a/airflow_code_editor/commons.py b/airflow_code_editor/commons.py index dc38f2b..a09d6d6 100644 --- a/airflow_code_editor/commons.py +++ b/airflow_code_editor/commons.py @@ -15,7 +15,7 @@ # limitations under the Licens # -import os +from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union __all__ = [ @@ -34,7 +34,6 @@ 'VERSION_FILE', 'VERSION', 'Args', - 'Path', 'GitOutput', 'TreeOutput', 'TreeFunc', @@ -99,13 +98,11 @@ 'airflow_code_editor.js', ] -VERSION_FILE = os.path.join(os.path.dirname(__file__), "VERSION") -with open(VERSION_FILE) as f: - VERSION = f.read().strip() +VERSION_FILE = Path(__file__).parent / "VERSION" +VERSION = VERSION_FILE.read_text().strip() Args = Dict[str, str] -Path = Optional[str] GitOutput = Union[None, bytes, str] TreeOutput = List[Dict[str, Any]] -TreeFunc = Callable[[Path, Args], TreeOutput] +TreeFunc = Callable[[Optional[str], Args], TreeOutput] diff --git a/airflow_code_editor/fs.py b/airflow_code_editor/fs.py new file mode 100644 index 0000000..b40f8c1 --- /dev/null +++ b/airflow_code_editor/fs.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python +# +# Copyright 2019 Andrea Bonomi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the Licens + +import os +import fs +from fs.mountfs import MountFS, MountError +from fs.multifs import MultiFS +from fs.path import abspath, forcedir, normpath +from typing import Any, List, Union +from flask import abort, send_file, stream_with_context, Response +from airflow_code_editor.utils import read_mount_points_config +from airflow_code_editor.commons import HTTP_404_NOT_FOUND + +__all__ = [ + 'RootFS', +] + +STAT_FIELDS = [ + "st_mode", + "st_ino", + "st_dev", + "st_nlink", + "st_uid", + "st_gid", + "st_size", + "st_atime", + "st_mtime", + "st_ctime", +] + +SEND_FILE_CHUNK_SIZE = 8192 + + +def split(pathname: str): + "Split a pathname, returns tuple (head, tail)" + pathname = pathname.rstrip("/") + i = pathname.rfind("/") + 1 + if i == 0: + return ("/", pathname) + else: + return pathname[: i - 1], pathname[i:] + + +class RootFS(MountFS): + "Root filesystem with mountpoints" + + def __init__(self): + super().__init__() + mounts = read_mount_points_config() + # Set default fs (root) + self.default_fs = MultiFS() + self.tmp_fs = fs.open_fs("mem://") + self.default_fs.add_fs('tmp', self.tmp_fs, write=False, priority=0) + self.root_fs = [fs.open_fs(v.path) for v in mounts.values() if v.default][0] + self.default_fs.add_fs('root', self.root_fs, write=True, priority=1) + # Mount other fs + for k, v in mounts.items(): + if not v.default: + self.mount("/~" + k, fs.open_fs(v.path)) + + def mount(self, path, fs_): + "Mounts a host FS object on a given path" + if isinstance(fs_, str): + fs_ = fs.open_fs(fs_) + path_ = forcedir(abspath(normpath(path))) + for mount_path, _ in self.mounts: + if path_.startswith(mount_path): + raise MountError("mount point overlaps existing mount") + self.mounts.append((path_, fs_)) + # Create mountpoint on the temporary filesystem + self.tmp_fs.makedirs(path_, recreate=True) + + def path(self, *parts: List[str]): + "Return a FSPath instance for the given path" + return FSPath(*parts, root_fs=self) + + +class FSPath(object): + def __init__(self, *parts: List[str], root_fs: RootFS) -> None: + self.root_fs = root_fs + self.path = os.path.join("/", *parts) + + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): + "Open the file pointed by this path and return a file object" + return self.root_fs.open( + self.path, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + ) + + @property + def name(self) -> str: + "The final path component" + return split(self.path)[1] + + @property + def parent(self): + "The logical parent of the path" + return self.root_fs.path(split(self.path)[0]) + + def touch(self, mode=0o666, exist_ok=True): + "Create this file" + return self.root_fs.touch(self.path) + + def rmdir(self) -> None: + "Remove this directory" + self.root_fs.removedir(self.path) + + def unlink(self, missing_ok: bool = False) -> None: + "Remove this file" + try: + self.root_fs.remove(self.path) + except fs.errors.ResourceNotFound: + if not missing_ok: + raise FileNotFoundError(self.path) + + def delete(self) -> None: + "Remove this file or directory" + if self.is_dir(): + self.rmdir() + else: + self.unlink() + + def stat(self): + "File stat" + info = self.root_fs.getinfo(self.path, namespaces=["stat"]) + if not info.has_namespace("stat"): + return os.stat_result([None for _ in STAT_FIELDS]) + return os.stat_result([info.raw["stat"].get(field) for field in STAT_FIELDS]) + + def is_dir(self) -> bool: + "Return True if this path is a directory" + try: + return self.root_fs.isdir(self.path) + except Exception: + return False + + def resolve(self): + "Make the path absolute" + return self.root_fs.path(os.path.realpath(self.path)) + + def exists(self): + "Check if this path exists" + return self.root_fs.exists(self.path) + + def iterdir(self): + "Iterate over the files in this directory" + try: + for name in sorted(self.root_fs.listdir(self.path)): + if name.startswith(".") or name == "__pycache__": + continue + yield self.root_fs.path(self.path, name) + except IOError: + yield from [] + + def size(self) -> int: + "Return file size for files and number of files for directories" + if self.is_dir(): + return len(self.root_fs.listdir(self.path)) + else: + return self.root_fs.getsize(self.path) + + def move(self, target) -> None: + "Move/rename a file or directory" + target = self.root_fs.path(target) + if target.is_dir(): + self.root_fs.move(self.path, (target / self.name).path) + else: + self.root_fs.move(self.path, target.path) + + def read_file_chunks(self, chunk_size: int = SEND_FILE_CHUNK_SIZE): + "Read file in chunks" + with self.root_fs.openbin(self.path) as f: + while True: + buffer = f.read(chunk_size) + if buffer: + yield buffer + else: + break + + def send_file(self, as_attachment: bool): + "Send the contents of a file to the client" + if not self.exists(): + abort(HTTP_404_NOT_FOUND) + elif self.root_fs.hassyspath(self.path): + # Local filesystem + return send_file( + self.root_fs.getsyspath(self.path), + as_attachment=as_attachment, + attachment_filename=self.name if as_attachment else None, + ) + else: + # Other filesystems + response = Response(stream_with_context(self.read_file_chunks())) + if as_attachment: + response.headers[ + 'Content-Disposition' + ] = 'attachment;filename={}'.format(self.name) + return response + + def write_file(self, data: Union[str, bytes], is_text: bool) -> None: + "Write data to a file" + self.root_fs.makedirs(self.parent.path, recreate=True) + if is_text: + self.root_fs.writetext(self.path, data) + else: + self.root_fs.writebytes(self.path, data) + + def read_text(self, encoding=None, errors=None) -> str: + "Get the contents of a file as a string" + return self.root_fs.readtext(self.path, encoding=encoding, errors=errors) + + def read_bytes(self) -> bytes: + "Get the contents of a file as bytes" + return self.root_fs.readbytes(self.path) + + def __str__(self) -> str: + return self.path + + def __truediv__(self, key): + try: + path = os.path.join(self.path, key) + return self.root_fs.path(path) + except TypeError: + return NotImplemented + + def __eq__(self, other) -> bool: + if not isinstance(other, FSPath): + return NotImplemented + return self.path == other.path + + def __hash__(self) -> int: + try: + return self._hash + except AttributeError: + self._hash = hash(self.path) + return self._hash + + def __lt__(self, other: Any): + if not isinstance(other, FSPath): + return NotImplemented + return self.path < other.path + + def __le__(self, other: Any) -> bool: + if not isinstance(other, FSPath): + return NotImplemented + return self.path <= other.path + + def __gt__(self, other: Any) -> bool: + if not isinstance(other, FSPath): + return NotImplemented + return self.path > other.path + + def __ge__(self, other: Any) -> bool: + if not isinstance(other, FSPath): + return NotImplemented + return self.path >= other.path diff --git a/airflow_code_editor/git.py b/airflow_code_editor/git.py new file mode 100644 index 0000000..b15b5b7 --- /dev/null +++ b/airflow_code_editor/git.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python +# +# Copyright 2019 Andrea Bonomi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the Licens + +import os +import logging +import subprocess +import threading +import shlex +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from datetime import datetime +from flask import make_response, Response +from flask_login import current_user # type: ignore +from airflow_code_editor.commons import ( + HTTP_200_OK, + HTTP_404_NOT_FOUND, + SUPPORTED_GIT_COMMANDS, + GitOutput, +) +from airflow_code_editor.utils import ( + normalize_path, + get_plugin_config, + get_plugin_boolean_config, + get_root_folder, + read_mount_points_config, +) +from airflow_code_editor.fs import RootFS + +__all__ = [ + 'git_enabled', + 'execute_git_command', +] + + +def git_enabled() -> bool: + "Return true if git is enabled in the configuration" + return get_plugin_boolean_config('git_enabled') + + +def prepare_git_response( + git_cmd: Optional[str], + result: GitOutput = None, + stderr: GitOutput = None, + returncode: int = 0, +) -> Response: + if result is None: + result = stderr + elif stderr: + result = result + stderr + if git_cmd == 'cat-file': + response = make_response( + result, HTTP_200_OK if returncode == 0 else HTTP_404_NOT_FOUND + ) + response.headers['Content-Type'] = 'text/plain' + else: + response = make_response(result) + response.headers['X-Git-Return-Code'] = str(returncode) + response.headers['X-Git-Stderr-Length'] = str(len(stderr or '')) + response.headers['Content-Type'] = 'text/plain' + return response + + +_execute_git_command_lock = threading.Lock() + + +def execute_git_command(git_args: List[str]) -> Response: + with _execute_git_command_lock: + logging.info(' '.join(git_args)) + git_cmd = git_args[0] if git_args else None + stdout: GitOutput = None + stderr: GitOutput = None + returncode = 0 + try: + # Init git repo + init_git_repo() + # Local commands + if git_cmd in LOCAL_COMMANDS: + handler = LOCAL_COMMANDS[git_cmd] + stdout = handler(git_args) + # Git commands + elif git_cmd in SUPPORTED_GIT_COMMANDS: + git_default_args = shlex.split(get_plugin_config('git_default_args')) + returncode, stdout, stderr = git_call( + git_default_args + git_args, capture_output=True + ) + else: + stdout = None + stderr = bytes( + 'Command not supported: git %s' % ' '.join(git_args), 'utf-8' + ) + returncode = 1 + except OSError as ex: + logging.error(ex) + stdout = None + stderr = ex.strerror + returncode = ex.errno + except Exception as ex: + logging.error(ex) + stdout = None + stderr = ex.message if hasattr(ex, 'message') else str(ex) + returncode = 1 + finally: + return prepare_git_response(git_cmd, stdout, stderr, returncode) + + +def git_ls_local(git_args: List[str]) -> str: + "'git ls-tree' like output for local folders" + long_ = False + if '-l' in git_args or '--long' in git_args: + git_args = [arg for arg in git_args if arg not in ('-l', '--long')] + long_ = True + path = git_args[1] if len(git_args) > 1 else '' + path = normalize_path(path.split('#', 1)[0]) + result = [] + root_fs = RootFS() + for item in root_fs.path(path).iterdir(): + if item.is_dir(): + type_ = 'tree' + else: + type_ = 'blob' + s = item.stat() + if long_: + mtime = datetime.utcfromtimestamp(s.st_mtime).isoformat()[:16] + result.append( + '%06o %s %s#%s %8s\t%s' + % (s.st_mode, type_, str(item), mtime, item.size(), item.name) + ) + else: + result.append('%06o %s %s\t%s' % (s.st_mode, type_, str(item), item.name)) + return '\n'.join(result) + + +def git_mounts(git_args: List[str]) -> str: + "List mountpoints" + mount_points = read_mount_points_config() + return '\n'.join(sorted(k for k, v in mount_points.items() if not v.default)) + + +def git_rm_local(git_args: List[str]) -> str: + "Delete local files/directories" + root_fs = RootFS() + for arg in git_args[1:]: + if arg: + root_fs.path(arg).delete() + return '' + + +def git_mv_local(git_args: List[str]) -> str: + "Rename/Move local files" + if len(git_args) < 3: + raise Exception('Missing source/destination args') + root_fs = RootFS() + target = git_args[-1] + for arg in git_args[1:-1]: + source = root_fs.path(arg) + source.move(target) + return '' + + +LOCAL_COMMANDS = { + 'mounts': git_mounts, + 'ls-local': git_ls_local, + 'rm-local': git_rm_local, + 'mv-local': git_mv_local, +} + + +def git_call( + argv: List[str], capture_output: bool = False +) -> Tuple[int, GitOutput, GitOutput]: + "Run git command. If capture_output is true, stdout and stderr will be captured." + if not git_enabled(): + return 1, '', 'Git is disabled' + cmd: List[str] = [get_plugin_config('git_cmd')] + argv + cwd: Path = get_root_folder() + env: Dict[str, str] = prepare_git_env() + if capture_output: + git = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + env=env, + ) + stdout, stderr = git.communicate() + returncode: int = git.returncode + else: + stdout = b'' + stderr = b'' + returncode = subprocess.call(cmd, cwd=cwd, env=env) + return returncode, stdout, stderr + + +def init_git_repo() -> None: + "Initialize the git repository in root folder" + cwd: Path = get_root_folder() + if ( + git_enabled() + and not (cwd / '.git').exists() + and get_plugin_boolean_config('git_init_repo') + ): + git_call(['init', '.']) + gitignore = cwd / '.gitignore' + if not gitignore.exists(): + with gitignore.open('w') as f: + f.write('__pycache__\n') + git_call(['add', '.gitignore']) + git_call(['commit', '-m', 'Initial commit']) + + +def prepare_git_env() -> Dict[str, str]: + "Prepare the environ for git" + env = dict(os.environ) + # Author + git_author_name = get_plugin_config('git_author_name') + if not git_author_name: + try: + git_author_name = '%s %s' % ( + current_user.first_name, + current_user.last_name, + ) + except Exception: + pass + if git_author_name: + env['GIT_AUTHOR_NAME'] = git_author_name + env['GIT_COMMITTER_NAME'] = git_author_name + # Email + git_author_email = get_plugin_config('git_author_email') + if not git_author_email: + try: + git_author_email = current_user.email + except Exception: + pass + if git_author_email: + env['GIT_AUTHOR_EMAIL'] = git_author_email + env['GIT_COMMITTER_EMAIL'] = git_author_email + return env diff --git a/airflow_code_editor/tree.py b/airflow_code_editor/tree.py index b1c2dbe..5da891b 100644 --- a/airflow_code_editor/tree.py +++ b/airflow_code_editor/tree.py @@ -14,26 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the Licens -import os -import os.path import re -import stat from datetime import datetime from typing import Any, Callable, Dict, List, NamedTuple, Optional from airflow_code_editor.commons import ( Args, - Path, TreeFunc, TreeOutput, ) from airflow_code_editor.utils import ( always, - git_absolute_path, + normalize_path, + read_mount_points_config, +) +from airflow_code_editor.git import ( git_enabled, execute_git_command, - mount_points, - normalize_path, ) +from airflow_code_editor.fs import RootFS __all__ = ['get_tree'] @@ -66,7 +64,7 @@ def f(func: TreeFunc) -> TreeFunc: @node(id=None, label='Root', leaf=False) -def get_root_node(path: Path, args: Args) -> TreeOutput: +def get_root_node(path: Optional[str], args: Args) -> TreeOutput: "Get tree root node" result = [] for id_, node in TREE_NODES.items(): @@ -79,6 +77,7 @@ def get_root_node(path: Path, args: Args) -> TreeOutput: ) # If the node is files, add the mount points if id_ == 'files': + mount_points = read_mount_points_config() for mount in sorted(k for k, v in mount_points.items() if not v.default): mount = mount.rstrip('/') result.append( @@ -93,42 +92,28 @@ def get_root_node(path: Path, args: Args) -> TreeOutput: @node(id='files', label='Files', leaf=False, icon='fa-home') -def get_files_node(path: Path, args: Args) -> TreeOutput: +def get_files_node(path: Optional[str], args: Args) -> TreeOutput: "Get tree files node" - - def try_listdir(path: str) -> List[str]: - try: - return os.listdir(path) - except IOError: - return [] - result = [] - dirpath: str = git_absolute_path(path) long_ = 'long' in args - for name in sorted(try_listdir(dirpath)): - if name.startswith('.') or name == '__pycache__': - continue - fullname = os.path.join(dirpath, name) - s = os.stat(fullname) - leaf = not stat.S_ISDIR(s.st_mode) + for item in RootFS().path(path).iterdir(): + s = item.stat() + leaf = not item.is_dir() if long_: # Long format - size = s.st_size if leaf else len(try_listdir(fullname)) + size = item.size() result.append( { - 'id': name, + 'id': item.name, 'leaf': leaf, 'size': size, 'mode': s.st_mode, 'mtime': datetime.fromtimestamp(int(s.st_mtime)).isoformat() + if s.st_mtime + else None, } ) else: # Short format - result.append( - { - 'id': name, - 'leaf': leaf - } - ) + result.append({'id': item.name, 'leaf': leaf}) return result @@ -157,19 +142,19 @@ def prepare_ls_tree_output(line: str) -> Dict[str, Any]: 'label': name, 'leaf': leaf, 'size': int(size) if leaf else None, - 'mode': int(mode, 8) + 'mode': int(mode, 8), } @node(id='git', label='Git Workspace', icon='fa-briefcase', condition=git_enabled) -def get_git_node(path: Path, args: Args) -> TreeOutput: +def get_git_node(path: Optional[str], args: Args) -> TreeOutput: "List the contents of a git tree object" output = git_command_output('ls-tree', '-l', path or 'HEAD') return [prepare_ls_tree_output(line) for line in output if line] @node(id='tags', label='Tags', leaf=False, icon='fa-tags', condition=git_enabled) -def get_tags_node(path: Path, args: Args) -> TreeOutput: +def get_tags_node(path: Optional[str], args: Args) -> TreeOutput: "Get tree tags node" if path: return get_git_node(path, args) @@ -184,7 +169,7 @@ def get_tags_node(path: Path, args: Args) -> TreeOutput: icon='fa-code-fork', condition=git_enabled, ) -def get_local_branches_node(path: Path, args: Args) -> TreeOutput: +def get_local_branches_node(path: Optional[str], args: Args) -> TreeOutput: "Get tree local branches node" if path: return get_git_node(path, args) @@ -199,7 +184,7 @@ def get_local_branches_node(path: Path, args: Args) -> TreeOutput: icon='fa-globe', condition=git_enabled, ) -def get_remote_branches_node(path: Path, args: Args) -> TreeOutput: +def get_remote_branches_node(path: Optional[str], args: Args) -> TreeOutput: "Get tree remote branches node" if path: return get_git_node(path, args) @@ -207,7 +192,7 @@ def get_remote_branches_node(path: Path, args: Args) -> TreeOutput: return [prepare_git_output(line, 'fa-globe') for line in output if line] -def get_tree(path: Path = None, args: Args = {}) -> TreeOutput: +def get_tree(path: Optional[str] = None, args: Args = {}) -> TreeOutput: "Get tree nodes at the given path" if not path: root = None diff --git a/airflow_code_editor/utils.py b/airflow_code_editor/utils.py index 941dccd..116dbf6 100644 --- a/airflow_code_editor/utils.py +++ b/airflow_code_editor/utils.py @@ -14,28 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the Licens -import os -import os.path -import logging -import subprocess -import threading -import shlex -import shutil -from typing import cast, Dict, List, Optional, Tuple -from datetime import datetime +import itertools +from pathlib import Path +from typing import cast, Dict, List, Optional from collections import namedtuple -from flask import jsonify, make_response, Response -from flask_login import current_user # type: ignore +from flask import jsonify from airflow import configuration from airflow_code_editor.commons import ( - HTTP_200_OK, - HTTP_404_NOT_FOUND, - SUPPORTED_GIT_COMMANDS, PLUGIN_NAME, PLUGIN_DEFAULT_CONFIG, ROOT_MOUNTPOUNT, - Path, - GitOutput, ) @@ -45,10 +33,7 @@ 'get_plugin_boolean_config', 'get_plugin_int_config', 'is_enabled', - 'git_enabled', 'get_root_folder', - 'git_absolute_path', - 'execute_git_command', 'error_message', 'prepare_api_response', 'always', @@ -58,11 +43,11 @@ # Create a new section in the configuration. try: configuration.conf.add_section(PLUGIN_NAME) -except: +except Exception: pass -def normalize_path(path: Path) -> str: +def normalize_path(path: Optional[str]) -> str: comps = (path or '/').split('/') result: List[str] = [] for comp in comps: @@ -105,241 +90,12 @@ def is_enabled() -> bool: return get_plugin_boolean_config('enabled') -def git_enabled() -> bool: - "Return true if git is enabled in the configuration" - return get_plugin_boolean_config('git_enabled') - - -def prepare_git_env() -> Dict[str, str]: - "Prepare the environ for git" - env = dict(os.environ) - # Author - git_author_name = get_plugin_config('git_author_name') - if not git_author_name: - try: - git_author_name = '%s %s' % ( - current_user.first_name, - current_user.last_name, - ) - except Exception: - pass - if git_author_name: - env['GIT_AUTHOR_NAME'] = git_author_name - env['GIT_COMMITTER_NAME'] = git_author_name - # Email - git_author_email = get_plugin_config('git_author_email') - if not git_author_email: - try: - git_author_email = current_user.email - except Exception: - pass - if git_author_email: - env['GIT_AUTHOR_EMAIL'] = git_author_email - env['GIT_COMMITTER_EMAIL'] = git_author_email - return env - - -def get_root_folder() -> str: +def get_root_folder() -> Path: "Return the configured root folder or Airflow DAGs folder" - return os.path.abspath( + return Path( get_plugin_config('root_directory') or cast(str, configuration.conf.get('core', 'dags_folder')) # type: ignore - ) - - -def git_absolute_path(git_path: Path) -> str: - "Git relative path to absolute path" - path: str = normalize_path(git_path) - if path.startswith('~'): - # Expand paths beginning with '~' - prefix, remain = path.split('/', 1) if '/' in path else (path, '') - try: - return os.path.join(mount_points[prefix[1:]].path, remain) - except KeyError: - pass - return os.path.join(get_root_folder(), path) - - -def prepare_git_response( - git_cmd: Optional[str], - result: GitOutput = None, - stderr: GitOutput = None, - returncode: int = 0, -) -> Response: - if result is None: - result = stderr - elif stderr: - result = result + stderr - if git_cmd == 'cat-file': - response = make_response( - result, HTTP_200_OK if returncode == 0 else HTTP_404_NOT_FOUND - ) - response.headers['Content-Type'] = 'text/plain' - else: - response = make_response(result) - response.headers['X-Git-Return-Code'] = str(returncode) - response.headers['X-Git-Stderr-Length'] = str(len(stderr or '')) - response.headers['Content-Type'] = 'text/plain' - return response - - -_execute_git_command_lock = threading.Lock() - - -def execute_git_command(git_args: List[str]) -> Response: - with _execute_git_command_lock: - logging.info(' '.join(git_args)) - git_cmd = git_args[0] if git_args else None - stdout: GitOutput = None - stderr: GitOutput = None - returncode = 0 - try: - # Init git repo - init_git_repo() - # Local commands - if git_cmd in LOCAL_COMMANDS: - handler = LOCAL_COMMANDS[git_cmd] - stdout = handler(git_args) - # Git commands - elif git_cmd in SUPPORTED_GIT_COMMANDS: - git_default_args = shlex.split(get_plugin_config('git_default_args')) - returncode, stdout, stderr = git_call( - git_default_args + git_args, capture_output=True - ) - else: - stdout = None - stderr = bytes( - 'Command not supported: git %s' % ' '.join(git_args), 'utf-8' - ) - returncode = 1 - except OSError as ex: - logging.error(ex) - stdout = None - stderr = ex.strerror - returncode = ex.errno - except Exception as ex: - logging.error(ex) - stdout = None - stderr = ex.message - returncode = 1 - finally: - return prepare_git_response(git_cmd, stdout, stderr, returncode) - - -def git_ls_local(git_args: List[str]) -> str: - "'git ls-tree' like output for local folders" - long_ = False - if '-l' in git_args or '--long' in git_args: - git_args = [arg for arg in git_args if arg not in ('-l', '--long')] - long_ = True - path = git_args[1] if len(git_args) > 1 else '' - path = normalize_path(path.split('#', 1)[0]) - dirpath = git_absolute_path(path) - result = [] - for name in sorted(os.listdir(dirpath)): - if name.startswith('.') or name == '__pycache__': - continue - fullname = os.path.join(dirpath, name) - s = os.stat(fullname) - if os.path.isdir(fullname): - type_ = 'tree' - try: - size_: Optional[int] = len(os.listdir(fullname)) - except Exception: - size_ = None - else: - type_ = 'blob' - size_ = s.st_size - relname = os.path.join('/', path, name) # fullname[len(cwd):] - if long_: - mtime = datetime.utcfromtimestamp(s.st_mtime).isoformat()[:16] - result.append( - '%06o %s %s#%s %8s\t%s' - % (s.st_mode, type_, relname, mtime, size_, name) - ) - else: - result.append('%06o %s %s\t%s' % (s.st_mode, type_, relname, name)) - return '\n'.join(result) - - -def git_mounts(git_args: List[str]) -> str: - "List mountpoints" - return '\n'.join(sorted(k for k, v in mount_points.items() if not v.default)) - - -def git_rm_local(git_args: List[str]) -> str: - "Delete local files/directories" - for arg in git_args[1:]: - if arg: - path = git_absolute_path(arg) - if os.path.isdir(path): - os.rmdir(path) - else: - os.unlink(path) - return '' - - -def git_mv_local(git_args: List[str]) -> str: - "Rename/Move local files" - if len(git_args) < 3: - raise Exception('Missing source/destination args') - target = git_absolute_path(git_args[-1]) - for arg in git_args[1:-1]: - source = git_absolute_path(arg) - shutil.move(source, target) - return '' - - -LOCAL_COMMANDS = { - 'mounts': git_mounts, - 'ls-local': git_ls_local, - 'rm-local': git_rm_local, - 'mv-local': git_mv_local, -} - - -def git_call( - argv: List[str], capture_output: bool = False -) -> Tuple[int, GitOutput, GitOutput]: - "Run git command. If capture_output is true, stdout and stderr will be captured." - if not git_enabled(): - return 1, '', 'Git is disabled' - cmd: List[str] = [get_plugin_config('git_cmd')] + argv - cwd: str = get_root_folder() - env: Dict[str, str] = prepare_git_env() - if capture_output: - git = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=cwd, - env=env, - ) - stdout, stderr = git.communicate() - returncode: int = git.returncode - else: - stdout = b'' - stderr = b'' - returncode = subprocess.call(cmd, cwd=cwd, env=env) - return returncode, stdout, stderr - - -def init_git_repo() -> None: - "Initialize the git repository in root folder" - cwd: str = get_root_folder() - if ( - git_enabled() - and not os.path.exists(os.path.join(cwd, '.git')) - and get_plugin_boolean_config('git_init_repo') - ): - git_call(['init', '.']) - gitignore = os.path.join(cwd, '.gitignore') - if not os.path.exists(gitignore): - with open(gitignore, 'w') as f: - f.write('__pycache__\n') - git_call(['add', '.gitignore']) - git_call(['commit', '-m', 'Initial commit']) + ).resolve() MountPoint = namedtuple('MountPoint', 'path default') @@ -347,13 +103,42 @@ def init_git_repo() -> None: def read_mount_points_config() -> Dict[str, MountPoint]: "Return the plugin configuration" - config = {ROOT_MOUNTPOUNT: MountPoint(path=get_root_folder(), default=True)} - i = 0 + path = str(get_root_folder()) + config = {ROOT_MOUNTPOUNT: MountPoint(path=path, default=True)} + # Iterate over the configurations - while True: - suffix = ( - str(i) if i != 0 else '' - ) # the first configuration doesn't have a suffix + for i in itertools.count(): + # the first configuration doesn't have a suffix + if i == 0: + suffix = '' + else: + suffix = str(i) + try: + if not configuration.conf.has_option(PLUGIN_NAME, 'mount{}'.format(suffix)): + break + conf = configuration.conf.get(PLUGIN_NAME, 'mount{}'.format(suffix)) + if conf is None: + break + except Exception: + break + try: + mount_conf = {} + for part in conf.split(','): + k, v = part.split('=') + mount_conf[k] = v + config[mount_conf['name']] = MountPoint( + path=mount_conf['path'], default=False + ) + except Exception: + pass + + # Old configuration format + for i in itertools.count(): + # the first configuration doesn't have a suffix + if i == 0: + suffix = '' + else: + suffix = str(i) try: if not configuration.conf.has_option( PLUGIN_NAME, 'mount{}_name'.format(suffix) @@ -364,13 +149,9 @@ def read_mount_points_config() -> Dict[str, MountPoint]: name = configuration.conf.get(PLUGIN_NAME, 'mount{}_name'.format(suffix)) path = configuration.conf.get(PLUGIN_NAME, 'mount{}_path'.format(suffix)) config[name] = MountPoint(path=path, default=False) - i = i + 1 return config -mount_points = read_mount_points_config() - - def error_message(ex: Exception) -> str: "Get exception error message" if ex is None: diff --git a/changelog.txt b/changelog.txt index 51cc9ce..00ddce6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -371,3 +371,15 @@ - upgrade axios from 0.26.1 to 0.27.2 - replace nose with pytest - Replace test configuration with environment variables + +## 6.0.0 + +2022-07-23 + +### Added + +- add support for remote filesystem (S3, GCP, etc...) in editor/file browser + +### Changed + +- mountpoint configuration changed diff --git a/requirements.txt b/requirements.txt index 84892d6..cd30bda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ apache-airflow +fs diff --git a/scripts/config.sh b/scripts/config.sh index 0a5f290..4fbcc68 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -15,7 +15,5 @@ export AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY="${AIRFLOW_HOME}/logs" # Code Editor config export AIRFLOW__CODE_EDITOR__ENABLED="True" -export AIRFLOW__CODE_EDITOR__MOUNT_NAME="airflow_home" -export AIRFLOW__CODE_EDITOR__MOUNT_PATH="${AIRFLOW_HOME}" -export AIRFLOW__CODE_EDITOR__MOUNT1_NAME="logs" -export AIRFLOW__CODE_EDITOR__MOUNT1_PATH="${AIRFLOW_HOME}/logs" +export AIRFLOW__CODE_EDITOR__MOUNT="name=airflow_home,path=${AIRFLOW_HOME}" +export AIRFLOW__CODE_EDITOR__MOUNT1="name=logs,path=${AIRFLOW_HOME}/logs" diff --git a/setup.py b/setup.py index 341db7a..6fa3cc7 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,12 @@ #!/usr/bin/env python -import os +from pathlib import Path from setuptools import find_packages, setup -HERE = os.path.dirname(__file__) -VERSION_FILE = os.path.join(HERE, 'airflow_code_editor', 'VERSION') - -with open(VERSION_FILE) as f: - version = f.read().strip() - -with open(os.path.join(HERE, "README.md"), "r") as f: - long_description = f.read() - -with open(os.path.join(HERE, "requirements.txt"), "r") as f: - install_requires = f.read().split("\n") +HERE = Path(__file__).parent +version = (HERE / "airflow_code_editor" / "VERSION").read_text().strip() +long_description = (HERE / "README.md").read_text() +install_requires = (HERE / "requirements.txt").read_text().split("\n") setup( name="airflow_code_editor", @@ -34,13 +27,12 @@ long_description_content_type="text/markdown", install_requires=install_requires, license="Apache License, Version 2.0", - python_requires=">=3.4", + python_requires=">=3.5", classifiers=[ "Operating System :: OS Independent", "Programming Language :: Python", "Topic :: Software Development :: Libraries :: Python Modules", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/sh.py b/sh.py new file mode 100644 index 0000000..4dd4b75 --- /dev/null +++ b/sh.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Copyright 2019 Andrea Bonomi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the Licens + +import cmd +import shlex +from airflow_code_editor.fs import RootFS + +class Shell(cmd.Cmd): + intro = 'Type "help" list commands.\n' + root_fs = RootFS() + cwd = root_fs.path("/") + + @property + def prompt(self): + return str(self.cwd) + '$ ' + + def emptyline(self): + pass + + def parseline(self, line): + """Parse the line into a command name and a string containing + the arguments. Returns a tuple containing (command, args, line). + 'command' and 'args' may be None if the line couldn't be parsed. + """ + parts = shlex.split(line) + if parts: + return parts[0], parts[1:], line + else: + return None, None, line + + def do_help(self, args): + 'List available commands with "help" or detailed help with "help cmd".' + super().do_help(args[0] if args else None) + + def do_cd(self, args): + "Change directory" + if args: + cwd = (self.cwd / args[0]).resolve() + else: + cwd = self.root_fs.path("/") + if cwd.exists(): + self.cwd = cwd + else: + print("cd: no such file or directory: {cwd}".format(cwd=cwd)) + + def do_cat(self, args): + "Print file content" + for arg in args: + try: + path = self.cwd / arg + print(path.read_text()) + except Exception: + print("cat: error") + + def do_pwd(self, args): + "Print current directory" + print(self.cwd) + + def do_ls(self, args): + "List directory" + for arg in args or ["."]: + path = self.cwd / arg + if not path.exists(): + print("ls: no such file or directory: {arg}".format(arg=arg)) + elif not path.is_dir(): + print(str(path.name)) + else: + for item in path.iterdir(): + if item.is_dir(): + print(str(item.name) + "/") + else: + print(str(item.name)) + + def do_mount(self, args): + "List mountpoints" + print("{0} on /".format(self.root_fs.default_fs)) + for item in self.root_fs.mounts: + print("{1} on {0}".format(*item)) + + def do_exit(self, args): + "Exit" + return True + +def main(): + try: + Shell().cmdloop() + except KeyboardInterrupt: + pass + +if __name__ == "__main__": + main() diff --git a/tests/test_fs.py b/tests/test_fs.py new file mode 100644 index 0000000..ebade74 --- /dev/null +++ b/tests/test_fs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import fs +from pathlib import Path +from flask import Flask +from airflow_code_editor.commons import PLUGIN_NAME +from airflow_code_editor.fs import split, RootFS +from airflow import configuration + +app = Flask(__name__) + + +def test_split(): + assert split("/aaa/bb/c") == ("/aaa/bb", "c") + assert split("/aaa/bb/") == ("/aaa", "bb") + assert split("/") == ("/", "") + assert split("ciccio") == ("/", "ciccio") + assert split("") == ("/", "") + + +def test_parent(): + root_fs = RootFS() + a = root_fs.path("/aaa/bbb/ccc") + assert a.name == "ccc" + assert a.parent == root_fs.path("/aaa/bbb") + + a = root_fs.path("aaa", "bbb") + assert a.name == "bbb" + assert a.parent == root_fs.path("/aaa") + + +def test_root_fs(): + root_dir = Path(__file__).parent + configuration.conf.set(PLUGIN_NAME, 'git_init_repo', 'False') + configuration.conf.set(PLUGIN_NAME, 'root_directory', str(root_dir)) + configuration.conf.set(PLUGIN_NAME, 'git_enabled', 'True') + + root_fs = RootFS() + name = "/folder/root_fs_test_1" + try: + root_fs.remove(name) + except fs.errors.ResourceNotFound: + pass + root_fs.writetext(name, "data") + assert root_fs.readtext(name) == "data" + root_fs.copy(name, name + ".new") + assert root_fs.readtext(name + ".new") == "data" + root_fs.remove(name) + root_fs.remove(name + ".new") + + name = "/~logs/logs_test_2" + try: + root_fs.remove(name) + except fs.errors.ResourceNotFound: + pass + try: + root_fs.remove(name + ".new") + except fs.errors.ResourceNotFound: + pass + root_fs.writetext(name, "data") + assert root_fs.readtext(name) == "data" + root_fs.copy(name, name + ".new") + assert root_fs.readtext(name + ".new") == "data" + assert len(root_fs.listdir("/~logs/")) + root_fs.remove(name) + root_fs.remove(name + ".new") + + +def test_mem(): + root_fs = RootFS() + root_fs.mount("/~mem", "mem://") + root_fs.path("/~mem/f.txt").write_file("data", is_text=True) + root_fs.path("/~mem/f.bin").write_file(b"data", is_text=False) diff --git a/tests/test_import.py b/tests/test_import.py index 3bd456f..aff5baf 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -3,29 +3,42 @@ import airflow import airflow.plugins_manager + def test_plugin_manager(): assert airflow.plugins_manager + def test_import_auth(): import airflow_code_editor.auth + assert airflow_code_editor.auth + def test_import_commons(): import airflow_code_editor.commons + assert airflow_code_editor.commons + def test_import_flask_admin_view(): import airflow_code_editor.flask_admin_view + assert airflow_code_editor.flask_admin_view + def test_import_app_builder_view(): import airflow_code_editor.app_builder_view + assert airflow_code_editor.app_builder_view + def test_import_code_editor_view(): import airflow_code_editor.code_editor_view + assert airflow_code_editor.code_editor_view + def test_import_airflow_code_editor(): import airflow_code_editor.airflow_code_editor + assert airflow_code_editor.airflow_code_editor diff --git a/tests/test_tree.py b/tests/test_tree.py index c8b2d8e..f29f676 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -import os -import os.path import stat import airflow import airflow.plugins_manager +from pathlib import Path from airflow import configuration from flask import Flask from unittest import TestCase @@ -19,90 +18,85 @@ class TestTree(TestCase): def setUp(self): - self.root_dir = os.path.dirname(os.path.realpath(__file__)) + self.root_dir = Path(__file__).parent configuration.conf.set(PLUGIN_NAME, 'git_init_repo', 'False') - configuration.conf.set(PLUGIN_NAME, 'root_directory', self.root_dir) + configuration.conf.set(PLUGIN_NAME, 'root_directory', str(self.root_dir)) def test_tree(self): with app.app_context(): t = get_tree() - self.assertTrue(len(t) > 0) - self.assertTrue('git' in (x['id'] for x in t)) + assert len(t) > 0 + assert 'git' in (x['id'] for x in t) def test_tags(self): with app.app_context(): t = get_tree("tags") - self.assertIsNotNone(t) + assert t is not None def test_local_branches(self): with app.app_context(): t = get_tree("local-branches") - self.assertIsNotNone(t) + assert t is not None def test_remote_branches(self): with app.app_context(): t = get_tree("remote-branches") - self.assertIsNotNone(t) + assert t is not None def test_files(self): with app.app_context(): t = get_tree("files") - self.assertTrue( - len([x.get('id') for x in t if x.get('id') == 'test_utils.py']) == 1 - ) + assert len([x.get('id') for x in t if x.get('id') == 'test_utils.py']) == 1 t = get_tree("files/folder") - self.assertTrue(len([x.get('id') for x in t if x.get('id') == '1']) == 1) + assert len([x.get('id') for x in t if x.get('id') == '1']) == 1 def test_files_long(self): with app.app_context(): t = get_tree("files", ["long"]) - self.assertEqual( - len([x.get('id') for x in t if x.get('id') == 'folder']), 1 - ) + assert len([x.get('id') for x in t if x.get('id') == 'folder']) == 1 folder = [x for x in t if x.get('id') == 'folder'][0] - self.assertFalse(folder['leaf']) - self.assertEqual(folder['size'], 3) - self.assertTrue(stat.S_ISDIR(folder['mode'])) + assert not folder['leaf'] + assert folder['size'] == 3 + assert stat.S_ISDIR(folder['mode']) self.assertEqual( len([x.get('id') for x in t if x.get('id') == 'test_utils.py']), 1 ) test_utils = [x for x in t if x.get('id') == 'test_utils.py'][0] - self.assertTrue(test_utils['leaf']) - self.assertFalse(stat.S_ISDIR(test_utils['mode'])) + assert test_utils['leaf'] + assert not stat.S_ISDIR(test_utils['mode']) t = get_tree("files/folder", ["long"]) - self.assertEqual(len([x.get('id') for x in t if x.get('id') == '1']), 1) + assert len([x.get('id') for x in t if x.get('id') == '1']) == 1 one = [x for x in t if x.get('id') == '1'][0] - self.assertTrue(one['leaf']) + assert one['leaf'] def test_git(self): with app.app_context(): t = get_tree("git/HEAD") - self.assertTrue(t is not None) + assert t is not None class TestTreeGitDisabled(TestCase): def setUp(self): - self.root_dir = os.path.dirname(os.path.realpath(__file__)) + self.root_dir = Path(__file__).parent configuration.conf.set(PLUGIN_NAME, 'git_init_repo', 'False') - configuration.conf.set(PLUGIN_NAME, 'root_directory', self.root_dir) + configuration.conf.set(PLUGIN_NAME, 'root_directory', str(self.root_dir)) configuration.conf.set(PLUGIN_NAME, 'git_enabled', 'False') def test_tree(self): with app.app_context(): t = get_tree() - self.assertTrue(len(t) > 0) - self.assertTrue('git' not in (x['id'] for x in t)) + assert len(t) > 0 + assert 'git' not in (x['id'] for x in t) t = get_tree("tags") - self.assertEqual(t, []) + assert t == [] t = get_tree("local-branches") - self.assertEqual(t, []) + assert t == [] t = get_tree("remote-branches") - self.assertEqual(t, []) + assert t == [] t = get_tree("files") - self.assertTrue( - len([x.get('id') for x in t if x.get('id') == 'test_utils.py']) == 1 - ) + print(t) + assert len([x.get('id') for x in t if x.get('id') == 'test_utils.py']) == 1 t = get_tree("files/folder") - self.assertTrue(len([x.get('id') for x in t if x.get('id') == '1']) == 1) + assert len([x.get('id') for x in t if x.get('id') == '1']) == 1 diff --git a/tests/test_utils.py b/tests/test_utils.py index 879a00b..14d0607 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -14,9 +14,10 @@ from airflow_code_editor.utils import ( get_plugin_config, get_root_folder, - mount_points, read_mount_points_config, normalize_path, +) +from airflow_code_editor.git import ( execute_git_command, ) @@ -29,102 +30,118 @@ def setUp(self): self.root_dir = os.path.dirname(os.path.realpath(__file__)) configuration.conf.set(PLUGIN_NAME, 'git_init_repo', 'False') configuration.conf.set(PLUGIN_NAME, 'root_directory', self.root_dir) + configuration.conf.set(PLUGIN_NAME, 'git_enabled', 'True') def test_get_root_folder(self): - self.assertIsNotNone(get_root_folder()) + assert get_root_folder() is not None def test_mount_points_config(self): - self.assertTrue('root' in mount_points) - self.assertTrue('airflow_home' in mount_points) - self.assertTrue('logs' in mount_points) + mount_points = read_mount_points_config() + assert 'root' in mount_points + assert 'airflow_home' in mount_points + assert 'logs' in mount_points def test_normalize_path(self): - self.assertEqual(normalize_path(None), '') - self.assertEqual(normalize_path('/'), '') - self.assertEqual(normalize_path('/../'), '') - self.assertEqual(normalize_path('../'), '') - self.assertEqual(normalize_path('../../'), '') - self.assertEqual(normalize_path('../..'), '') - self.assertEqual(normalize_path('/..'), '') - - self.assertEqual(normalize_path('//'), '') - self.assertEqual(normalize_path('////../'), '') - self.assertEqual(normalize_path('..///'), '') - self.assertEqual(normalize_path('..///../'), '') - self.assertEqual(normalize_path('..///..'), '') - self.assertEqual(normalize_path('//..'), '') - - self.assertEqual(normalize_path('/aaa'), 'aaa') - self.assertEqual(normalize_path('/../aaa'), 'aaa') - self.assertEqual(normalize_path('../aaa'), 'aaa') - self.assertEqual(normalize_path('../../aaa'), 'aaa') - self.assertEqual(normalize_path('../../aaa'), 'aaa') - self.assertEqual(normalize_path('/../aaa'), 'aaa') - - self.assertEqual(normalize_path('/aaa'), 'aaa') - self.assertEqual(normalize_path('aaa'), 'aaa') + assert normalize_path(None) == '' + assert normalize_path('/') == '' + assert normalize_path('/../') == '' + assert normalize_path('../') == '' + assert normalize_path('../../') == '' + assert normalize_path('../..') == '' + assert normalize_path('/..') == '' + + assert normalize_path('//') == '' + assert normalize_path('////../') == '' + assert normalize_path('..///') == '' + assert normalize_path('..///../') == '' + assert normalize_path('..///..') == '' + assert normalize_path('//..') == '' + + assert normalize_path('/aaa') == 'aaa' + assert normalize_path('/../aaa') == 'aaa' + assert normalize_path('../aaa') == 'aaa' + assert normalize_path('../../aaa') == 'aaa' + assert normalize_path('../../aaa') == 'aaa' + assert normalize_path('/../aaa') == 'aaa' + + assert normalize_path('/aaa') == 'aaa' + assert normalize_path('aaa') == 'aaa' def test_invalid_command(self): with app.app_context(): r = execute_git_command(['invalid-command']) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertTrue('Command not supported' in t) + assert r.status_code == 200 + assert 'Command not supported' in t def test_ls_tree(self): with app.app_context(): r = execute_git_command(['ls-tree', 'HEAD', '-l']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertIsNotNone(t) + assert r.status_code == 200 + assert t is not None def test_mounts(self): with app.app_context(): r = execute_git_command(['mounts']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertEqual(t, 'airflow_home\nlogs') + assert r.status_code == 200 + assert t == 'airflow_home\nlogs' def test_ls_local_logs(self): with app.app_context(): r = execute_git_command(['ls-local', '-l', '~logs']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertIsNotNone(t) + assert r.status_code == 200 + assert t is not None - def test_ls_local_airflow_hone(self): + def test_ls_local_airflow_home(self): with app.app_context(): r = execute_git_command(['ls-local', '-l', '~airflow_home']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertIsNotNone(t) + assert r.status_code == 200 + assert t is not None + print(t) for line in t.split('\n'): + print('>>>>', line) i = line.split() - self.assertTrue(i[1] in ['tree', 'blob']) - self.assertTrue(i[2].startswith('/~airflow_home/')) + assert i[1] in ['tree', 'blob'] + assert i[2].startswith('/~airflow_home/') def test_ls_local_folder(self): with app.app_context(): r = execute_git_command(['ls-local', '-l', 'folder']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertIsNotNone(t) + assert r.status_code == 200 + assert t is not None for line in t.split('\n'): i = line.split() - self.assertEqual(i[1], 'blob') - self.assertTrue(i[2].startswith('/folder/')) - self.assertEqual(i[3], '2') - self.assertTrue(i[4] in ['1', '2', '3']) + assert i[1] == 'blob' + assert i[2].startswith('/folder/') + assert i[3] == '2' + assert i[4] in ['1', '2', '3'] def test_rm_local(self): try: source = os.path.join(self.root_dir, 'new.file') with open(source, 'w') as f: f.write('test') - self.assertTrue(os.path.exists(source)) + assert os.path.exists(source) with app.app_context(): - execute_git_command(['rm-local', 'new.file']) - self.assertFalse(os.path.exists(source)) + r = execute_git_command(['rm-local', 'new.file']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) + assert not os.path.exists(source) finally: try: os.unlink(source) @@ -135,12 +152,14 @@ def test_mv_local(self): try: source = os.path.join(self.root_dir, 'new.file') target = os.path.join(self.root_dir, 'folder', 'new.file') - self.assertFalse(os.path.exists(target)) + assert not os.path.exists(target) with open(source, 'w') as f: f.write('test') with app.app_context(): - execute_git_command(['mv-local', 'new.file', 'folder']) - self.assertTrue(os.path.exists(target)) + r = execute_git_command(['mv-local', 'new.file', 'folder']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) + assert os.path.exists(target) finally: try: os.unlink(source) @@ -164,9 +183,11 @@ def tearDown(self): def test_ls_tree(self): with app.app_context(): r = execute_git_command(['ls-tree', 'HEAD', '-l']) + if r.headers.get('X-Git-Return-Code') != '0': + raise Exception(r.data.decode('utf-8').split('\n')) t = r.data.decode('utf-8') - self.assertEqual(r.status_code, 200) - self.assertIsNotNone(t) + assert r.status_code == 200 + assert t is not None class TestConfig(TestCase): @@ -196,10 +217,10 @@ def env_vars(self, overrides): os.environ.pop(env) def test_env_config(self): - self.assertEqual(get_plugin_config('git_cmd'), PLUGIN_DEFAULT_CONFIG['git_cmd']) + assert get_plugin_config('git_cmd') == PLUGIN_DEFAULT_CONFIG['git_cmd'] with self.env_vars({'git_cmd': '--test--'}): - self.assertEqual(get_plugin_config('git_cmd'), '--test--') - self.assertEqual(get_plugin_config('git_cmd'), PLUGIN_DEFAULT_CONFIG['git_cmd']) + assert get_plugin_config('git_cmd') == '--test--' + assert get_plugin_config('git_cmd') == PLUGIN_DEFAULT_CONFIG['git_cmd'] # for key in PLUGIN_DEFAULT_CONFIG: # print(configuration.conf._env_var_name(PLUGIN_NAME, key)) @@ -215,6 +236,6 @@ def test_mount_points(self): } ): m = read_mount_points_config() - self.assertEqual(m['test'].path, '/tmp/test') - self.assertEqual(m['test1'].path, '/tmp/test1') - self.assertEqual(m['test2'].path, '/tmp/test2') + assert str(m['test'].path) == '/tmp/test' + assert str(m['test1'].path) == '/tmp/test1' + assert str(m['test2'].path) == '/tmp/test2' From d9205ba64c6ea65de6b0e16002be81daa9fe9f86 Mon Sep 17 00:00:00 2001 From: Andrea Bonomi Date: Fri, 22 Jul 2022 22:16:42 +0000 Subject: [PATCH 2/7] Rename default branch to main --- airflow_code_editor/commons.py | 2 ++ airflow_code_editor/git.py | 9 ++++++++- tests/test_tree.py | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airflow_code_editor/commons.py b/airflow_code_editor/commons.py index a09d6d6..ad25c38 100644 --- a/airflow_code_editor/commons.py +++ b/airflow_code_editor/commons.py @@ -25,6 +25,7 @@ 'ROUTE', 'STATIC', 'CONFIG_SECTION', + 'DEFAULT_GIT_BRANCH', 'SUPPORTED_GIT_COMMANDS', 'HTTP_200_OK', 'HTTP_404_NOT_FOUND', @@ -45,6 +46,7 @@ ROUTE = '/' + PLUGIN_NAME STATIC = '/static/' + PLUGIN_NAME CONFIG_SECTION = PLUGIN_NAME + '_plugin' +DEFAULT_GIT_BRANCH = 'main' HTTP_200_OK = 200 HTTP_404_NOT_FOUND = 404 SUPPORTED_GIT_COMMANDS = [ diff --git a/airflow_code_editor/git.py b/airflow_code_editor/git.py index b15b5b7..77dcd38 100644 --- a/airflow_code_editor/git.py +++ b/airflow_code_editor/git.py @@ -25,6 +25,7 @@ from flask import make_response, Response from flask_login import current_user # type: ignore from airflow_code_editor.commons import ( + DEFAULT_GIT_BRANCH, HTTP_200_OK, HTTP_404_NOT_FOUND, SUPPORTED_GIT_COMMANDS, @@ -205,6 +206,12 @@ def git_call( return returncode, stdout, stderr +def get_default_branch() -> str: + stdout = git_call(['config', '--global', 'init.defaultBranch'], capture_output=True)[1] + default_branch = stdout.decode('utf8').strip('\n') + return default_branch or DEFAULT_GIT_BRANCH + + def init_git_repo() -> None: "Initialize the git repository in root folder" cwd: Path = get_root_folder() @@ -213,7 +220,7 @@ def init_git_repo() -> None: and not (cwd / '.git').exists() and get_plugin_boolean_config('git_init_repo') ): - git_call(['init', '.']) + git_call(['init', '-b', get_default_branch(), '.']) gitignore = cwd / '.gitignore' if not gitignore.exists(): with gitignore.open('w') as f: diff --git a/tests/test_tree.py b/tests/test_tree.py index f29f676..04797b3 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import os import stat import airflow import airflow.plugins_manager @@ -83,6 +84,8 @@ def setUp(self): configuration.conf.set(PLUGIN_NAME, 'git_init_repo', 'False') configuration.conf.set(PLUGIN_NAME, 'root_directory', str(self.root_dir)) configuration.conf.set(PLUGIN_NAME, 'git_enabled', 'False') + os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'git_author_name' + os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'git_author_email' def test_tree(self): with app.app_context(): From 94646cda97c54bc008cd3feb9298e31115f0d6ac Mon Sep 17 00:00:00 2001 From: Andrea Bonomi Date: Sat, 23 Jul 2022 07:57:05 +0000 Subject: [PATCH 3/7] _git_repo_get and _git_repo_post refactoring --- airflow_code_editor/code_editor_view.py | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/airflow_code_editor/code_editor_view.py b/airflow_code_editor/code_editor_view.py index 04f9e8d..ea4331d 100644 --- a/airflow_code_editor/code_editor_view.py +++ b/airflow_code_editor/code_editor_view.py @@ -76,7 +76,24 @@ def _git_repo(self, path): def _git_repo_get(self, path): "Get a file from GIT (invoked by the HTTP GET method)" - return execute_git_command(["cat-file", "-p", path]) + try: + # Download git blob - path = '/' + path, attachment_filename = path.split('/', 1) + except: + # No attachment filename + attachment_filename = None + response = execute_git_command(["cat-file", "-p", path]) + if attachment_filename: + response.headers["Content-Disposition"] = ( + 'attachment; filename="{0}"'.format(attachment_filename) + ) + try: + content_type = mimetypes.guess_type(attachment_filename)[0] + if content_type: + response.headers["Content-Type"] = content_type + except Exception: + pass + return response def _git_repo_post(self, path): "Execute a GIT command (invoked by the HTTP POST method)" @@ -87,23 +104,13 @@ def _load(self, path): "Send the contents of a file to the client" try: path = normalize_path(path) - root_fs = RootFS() if path.startswith("~git/"): # Download git blob - path = '~git//' - _, path, filename = path.split("/", 3) - response = execute_git_command(["cat-file", "-p", path]) - response.headers["Content-Disposition"] = ( - 'attachment; filename="%s"' % filename - ) - try: - content_type = mimetypes.guess_type(filename)[0] - if content_type: - response.headers["Content-Type"] = content_type - except Exception: - pass - return response + _, path = path.split("/", 1) + return self._git_repo_get(path) else: # Download file + root_fs = RootFS() return root_fs.path(path).send_file(as_attachment=True) except Exception as ex: logging.error(ex) From 482c224dbc90d68a0cb213219b8f8b0831d8bf4c Mon Sep 17 00:00:00 2001 From: Andrea Bonomi Date: Sat, 23 Jul 2022 07:58:37 +0000 Subject: [PATCH 4/7] download git objects from /repo/ instead of /files/~git/ --- src/commons.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commons.js b/src/commons.js index 6db6ece..9aca046 100644 --- a/src/commons.js +++ b/src/commons.js @@ -91,7 +91,7 @@ export function TreeEntry(data, isGit, path) { self.icon = getIcon(self.type, self.name); // href if (self.isGit) { // git blob - self.href = prepareHref('files/~git/' + self.object + '/' + self.name); + self.href = prepareHref('repo/' + self.object + '/' + self.name); } else { // local file/dir if (self.type == 'tree') { self.href = '#files' + encodeURI(self.object); @@ -103,7 +103,7 @@ export function TreeEntry(data, isGit, path) { if (self.type == 'tree') { // tree self.downloadHref = '#'; } else if (self.isGit) { // git blob - self.downloadHref = prepareHref('files/~git/' + self.object + '/' + self.name); + self.downloadHref = prepareHref('repo/' + self.object + '/' + self.name); } else { // local file self.downloadHref = prepareHref('files/' + self.object); } From 35434e5a81d4804f29635463b1be01a54e3e9202 Mon Sep 17 00:00:00 2001 From: Andrea Bonomi Date: Sat, 23 Jul 2022 08:01:05 +0000 Subject: [PATCH 5/7] node packages upgrade --- Makefile | 2 +- .../static/airflow_code_editor.js | 2 +- .../static/airflow_code_editor.js.LICENSE.txt | 4 +- changelog.txt | 2 + package-lock.json | 6061 ++++++----------- package.json | 8 +- 6 files changed, 2209 insertions(+), 3870 deletions(-) diff --git a/Makefile b/Makefile index dd851eb..4486878 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ codemirror: @python3 update_themes_js.py npm-build: - @NODE_OPTIONS=--openssl-legacy-provider npm run build + @npm run build npm-watch: @NODE_OPTIONS=--openssl-legacy-provider npm run watch diff --git a/airflow_code_editor/static/airflow_code_editor.js b/airflow_code_editor/static/airflow_code_editor.js index ce9437a..7262529 100644 --- a/airflow_code_editor/static/airflow_code_editor.js +++ b/airflow_code_editor/static/airflow_code_editor.js @@ -1,2 +1,2 @@ /*! For license information please see airflow_code_editor.js.LICENSE.txt */ -!function(){var e={9669:function(e,t,n){e.exports=n(1609)},5448:function(e,t,n){"use strict";var r=n(4867),i=n(6026),o=n(4372),a=n(5327),s=n(4097),l=n(4109),c=n(7985),u=n(7874),d=n(2648),f=n(644),p=n(205);e.exports=function(e){return new Promise((function(t,n){var h,v=e.data,g=e.headers,m=e.responseType;function y(){e.cancelToken&&e.cancelToken.unsubscribe(h),e.signal&&e.signal.removeEventListener("abort",h)}r.isFormData(v)&&r.isStandardBrowserEnv()&&delete g["Content-Type"];var b=new XMLHttpRequest;if(e.auth){var w=e.auth.username||"",_=e.auth.password?unescape(encodeURIComponent(e.auth.password)):"";g.Authorization="Basic "+btoa(w+":"+_)}var x=s(e.baseURL,e.url);function C(){if(b){var r="getAllResponseHeaders"in b?l(b.getAllResponseHeaders()):null,o={data:m&&"text"!==m&&"json"!==m?b.response:b.responseText,status:b.status,statusText:b.statusText,headers:r,config:e,request:b};i((function(e){t(e),y()}),(function(e){n(e),y()}),o),b=null}}if(b.open(e.method.toUpperCase(),a(x,e.params,e.paramsSerializer),!0),b.timeout=e.timeout,"onloadend"in b?b.onloadend=C:b.onreadystatechange=function(){b&&4===b.readyState&&(0!==b.status||b.responseURL&&0===b.responseURL.indexOf("file:"))&&setTimeout(C)},b.onabort=function(){b&&(n(new d("Request aborted",d.ECONNABORTED,e,b)),b=null)},b.onerror=function(){n(new d("Network Error",d.ERR_NETWORK,e,b,b)),b=null},b.ontimeout=function(){var t=e.timeout?"timeout of "+e.timeout+"ms exceeded":"timeout exceeded",r=e.transitional||u;e.timeoutErrorMessage&&(t=e.timeoutErrorMessage),n(new d(t,r.clarifyTimeoutError?d.ETIMEDOUT:d.ECONNABORTED,e,b)),b=null},r.isStandardBrowserEnv()){var S=(e.withCredentials||c(x))&&e.xsrfCookieName?o.read(e.xsrfCookieName):void 0;S&&(g[e.xsrfHeaderName]=S)}"setRequestHeader"in b&&r.forEach(g,(function(e,t){void 0===v&&"content-type"===t.toLowerCase()?delete g[t]:b.setRequestHeader(t,e)})),r.isUndefined(e.withCredentials)||(b.withCredentials=!!e.withCredentials),m&&"json"!==m&&(b.responseType=e.responseType),"function"==typeof e.onDownloadProgress&&b.addEventListener("progress",e.onDownloadProgress),"function"==typeof e.onUploadProgress&&b.upload&&b.upload.addEventListener("progress",e.onUploadProgress),(e.cancelToken||e.signal)&&(h=function(e){b&&(n(!e||e&&e.type?new f:e),b.abort(),b=null)},e.cancelToken&&e.cancelToken.subscribe(h),e.signal&&(e.signal.aborted?h():e.signal.addEventListener("abort",h))),v||(v=null);var T=p(x);T&&-1===["http","https","file"].indexOf(T)?n(new d("Unsupported protocol "+T+":",d.ERR_BAD_REQUEST,e)):b.send(v)}))}},1609:function(e,t,n){"use strict";var r=n(4867),i=n(1849),o=n(321),a=n(7185),s=function e(t){var n=new o(t),s=i(o.prototype.request,n);return r.extend(s,o.prototype,n),r.extend(s,n),s.create=function(n){return e(a(t,n))},s}(n(5546));s.Axios=o,s.CanceledError=n(644),s.CancelToken=n(4972),s.isCancel=n(6502),s.VERSION=n(7288).version,s.toFormData=n(7675),s.AxiosError=n(2648),s.Cancel=s.CanceledError,s.all=function(e){return Promise.all(e)},s.spread=n(8713),s.isAxiosError=n(6268),e.exports=s,e.exports.default=s},4972:function(e,t,n){"use strict";var r=n(644);function i(e){if("function"!=typeof e)throw new TypeError("executor must be a function.");var t;this.promise=new Promise((function(e){t=e}));var n=this;this.promise.then((function(e){if(n._listeners){var t,r=n._listeners.length;for(t=0;t=200&&e<300},headers:{common:{Accept:"application/json, text/plain, */*"}}};r.forEach(["delete","get","head"],(function(e){d.headers[e]={}})),r.forEach(["post","put","patch"],(function(e){d.headers[e]=r.merge(l)})),e.exports=d},7874:function(e){"use strict";e.exports={silentJSONParsing:!0,forcedJSONParsing:!0,clarifyTimeoutError:!1}},7288:function(e){e.exports={version:"0.27.2"}},1849:function(e){"use strict";e.exports=function(e,t){return function(){for(var n=new Array(arguments.length),r=0;r=0)return;a[t]="set-cookie"===t?(a[t]?a[t]:[]).concat([n]):a[t]?a[t]+", "+n:n}})),a):a}},205:function(e){"use strict";e.exports=function(e){var t=/^([-+\w]{1,25})(:?\/\/|:)/.exec(e);return t&&t[1]||""}},8713:function(e){"use strict";e.exports=function(e){return function(t){return e.apply(null,t)}}},7675:function(e,t,n){"use strict";var r=n(4867);e.exports=function(e,t){t=t||new FormData;var n=[];function i(e){return null===e?"":r.isDate(e)?e.toISOString():r.isArrayBuffer(e)||r.isTypedArray(e)?"function"==typeof Blob?new Blob([e]):Buffer.from(e):e}return function e(o,a){if(r.isPlainObject(o)||r.isArray(o)){if(-1!==n.indexOf(o))throw Error("Circular reference detected in "+a);n.push(o),r.forEach(o,(function(n,o){if(!r.isUndefined(n)){var s,l=a?a+"."+o:o;if(n&&!a&&"object"==typeof n)if(r.endsWith(o,"{}"))n=JSON.stringify(n);else if(r.endsWith(o,"[]")&&(s=r.toArray(n)))return void s.forEach((function(e){!r.isUndefined(e)&&t.append(l,i(e))}));e(n,l)}})),n.pop()}else t.append(a,i(o))}(e),t}},4875:function(e,t,n){"use strict";var r=n(7288).version,i=n(2648),o={};["object","boolean","number","function","string","symbol"].forEach((function(e,t){o[e]=function(n){return typeof n===e||"a"+(t<1?"n ":" ")+e}}));var a={};o.transitional=function(e,t,n){function o(e,t){return"[Axios v"+r+"] Transitional option '"+e+"'"+t+(n?". "+n:"")}return function(n,r,s){if(!1===e)throw new i(o(r," has been removed"+(t?" in "+t:"")),i.ERR_DEPRECATED);return t&&!a[r]&&(a[r]=!0,console.warn(o(r," has been deprecated since v"+t+" and will be removed in the near future"))),!e||e(n,r,s)}},e.exports={assertOptions:function(e,t,n){if("object"!=typeof e)throw new i("options must be an object",i.ERR_BAD_OPTION_VALUE);for(var r=Object.keys(e),o=r.length;o-- >0;){var a=r[o],s=t[a];if(s){var l=e[a],c=void 0===l||s(l,a,e);if(!0!==c)throw new i("option "+a+" must be "+c,i.ERR_BAD_OPTION_VALUE)}else if(!0!==n)throw new i("Unknown option "+a,i.ERR_BAD_OPTION)}},validators:o}},4867:function(e,t,n){"use strict";var r,i=n(1849),o=Object.prototype.toString,a=(r=Object.create(null),function(e){var t=o.call(e);return r[t]||(r[t]=t.slice(8,-1).toLowerCase())});function s(e){return e=e.toLowerCase(),function(t){return a(t)===e}}function l(e){return Array.isArray(e)}function c(e){return void 0===e}var u=s("ArrayBuffer");function d(e){return null!==e&&"object"==typeof e}function f(e){if("object"!==a(e))return!1;var t=Object.getPrototypeOf(e);return null===t||t===Object.prototype}var p=s("Date"),h=s("File"),v=s("Blob"),g=s("FileList");function m(e){return"[object Function]"===o.call(e)}var y=s("URLSearchParams");function b(e,t){if(null!=e)if("object"!=typeof e&&(e=[e]),l(e))for(var n=0,r=e.length;n0;)a[o=r[i]]||(t[o]=e[o],a[o]=!0);e=Object.getPrototypeOf(e)}while(e&&(!n||n(e,t))&&e!==Object.prototype);return t},kindOf:a,kindOfTest:s,endsWith:function(e,t,n){e=String(e),(void 0===n||n>e.length)&&(n=e.length),n-=t.length;var r=e.indexOf(t,n);return-1!==r&&r===n},toArray:function(e){if(!e)return null;var t=e.length;if(c(t))return null;for(var n=new Array(t);t-- >0;)n[t]=e[t];return n},isTypedArray:_,isFileList:g}},3099:function(e){e.exports=function(e){if("function"!=typeof e)throw TypeError(String(e)+" is not a function");return e}},6077:function(e,t,n){var r=n(111);e.exports=function(e){if(!r(e)&&null!==e)throw TypeError("Can't set "+String(e)+" as a prototype");return e}},1223:function(e,t,n){var r=n(5112),i=n(30),o=n(3070),a=r("unscopables"),s=Array.prototype;null==s[a]&&o.f(s,a,{configurable:!0,value:i(null)}),e.exports=function(e){s[a][e]=!0}},1530:function(e,t,n){"use strict";var r=n(8710).charAt;e.exports=function(e,t,n){return t+(n?r(e,t).length:1)}},5787:function(e){e.exports=function(e,t,n){if(!(e instanceof t))throw TypeError("Incorrect "+(n?n+" ":"")+"invocation");return e}},9670:function(e,t,n){var r=n(111);e.exports=function(e){if(!r(e))throw TypeError(String(e)+" is not an object");return e}},8533:function(e,t,n){"use strict";var r=n(2092).forEach,i=n(9341)("forEach");e.exports=i?[].forEach:function(e){return r(this,e,arguments.length>1?arguments[1]:void 0)}},8457:function(e,t,n){"use strict";var r=n(9974),i=n(7908),o=n(3411),a=n(7659),s=n(7466),l=n(6135),c=n(1246);e.exports=function(e){var t,n,u,d,f,p,h=i(e),v="function"==typeof this?this:Array,g=arguments.length,m=g>1?arguments[1]:void 0,y=void 0!==m,b=c(h),w=0;if(y&&(m=r(m,g>2?arguments[2]:void 0,2)),null==b||v==Array&&a(b))for(n=new v(t=s(h.length));t>w;w++)p=y?m(h[w],w):h[w],l(n,w,p);else for(f=(d=b.call(h)).next,n=new v;!(u=f.call(d)).done;w++)p=y?o(d,m,[u.value,w],!0):u.value,l(n,w,p);return n.length=w,n}},1318:function(e,t,n){var r=n(5656),i=n(7466),o=n(1400),a=function(e){return function(t,n,a){var s,l=r(t),c=i(l.length),u=o(a,c);if(e&&n!=n){for(;c>u;)if((s=l[u++])!=s)return!0}else for(;c>u;u++)if((e||u in l)&&l[u]===n)return e||u||0;return!e&&-1}};e.exports={includes:a(!0),indexOf:a(!1)}},2092:function(e,t,n){var r=n(9974),i=n(8361),o=n(7908),a=n(7466),s=n(5417),l=[].push,c=function(e){var t=1==e,n=2==e,c=3==e,u=4==e,d=6==e,f=7==e,p=5==e||d;return function(h,v,g,m){for(var y,b,w=o(h),_=i(w),x=r(v,g,3),C=a(_.length),S=0,T=m||s,k=t?T(h,C):n||f?T(h,0):void 0;C>S;S++)if((p||S in _)&&(b=x(y=_[S],S,w),e))if(t)k[S]=b;else if(b)switch(e){case 3:return!0;case 5:return y;case 6:return S;case 2:l.call(k,y)}else switch(e){case 4:return!1;case 7:l.call(k,y)}return d?-1:c||u?u:k}};e.exports={forEach:c(0),map:c(1),filter:c(2),some:c(3),every:c(4),find:c(5),findIndex:c(6),filterOut:c(7)}},1194:function(e,t,n){var r=n(7293),i=n(5112),o=n(7392),a=i("species");e.exports=function(e){return o>=51||!r((function(){var t=[];return(t.constructor={})[a]=function(){return{foo:1}},1!==t[e](Boolean).foo}))}},9341:function(e,t,n){"use strict";var r=n(7293);e.exports=function(e,t){var n=[][e];return!!n&&r((function(){n.call(null,t||function(){throw 1},1)}))}},4362:function(e){var t=Math.floor,n=function(e,o){var a=e.length,s=t(a/2);return a<8?r(e,o):i(n(e.slice(0,s),o),n(e.slice(s),o),o)},r=function(e,t){for(var n,r,i=e.length,o=1;o0;)e[r]=e[--r];r!==o++&&(e[r]=n)}return e},i=function(e,t,n){for(var r=e.length,i=t.length,o=0,a=0,s=[];o=74)&&(r=a.match(/Chrome\/(\d+)/))&&(i=r[1]),e.exports=i&&+i},8008:function(e,t,n){var r=n(8113).match(/AppleWebKit\/(\d+)\./);e.exports=!!r&&+r[1]},748:function(e){e.exports=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"]},2109:function(e,t,n){var r=n(7854),i=n(1236).f,o=n(8880),a=n(1320),s=n(3505),l=n(9920),c=n(4705);e.exports=function(e,t){var n,u,d,f,p,h=e.target,v=e.global,g=e.stat;if(n=v?r:g?r[h]||s(h,{}):(r[h]||{}).prototype)for(u in t){if(f=t[u],d=e.noTargetGet?(p=i(n,u))&&p.value:n[u],!c(v?u:h+(g?".":"#")+u,e.forced)&&void 0!==d){if(typeof f==typeof d)continue;l(f,d)}(e.sham||d&&d.sham)&&o(f,"sham",!0),a(n,u,f,e)}}},7293:function(e){e.exports=function(e){try{return!!e()}catch(e){return!0}}},7007:function(e,t,n){"use strict";n(4916);var r=n(1320),i=n(2261),o=n(7293),a=n(5112),s=n(8880),l=a("species"),c=RegExp.prototype;e.exports=function(e,t,n,u){var d=a(e),f=!o((function(){var t={};return t[d]=function(){return 7},7!=""[e](t)})),p=f&&!o((function(){var t=!1,n=/a/;return"split"===e&&((n={}).constructor={},n.constructor[l]=function(){return n},n.flags="",n[d]=/./[d]),n.exec=function(){return t=!0,null},n[d](""),!t}));if(!f||!p||n){var h=/./[d],v=t(d,""[e],(function(e,t,n,r,o){var a=t.exec;return a===i||a===c.exec?f&&!o?{done:!0,value:h.call(t,n,r)}:{done:!0,value:e.call(n,t,r)}:{done:!1}}));r(String.prototype,e,v[0]),r(c,d,v[1])}u&&s(c[d],"sham",!0)}},9974:function(e,t,n){var r=n(3099);e.exports=function(e,t,n){if(r(e),void 0===t)return e;switch(n){case 0:return function(){return e.call(t)};case 1:return function(n){return e.call(t,n)};case 2:return function(n,r){return e.call(t,n,r)};case 3:return function(n,r,i){return e.call(t,n,r,i)}}return function(){return e.apply(t,arguments)}}},5005:function(e,t,n){var r=n(857),i=n(7854),o=function(e){return"function"==typeof e?e:void 0};e.exports=function(e,t){return arguments.length<2?o(r[e])||o(i[e]):r[e]&&r[e][t]||i[e]&&i[e][t]}},1246:function(e,t,n){var r=n(648),i=n(7497),o=n(5112)("iterator");e.exports=function(e){if(null!=e)return e[o]||e["@@iterator"]||i[r(e)]}},647:function(e,t,n){var r=n(7908),i=Math.floor,o="".replace,a=/\$([$&'`]|\d{1,2}|<[^>]*>)/g,s=/\$([$&'`]|\d{1,2})/g;e.exports=function(e,t,n,l,c,u){var d=n+e.length,f=l.length,p=s;return void 0!==c&&(c=r(c),p=a),o.call(u,p,(function(r,o){var a;switch(o.charAt(0)){case"$":return"$";case"&":return e;case"`":return t.slice(0,n);case"'":return t.slice(d);case"<":a=c[o.slice(1,-1)];break;default:var s=+o;if(0===s)return r;if(s>f){var u=i(s/10);return 0===u?r:u<=f?void 0===l[u-1]?o.charAt(1):l[u-1]+o.charAt(1):r}a=l[s-1]}return void 0===a?"":a}))}},7854:function(e,t,n){var r=function(e){return e&&e.Math==Math&&e};e.exports=r("object"==typeof globalThis&&globalThis)||r("object"==typeof window&&window)||r("object"==typeof self&&self)||r("object"==typeof n.g&&n.g)||function(){return this}()||Function("return this")()},6656:function(e,t,n){var r=n(7908),i={}.hasOwnProperty;e.exports=Object.hasOwn||function(e,t){return i.call(r(e),t)}},3501:function(e){e.exports={}},842:function(e,t,n){var r=n(7854);e.exports=function(e,t){var n=r.console;n&&n.error&&(1===arguments.length?n.error(e):n.error(e,t))}},490:function(e,t,n){var r=n(5005);e.exports=r("document","documentElement")},4664:function(e,t,n){var r=n(9781),i=n(7293),o=n(317);e.exports=!r&&!i((function(){return 7!=Object.defineProperty(o("div"),"a",{get:function(){return 7}}).a}))},8361:function(e,t,n){var r=n(7293),i=n(4326),o="".split;e.exports=r((function(){return!Object("z").propertyIsEnumerable(0)}))?function(e){return"String"==i(e)?o.call(e,""):Object(e)}:Object},2788:function(e,t,n){var r=n(5465),i=Function.toString;"function"!=typeof r.inspectSource&&(r.inspectSource=function(e){return i.call(e)}),e.exports=r.inspectSource},9909:function(e,t,n){var r,i,o,a=n(8536),s=n(7854),l=n(111),c=n(8880),u=n(6656),d=n(5465),f=n(6200),p=n(3501),h="Object already initialized",v=s.WeakMap;if(a||d.state){var g=d.state||(d.state=new v),m=g.get,y=g.has,b=g.set;r=function(e,t){if(y.call(g,e))throw new TypeError(h);return t.facade=e,b.call(g,e,t),t},i=function(e){return m.call(g,e)||{}},o=function(e){return y.call(g,e)}}else{var w=f("state");p[w]=!0,r=function(e,t){if(u(e,w))throw new TypeError(h);return t.facade=e,c(e,w,t),t},i=function(e){return u(e,w)?e[w]:{}},o=function(e){return u(e,w)}}e.exports={set:r,get:i,has:o,enforce:function(e){return o(e)?i(e):r(e,{})},getterFor:function(e){return function(t){var n;if(!l(t)||(n=i(t)).type!==e)throw TypeError("Incompatible receiver, "+e+" required");return n}}}},7659:function(e,t,n){var r=n(5112),i=n(7497),o=r("iterator"),a=Array.prototype;e.exports=function(e){return void 0!==e&&(i.Array===e||a[o]===e)}},3157:function(e,t,n){var r=n(4326);e.exports=Array.isArray||function(e){return"Array"==r(e)}},4705:function(e,t,n){var r=n(7293),i=/#|\.prototype\./,o=function(e,t){var n=s[a(e)];return n==c||n!=l&&("function"==typeof t?r(t):!!t)},a=o.normalize=function(e){return String(e).replace(i,".").toLowerCase()},s=o.data={},l=o.NATIVE="N",c=o.POLYFILL="P";e.exports=o},111:function(e){e.exports=function(e){return"object"==typeof e?null!==e:"function"==typeof e}},1913:function(e){e.exports=!1},7850:function(e,t,n){var r=n(111),i=n(4326),o=n(5112)("match");e.exports=function(e){var t;return r(e)&&(void 0!==(t=e[o])?!!t:"RegExp"==i(e))}},408:function(e,t,n){var r=n(9670),i=n(7659),o=n(7466),a=n(9974),s=n(1246),l=n(9212),c=function(e,t){this.stopped=e,this.result=t};e.exports=function(e,t,n){var u,d,f,p,h,v,g,m=n&&n.that,y=!(!n||!n.AS_ENTRIES),b=!(!n||!n.IS_ITERATOR),w=!(!n||!n.INTERRUPTED),_=a(t,m,1+y+w),x=function(e){return u&&l(u),new c(!0,e)},C=function(e){return y?(r(e),w?_(e[0],e[1],x):_(e[0],e[1])):w?_(e,x):_(e)};if(b)u=e;else{if("function"!=typeof(d=s(e)))throw TypeError("Target is not iterable");if(i(d)){for(f=0,p=o(e.length);p>f;f++)if((h=C(e[f]))&&h instanceof c)return h;return new c(!1)}u=d.call(e)}for(v=u.next;!(g=v.call(u)).done;){try{h=C(g.value)}catch(e){throw l(u),e}if("object"==typeof h&&h&&h instanceof c)return h}return new c(!1)}},9212:function(e,t,n){var r=n(9670);e.exports=function(e){var t=e.return;if(void 0!==t)return r(t.call(e)).value}},3383:function(e,t,n){"use strict";var r,i,o,a=n(7293),s=n(9518),l=n(8880),c=n(6656),u=n(5112),d=n(1913),f=u("iterator"),p=!1;[].keys&&("next"in(o=[].keys())?(i=s(s(o)))!==Object.prototype&&(r=i):p=!0);var h=null==r||a((function(){var e={};return r[f].call(e)!==e}));h&&(r={}),d&&!h||c(r,f)||l(r,f,(function(){return this})),e.exports={IteratorPrototype:r,BUGGY_SAFARI_ITERATORS:p}},7497:function(e){e.exports={}},5948:function(e,t,n){var r,i,o,a,s,l,c,u,d=n(7854),f=n(1236).f,p=n(261).set,h=n(6833),v=n(1036),g=n(5268),m=d.MutationObserver||d.WebKitMutationObserver,y=d.document,b=d.process,w=d.Promise,_=f(d,"queueMicrotask"),x=_&&_.value;x||(r=function(){var e,t;for(g&&(e=b.domain)&&e.exit();i;){t=i.fn,i=i.next;try{t()}catch(e){throw i?a():o=void 0,e}}o=void 0,e&&e.enter()},h||g||v||!m||!y?w&&w.resolve?((c=w.resolve(void 0)).constructor=w,u=c.then,a=function(){u.call(c,r)}):a=g?function(){b.nextTick(r)}:function(){p.call(d,r)}:(s=!0,l=y.createTextNode(""),new m(r).observe(l,{characterData:!0}),a=function(){l.data=s=!s})),e.exports=x||function(e){var t={fn:e,next:void 0};o&&(o.next=t),i||(i=t,a()),o=t}},3366:function(e,t,n){var r=n(7854);e.exports=r.Promise},133:function(e,t,n){var r=n(7392),i=n(7293);e.exports=!!Object.getOwnPropertySymbols&&!i((function(){var e=Symbol();return!String(e)||!(Object(e)instanceof Symbol)||!Symbol.sham&&r&&r<41}))},8536:function(e,t,n){var r=n(7854),i=n(2788),o=r.WeakMap;e.exports="function"==typeof o&&/native code/.test(i(o))},8523:function(e,t,n){"use strict";var r=n(3099),i=function(e){var t,n;this.promise=new e((function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r})),this.resolve=r(t),this.reject=r(n)};e.exports.f=function(e){return new i(e)}},3929:function(e,t,n){var r=n(7850);e.exports=function(e){if(r(e))throw TypeError("The method doesn't accept regular expressions");return e}},30:function(e,t,n){var r,i=n(9670),o=n(6048),a=n(748),s=n(3501),l=n(490),c=n(317),u=n(6200)("IE_PROTO"),d=function(){},f=function(e){return"