Merge pull request #142 from zizzic/develop

Merge to main branch

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @zizzic/programmers

.github/ISSUE_TEMPLATE/bug_issue_template.md

@@ -0,0 +1,16 @@
+---
+name: 버그 관련 이슈 템플릿
+about: 버그와 관련된 이슈
+title: "[BUG] XXXXX"
+labels: bug
+assignees: ''
+
+---
+
+<!-- 버그 관련 이슈 제목은 [BUG]를 붙여 생성하는 것을 추천드립니다.-->
+### Description
+<!--
+ 작업이 필요한 내용을 적어주세요.
+ ex.)
+ - Spark 실행을 위한 GlueJobOperator dag 수정
+-->

.github/ISSUE_TEMPLATE/feature_issue_template.md

@@ -0,0 +1,16 @@
+---
+name: 새로운 기능 이슈 템플릿
+about: 새로운 기능 이슈와 관련된 템플릿
+title: "[FEAT] XXXXXXX"
+labels: enhancement
+assignees: ''
+
+---
+
+<!-- 새로운 기능 관련 이슈 제목은 [FEAT]를 붙여 생성하는 것을 추천드립니다.-->
+### Description
+<!--
+ 작업이 필요한 내용을 적어주세요.
+ ex.)
+ - Spark 실행을 위한 GlueJobOperator dag 작성
+-->

.github/pull_request_template.md

@@ -0,0 +1,17 @@
+<!-- 제목 : [#{issue number}] {이슈 제목}-->
+### 작업 분류
+
+- [x] 신규 기능
+- [ ] 기능 수정
+- [ ] 버그 수정
+- [ ] 프로젝트 구조 변경
+
+### 작업 상세 내용
+<!--  작업한 내용을 적어주세요. -->
+
+
+### 생각해볼 문제
+<!-- 생략 가능 -->
+
+
+resolve #{issue number}

.github/workflows/sync_dag_code.yml

@@ -0,0 +1,94 @@
+name: CI/CD Workflow
+
+on:
+    push:
+        branches: develop
+    pull_request:
+        branches: develop
+
+permissions:
+    contents: read
+    pull-requests: read
+
+jobs:
+    path-filter:
+        runs-on: ubuntu-latest
+        outputs:
+            dags-changed: ${{ steps.filter.outputs.dags }}
+            docker-changed: ${{ steps.filter.outputs.docker }}
+            plugin-created: ${{ steps.filter.outputs.plugin}}
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Path Filter
+              id: filter
+              uses: dorny/paths-filter@v3
+              with:
+                  filters: |
+                      dags:
+                        - 'dags/**/*.py'
+                        - 'tests/dags/**/*.py' 
+                      docker:
+                        - 'docker-compose.yaml'
+                        - 'Dockerfile'
+                      plugin:
+                        - 'plugins/***'
+
+    build:
+        needs: path-filter
+        if: (needs.path-filter.outputs.dags-changed == 'true'|| needs.path-filter.outputs.plugin-created == 'true') && github.event_name == 'pull_request'
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Setup Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.11"
+
+            - name: Set Environment Variables
+              run: |
+                  echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
+                  echo "AIRFLOW_VERSION=2.7.2" >> $GITHUB_ENV
+
+            - name: Install dependencies
+              run: |
+                  CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
+                  echo "Installing Apache Airflow version ${AIRFLOW_VERSION} with constraints from ${CONSTRAINT_URL}"
+                  pip install "apache-airflow[amazon,mysql]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
+                  pip install pytest
+
+            - name: Install Pylint
+              run: pip install pylint
+            - name: Run Pylint
+              env:
+                  PYTHONPATH: plugins
+              run: pylint --output-format=colorized --disable=missing-docstring,invalid-name,W,C0301,C0411 $(find dags/ tests/dags/ plugins/ -name "*.py") || true
+            - name: Test DAG integrity
+              run: pytest tests/
+
+    deploy:
+        needs: path-filter
+        if: github.event_name == 'push'
+        runs-on: self-hosted
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v4
+
+            - name: Extract repository name
+              run: echo "REPO_NAME=$(echo $GITHUB_REPOSITORY | cut -d '/' -f 2)" >> $GITHUB_ENV
+
+            - name: Sync specific files
+              env:
+                  DEPLOY_DIRECTORY: /home/zizzicteam2/${{ env.REPO_NAME }}
+              run: |
+                  rsync -av --include='dags/***' --include='plugins/***' --include='docker-compose.yaml' --include='Dockerfile' --exclude='*' ./ $DEPLOY_DIRECTORY
+            - name: Restart Docker Services
+              if: ${{ needs.path-filter.outputs.docker-changed == 'true' || needs.path-filter.outputs.plugin-created == 'true'}}
+              env:
+                  DEPLOY_DIRECTORY: /home/zizzicteam2/${{ env.REPO_NAME }}
+              run: |
+                  cd $DEPLOY_DIRECTORY
+
+                  docker compose down
+                  docker compose up -d

.gitignore

@@ -0,0 +1,181 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+# airflow
+logs/
+
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+airflowvenv/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+    - repo: https://github.com/psf/black
+      rev: 24.2.0
+      hooks:
+          - id: black
+            language_version: python3.11
+            # Black doesn't use file filters for inclusion, it formats all Python files by default.
+
+    - repo: https://github.com/PyCQA/flake8
+      rev: "7.0.0"
+      hooks:
+          - id: flake8
+            # Use a regex pattern to include dags and tests/dags directories
+            files: ^(dags|tests/dags)/.*\.py$
+            # Add errors that would be ignored
+            args: ["--ignore=E501,W503"]

Dockerfile

@@ -0,0 +1,13 @@
+# 기본 이미지 지정. 버전은 2.7.2
+FROM apache/airflow:2.7.2-python3.11
+
+# root 사용자로 전환 (필요한 경우)
+USER root
+
+# 패키지 목록 업데이트 및 필요한 패키지 설치
+RUN apt-get update && \
+    apt-get install -y \
+    vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# 이후 필요한 작업 수행. 예를 들어, 환경 설정 변경, 파일 복사 등