From d2249ea78d8c4763db0561450e99f164109f464b Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 2 Sep 2024 12:09:57 +0200 Subject: [PATCH] Refactoring/13 documentation and formatting (#30) Release notes * method documentation * module documentation * class documentation * implementing Black format tool * Black tool check into CI * all python code formatted * README.md updated with using new tools --- .../workflows/static_analysis_and_tests.yml | 24 +- .pylintrc | 8 +- README.md | 72 +++- .../action_inputs.py | 55 +++- living_documentation_generator/generator.py | 309 ++++++++++++------ .../github_projects.py | 109 +++--- .../model/config_repository.py | 42 ++- .../model/consolidated_issue.py | 57 +++- .../model/github_project.py | 61 +++- .../model/project_issue.py | 56 +++- .../model/project_status.py | 30 ++ .../utils/constants.py | 133 +++++--- .../utils/decorators.py | 34 +- .../utils/github_project_queries.py | 46 ++- .../utils/github_rate_limiter.py | 52 ++- .../utils/logging_config.py | 37 ++- living_documentation_generator/utils/utils.py | 85 +++-- main.py | 35 +- pyproject.toml | 3 + requirements.txt | 1 + tests/test_hello.py | 5 +- 21 files changed, 972 insertions(+), 282 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 938d96c..9999f33 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -38,9 +38,31 @@ jobs: echo "Success: Pylint score is above 9.5 (project score: $PYLINT_SCORE)." fi + code-format-check: + runs-on: ubuntu-latest + name: Black Format Check + steps: + - name: Checkout repository + uses: actions/checkout@v4.1.5 + + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Check code format with Black + id: check-format + run: | + black --check $(git ls-files '*.py') + python-tests: runs-on: ubuntu-latest - name: Python tests + name: Python Tests steps: - name: Checkout repository uses: actions/checkout@v4.1.5 diff --git a/.pylintrc b/.pylintrc index 131415b..3a587bb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -179,7 +179,7 @@ const-naming-style=UPPER_CASE # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. -docstring-min-length=-1 +docstring-min-length=5 # Naming style matching correct function names. function-naming-style=snake_case @@ -426,11 +426,7 @@ confidence=HIGH, # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -# TODO: Remove after this PR -disable=C0116, - C0115, - C0114, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, diff --git a/README.md b/README.md index 663fc77..c8751af 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ - [Project Setup](#project-setup) - [Run Scripts Locally](#run-scripts-locally) - [Run Pylint Check Locally](#run-pylint-check-locally) +- [Run Black Tool Locally](#run-black-tool-locally) - [Run unit test](#run-unit-test) - [Deployment](#deployment) - [Features](#features) @@ -150,7 +151,7 @@ Configure the action by customizing the following parameters based on your needs ]' ``` -### Features de/activation +### Features De/Activation - **project-state-mining** (optional, `default: false`) - **Description**: Enables or disables the mining of project state data from [GitHub Projects](https://docs.github.com/en/issues/planning-and-tracking-with-projects/learning-about-projects/about-projects). - **Usage**: Set to true to activate. @@ -187,7 +188,7 @@ The Living Documentation Generator action provides a key output that allows user ## Expected Output The Living Documentation Generator is designed to produce an Issue Summary page (index.md) along with multiple detailed single issue pages. -### Index page example +### Index Page Example ```markdown # Issue Summary page @@ -209,7 +210,7 @@ Here, you'll find a summarized list of all these issues, their brief description These values can vary from project to project. - The `---` symbol is used to indicate that an issue has no required project data. -### Issue page example +### Issue Page Example ```markdown # FEAT: Advanced Book Search @@ -255,7 +256,7 @@ pip install -r requirements.txt ## Run Scripts Locally If you need to run the scripts locally, follow these steps: -### Create the shell script +### Create the Shell Script Create the shell file in the root directory. We will use `run_script.sh`. ```shell touch run_script.sh @@ -265,7 +266,7 @@ Add the shebang line at the top of the sh script file. #!/bin/sh ``` -### Set the environment variables +### Set the Environment Variables Set the configuration environment variables in the shell script following the structure below. Also make sure that the GITHUB_TOKEN is configured in your environment variables. ``` @@ -305,7 +306,7 @@ chmod +x run_script.sh ``` ## Run Pylint Check Locally -This project uses Pylint tool for static code analysis. +This project uses [Pylint](https://pypi.org/project/pylint/) tool for static code analysis. Pylint analyses your code without actually running it. It checks for errors, enforces, coding standards, looks for code smells etc. @@ -338,20 +339,71 @@ Example: pylint living_documentation_generator/generator.py ``` -## Run unit test +### Expected Output +This is the console expected output example after running the tool: +``` +************* Module main +main.py:30:0: C0116: Missing function or method docstring (missing-function-docstring) + +------------------------------------------------------------------ +Your code has been rated at 9.41/10 (previous run: 8.82/10, +0.59) +``` + +## Run Black Tool Locally +This project uses the [Black](https://github.com/psf/black) tool for code formatting. +Black aims for consistency, generality, readability and reducing git diffs. +The coding style used can be viewed as a strict subset of PEP 8. + +The project root file `pyproject.toml` defines the Black tool configuration. +In this project we are accepting the line length of 120 characters. + +Follow these steps to format your code with Black locally: + +### Set Up Python Environment +From terminal in the root of the project, run the following command: + +```shell +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +This command will also install a Black tool, since it is listed in the project requirements. + +### Run Black +Run Black on all files that are currently tracked by Git in the project. +```shell +black $(git ls-files '*.py') +``` + +To run Black on a specific file, follow the pattern `black /.py`. + +Example: +```shell +black living_documentation_generator/generator.py +``` + +### Expected Output +This is the console expected output example after running the tool: +``` +All done! ✨ 🍰 ✨ +1 file reformatted. +``` + +## Run Unit Test TODO - check this chapter and update by latest state -### Launch unit tests +### Launch Unit Tests ``` pytest ``` -### To run specific tests or get verbose output: +### To Run Specific Tests or Get Verbose Output: ``` pytest -v # Verbose mode pytest path/to/test_file.py # Run specific test file ``` -### To check Test Coverage: +### To Check Test Coverage: ``` pytest --cov=../scripts ``` diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 82c1ae4..9634bc0 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -1,3 +1,24 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains an Action Inputs class methods, +which are essential for running the GH action. +""" + import json import logging import sys @@ -10,6 +31,10 @@ class ActionInputs: + """ + A class representing all the action inputs. It is responsible for loading, managing + and validating the inputs required for running the GH Action. + """ def __init__(self): self.__github_token: str = "" @@ -19,31 +44,40 @@ def __init__(self): @property def github_token(self) -> str: + """Getter of the GitHub authorization token.""" return self.__github_token @property def is_project_state_mining_enabled(self) -> bool: + """Getter of the project state mining switch.""" return self.__is_project_state_mining_enabled @property def repositories(self) -> list[ConfigRepository]: + """Getter of the list of repositories to fetch from.""" return self.__repositories @property def output_directory(self) -> str: + """Getter of the output directory.""" return self.__output_directory - def load_from_environment(self, validate: bool = True) -> 'ActionInputs': + def load_from_environment(self, validate: bool = True) -> "ActionInputs": + """ + Load the action inputs from the environment variables and validate them if needed. + + @param validate: Switch indicating if the inputs should be validated. + @return: The instance of the ActionInputs class. + """ self.__github_token = get_action_input(GITHUB_TOKEN) - self.__is_project_state_mining_enabled = (get_action_input(PROJECT_STATE_MINING, "false") - .lower() == "true") - out_path = get_action_input(OUTPUT_PATH, './output') + self.__is_project_state_mining_enabled = get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" + out_path = get_action_input(OUTPUT_PATH, "./output") self.__output_directory = make_absolute_path(out_path) repositories_json = get_action_input(REPOSITORIES, "") - logger.debug('Is project state mining allowed: %s.', self.is_project_state_mining_enabled) - logger.debug('JSON repositories to fetch from: %s.', repositories_json) - logger.debug('Output directory: %s.', self.output_directory) + logger.debug("Is project state mining allowed: %s.", self.is_project_state_mining_enabled) + logger.debug("JSON repositories to fetch from: %s.", repositories_json) + logger.debug("Output directory: %s.", self.output_directory) # Validate inputs if validate: @@ -64,6 +98,13 @@ def load_from_environment(self, validate: bool = True) -> 'ActionInputs': return self def validate_inputs(self, repositories_json: str) -> None: + """ + Validate the input attributes of the action. + + @param repositories_json: The JSON string containing the repositories to fetch. + @return: None + """ + # Validate correct format of input repositories_json try: json.loads(repositories_json) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index bc21857..82924ae 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -1,39 +1,77 @@ +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains the LivingDocumentationGenerator class which is responsible for generating +the Living Documentation output. +""" + import logging import os import shutil from datetime import datetime +from typing import Callable from github import Github, Auth from github.Issue import Issue from living_documentation_generator.github_projects import GithubProjects +from living_documentation_generator.model.github_project import GithubProject from living_documentation_generator.model.config_repository import ConfigRepository from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue from living_documentation_generator.model.project_issue import ProjectIssue from living_documentation_generator.utils.decorators import safe_call_decorator from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter from living_documentation_generator.utils.utils import make_issue_key -from living_documentation_generator.utils.constants import (ISSUES_PER_PAGE_LIMIT, ISSUE_STATE_ALL, - LINKED_TO_PROJECT_TRUE, LINKED_TO_PROJECT_FALSE) +from living_documentation_generator.utils.constants import ( + ISSUES_PER_PAGE_LIMIT, + ISSUE_STATE_ALL, + LINKED_TO_PROJECT_TRUE, + LINKED_TO_PROJECT_FALSE, + TABLE_HEADER_WITH_PROJECT_DATA, + TABLE_HEADER_WITHOUT_PROJECT_DATA, +) logger = logging.getLogger(__name__) class LivingDocumentationGenerator: + """ + A class representing the Living Documentation Generator. + The class uses several helper methods to fetch required data from GitHub, consolidate the data + and generate the markdown pages as the output of Living Documentation action. + """ PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) - ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, "..", "templates", "issue_detail_page_template.md") - INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, "..", "templates", "_index_page_template.md") + ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "issue_detail_page_template.md") + INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_template.md") - def __init__(self, github_token: str, repositories: list[ConfigRepository], project_state_mining_enabled: bool, - output_path: str): + def __init__( + self, + github_token: str, + repositories: list[ConfigRepository], + project_state_mining_enabled: bool, + output_path: str, + ): - self.github_instance = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) - self.github_projects_instance = GithubProjects(token=github_token) - self.rate_limiter = GithubRateLimiter(self.github_instance) - self.safe_call = safe_call_decorator(self.rate_limiter) + self.__github_instance: Github = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) + self.__github_projects_instance: GithubProjects = GithubProjects(token=github_token) + self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) + self.__safe_call: Callable = safe_call_decorator(self.__rate_limiter) # data self.__repositories: list[ConfigRepository] = repositories @@ -44,19 +82,32 @@ def __init__(self, github_token: str, repositories: list[ConfigRepository], proj # paths self.__output_path: str = output_path + @property + def github_instance(self) -> Github: + """Getter of the GitHub instance.""" + return self.__github_instance + @property def repositories(self) -> list[ConfigRepository]: + """Getter of the list of config repository objects to fetch from.""" return self.__repositories @property def project_state_mining_enabled(self) -> bool: + """Getter of the project state mining switch.""" return self.__project_state_mining_enabled @property def output_path(self) -> str: + """Getter of the output directory.""" return self.__output_path - def generate(self): + def generate(self) -> None: + """ + Generate the Living Documentation markdown pages output. + + @return: None + """ self._clean_output_directory() logger.debug("Output directory cleaned.") @@ -72,59 +123,87 @@ def generate(self): # Note: got dict of project issues with unique string key defying the issue logger.info("Fetching GitHub project data - finished.") - # Consolidate all issues data together + # Consolidate all issue data together logger.info("Issue and project data consolidation - started.") - projects_issues = self._consolidate_issues_data(repository_issues, project_issues) + consolidated_issues = self._consolidate_issues_data(repository_issues, project_issues) logger.info("Issue and project data consolidation - finished.") # Generate markdown pages logger.info("Markdown page generation - started.") - self._generate_markdown_pages(projects_issues) + self._generate_markdown_pages(consolidated_issues) logger.info("Markdown page generation - finished.") - def _clean_output_directory(self): + def _clean_output_directory(self) -> None: + """ + Clean the output directory from the previous run. + + @return: None + """ if os.path.exists(self.output_path): shutil.rmtree(self.output_path) os.makedirs(self.output_path) def _fetch_github_issues(self) -> dict[str, list[Issue]]: + """ + Fetch GitHub repository issues using the GitHub library. Only issues with correct labels are fetched, + if no labels are defined in the configuration, all repository issues are fetched. + + @return: A dictionary containing repository issue objects with unique key. + """ issues = {} total_issues_number = 0 - # Mine issues from every config repository + # Run the fetching logic for every config repository for config_repository in self.repositories: repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" - repository = self.safe_call(self.github_instance.get_repo)(repository_id) + repository = self.__safe_call(self.github_instance.get_repo)(repository_id) if repository is None: return {} logger.info("Fetching repository GitHub issues - from `%s`.", repository.full_name) - # Load all issues from one repository (unique for each repository) and save it under repository id + # If the query labels are not defined, fetch all issues from the repository if not config_repository.query_labels: logger.debug("Fetching all issues in the repository") - issues[repository_id] = self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) + issues[repository_id] = self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) amount_of_issues_per_repo = len(issues[repository_id]) - logger.debug("Fetched `%s` repository issues (%s)`.", amount_of_issues_per_repo, repository.full_name) + logger.debug( + "Fetched `%s` repository issues (%s)`.", + amount_of_issues_per_repo, + repository.full_name, + ) else: + # Fetch only issues with required labels from configuration issues[repository_id] = [] logger.debug("Labels to be fetched from: %s.", config_repository.query_labels) for label in config_repository.query_labels: logger.debug("Fetching issues with label `%s`.", label) - issues[repository_id].extend(self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, - labels=[label])) + issues[repository_id].extend( + self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, labels=[label]) + ) amount_of_issues_per_repo = len(issues[repository_id]) # Accumulate the count of issues total_issues_number += amount_of_issues_per_repo - logger.info("Fetching repository GitHub issues - fetched `%s` repository issues (%s).", - amount_of_issues_per_repo, repository.full_name) - - logger.info("Fetching repository GitHub issues - loaded `%s` repository issues in total.", total_issues_number) + logger.info( + "Fetching repository GitHub issues - fetched `%s` repository issues (%s).", + amount_of_issues_per_repo, + repository.full_name, + ) + + logger.info( + "Fetching repository GitHub issues - loaded `%s` repository issues in total.", + total_issues_number, + ) return issues def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: + """ + Fetch GitHub project issues using the GraphQL API. + + @return: A dictionary containing project issue objects with unique key. + """ if not self.project_state_mining_enabled: logger.info("Fetching GitHub project data - project mining is not allowed.") return {} @@ -139,32 +218,45 @@ def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: projects_title_filter = config_repository.projects_title_filter logger.debug("Filtering projects: %s. If filter is empty, fetching all.", projects_title_filter) - repository = self.safe_call(self.github_instance.get_repo)(repository_id) + repository = self.__safe_call(self.github_instance.get_repo)(repository_id) if repository is None: return {} # Fetch all projects_buffer attached to the repository logger.info("Fetching GitHub project data - looking for repository `%s` projects.", repository_id) - projects = self.safe_call(self.github_projects_instance.get_repository_projects)( - repository=repository, projects_title_filter=projects_title_filter) + projects: list[GithubProject] = self.__safe_call(self.__github_projects_instance.get_repository_projects)( + repository=repository, projects_title_filter=projects_title_filter + ) # Update every project with project issue related data for project in projects: logger.info("Fetching GitHub project data - from `%s`.", project.title) - project_issues: list[ProjectIssue] = (self.safe_call(self.github_projects_instance.get_project_issues) - (project=project)) + project_issues: list[ProjectIssue] = self.__safe_call( + self.__github_projects_instance.get_project_issues + )(project=project) for project_issue in project_issues: - key = make_issue_key(project_issue.organization_name, project_issue.repository_name, - project_issue.number) + key = make_issue_key( + project_issue.organization_name, + project_issue.repository_name, + project_issue.number, + ) all_project_issues[key] = project_issue logger.info("Fetching GitHub project data - fetched project data (%s).", project.title) return all_project_issues @staticmethod - def _consolidate_issues_data(repository_issues: dict[str, list[Issue]], - projects_issues: dict[str, ProjectIssue]) -> dict[str, ConsolidatedIssue]: + def _consolidate_issues_data( + repository_issues: dict[str, list[Issue]], projects_issues: dict[str, ProjectIssue] + ) -> dict[str, ConsolidatedIssue]: + """ + Consolidate the fetched issues and extra project data into a one consolidated object. + + @param repository_issues: A dictionary containing repository issue objects with unique key. + @param projects_issues: A dictionary containing project issue objects with unique key. + @return: A dictionary containing all consolidated issues. + """ consolidated_issues = {} @@ -173,8 +265,9 @@ def _consolidate_issues_data(repository_issues: dict[str, list[Issue]], for repository_issue in repository_issues[repository_id]: repo_id_parts = repository_id.split("/") unique_key = make_issue_key(repo_id_parts[0], repo_id_parts[1], repository_issue.number) - consolidated_issues[unique_key] = ConsolidatedIssue(repository_id=repository_id, - repository_issue=repository_issue) + consolidated_issues[unique_key] = ConsolidatedIssue( + repository_id=repository_id, repository_issue=repository_issue + ) # Update consolidated issue structures with project data logger.debug("Updating consolidated issue structure with project data.") @@ -182,30 +275,50 @@ def _consolidate_issues_data(repository_issues: dict[str, list[Issue]], if key in projects_issues: consolidated_issue.update_with_project_data(projects_issues[key].project_status) - logging.info("Issue and project data consolidation - consolidated `%s` repository issues" - " with extra project data.", - len(consolidated_issues)) + logging.info( + "Issue and project data consolidation - consolidated `%s` repository issues" " with extra project data.", + len(consolidated_issues), + ) return consolidated_issues - def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]): - with open(LivingDocumentationGenerator.ISSUE_PAGE_TEMPLATE_FILE, 'r', encoding='utf-8') as f: - issue_page_detail_template = f.read() - - with open(LivingDocumentationGenerator.INDEX_PAGE_TEMPLATE_FILE, 'r', encoding='utf-8') as f: - issue_index_page_template = f.read() + def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None: + """ + Generate the Markdown pages for all consolidated issues, create a summary index page and + save it all to the output directory. + @param issues: A dictionary containing all consolidated issues. + """ + issue_page_detail_template = None + issue_index_page_template = None + + # Load the template files for generating the Markdown pages + try: + with open(LivingDocumentationGenerator.ISSUE_PAGE_TEMPLATE_FILE, "r", encoding="utf-8") as f: + issue_page_detail_template = f.read() + except IOError: + logger.error("Issue page template file was not successfully loaded.", exc_info=True) + + try: + with open(LivingDocumentationGenerator.INDEX_PAGE_TEMPLATE_FILE, "r", encoding="utf-8") as f: + issue_index_page_template = f.read() + except IOError: + logger.error("Index page template file was not successfully loaded.", exc_info=True) + + # Generate a markdown page for every issue for consolidated_issue in issues.values(): self._generate_md_issue_page(issue_page_detail_template, consolidated_issue) logger.info("Markdown page generation - generated `%s` issue pages.", len(issues)) + # Generate an index page with a summary table about all issues self._generate_index_page(issue_index_page_template, issues) def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: ConsolidatedIssue) -> None: """ - Generates a single issue Markdown page from a template and save to the output directory. + Generates a single issue Markdown page from a template and save to the output directory. - @param issue_page_template: The template string for generating a single issue page. - @param consolidated_issue: The dictionary containing issue data. + @param issue_page_template: The template string for generating the single markdown issue page. + @param consolidated_issue: The ConsolidatedIssue object containing the issue data. + @return: None """ # Get all replacements for generating single issue page from a template @@ -222,7 +335,7 @@ def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: "date": date, "page_issue_title": title, "issue_summary_table": issue_table, - "issue_content": issue_content + "issue_content": issue_content, } # Run through all replacements and update template keys with adequate content @@ -230,30 +343,26 @@ def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: # Save the single issue Markdown page page_filename = consolidated_issue.generate_page_filename() - with open(os.path.join(self.output_path, page_filename), 'w', encoding='utf-8') as f: + with open(os.path.join(self.output_path, page_filename), "w", encoding="utf-8") as f: f.write(issue_md_page_content) logger.debug("Generated Markdown page: %s.", page_filename) - def _generate_index_page(self, issue_index_page_template: str, consolidated_issues: dict[str, ConsolidatedIssue])\ - -> None: + def _generate_index_page( + self, issue_index_page_template: str, consolidated_issues: dict[str, ConsolidatedIssue] + ) -> None: """ Generates an index page with a summary of all issues and save it to the output directory. - :param issue_index_page_template: The template string for generating the index page. - :param consolidated_issues: The dictionary containing all issues data. + @param issue_index_page_template: The template string for generating the index markdown page. + @param consolidated_issues: A dictionary containing all consolidated issues. + @return: None """ # Initializing the issue table header based on the project mining state - if self.__project_state_mining_enabled: - issue_table = ("| Organization name | Repository name | Issue 'Number - Title' |" - " Linked to project | Project status | Issue URL |\n" - "|-------------------|-----------------|------------------------|" - "-------------------|----------------|-----------|\n") - else: - issue_table = """| Organization name | Repository name | Issue 'Number - Title' | Issue state | Issue URL | - |-------------------|-----------------|------------------------|-------------|-----------| - """ + issue_table = ( + TABLE_HEADER_WITH_PROJECT_DATA if self.__project_state_mining_enabled else TABLE_HEADER_WITHOUT_PROJECT_DATA + ) # Create an issue summary table for every issue for consolidated_issue in consolidated_issues.values(): @@ -262,28 +371,25 @@ def _generate_index_page(self, issue_index_page_template: str, consolidated_issu # Prepare issues replacement for the index page replacement = { "date": datetime.now().strftime("%Y-%m-%d"), - "issues": issue_table + "issues": issue_table, } # Replace the issue placeholders in the index template index_page = issue_index_page_template.format(**replacement) # Create an index page file - with open(os.path.join(self.output_path, "_index.md"), 'w', encoding='utf-8') as f: + with open(os.path.join(self.output_path, "_index.md"), "w", encoding="utf-8") as f: f.write(index_page) logger.info("Markdown page generation - generated `_index.md`.") def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: """ - Generates a markdown summary line for a given feature. + Generates a markdown summary line for a single issue. - @param consolidated_issue: The dictionary containing feature data. - - @return: A string representing the markdown line for the feature. + @param consolidated_issue: The ConsolidatedIssue object containing the issue data. + @return: The markdown line for the issue. """ - - # Extract issue details from the consolidated issue object organization_name = consolidated_issue.organization_name repository_name = consolidated_issue.repository_name number = consolidated_issue.number @@ -302,26 +408,29 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: linked_to_project = LINKED_TO_PROJECT_FALSE # Generate the Markdown issue line WITH extra project data - md_issue_line = (f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" - f" {linked_to_project} | {status} |[GitHub link]({url}) |\n") + md_issue_line = ( + f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" + f" {linked_to_project} | {status} |[GitHub link]({url}) |\n" + ) else: # Generate the Markdown issue line WITHOUT project data - md_issue_line = (f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" - f" {state} |[GitHub link]({url}) |\n") + md_issue_line = ( + f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" + f" {state} |[GitHub link]({url}) |\n" + ) return md_issue_line def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) -> str: """ - Generates a string representation of feature info in a table format. - - @param consolidated_issue: The dictionary containing feature data. + Generates a string representation of feature info in a table format. - @return: A string representing the feature information in a table format. + @param consolidated_issue: The ConsolidatedIssue object containing the issue data. + @return: The string representation of the issue info in a table format. """ # Join issue labels into one string labels = consolidated_issue.labels - labels = ', '.join(labels) if labels else None + labels = ", ".join(labels) if labels else None # Format issue URL as a Markdown link issue_url = consolidated_issue.html_url @@ -337,7 +446,7 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - "Created at", "Updated at", "Closed at", - "Labels" + "Labels", ] # Define the values for the issue summary table @@ -350,7 +459,7 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - consolidated_issue.created_at, consolidated_issue.updated_at, consolidated_issue.closed_at, - labels + labels, ] # Update the summary table, based on the project data mining situation @@ -358,21 +467,25 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - project_status = consolidated_issue.project_status if consolidated_issue.linked_to_project: - headers.extend([ - "Project title", - "Status", - "Priority", - "Size", - "MoSCoW" - ]) - - values.extend([ - project_status.project_title, - project_status.status, - project_status.priority, - project_status.size, - project_status.moscow - ]) + headers.extend( + [ + "Project title", + "Status", + "Priority", + "Size", + "MoSCoW", + ] + ) + + values.extend( + [ + project_status.project_title, + project_status.status, + project_status.priority, + project_status.size, + project_status.moscow, + ] + ) else: headers.append("Linked to project") linked_to_project = LINKED_TO_PROJECT_FALSE diff --git a/living_documentation_generator/github_projects.py b/living_documentation_generator/github_projects.py index 86aa485..d199c93 100644 --- a/living_documentation_generator/github_projects.py +++ b/living_documentation_generator/github_projects.py @@ -1,3 +1,22 @@ +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains the GithubProjects class which is responsible for mining data for GitHub Projects. +""" + import logging from typing import Optional import requests @@ -16,24 +35,27 @@ class GithubProjects: + """ + A class representing all the logic for mining data for GitHub Projects. + The class handles the logic of initializing request session, sending GraphQL queries + and processes the responses. + """ def __init__(self, token: str): self.__token = token self.__session = None - def __initialize_request_session(self): + def __initialize_request_session(self) -> requests.Session: """ Initializes the request Session and updates the headers. - @param github_token: The GitHub user token for authentication. - - @return: A configured request session. + @return: The request session object. """ self.__session = requests.Session() headers = { "Authorization": f"Bearer {self.__token}", - "User-Agent": "IssueFetcher/1.0" + "User-Agent": "IssueFetcher/1.0", } self.__session.headers.update(headers) @@ -41,19 +63,18 @@ def __initialize_request_session(self): def __send_graphql_query(self, query: str) -> Optional[dict[str, dict]]: """ - Sends a GraphQL query to the GitHub API and returns the response. - If an HTTP error occurs, it prints the error and returns an empty dictionary. + Send a GraphQL query to the GitHub API and returns the response. + If an HTTP error occurs, it prints the error and returns None instead. - @param query: The GraphQL query to be sent in f string format. - - @return: The response from the GitHub GraphQL API in a dictionary format. + @param query: The formated GraphQL query to send to the GitHub API. + @return: The response from the GitHub API. """ try: if self.__session is None: self.__initialize_request_session() # Fetch the response from the API - response = self.__session.post('https://api.github.com/graphql', json={'query': query}) + response = self.__session.post("https://api.github.com/graphql", json={"query": query}) # Check if the request was successful response.raise_for_status() @@ -69,35 +90,41 @@ def __send_graphql_query(self, query: str) -> Optional[dict[str, dict]]: return None def get_repository_projects(self, repository: Repository, projects_title_filter: list[str]) -> list[GithubProject]: + """ + Fetch all projects attached to a given repository using a GraphQL query. Based on the response create + GitHub project instances and return them in a list. + + @param repository: The repository instance to fetch projects from. + @param projects_title_filter: The list of project titles to filter for. + @return: A list of GitHub project instances. + """ projects = [] # Fetch the project response from the GraphQL API - projects_from_repo_query = get_projects_from_repo_query(organization_name=repository.owner.login, - repository_name=repository.name) + projects_from_repo_query = get_projects_from_repo_query( + organization_name=repository.owner.login, repository_name=repository.name + ) projects_from_repo_response = self.__send_graphql_query(projects_from_repo_query) if projects_from_repo_response is None: - logger.warning("Fetching GitHub project data - no project data for repository %s. No data received.", - repository.full_name - ) + logger.warning( + "Fetching GitHub project data - no project data for repository %s. No data received.", + repository.full_name, + ) return projects # This will return `None` at any point if a key is missing or if the data is not found - projects_from_repo_nodes = ( - projects_from_repo_response - .get("repository", {}) - .get("projectsV2", {}) - .get("nodes")) + projects_from_repo_nodes = projects_from_repo_response.get("repository", {}).get("projectsV2", {}).get("nodes") # If response is not None, parse the project response if projects_from_repo_nodes is not None: - projects_from_repo_nodes = projects_from_repo_response['repository']['projectsV2']['nodes'] + projects_from_repo_nodes = projects_from_repo_response["repository"]["projectsV2"]["nodes"] for project_json in projects_from_repo_nodes: # Check if the project is required based on the configuration filter - project_title = project_json['title'] - project_number = project_json['number'] + project_title = project_json["title"] + project_number = project_json["number"] # If no filter is provided, all projects are required is_project_required = True if not projects_title_filter else project_title in projects_title_filter @@ -111,11 +138,12 @@ def get_repository_projects(self, repository: Repository, projects_title_filter: project_field_options_query = get_project_field_options_query( organization_name=repository.owner.login, repository_name=repository.name, - project_number=project_number) + project_number=project_number, + ) field_option_response = self.__send_graphql_query(project_field_options_query) - project = GithubProject().load_from_json(project_json, repository, field_option_response) - + # Create the GitHub project instance and add it to the output list + project = GithubProject().loads(project_json, repository, field_option_response) if project not in projects: projects.append(project) @@ -129,21 +157,24 @@ def get_repository_projects(self, repository: Repository, projects_title_filter: def get_project_issues(self, project: GithubProject) -> list[ProjectIssue]: """ - Fetches all issues from a given project using a GraphQL query. - The issues are fetched supported by pagination. + Fetch all issues that are attached to a GitHub Project using a GraphQL query. + Fetching is supported by pagination. Based on the response create project issue objects + and return them in a list. - @return: The list of all issues in the project. + @param project: The GitHub project object to fetch issues from. + @return: A list of project issue objects. """ project_issues_raw = [] cursor = None while True: # Add the after argument to the query if a cursor is provided - after_argument = f'after: "{cursor}"' if cursor else '' + after_argument = f'after: "{cursor}"' if cursor else "" # Fetch project issues via GraphQL query - issues_from_project_query = get_issues_from_project_query(project_id=project.id, - after_argument=after_argument) + issues_from_project_query = get_issues_from_project_query( + project_id=project.id, after_argument=after_argument + ) project_issues_response = self.__send_graphql_query(issues_from_project_query) @@ -151,19 +182,19 @@ def get_project_issues(self, project: GithubProject) -> list[ProjectIssue]: if not project_issues_response: return [] - general_response_structure = project_issues_response['node']['items'] - project_issue_data = general_response_structure['nodes'] - page_info = general_response_structure['pageInfo'] + general_response_structure = project_issues_response["node"]["items"] + project_issue_data = general_response_structure["nodes"] + page_info = general_response_structure["pageInfo"] # Extend project issues list per every page during pagination project_issues_raw.extend(project_issue_data) logger.debug("Loaded `%s` issues from project: %s.", len(project_issue_data), project.title) # Check for closing the pagination process - if not page_info['hasNextPage']: + if not page_info["hasNextPage"]: break - cursor = page_info['endCursor'] + cursor = page_info["endCursor"] - project_issues = [ProjectIssue().load_from_json(issue_json, project) for issue_json in project_issues_raw] + project_issues = [ProjectIssue().loads(issue_json, project) for issue_json in project_issues_raw] return project_issues diff --git a/living_documentation_generator/model/config_repository.py b/living_documentation_generator/model/config_repository.py index 619a06d..6669a3b 100644 --- a/living_documentation_generator/model/config_repository.py +++ b/living_documentation_generator/model/config_repository.py @@ -1,4 +1,30 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a data container for Config Repository, which holds all the essential logic. +""" + + class ConfigRepository: + """ + A class representing the configuration for a GH repository to fetch data from. + The class provides loading logic and properties to access all the details. + """ + def __init__(self): self.__organization_name: str = "" self.__repository_name: str = "" @@ -7,21 +33,31 @@ def __init__(self): @property def organization_name(self) -> str: + """Getter of the repository organization name.""" return self.__organization_name @property def repository_name(self) -> str: + """Getter of the repository name.""" return self.__repository_name @property def query_labels(self) -> list[str]: - return self.__query_labels if self.__query_labels is not None else [] + """Getter of the query labels.""" + return self.__query_labels @property def projects_title_filter(self) -> list[str]: - return self.__projects_title_filter if self.__projects_title_filter is not None else [] + """Getter of the project title filter.""" + return self.__projects_title_filter + + def load_from_json(self, repository_json: dict) -> None: + """ + Load the configuration from a JSON object. - def load_from_json(self, repository_json: dict): + @param repository_json: The JSON object containing the repository configuration. + @return: None + """ self.__organization_name = repository_json["organization-name"] self.__repository_name = repository_json["repository-name"] self.__query_labels = repository_json["query-labels"] diff --git a/living_documentation_generator/model/consolidated_issue.py b/living_documentation_generator/model/consolidated_issue.py index 365a8b7..e38b1db 100644 --- a/living_documentation_generator/model/consolidated_issue.py +++ b/living_documentation_generator/model/consolidated_issue.py @@ -1,3 +1,23 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a data container for Consolidated Issue, which holds all the essential logic. +""" + from typing import Optional from github.Issue import Issue @@ -7,10 +27,16 @@ class ConsolidatedIssue: + """ + A class representing a consolidated issue from the repository and project data. + It provides methods for updating project data and generating page filenames, + along with properties to access consolidated issue details. + """ + def __init__(self, repository_id: str, repository_issue: Issue = None): # save issue from repository (got from GitHub library & keep connection to repository for lazy loading) # Warning: several issue properties requires additional API calls - use wisely to keep low API usage - self.__issue = repository_issue + self.__issue: Issue = repository_issue parts = repository_id.split("/") self.__organization_name: str = parts[0] if len(parts) == 2 else "" @@ -25,46 +51,57 @@ def __init__(self, repository_id: str, repository_issue: Issue = None): # Issue properties @property def number(self) -> int: + """Getter of the issue number.""" return self.__issue.number if self.__issue else 0 @property def organization_name(self) -> str: + """Getter of the organization where the issue was fetched from.""" return self.__organization_name @property def repository_name(self) -> str: + """Getter of the repository name where the issue was fetched from.""" return self.__repository_name @property def title(self) -> str: + """Getter of the issue title.""" return self.__issue.title if self.__issue else "" @property def state(self) -> str: + """Getter of the issue state.""" return self.__issue.state if self.__issue else "" @property def created_at(self) -> str: + """Getter of the info when issue was created.""" return self.__issue.created_at if self.__issue else "" @property def updated_at(self) -> str: + """Getter of the info when issue was updated""" return self.__issue.updated_at if self.__issue else "" @property def closed_at(self) -> str: + """Getter of the info when issue was closed.""" return self.__issue.closed_at if self.__issue else "" @property def html_url(self) -> str: + """Getter of the issue GitHub html URL.""" return self.__issue.html_url if self.__issue else "" @property def body(self) -> str: + """Getter of the issue description.""" return self.__issue.body if self.__issue else "" @property def labels(self) -> list[str]: + """Getter of the issue labels.""" if self.__issue: return [label.name for label in self.__issue.labels] return [] @@ -72,18 +109,27 @@ def labels(self) -> list[str]: # Project properties @property def linked_to_project(self) -> bool: + """Getter of the info if the issue is linked to a project.""" return self.__linked_to_project @property def project_status(self) -> ProjectStatus: + """Getter of the project status.""" return self.__project_status # Error property @property def error(self) -> Optional[str]: + """Getter of the error message.""" return self.__error - def update_with_project_data(self, project_status: ProjectStatus): + def update_with_project_data(self, project_status: ProjectStatus) -> None: + """ + Update the consolidated issue with attached project data. + + @param project_status: The extra issue project data. + @return: None + """ self.__linked_to_project = True self.__project_status.project_title = project_status.project_title self.__project_status.status = project_status.status @@ -91,7 +137,12 @@ def update_with_project_data(self, project_status: ProjectStatus): self.__project_status.size = project_status.size self.__project_status.moscow = project_status.moscow - def generate_page_filename(self): + def generate_page_filename(self) -> str: + """ + Generate a filename page naming based on the issue number and title. + + @return: The generated page filename. + """ md_filename_base = f"{self.number}_{self.title.lower()}.md" page_filename = sanitize_filename(md_filename_base) diff --git a/living_documentation_generator/model/github_project.py b/living_documentation_generator/model/github_project.py index bc7223f..e58c16f 100644 --- a/living_documentation_generator/model/github_project.py +++ b/living_documentation_generator/model/github_project.py @@ -1,59 +1,102 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a data container for GitHub Project, which holds all the essential logic. +""" + import logging -from living_documentation_generator.model.config_repository import ConfigRepository +from github.Repository import Repository logger = logging.getLogger(__name__) class GithubProject: + """ + A class representing GitHub Project is responsible for loading JSON format data, + fetching project field options, along with properties to access project specifics. + """ + def __init__(self): self.__id: str = "" self.__number: int = 0 self.__title: str = "" self.__organization_name: str = "" - self.__config_repositories: list[ConfigRepository] = [] self.__field_options: dict[str, str] = {} @property def id(self) -> str: + """Getter of the project ID.""" return self.__id @property def number(self) -> int: + """Getter of the project number.""" return self.__number @property def title(self) -> str: + """Getter of the project title.""" return self.__title @property def organization_name(self) -> str: + """Getter of the organization name.""" return self.__organization_name @property def field_options(self) -> dict[str, str]: + """Getter of the project field options.""" return self.__field_options - def load_from_json(self, project_json, repository, field_option_response): + def loads(self, project_json: dict, repository: Repository, field_option_response: dict) -> "GithubProject": + """ + Load the project data from several inputs. + + @param project_json: The JSON object containing the data about the project. + @param repository: The GH repository object where the project is located. + @param field_option_response: The response containing the field options for the project. + @return: The GithubProject object with the loaded data. + """ self.__id = project_json["id"] self.__number = project_json["number"] self.__title = project_json["title"] self.__organization_name = repository.owner.login - self.__config_repositories.append(repository.name) logger.debug("Updating field options for projects in repository `%s`.", repository.full_name) self.__update_field_options(field_option_response) return self - def __update_field_options(self, field_option_response: dict): + def __update_field_options(self, field_option_response: dict) -> None: + """ + Parse and update the field options of the project from a JSON response. + + @param field_option_response: The JSON API response containing the field options. + @return: None + """ try: field_options_nodes = field_option_response["repository"]["projectV2"]["fields"]["nodes"] except KeyError: - logger.error("There is no expected response structure for field options fetched from project: %s", - self.title, - exc_info=True - ) + logger.error( + "There is no expected response structure for field options fetched from project: %s", + self.title, + exc_info=True, + ) return for field_option in field_options_nodes: diff --git a/living_documentation_generator/model/project_issue.py b/living_documentation_generator/model/project_issue.py index c28ef06..dc534fd 100644 --- a/living_documentation_generator/model/project_issue.py +++ b/living_documentation_generator/model/project_issue.py @@ -1,8 +1,33 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a data container for Project Issue, which holds all the essential logic. +""" + from living_documentation_generator.model.github_project import GithubProject from living_documentation_generator.model.project_status import ProjectStatus class ProjectIssue: + """ + A class representing a Project Issue and is responsible for receiving, managing + response data, along with properties to access project issue specifics. + """ + def __init__(self): self.__number: int = 0 self.__organization_name: str = "" @@ -11,40 +36,53 @@ def __init__(self): @property def number(self) -> int: + """Getter of the project issue number.""" return self.__number @property def organization_name(self) -> str: + """Getter of the organization where the issue was fetched from.""" return self.__organization_name @property def repository_name(self) -> str: + """Getter of the repository name where the issue was fetched from.""" return self.__repository_name @property def project_status(self) -> ProjectStatus: + """Getter of the project issue status.""" return self.__project_status - def load_from_json(self, issue_json: dict, project: GithubProject): + def loads(self, issue_json: dict, project: GithubProject) -> "ProjectIssue": + """ + Loads the project issue data from the provided JSON and GithubProject object. + + @param: issue_json: The JSON data of the project issue. + @param: project: The GithubProject object representing the project the issue belongs to. + @return: The ProjectIssue object with the loaded data. + """ self.__number = issue_json["content"]["number"] self.__organization_name = issue_json["content"]["repository"]["owner"]["login"] self.__repository_name = issue_json["content"]["repository"]["name"] self.__project_status.project_title = project.title + # Parse the field types from the response field_types = [] - if 'fieldValues' in issue_json: - for node in issue_json['fieldValues']['nodes']: - if node['__typename'] == 'ProjectV2ItemFieldSingleSelectValue': - field_types.append(node['name']) + if "fieldValues" in issue_json: + for node in issue_json["fieldValues"]["nodes"]: + if node["__typename"] == "ProjectV2ItemFieldSingleSelectValue": + field_types.append(node["name"]) + # Update the project status with the field type values for field_type in field_types: - if field_type in project.field_options.get('Status', []): + if field_type in project.field_options.get("Status", []): self.__project_status.status = field_type - elif field_type in project.field_options.get('Priority', []): + elif field_type in project.field_options.get("Priority", []): self.__project_status.priority = field_type - elif field_type in project.field_options.get('Size', []): + elif field_type in project.field_options.get("Size", []): self.__project_status.size = field_type - elif field_type in project.field_options.get('MoSCoW', []): + elif field_type in project.field_options.get("MoSCoW", []): self.__project_status.moscow = field_type return self diff --git a/living_documentation_generator/model/project_status.py b/living_documentation_generator/model/project_status.py index 4867f3c..edc89c4 100644 --- a/living_documentation_generator/model/project_status.py +++ b/living_documentation_generator/model/project_status.py @@ -1,7 +1,32 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a data container for issue Project Status. +""" + from living_documentation_generator.utils.constants import NO_PROJECT_DATA class ProjectStatus: + """ + A class representing the project status of an issue is responsible for access + and change project issue status specifics. + """ + def __init__(self): self.__project_title: str = "" self.__status: str = NO_PROJECT_DATA @@ -11,6 +36,7 @@ def __init__(self): @property def project_title(self) -> str: + """Getter of the issue attached project title.""" return self.__project_title @project_title.setter @@ -19,6 +45,7 @@ def project_title(self, value: str): @property def status(self) -> str: + """Getter of the issue project status.""" return self.__status @status.setter @@ -27,6 +54,7 @@ def status(self, value: str): @property def priority(self) -> str: + """Getter of the issue project priority.""" return self.__priority @priority.setter @@ -35,6 +63,7 @@ def priority(self, value: str): @property def size(self) -> str: + """Getter of the issue project difficulty.""" return self.__size @size.setter @@ -43,6 +72,7 @@ def size(self, value: str): @property def moscow(self) -> str: + """Getter of the issue project MoSCoW prioritization.""" return self.__moscow @moscow.setter diff --git a/living_documentation_generator/utils/constants.py b/living_documentation_generator/utils/constants.py index 52310ad..1775c18 100644 --- a/living_documentation_generator/utils/constants.py +++ b/living_documentation_generator/utils/constants.py @@ -1,12 +1,32 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains all constants used across the project. +""" + # Action inputs environment variables -GITHUB_TOKEN = 'GITHUB_TOKEN' -PROJECT_STATE_MINING = 'PROJECT_STATE_MINING' -REPOSITORIES = 'REPOSITORIES' -OUTPUT_PATH = 'OUTPUT_PATH' +GITHUB_TOKEN = "GITHUB_TOKEN" +PROJECT_STATE_MINING = "PROJECT_STATE_MINING" +REPOSITORIES = "REPOSITORIES" +OUTPUT_PATH = "OUTPUT_PATH" # GitHub API constants ISSUES_PER_PAGE_LIMIT = 100 -ISSUE_STATE_ALL = 'all' +ISSUE_STATE_ALL = "all" PROJECTS_FROM_REPO_QUERY = """ query {{ repository(owner: "{organization_name}", name: "{repository_name}") {{ @@ -21,65 +41,76 @@ }} """ ISSUES_FROM_PROJECT_QUERY = """ - query {{ - node(id: "{project_id}") {{ - ... on ProjectV2 {{ - items(first: {issues_per_page}, {after_argument}) {{ - pageInfo {{ - endCursor - hasNextPage - }} - nodes {{ - content {{ - ... on Issue {{ - title - state - number - repository {{ - name - owner {{ - login + query {{ + node(id: "{project_id}") {{ + ... on ProjectV2 {{ + items(first: {issues_per_page}, {after_argument}) {{ + pageInfo {{ + endCursor + hasNextPage + }} + nodes {{ + content {{ + ... on Issue {{ + title + state + number + repository {{ + name + owner {{ + login + }} + }} + }} + }} + fieldValues(first: 100) {{ + nodes {{ + __typename + ... on ProjectV2ItemFieldSingleSelectValue {{ + name + }} + }} }} }} }} }} - fieldValues(first: 100) {{ - nodes {{ - __typename - ... on ProjectV2ItemFieldSingleSelectValue {{ - name - }} - }} }} }} - }} - }} - }} - }} - """ + """ PROJECT_FIELD_OPTIONS_QUERY = """ - query {{ - repository(owner: "{organization_name}", name: "{repository_name}") {{ - projectV2(number: {project_number}) {{ - title - fields(first: 100) {{ - nodes {{ - ... on ProjectV2SingleSelectField {{ - name - options {{ - name + query {{ + repository(owner: "{organization_name}", name: "{repository_name}") {{ + projectV2(number: {project_number}) {{ + title + fields(first: 100) {{ + nodes {{ + ... on ProjectV2SingleSelectField {{ + name + options {{ + name + }} + }} }} }} }} }} }} - }} - }} - """ + """ + +# Table headers for Index page +TABLE_HEADER_WITH_PROJECT_DATA = """ +| Organization name | Repository name | Issue 'Number - Title' |Linked to project | Project status | Issue URL | +|-------------------|-----------------|------------------------|------------------|----------------|-----------| +""" +TABLE_HEADER_WITHOUT_PROJECT_DATA = """ +| Organization name | Repository name | Issue 'Number - Title' | Issue state | Issue URL | +|-------------------|-----------------|------------------------|-------------|-----------| +""" + # Symbol, when no project is attached to an issue -NO_PROJECT_DATA = '---' +NO_PROJECT_DATA = "---" # Constant to symbolize if issue is linked to a project -LINKED_TO_PROJECT_TRUE = '🟢' -LINKED_TO_PROJECT_FALSE = '🔴' +LINKED_TO_PROJECT_TRUE = "🟢" +LINKED_TO_PROJECT_FALSE = "🔴" diff --git a/living_documentation_generator/utils/decorators.py b/living_documentation_generator/utils/decorators.py index dc0e7ce..44939ce 100644 --- a/living_documentation_generator/utils/decorators.py +++ b/living_documentation_generator/utils/decorators.py @@ -1,3 +1,24 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains decorators for adding debug logging to method calls +and for creating rate-limited safe call functions. +""" + import logging from typing import Callable, Optional, Any @@ -11,20 +32,29 @@ def debug_log_decorator(method: Callable) -> Callable: """ Decorator to add debug logging for a method call. + + @param method: The method to decorate. + @return: The decorated method. """ + @wraps(method) def wrapped(*args, **kwargs) -> Optional[Any]: logger.debug("Calling method %s with args: %s and kwargs: %s.", method.__name__, args, kwargs) result = method(*args, **kwargs) logger.debug("Method %s returned %s.", method.__name__, result) return result + return wrapped -def safe_call_decorator(rate_limiter: GithubRateLimiter): +def safe_call_decorator(rate_limiter: GithubRateLimiter) -> Callable: """ Decorator factory to create a rate-limited safe call function. + + @param rate_limiter: The rate limiter to use. + @return: The decorator. """ + def decorator(method: Callable) -> Callable: # Note: Keep log decorator first to log correct method name. @debug_log_decorator @@ -36,5 +66,7 @@ def wrapped(*args, **kwargs) -> Optional[Any]: except (ValueError, TypeError) as e: logger.error("Error calling %s: %s", method.__name__, e, exc_info=True) return None + return wrapped + return decorator diff --git a/living_documentation_generator/utils/github_project_queries.py b/living_documentation_generator/utils/github_project_queries.py index 074c142..3cd5ebc 100644 --- a/living_documentation_generator/utils/github_project_queries.py +++ b/living_documentation_generator/utils/github_project_queries.py @@ -1,19 +1,45 @@ -from living_documentation_generator.utils.constants import (PROJECTS_FROM_REPO_QUERY, ISSUES_FROM_PROJECT_QUERY, - PROJECT_FIELD_OPTIONS_QUERY, ISSUES_PER_PAGE_LIMIT) +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains methods for formating the GitHub GraphQL queries. +""" + +from living_documentation_generator.utils.constants import ( + PROJECTS_FROM_REPO_QUERY, + ISSUES_FROM_PROJECT_QUERY, + PROJECT_FIELD_OPTIONS_QUERY, + ISSUES_PER_PAGE_LIMIT, +) def get_projects_from_repo_query(organization_name: str, repository_name: str) -> str: - return PROJECTS_FROM_REPO_QUERY.format(organization_name=organization_name, - repository_name=repository_name) + """Update the placeholder values and formate the graphQL query""" + return PROJECTS_FROM_REPO_QUERY.format(organization_name=organization_name, repository_name=repository_name) def get_issues_from_project_query(project_id: str, after_argument: str) -> str: - return ISSUES_FROM_PROJECT_QUERY.format(project_id=project_id, - issues_per_page=ISSUES_PER_PAGE_LIMIT, - after_argument=after_argument) + """Update the placeholder values and formate the graphQL query""" + return ISSUES_FROM_PROJECT_QUERY.format( + project_id=project_id, issues_per_page=ISSUES_PER_PAGE_LIMIT, after_argument=after_argument + ) def get_project_field_options_query(organization_name: str, repository_name: str, project_number: int) -> str: - return PROJECT_FIELD_OPTIONS_QUERY.format(organization_name=organization_name, - repository_name=repository_name, - project_number=project_number) + """Update the placeholder values and formate the graphQL query""" + return PROJECT_FIELD_OPTIONS_QUERY.format( + organization_name=organization_name, repository_name=repository_name, project_number=project_number + ) diff --git a/living_documentation_generator/utils/github_rate_limiter.py b/living_documentation_generator/utils/github_rate_limiter.py index d12fad5..ee38207 100644 --- a/living_documentation_generator/utils/github_rate_limiter.py +++ b/living_documentation_generator/utils/github_rate_limiter.py @@ -1,3 +1,24 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a GitHub Rate Limiter class methods, +which acts as a rate limiter for GitHub API calls. +""" + import logging import time from datetime import datetime @@ -10,10 +31,29 @@ # pylint: disable=too-few-public-methods # It is fine to have a single method in this class, since we use it as a callable class class GithubRateLimiter: + """ + A class that acts as a rate limiter for GitHub API calls. + + Note: + This class is used as a callable class, hence the `__call__` method. + """ + def __init__(self, github_client: Github): - self.github_client = github_client + self.__github_client: Github = github_client + + @property + def github_client(self) -> Github: + """Getter of the GitHub client.""" + return self.__github_client def __call__(self, method: Callable) -> Callable: + """ + Wraps the provided method to ensure it respects the GitHub API rate limit. + + @param method: The method to wrap. + @return: The wrapped method. + """ + def wrapped_method(*args, **kwargs) -> Optional[Any]: rate_limit = self.github_client.get_rate_limit().core remaining_calls = rate_limit.remaining @@ -31,9 +71,13 @@ def wrapped_method(*args, **kwargs) -> Optional[Any]: hours, remainder = divmod(total_sleep_time, 3600) minutes, seconds = divmod(remainder, 60) - logger.info("Sleeping for %s hours, %s minutes, and %s seconds until %s.", - int(hours), int(minutes), int(seconds), - datetime.fromtimestamp(reset_time).strftime('%Y-%m-%d %H:%M:%S')) + logger.info( + "Sleeping for %s hours, %s minutes, and %s seconds until %s.", + int(hours), + int(minutes), + int(seconds), + datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S"), + ) time.sleep(sleep_time + 5) # Sleep for the calculated time plus 5 seconds return method(*args, **kwargs) diff --git a/living_documentation_generator/utils/logging_config.py b/living_documentation_generator/utils/logging_config.py index 26e0846..a6aae9a 100644 --- a/living_documentation_generator/utils/logging_config.py +++ b/living_documentation_generator/utils/logging_config.py @@ -1,14 +1,41 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This module contains a method to set up logging in the project. +""" + import logging import os def setup_logging() -> None: - is_verbose_logging: bool = os.getenv("INPUT_VERBOSE_LOGGING", 'false').lower() == "true" - is_debug_mode = os.getenv("RUNNER_DEBUG", '0') == '1' + """ + Set up the logging configuration in the project + + @return: None + """ + # Load logging configuration from the environment variables + is_verbose_logging: bool = os.getenv("INPUT_VERBOSE_LOGGING", "false").lower() == "true" + is_debug_mode = os.getenv("RUNNER_DEBUG", "0") == "1" level = logging.DEBUG if is_verbose_logging or is_debug_mode else logging.INFO + # Set up the logging configuration logging.basicConfig( - level=level, - format='%(asctime)s - %(levelname)s - %(message)s', - datefmt="%Y-%m-%d %H:%M:%S" + level=level, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 2bf7dd2..40b09b0 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -1,9 +1,23 @@ -"""Utility Functions +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -This script contains helper functions that are used across multiple living_documentation_generator in this project. - -These functions can be imported and used in other living_documentation_generator as needed. """ +This module contains utility functions used across the project. +""" + import os import re import sys @@ -14,50 +28,75 @@ def make_issue_key(organization_name: str, repository_name: str, issue_number: int) -> str: """ - Creates a unique 3way string key for identifying every unique feature. + Create a unique string key for identifying the issue. - @return: The unique string key for the feature. + @param organization_name: The name of the organization where issue is located at. + @param repository_name: The name of the repository where issue is located at. + @param issue_number: The number of the issue. + @return: The unique string key for the issue. """ return f"{organization_name}/{repository_name}/{issue_number}" def sanitize_filename(filename: str) -> str: """ - Sanitizes the provided filename by removing invalid characters and replacing spaces with underscores. + Sanitize the provided filename by removing invalid characters. - @param filename: The filename to be sanitized. - - @return: The sanitized filename. + @param filename: The filename to sanitize. + @return: The sanitized filename """ # Remove invalid characters for Windows filenames - sanitized_name = re.sub(r'[<>:"/|?*`]', '', filename) + sanitized_name = re.sub(r'[<>:"/|?*`]', "", filename) # Reduce consecutive periods - sanitized_name = re.sub(r'\.{2,}', '.', sanitized_name) + sanitized_name = re.sub(r"\.{2,}", ".", sanitized_name) # Reduce consecutive spaces to a single space - sanitized_name = re.sub(r' {2,}', ' ', sanitized_name) + sanitized_name = re.sub(r" {2,}", " ", sanitized_name) # Replace space with '_' - sanitized_name = sanitized_name.replace(' ', '_') + sanitized_name = sanitized_name.replace(" ", "_") return sanitized_name -def make_absolute_path(path): +def make_absolute_path(path: str) -> str: + """ + Convert the provided path to an absolute path. + + @param path: The path to convert. + @return: The absolute path. + """ # If the path is already absolute, return it as is if os.path.isabs(path): return path # Otherwise, convert the relative path to an absolute path return os.path.abspath(path) -# Github - +# Github def get_action_input(name: str, default: str = None) -> str: + """ + Get the input value from the environment variables. + + @param name: The name of the input parameter. + @param default: The default value to return if the environment variable is not set. + @return: The value of the specified input parameter, or an empty string + """ return os.getenv(f"INPUT_{name}", default) -def set_action_output(name: str, value: str, default_output_path: str = "default_output.txt"): +def set_action_output(name: str, value: str, default_output_path: str = "default_output.txt") -> None: + """ + Write an action output to a file in the format expected by GitHub Actions. + + This function writes the output in a specific format that includes the name of the + output and its value. The output is appended to the specified file. + + @param name: The name of the output parameter. + @param value: The value of the output parameter. + @param default_output_path: The default file path to which the output is written if the + @return: None + """ output_file = os.getenv("GITHUB_OUTPUT", default_output_path) - with open(output_file, 'a', encoding='utf-8') as f: + with open(output_file, "a", encoding="utf-8") as f: # Write the multiline output to the file f.write(f"{name}< None: + """ + The main function to run the Living Documentation Generator. + + @return: None + """ setup_logging() logger = logging.getLogger(__name__) @@ -17,17 +43,18 @@ def run(): github_token=action_inputs.github_token, repositories=action_inputs.repositories, project_state_mining_enabled=action_inputs.is_project_state_mining_enabled, - output_path=action_inputs.output_directory + output_path=action_inputs.output_directory, ) + # Generate the Living Documentation generator.generate() # Set the output for the GitHub Action - set_action_output('output-path', generator.output_path) + set_action_output("output-path", generator.output_path) logger.info("Living Documentation generation - output path set to `%s`.", generator.output_path) logger.info("Living Documentation generation completed.") -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..474d0af --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target-version = ['py311'] diff --git a/requirements.txt b/requirements.txt index f7074a0..4f68a73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ requests~=2.31.0 typing_extensions~=4.12.2 PyGithub~=2.3.0 pylint~=3.2.6 +black~=24.8.0 diff --git a/tests/test_hello.py b/tests/test_hello.py index eb45a82..a52abe8 100644 --- a/tests/test_hello.py +++ b/tests/test_hello.py @@ -1,6 +1,7 @@ import unittest import sys -sys.path.append('living_documentation_generator') # Adjust path to include the directory where hello.py is located + +sys.path.append("living_documentation_generator") # Adjust path to include the directory where hello.py is located class TestHelloWorld(unittest.TestCase): @@ -11,5 +12,5 @@ def test_hello_world(self): self.assertEqual(result, expected) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()