From dc0e483cf83e316b7652a77eb33b2cbbd4a23913 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:26:59 +0200 Subject: [PATCH 01/60] Logic for having more project data for one issue half way done. --- living_documentation_generator/generator.py | 22 ++++--- .../model/project_status.py | 61 +++++++++++-------- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index cb81caf..11515c6 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -33,6 +33,7 @@ from living_documentation_generator.model.config_repository import ConfigRepository from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue from living_documentation_generator.model.project_issue import ProjectIssue +from living_documentation_generator.model.project_status import ProjectStatus from living_documentation_generator.utils.decorators import safe_call_decorator from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter from living_documentation_generator.utils.utils import make_issue_key @@ -119,13 +120,13 @@ def generate(self) -> None: # Data mine GitHub project's issues logger.info("Fetching GitHub project data - started.") - project_issues: dict[str, ProjectIssue] = self._fetch_github_project_issues() + project_issues: dict[str, dict[str, ProjectStatus]] = self._fetch_github_project_issues() # Note: got dict of project issues with unique string key defying the issue logger.info("Fetching GitHub project data - finished.") # Consolidate all issue data together logger.info("Issue and project data consolidation - started.") - consolidated_issues = self._consolidate_issues_data(repository_issues, project_issues) + consolidated_issues: dict[str, ConsolidatedIssue] = self._consolidate_issues_data(repository_issues, project_issues) logger.info("Issue and project data consolidation - finished.") # Generate markdown pages @@ -198,7 +199,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: ) return issues - def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: + def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: """ Fetch GitHub project issues using the GraphQL API. @@ -211,7 +212,7 @@ def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: logger.debug("Project data mining allowed.") # Mine project issues for every repository - all_project_issues: dict[str, ProjectIssue] = {} + all_project_issues: dict[str, dict[str, ProjectStatus]] = {} for config_repository in self.repositories: repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" @@ -252,7 +253,12 @@ def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: project_issue.repository_name, project_issue.number, ) - all_project_issues[key] = project_issue + + if key not in all_project_issues: + all_project_issues[key] = {} + all_project_issues[key][project.id] = project_issue.project_status + else: + all_project_issues[key][project.id] = project_issue.project_status logger.info( "Fetching GitHub project data - successfully fetched project data from `%s`.", project.title ) @@ -261,7 +267,7 @@ def _fetch_github_project_issues(self) -> dict[str, ProjectIssue]: @staticmethod def _consolidate_issues_data( - repository_issues: dict[str, list[Issue]], projects_issues: dict[str, ProjectIssue] + repository_issues: dict[str, list[Issue]], projects_issues: dict[str, dict[str, ProjectStatus]] ) -> dict[str, ConsolidatedIssue]: """ Consolidate the fetched issues and extra project data into a one consolidated object. @@ -286,7 +292,9 @@ def _consolidate_issues_data( logger.debug("Updating consolidated issue structure with project data.") for key, consolidated_issue in consolidated_issues.items(): if key in projects_issues: - consolidated_issue.update_with_project_data(projects_issues[key].project_status) + # Update the consolidated issue with project status data from all projects + for project_status in projects_issues[key].values(): + consolidated_issue.update_with_project_data(project_status) logging.info( "Issue and project data consolidation - consolidated `%s` repository issues" " with extra project data.", diff --git a/living_documentation_generator/model/project_status.py b/living_documentation_generator/model/project_status.py index edc89c4..35bf073 100644 --- a/living_documentation_generator/model/project_status.py +++ b/living_documentation_generator/model/project_status.py @@ -17,6 +17,7 @@ """ This module contains a data container for issue Project Status. """ +from typing import Union from living_documentation_generator.utils.constants import NO_PROJECT_DATA @@ -28,53 +29,63 @@ class ProjectStatus: """ def __init__(self): - self.__project_title: str = "" - self.__status: str = NO_PROJECT_DATA - self.__priority: str = NO_PROJECT_DATA - self.__size: str = NO_PROJECT_DATA - self.__moscow: str = NO_PROJECT_DATA + self.__project_title = [NO_PROJECT_DATA] + self.__status = [NO_PROJECT_DATA] + self.__priority = [NO_PROJECT_DATA] + self.__size = [NO_PROJECT_DATA] + self.__moscow = [NO_PROJECT_DATA] @property - def project_title(self) -> str: - """Getter of the issue attached project title.""" + def project_title(self) -> list: return self.__project_title @project_title.setter - def project_title(self, value: str): - self.__project_title = value + def project_title(self, value: Union[str, list]): + if self.__project_title == [NO_PROJECT_DATA]: + self.__project_title = [value] + else: + self.__project_title.append(value) @property - def status(self) -> str: - """Getter of the issue project status.""" + def status(self) -> list: return self.__status @status.setter - def status(self, value: str): - self.__status = value + def status(self, value: Union[str, list]): + if self.__status == [NO_PROJECT_DATA]: + self.__status = [value] + else: + self.__status.append(value) @property - def priority(self) -> str: - """Getter of the issue project priority.""" + def priority(self) -> list: return self.__priority @priority.setter - def priority(self, value: str): - self.__priority = value + def priority(self, value: Union[str, list]): + if self.__priority == [NO_PROJECT_DATA]: + self.__priority = [value] + else: + self.__priority.append(value) @property - def size(self) -> str: - """Getter of the issue project difficulty.""" + def size(self) -> list: return self.__size @size.setter - def size(self, value: str): - self.__size = value + def size(self, value: Union[str, list]): + if self.__size == [NO_PROJECT_DATA]: + self.__size = [value] + else: + self.__size.append(value) @property - def moscow(self) -> str: - """Getter of the issue project MoSCoW prioritization.""" + def moscow(self) -> list: return self.__moscow @moscow.setter - def moscow(self, value: str): - self.__moscow = value + def moscow(self, value: Union[str, list]): + if self.__moscow == [NO_PROJECT_DATA]: + self.__moscow = [value] + else: + self.__moscow.append(value) From 37440492a91c666148840dfe60a0f052d4b1dd7e Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:48:59 +0200 Subject: [PATCH 02/60] Logic that handles more projects attached to one repository issue. Pylint settings updated with not checking test files. --- living_documentation_generator/generator.py | 67 +++++++++++-------- .../model/consolidated_issue.py | 20 +++--- .../model/project_issue.py | 6 ++ .../model/project_status.py | 61 +++++++---------- pyproject.toml | 1 + 5 files changed, 78 insertions(+), 77 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 11515c6..cea02c8 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -33,7 +33,6 @@ from living_documentation_generator.model.config_repository import ConfigRepository from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue from living_documentation_generator.model.project_issue import ProjectIssue -from living_documentation_generator.model.project_status import ProjectStatus from living_documentation_generator.utils.decorators import safe_call_decorator from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter from living_documentation_generator.utils.utils import make_issue_key @@ -120,13 +119,15 @@ def generate(self) -> None: # Data mine GitHub project's issues logger.info("Fetching GitHub project data - started.") - project_issues: dict[str, dict[str, ProjectStatus]] = self._fetch_github_project_issues() + project_issues: dict[str, list[ProjectIssue]] = self._fetch_github_project_issues() # Note: got dict of project issues with unique string key defying the issue logger.info("Fetching GitHub project data - finished.") # Consolidate all issue data together logger.info("Issue and project data consolidation - started.") - consolidated_issues: dict[str, ConsolidatedIssue] = self._consolidate_issues_data(repository_issues, project_issues) + consolidated_issues: dict[str, ConsolidatedIssue] = self._consolidate_issues_data( + repository_issues, project_issues + ) logger.info("Issue and project data consolidation - finished.") # Generate markdown pages @@ -199,7 +200,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: ) return issues - def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: + def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: """ Fetch GitHub project issues using the GraphQL API. @@ -212,7 +213,7 @@ def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: logger.debug("Project data mining allowed.") # Mine project issues for every repository - all_project_issues: dict[str, dict[str, ProjectStatus]] = {} + all_project_issues: dict[str, list[ProjectIssue]] = {} for config_repository in self.repositories: repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" @@ -229,6 +230,10 @@ def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: repository=repository, projects_title_filter=projects_title_filter ) + print("PROJECTS FOR REPOSITORY") + for project in projects: + print(project.title) + if projects: logger.info( "Fetching GitHub project data - for repository `%s` found `%s` project/s.", @@ -254,11 +259,13 @@ def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: project_issue.number, ) + # If the key is unique, add the project data status to the repository issue if key not in all_project_issues: - all_project_issues[key] = {} - all_project_issues[key][project.id] = project_issue.project_status + all_project_issues[key] = [] + all_project_issues[key].append(project_issue) else: - all_project_issues[key][project.id] = project_issue.project_status + # If the repository issue is already present, add another project data from other projects + all_project_issues[key].append(project_issue) logger.info( "Fetching GitHub project data - successfully fetched project data from `%s`.", project.title ) @@ -267,13 +274,13 @@ def _fetch_github_project_issues(self) -> dict[str, dict[str, ProjectStatus]]: @staticmethod def _consolidate_issues_data( - repository_issues: dict[str, list[Issue]], projects_issues: dict[str, dict[str, ProjectStatus]] + repository_issues: dict[str, list[Issue]], project_issues: dict[str, list[ProjectIssue]] ) -> dict[str, ConsolidatedIssue]: """ Consolidate the fetched issues and extra project data into a one consolidated object. @param repository_issues: A dictionary containing repository issue objects with unique key. - @param projects_issues: A dictionary containing project issue objects with unique key. + @param project_issues: A dictionary containing project issue objects with unique key. @return: A dictionary containing all consolidated issues. """ @@ -291,10 +298,9 @@ def _consolidate_issues_data( # Update consolidated issue structures with project data logger.debug("Updating consolidated issue structure with project data.") for key, consolidated_issue in consolidated_issues.items(): - if key in projects_issues: - # Update the consolidated issue with project status data from all projects - for project_status in projects_issues[key].values(): - consolidated_issue.update_with_project_data(project_status) + if key in project_issues: + for project_issue in project_issues[key]: + consolidated_issue.update_with_project_data(project_issue.project_status) logging.info( "Issue and project data consolidation - consolidated `%s` repository issues" " with extra project data.", @@ -417,10 +423,12 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: title = consolidated_issue.title title = title.replace("|", " _ ") page_filename = consolidated_issue.generate_page_filename() - status = consolidated_issue.project_status.status url = consolidated_issue.html_url state = consolidated_issue.state + status_list = [project_status.status for project_status in consolidated_issue.project_issue_statuses] + status = ", ".join(status_list) + # Change the bool values to more user-friendly characters if self.__project_state_mining_enabled: if consolidated_issue.linked_to_project: @@ -485,28 +493,29 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - # Update the summary table, based on the project data mining situation if self.__project_state_mining_enabled: - project_status = consolidated_issue.project_status + project_statuses = consolidated_issue.project_issue_statuses if consolidated_issue.linked_to_project: - headers.extend( - [ - "Project title", - "Status", - "Priority", - "Size", - "MoSCoW", - ] - ) - - values.extend( - [ + project_data_header = [ + "Project title", + "Status", + "Priority", + "Size", + "MoSCoW", + ] + + for project_status in project_statuses: + # Update the summary data table for every project attached to repository issue + project_data_values = [ project_status.project_title, project_status.status, project_status.priority, project_status.size, project_status.moscow, ] - ) + + headers.extend(project_data_header) + values.extend(project_data_values) else: headers.append("Linked to project") linked_to_project = LINKED_TO_PROJECT_FALSE diff --git a/living_documentation_generator/model/consolidated_issue.py b/living_documentation_generator/model/consolidated_issue.py index e38b1db..c57a97f 100644 --- a/living_documentation_generator/model/consolidated_issue.py +++ b/living_documentation_generator/model/consolidated_issue.py @@ -44,7 +44,7 @@ def __init__(self, repository_id: str, repository_issue: Issue = None): # Extra project data (optionally provided from GithubProjects class) self.__linked_to_project: bool = False - self.__project_status: ProjectStatus = ProjectStatus() + self.__project_issue_statuses: list[ProjectStatus] = [] self.__error: Optional[str] = None @@ -113,9 +113,9 @@ def linked_to_project(self) -> bool: return self.__linked_to_project @property - def project_status(self) -> ProjectStatus: - """Getter of the project status.""" - return self.__project_status + def project_issue_statuses(self) -> list[ProjectStatus]: + """Getter of the project issue statuses.""" + return self.__project_issue_statuses # Error property @property @@ -123,19 +123,15 @@ def error(self) -> Optional[str]: """Getter of the error message.""" return self.__error - def update_with_project_data(self, project_status: ProjectStatus) -> None: + def update_with_project_data(self, project_issue_status: ProjectStatus) -> None: """ - Update the consolidated issue with attached project data. + Update the consolidated issue with Project Status data. - @param project_status: The extra issue project data. + @param project_issue_status: The extra issue project data per one project. @return: None """ self.__linked_to_project = True - self.__project_status.project_title = project_status.project_title - self.__project_status.status = project_status.status - self.__project_status.priority = project_status.priority - self.__project_status.size = project_status.size - self.__project_status.moscow = project_status.moscow + self.__project_issue_statuses.append(project_issue_status) def generate_page_filename(self) -> str: """ diff --git a/living_documentation_generator/model/project_issue.py b/living_documentation_generator/model/project_issue.py index fe2d260..a542946 100644 --- a/living_documentation_generator/model/project_issue.py +++ b/living_documentation_generator/model/project_issue.py @@ -37,6 +37,7 @@ def __init__(self): self.__number: int = 0 self.__organization_name: str = "" self.__repository_name: str = "" + self.__project_id: str = "" self.__project_status: ProjectStatus = ProjectStatus() @property @@ -54,6 +55,11 @@ def repository_name(self) -> str: """Getter of the repository name where the issue was fetched from.""" return self.__repository_name + @property + def project_id(self) -> str: + """Getter of the project ID where the issue belongs.""" + return self.__project_id + @property def project_status(self) -> ProjectStatus: """Getter of the project issue status.""" diff --git a/living_documentation_generator/model/project_status.py b/living_documentation_generator/model/project_status.py index 35bf073..64a0e8f 100644 --- a/living_documentation_generator/model/project_status.py +++ b/living_documentation_generator/model/project_status.py @@ -17,7 +17,6 @@ """ This module contains a data container for issue Project Status. """ -from typing import Union from living_documentation_generator.utils.constants import NO_PROJECT_DATA @@ -29,63 +28,53 @@ class ProjectStatus: """ def __init__(self): - self.__project_title = [NO_PROJECT_DATA] - self.__status = [NO_PROJECT_DATA] - self.__priority = [NO_PROJECT_DATA] - self.__size = [NO_PROJECT_DATA] - self.__moscow = [NO_PROJECT_DATA] + self.__project_title: str = NO_PROJECT_DATA + self.__status: str = NO_PROJECT_DATA + self.__priority: str = NO_PROJECT_DATA + self.__size: str = NO_PROJECT_DATA + self.__moscow: str = NO_PROJECT_DATA @property - def project_title(self) -> list: + def project_title(self) -> str: + """Getter of the issue attached project title.""" return self.__project_title @project_title.setter - def project_title(self, value: Union[str, list]): - if self.__project_title == [NO_PROJECT_DATA]: - self.__project_title = [value] - else: - self.__project_title.append(value) + def project_title(self, value: str): + self.__project_title = value @property - def status(self) -> list: + def status(self) -> str: + """Getter of the issue project status.""" return self.__status @status.setter - def status(self, value: Union[str, list]): - if self.__status == [NO_PROJECT_DATA]: - self.__status = [value] - else: - self.__status.append(value) + def status(self, value: str): + self.__status = value @property - def priority(self) -> list: + def priority(self) -> str: + """Getter of the issue project priority.""" return self.__priority @priority.setter - def priority(self, value: Union[str, list]): - if self.__priority == [NO_PROJECT_DATA]: - self.__priority = [value] - else: - self.__priority.append(value) + def priority(self, value: str): + self.__priority = value @property - def size(self) -> list: + def size(self) -> str: + """Getter of the issue project difficulty.""" return self.__size @size.setter - def size(self, value: Union[str, list]): - if self.__size == [NO_PROJECT_DATA]: - self.__size = [value] - else: - self.__size.append(value) + def size(self, value: str): + self.__size = value @property - def moscow(self) -> list: + def moscow(self) -> str: + """Getter of the issue project MoSCoW prioritization.""" return self.__moscow @moscow.setter - def moscow(self, value: Union[str, list]): - if self.__moscow == [NO_PROJECT_DATA]: - self.__moscow = [value] - else: - self.__moscow.append(value) + def moscow(self, value: str): + self.__moscow = value diff --git a/pyproject.toml b/pyproject.toml index 474d0af..68eb39c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,4 @@ [tool.black] line-length = 120 target-version = ['py311'] +force-exclude = '''test''' From 5bd1fe6f205f6fb7fc47d9aa629f87e16abf5c4c Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:00:40 +0200 Subject: [PATCH 03/60] Bug fix. --- living_documentation_generator/generator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index cea02c8..3808b05 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -230,10 +230,6 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: repository=repository, projects_title_filter=projects_title_filter ) - print("PROJECTS FOR REPOSITORY") - for project in projects: - print(project.title) - if projects: logger.info( "Fetching GitHub project data - for repository `%s` found `%s` project/s.", From c2e02ff81cf0361d90631188475c59dc75b00af0 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:04:47 +0200 Subject: [PATCH 04/60] Comments updated. --- living_documentation_generator/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 3808b05..71a9c35 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -255,12 +255,12 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: project_issue.number, ) - # If the key is unique, add the project data status to the repository issue + # If the key is unique, add the project issue to the dictionary if key not in all_project_issues: all_project_issues[key] = [] all_project_issues[key].append(project_issue) else: - # If the repository issue is already present, add another project data from other projects + # If the project issue key is already present, add another project data from other projects all_project_issues[key].append(project_issue) logger.info( "Fetching GitHub project data - successfully fetched project data from `%s`.", project.title From 1022b331161f6ce45385ad6edabca09001d9246f Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:05:56 +0200 Subject: [PATCH 05/60] Bug fix. --- living_documentation_generator/model/project_issue.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/living_documentation_generator/model/project_issue.py b/living_documentation_generator/model/project_issue.py index a542946..fe2d260 100644 --- a/living_documentation_generator/model/project_issue.py +++ b/living_documentation_generator/model/project_issue.py @@ -37,7 +37,6 @@ def __init__(self): self.__number: int = 0 self.__organization_name: str = "" self.__repository_name: str = "" - self.__project_id: str = "" self.__project_status: ProjectStatus = ProjectStatus() @property @@ -55,11 +54,6 @@ def repository_name(self) -> str: """Getter of the repository name where the issue was fetched from.""" return self.__repository_name - @property - def project_id(self) -> str: - """Getter of the project ID where the issue belongs.""" - return self.__project_id - @property def project_status(self) -> ProjectStatus: """Getter of the project issue status.""" From b161cc02bf9080450f5a74f06aaac733b1c6314d Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:14:41 +0200 Subject: [PATCH 06/60] Index page bug fix. --- living_documentation_generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 71a9c35..19a5eb6 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -423,7 +423,7 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: state = consolidated_issue.state status_list = [project_status.status for project_status in consolidated_issue.project_issue_statuses] - status = ", ".join(status_list) + status = ", ".join(status_list) if status_list else "---" # Change the bool values to more user-friendly characters if self.__project_state_mining_enabled: From 71ab3adf50fb84fb54c7cba8fe23f641f2ab8f36 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:44:57 +0200 Subject: [PATCH 07/60] New feature for having a structured output added. --- README.md | 35 +++++++- action.yml | 5 ++ .../action_inputs.py | 16 +++- living_documentation_generator/generator.py | 80 ++++++++++++++++--- .../model/consolidated_issue.py | 6 ++ .../utils/constants.py | 1 + living_documentation_generator/utils/utils.py | 2 +- main.py | 1 + 8 files changed, 131 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 301d19a..edec492 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ - [Data Mining from GitHub Repositories](#data-mining-from-github-repositories) - [Data Mining from GitHub Projects](#data-mining-from-github-projects) - [Living Documentation Page Generation](#living-documentation-page-generation) + - [Structured Output](#structured-output) - [Contribution Guidelines](#contribution-guidelines) - [License Information](#license-information) - [Contact or Support Information](#contact-or-support-information) @@ -107,6 +108,9 @@ See the full example of action step definition (in example are used non-default # project verbose (debug) logging feature de/activation verbose-logging: true + + # structured output feature de/activation + structured-output: true ``` ## Action Configuration @@ -203,6 +207,15 @@ Configure the action by customizing the following parameters based on your needs with: verbose-logging: true ``` + +- **structured-output** (optional, `default: false`) + - **Description**: Enables or disables structured output. + - **Usage**: Set to true to activate. + - **Example**: + ```yaml + with: + structured-output: true + ``` ## Action Outputs The Living Documentation Generator action provides a key output that allows users to locate and access the generated documentation easily. This output can be utilized in various ways within your CI/CD pipeline to ensure the documentation is effectively distributed and accessible. @@ -496,7 +509,27 @@ The goal is to provide a straightforward view of all issues in a single table, m - **Default Behavior**: By default, the action generates a single table that lists all issues from the defined repositories. ---- +### Structured Output + +This feature allows you to generate structured output for the living documentation. Enabling this feature allows you to see a summary `index.md` page for all fetched repository. + +- **Default Behavior**: By default, the action generates all the documentation in a single directory. +- **Non-default Example**: Use the structured output feature to organize the generated documentation by organization and repository name. + - `structured-output: true` activates the structured output feature. + ``` + output + |_ org 1 + |__repo 1 + |__ issue md page 1 + |__ issue md page 2 + |__ _index.md + |_ org 2 + |__repo 1 + |__ issue md page 1 + |__ _index.md + |__repo 2 + ``` + ## Contribution Guidelines We welcome contributions to the Living Documentation Generator! Whether you're fixing bugs, improving documentation, or proposing new features, your help is appreciated. diff --git a/action.yml b/action.yml index ece267b..e484f03 100644 --- a/action.yml +++ b/action.yml @@ -19,6 +19,10 @@ inputs: description: 'Path to the generated living documentation files.' required: false default: './output' + structured-output: + description: 'Enable or disable structured output.' + required: false + default: 'false' outputs: output-path: description: 'Path to the generated living documentation files' @@ -51,6 +55,7 @@ runs: INPUT_PROJECT_STATE_MINING: ${{ inputs.project-state-mining }} INPUT_VERBOSE_LOGGING: ${{ inputs.verbose-logging }} INPUT_OUTPUT_PATH: ${{ inputs.output-path }} + INPUT_STRUCTURED_OUTPUT: ${{ inputs.structured-output }} run: | python ${{ github.action_path }}/main.py diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 9634bc0..7ada813 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -25,7 +25,13 @@ from living_documentation_generator.model.config_repository import ConfigRepository from living_documentation_generator.utils.utils import get_action_input, make_absolute_path -from living_documentation_generator.utils.constants import GITHUB_TOKEN, PROJECT_STATE_MINING, REPOSITORIES, OUTPUT_PATH +from living_documentation_generator.utils.constants import ( + GITHUB_TOKEN, + PROJECT_STATE_MINING, + REPOSITORIES, + OUTPUT_PATH, + STRUCTURED_OUTPUT, +) logger = logging.getLogger(__name__) @@ -41,6 +47,7 @@ def __init__(self): self.__is_project_state_mining_enabled: bool = False self.__repositories: list[ConfigRepository] = [] self.__output_directory: str = "" + self.__structured_output: bool = False @property def github_token(self) -> str: @@ -62,6 +69,11 @@ def output_directory(self) -> str: """Getter of the output directory.""" return self.__output_directory + @property + def structured_output(self) -> bool: + """Getter of the structured output switch.""" + return self.__structured_output + def load_from_environment(self, validate: bool = True) -> "ActionInputs": """ Load the action inputs from the environment variables and validate them if needed. @@ -71,6 +83,7 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": """ self.__github_token = get_action_input(GITHUB_TOKEN) self.__is_project_state_mining_enabled = get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" + self.__structured_output = get_action_input(STRUCTURED_OUTPUT, "false").lower() == "true" out_path = get_action_input(OUTPUT_PATH, "./output") self.__output_directory = make_absolute_path(out_path) repositories_json = get_action_input(REPOSITORIES, "") @@ -78,6 +91,7 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": logger.debug("Is project state mining allowed: %s.", self.is_project_state_mining_enabled) logger.debug("JSON repositories to fetch from: %s.", repositories_json) logger.debug("Output directory: %s.", self.output_directory) + logger.debug("Is output directory structured: %s.", self.structured_output) # Validate inputs if validate: diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 19a5eb6..f8a6c67 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -23,7 +23,7 @@ import shutil from datetime import datetime -from typing import Callable +from typing import Callable, Optional from github import Github, Auth from github.Issue import Issue @@ -65,6 +65,7 @@ def __init__( github_token: str, repositories: list[ConfigRepository], project_state_mining_enabled: bool, + structured_output: bool, output_path: str, ): @@ -78,6 +79,7 @@ def __init__( # features control self.__project_state_mining_enabled: bool = project_state_mining_enabled + self.__structured_output: bool = structured_output # paths self.__output_path: str = output_path @@ -97,6 +99,11 @@ def project_state_mining_enabled(self) -> bool: """Getter of the project state mining switch.""" return self.__project_state_mining_enabled + @property + def structured_output(self) -> bool: + """Getter of the structured output switch.""" + return self.__structured_output + @property def output_path(self) -> str: """Getter of the output directory.""" @@ -257,8 +264,7 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: # If the key is unique, add the project issue to the dictionary if key not in all_project_issues: - all_project_issues[key] = [] - all_project_issues[key].append(project_issue) + all_project_issues[key] = [project_issue] else: # If the project issue key is already present, add another project data from other projects all_project_issues[key].append(project_issue) @@ -299,7 +305,7 @@ def _consolidate_issues_data( consolidated_issue.update_with_project_data(project_issue.project_status) logging.info( - "Issue and project data consolidation - consolidated `%s` repository issues" " with extra project data.", + "Issue and project data consolidation - consolidated `%s` repository issues with extra project data.", len(consolidated_issues), ) return consolidated_issues @@ -333,7 +339,12 @@ def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None logger.info("Markdown page generation - generated `%s` issue pages.", len(issues)) # Generate an index page with a summary table about all issues - self._generate_index_page(issue_index_page_template, issues) + if self.structured_output: + self._generate_structured_index_page(issue_index_page_template, issues) + else: + issues = list(issues.values()) + self._generate_index_page(issue_index_page_template, issues) + logger.info("Markdown page generation - generated `_index.md`") def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: ConsolidatedIssue) -> None: """ @@ -364,31 +375,57 @@ def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: # Run through all replacements and update template keys with adequate content issue_md_page_content = issue_page_template.format(**replacements) + # Create a directory structure path for the issue page + page_directory_path = self._generate_directory_path(consolidated_issue.repository_id) + # Save the single issue Markdown page page_filename = consolidated_issue.generate_page_filename() - with open(os.path.join(self.output_path, page_filename), "w", encoding="utf-8") as f: + with open(os.path.join(page_directory_path, page_filename), "w", encoding="utf-8") as f: f.write(issue_md_page_content) logger.debug("Generated Markdown page: %s.", page_filename) - def _generate_index_page( + def _generate_structured_index_page( self, issue_index_page_template: str, consolidated_issues: dict[str, ConsolidatedIssue] ) -> None: """ - Generates an index page with a summary of all issues and save it to the output directory. + Generates a structured index page with a summary of one repository issues. @param issue_index_page_template: The template string for generating the index markdown page. @param consolidated_issues: A dictionary containing all consolidated issues. @return: None """ + # Group issues by repository for structured index page content + issues_by_repository = {} + for consolidated_issue in consolidated_issues.values(): + repository_id = consolidated_issue.repository_id + if repository_id not in issues_by_repository: + issues_by_repository[repository_id] = [] + issues_by_repository[repository_id].append(consolidated_issue) + + # Generate an index page for each repository + for repository_id, issues in issues_by_repository.items(): + self._generate_index_page(issue_index_page_template, issues, repository_id) + logger.info("Markdown page generation - generated `_index.md` for %s.", repository_id) + + def _generate_index_page( + self, issue_index_page_template: str, consolidated_issues: list[ConsolidatedIssue], repository_id: str = None + ) -> None: + """ + Generates an index page with a summary of all issues and save it to the output directory. + @param issue_index_page_template: The template string for generating the index markdown page. + @param consolidated_issues: A dictionary containing all consolidated issues. + @param repository_id: The repository id used if the structured output is generated. + @return: None + """ # Initializing the issue table header based on the project mining state issue_table = ( TABLE_HEADER_WITH_PROJECT_DATA if self.__project_state_mining_enabled else TABLE_HEADER_WITHOUT_PROJECT_DATA ) # Create an issue summary table for every issue - for consolidated_issue in consolidated_issues.values(): + for consolidated_issue in consolidated_issues: issue_table += self._generate_markdown_line(consolidated_issue) # Prepare issues replacement for the index page @@ -400,12 +437,14 @@ def _generate_index_page( # Replace the issue placeholders in the index template index_page = issue_index_page_template.format(**replacement) + # Generate a directory structure path for the index page + # Note: repository_id is used only, if the structured output is generated + index_directory_path = self._generate_directory_path(repository_id) + # Create an index page file - with open(os.path.join(self.output_path, "_index.md"), "w", encoding="utf-8") as f: + with open(os.path.join(index_directory_path, "_index.md"), "w", encoding="utf-8") as f: f.write(index_page) - logger.info("Markdown page generation - generated `_index.md`.") - def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: """ Generates a markdown summary line for a single issue. @@ -525,3 +564,20 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - issue_info += f"| {attribute} | {content} |\n" return issue_info + + def _generate_directory_path(self, repository_id: Optional[str]) -> str: + """ + Generate a directory path based on the repository id. + + @param repository_id: The repository id. + @return: The generated directory path. + """ + if self.structured_output and repository_id: + organization_name, repository_name = repository_id.split("/") + output_path = os.path.join(self.output_path, organization_name, repository_name) + else: + output_path = self.output_path + + os.makedirs(output_path, exist_ok=True) + + return output_path diff --git a/living_documentation_generator/model/consolidated_issue.py b/living_documentation_generator/model/consolidated_issue.py index c57a97f..8fd73f2 100644 --- a/living_documentation_generator/model/consolidated_issue.py +++ b/living_documentation_generator/model/consolidated_issue.py @@ -38,6 +38,7 @@ def __init__(self, repository_id: str, repository_issue: Issue = None): # Warning: several issue properties requires additional API calls - use wisely to keep low API usage self.__issue: Issue = repository_issue + self.__repository_id: str = repository_id parts = repository_id.split("/") self.__organization_name: str = parts[0] if len(parts) == 2 else "" self.__repository_name: str = parts[1] if len(parts) == 2 else "" @@ -54,6 +55,11 @@ def number(self) -> int: """Getter of the issue number.""" return self.__issue.number if self.__issue else 0 + @property + def repository_id(self) -> str: + """Getter of the repository id.""" + return self.__repository_id + @property def organization_name(self) -> str: """Getter of the organization where the issue was fetched from.""" diff --git a/living_documentation_generator/utils/constants.py b/living_documentation_generator/utils/constants.py index 1775c18..4b8e865 100644 --- a/living_documentation_generator/utils/constants.py +++ b/living_documentation_generator/utils/constants.py @@ -23,6 +23,7 @@ PROJECT_STATE_MINING = "PROJECT_STATE_MINING" REPOSITORIES = "REPOSITORIES" OUTPUT_PATH = "OUTPUT_PATH" +STRUCTURED_OUTPUT = "STRUCTURED_OUTPUT" # GitHub API constants ISSUES_PER_PAGE_LIMIT = 100 diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index f9181b9..52cf5b2 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -96,7 +96,7 @@ def set_action_output(name: str, value: str, default_output_path: str = "default @return: None """ output_file = os.getenv("GITHUB_OUTPUT", default_output_path) - with open(output_file, "a") as f: + with open(output_file, "a", encoding="utf-8") as f: f.write(f"{name}={value}\n") diff --git a/main.py b/main.py index 98d850e..54e0993 100644 --- a/main.py +++ b/main.py @@ -44,6 +44,7 @@ def run() -> None: repositories=action_inputs.repositories, project_state_mining_enabled=action_inputs.is_project_state_mining_enabled, output_path=action_inputs.output_directory, + structured_output=action_inputs.structured_output, ) # Generate the Living Documentation From ea9ac2feb14a0954272c1520665c7c605cbc0285 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:52:46 +0200 Subject: [PATCH 08/60] Typo fix. --- living_documentation_generator/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index f8a6c67..4f9d17e 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -567,12 +567,12 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - def _generate_directory_path(self, repository_id: Optional[str]) -> str: """ - Generate a directory path based on the repository id. + Generates a directory path based on if structured output is required. @param repository_id: The repository id. @return: The generated directory path. """ - if self.structured_output and repository_id: + if self.structured_output: organization_name, repository_name = repository_id.split("/") output_path = os.path.join(self.output_path, organization_name, repository_name) else: From 9a038a3efebc09b31077ce03bc352e82f32e2728 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:03:21 +0200 Subject: [PATCH 09/60] README.md comments implemented. --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index edec492..2188594 100644 --- a/README.md +++ b/README.md @@ -511,25 +511,25 @@ The goal is to provide a straightforward view of all issues in a single table, m ### Structured Output -This feature allows you to generate structured output for the living documentation. Enabling this feature allows you to see a summary `index.md` page for all fetched repository. +This feature allows you to generate structured output for the living documentation and see a summary `index.md` page for each fetched repository. - **Default Behavior**: By default, the action generates all the documentation in a single directory. - **Non-default Example**: Use the structured output feature to organize the generated documentation by organization and repository name. - `structured-output: true` activates the structured output feature. ``` output - |_ org 1 - |__repo 1 - |__ issue md page 1 - |__ issue md page 2 - |__ _index.md - |_ org 2 - |__repo 1 - |__ issue md page 1 - |__ _index.md - |__repo 2 + |- org 1 + |--repo 1 + |-- issue md page 1 + |-- issue md page 2 + |-- _index.md + |- org 2 + |--repo 1 + |-- issue md page 1 + |-- _index.md + |--repo 2 ``` - +--- ## Contribution Guidelines We welcome contributions to the Living Documentation Generator! Whether you're fixing bugs, improving documentation, or proposing new features, your help is appreciated. From 300fea50d6d276af3f5fe7173e4ad4d66c12e40a Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:16:47 +0200 Subject: [PATCH 10/60] ActionInput comment implemented. --- living_documentation_generator/generator.py | 22 ++++++++------------- main.py | 9 ++------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 4f9d17e..f701ba7 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -28,6 +28,7 @@ from github import Github, Auth from github.Issue import Issue +from living_documentation_generator.action_inputs import ActionInputs from living_documentation_generator.github_projects import GithubProjects from living_documentation_generator.model.github_project import GithubProject from living_documentation_generator.model.config_repository import ConfigRepository @@ -60,29 +61,22 @@ class LivingDocumentationGenerator: ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "issue_detail_page_template.md") INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_template.md") - def __init__( - self, - github_token: str, - repositories: list[ConfigRepository], - project_state_mining_enabled: bool, - structured_output: bool, - output_path: str, - ): - + def __init__(self, action_inputs: ActionInputs): + github_token = action_inputs.github_token self.__github_instance: Github = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) self.__github_projects_instance: GithubProjects = GithubProjects(token=github_token) self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) self.__safe_call: Callable = safe_call_decorator(self.__rate_limiter) # data - self.__repositories: list[ConfigRepository] = repositories + self.__repositories: list[ConfigRepository] = action_inputs.repositories # features control - self.__project_state_mining_enabled: bool = project_state_mining_enabled - self.__structured_output: bool = structured_output + self.__project_state_mining_enabled: bool = action_inputs.is_project_state_mining_enabled + self.__structured_output: bool = action_inputs.structured_output # paths - self.__output_path: str = output_path + self.__output_path: str = action_inputs.output_directory @property def github_instance(self) -> Github: @@ -572,7 +566,7 @@ def _generate_directory_path(self, repository_id: Optional[str]) -> str: @param repository_id: The repository id. @return: The generated directory path. """ - if self.structured_output: + if self.structured_output and repository_id: organization_name, repository_name = repository_id.split("/") output_path = os.path.join(self.output_path, organization_name, repository_name) else: diff --git a/main.py b/main.py index 54e0993..d958413 100644 --- a/main.py +++ b/main.py @@ -39,13 +39,8 @@ def run() -> None: logger.info("Starting Living Documentation generation.") action_inputs = ActionInputs().load_from_environment() - generator = LivingDocumentationGenerator( - github_token=action_inputs.github_token, - repositories=action_inputs.repositories, - project_state_mining_enabled=action_inputs.is_project_state_mining_enabled, - output_path=action_inputs.output_directory, - structured_output=action_inputs.structured_output, - ) + # Create the Living Documentation Generator + generator = LivingDocumentationGenerator(action_inputs=action_inputs) # Generate the Living Documentation generator.generate() From 2f57dd5128625b472d3889280f63adab2132230a Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:42:12 +0200 Subject: [PATCH 11/60] ActionInput new structure implemented. --- living_documentation_generator/__init__.py | 15 ++++++ living_documentation_generator/generator.py | 52 ++++++++++--------- .../model/__init__.py | 15 ++++++ .../utils/__init__.py | 15 ++++++ 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/living_documentation_generator/__init__.py b/living_documentation_generator/__init__.py index e69de29..6f9c372 100644 --- a/living_documentation_generator/__init__.py +++ b/living_documentation_generator/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index f701ba7..fef4397 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -62,22 +62,14 @@ class LivingDocumentationGenerator: INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_template.md") def __init__(self, action_inputs: ActionInputs): - github_token = action_inputs.github_token + self.__action_inputs = action_inputs + + github_token = self.__action_inputs.github_token self.__github_instance: Github = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) self.__github_projects_instance: GithubProjects = GithubProjects(token=github_token) self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) self.__safe_call: Callable = safe_call_decorator(self.__rate_limiter) - # data - self.__repositories: list[ConfigRepository] = action_inputs.repositories - - # features control - self.__project_state_mining_enabled: bool = action_inputs.is_project_state_mining_enabled - self.__structured_output: bool = action_inputs.structured_output - - # paths - self.__output_path: str = action_inputs.output_directory - @property def github_instance(self) -> Github: """Getter of the GitHub instance.""" @@ -86,22 +78,32 @@ def github_instance(self) -> Github: @property def repositories(self) -> list[ConfigRepository]: """Getter of the list of config repository objects to fetch from.""" - return self.__repositories + return self.__action_inputs.repositories @property def project_state_mining_enabled(self) -> bool: """Getter of the project state mining switch.""" - return self.__project_state_mining_enabled + return self.__action_inputs.is_project_state_mining_enabled @property def structured_output(self) -> bool: """Getter of the structured output switch.""" - return self.__structured_output + return self.__action_inputs.structured_output @property def output_path(self) -> str: """Getter of the output directory.""" - return self.__output_path + return self.__action_inputs.output_directory + + @property + def github_projects_instance(self) -> GithubProjects: + """Getter of the GitHub projects instance.""" + return self.__github_projects_instance + + @property + def safe_call(self) -> Callable: + """Getter of the safe call decorator.""" + return self.__safe_call def generate(self) -> None: """ @@ -160,7 +162,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: for config_repository in self.repositories: repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" - repository = self.__safe_call(self.github_instance.get_repo)(repository_id) + repository = self.safe_call(self.github_instance.get_repo)(repository_id) if repository is None: return {} @@ -169,7 +171,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: # If the query labels are not defined, fetch all issues from the repository if not config_repository.query_labels: logger.debug("Fetching all issues in the repository") - issues[repository_id] = self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) + issues[repository_id] = self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) amount_of_issues_per_repo = len(list(issues[repository_id])) logger.debug( "Fetched `%s` repository issues (%s)`.", @@ -183,7 +185,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: for label in config_repository.query_labels: logger.debug("Fetching issues with label `%s`.", label) issues[repository_id].extend( - self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, labels=[label]) + self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, labels=[label]) ) amount_of_issues_per_repo = len(issues[repository_id]) @@ -221,13 +223,13 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: projects_title_filter = config_repository.projects_title_filter logger.debug("Filtering projects: %s. If filter is empty, fetching all.", projects_title_filter) - repository = self.__safe_call(self.github_instance.get_repo)(repository_id) + repository = self.safe_call(self.github_instance.get_repo)(repository_id) if repository is None: return {} # Fetch all projects_buffer attached to the repository logger.debug("Fetching GitHub project data - looking for repository `%s` projects.", repository_id) - projects: list[GithubProject] = self.__safe_call(self.__github_projects_instance.get_repository_projects)( + projects: list[GithubProject] = self.safe_call(self.github_projects_instance.get_repository_projects)( repository=repository, projects_title_filter=projects_title_filter ) @@ -245,8 +247,8 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: # Update every project with project issue related data for project in projects: logger.info("Fetching GitHub project data - fetching project data from `%s`.", project.title) - project_issues: list[ProjectIssue] = self.__safe_call( - self.__github_projects_instance.get_project_issues + project_issues: list[ProjectIssue] = self.safe_call( + self.github_projects_instance.get_project_issues )(project=project) for project_issue in project_issues: @@ -415,7 +417,7 @@ def _generate_index_page( """ # Initializing the issue table header based on the project mining state issue_table = ( - TABLE_HEADER_WITH_PROJECT_DATA if self.__project_state_mining_enabled else TABLE_HEADER_WITHOUT_PROJECT_DATA + TABLE_HEADER_WITH_PROJECT_DATA if self.project_state_mining_enabled else TABLE_HEADER_WITHOUT_PROJECT_DATA ) # Create an issue summary table for every issue @@ -459,7 +461,7 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: status = ", ".join(status_list) if status_list else "---" # Change the bool values to more user-friendly characters - if self.__project_state_mining_enabled: + if self.project_state_mining_enabled: if consolidated_issue.linked_to_project: linked_to_project = LINKED_TO_PROJECT_TRUE else: @@ -521,7 +523,7 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - ] # Update the summary table, based on the project data mining situation - if self.__project_state_mining_enabled: + if self.project_state_mining_enabled: project_statuses = consolidated_issue.project_issue_statuses if consolidated_issue.linked_to_project: diff --git a/living_documentation_generator/model/__init__.py b/living_documentation_generator/model/__init__.py index e69de29..6f9c372 100644 --- a/living_documentation_generator/model/__init__.py +++ b/living_documentation_generator/model/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/living_documentation_generator/utils/__init__.py b/living_documentation_generator/utils/__init__.py index e69de29..6f9c372 100644 --- a/living_documentation_generator/utils/__init__.py +++ b/living_documentation_generator/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From f1ca7ac917d39492a2003f364b46932a2a60cd09 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:49:02 +0200 Subject: [PATCH 12/60] Black tool check fix. --- living_documentation_generator/generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index fef4397..0a90d9a 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -247,9 +247,9 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: # Update every project with project issue related data for project in projects: logger.info("Fetching GitHub project data - fetching project data from `%s`.", project.title) - project_issues: list[ProjectIssue] = self.safe_call( - self.github_projects_instance.get_project_issues - )(project=project) + project_issues: list[ProjectIssue] = self.safe_call(self.github_projects_instance.get_project_issues)( + project=project + ) for project_issue in project_issues: key = make_issue_key( From 0d7be2743ed42800deb71da0793a64636f075e85 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:02:42 +0200 Subject: [PATCH 13/60] Private var implemented. --- living_documentation_generator/generator.py | 31 ++++++--------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 0a90d9a..3913d82 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -70,11 +70,6 @@ def __init__(self, action_inputs: ActionInputs): self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) self.__safe_call: Callable = safe_call_decorator(self.__rate_limiter) - @property - def github_instance(self) -> Github: - """Getter of the GitHub instance.""" - return self.__github_instance - @property def repositories(self) -> list[ConfigRepository]: """Getter of the list of config repository objects to fetch from.""" @@ -95,16 +90,6 @@ def output_path(self) -> str: """Getter of the output directory.""" return self.__action_inputs.output_directory - @property - def github_projects_instance(self) -> GithubProjects: - """Getter of the GitHub projects instance.""" - return self.__github_projects_instance - - @property - def safe_call(self) -> Callable: - """Getter of the safe call decorator.""" - return self.__safe_call - def generate(self) -> None: """ Generate the Living Documentation markdown pages output. @@ -162,7 +147,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: for config_repository in self.repositories: repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" - repository = self.safe_call(self.github_instance.get_repo)(repository_id) + repository = self.__safe_call(self.__github_instance.get_repo)(repository_id) if repository is None: return {} @@ -171,7 +156,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: # If the query labels are not defined, fetch all issues from the repository if not config_repository.query_labels: logger.debug("Fetching all issues in the repository") - issues[repository_id] = self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) + issues[repository_id] = self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL) amount_of_issues_per_repo = len(list(issues[repository_id])) logger.debug( "Fetched `%s` repository issues (%s)`.", @@ -185,7 +170,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: for label in config_repository.query_labels: logger.debug("Fetching issues with label `%s`.", label) issues[repository_id].extend( - self.safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, labels=[label]) + self.__safe_call(repository.get_issues)(state=ISSUE_STATE_ALL, labels=[label]) ) amount_of_issues_per_repo = len(issues[repository_id]) @@ -223,13 +208,13 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: projects_title_filter = config_repository.projects_title_filter logger.debug("Filtering projects: %s. If filter is empty, fetching all.", projects_title_filter) - repository = self.safe_call(self.github_instance.get_repo)(repository_id) + repository = self.__safe_call(self.__github_instance.get_repo)(repository_id) if repository is None: return {} # Fetch all projects_buffer attached to the repository logger.debug("Fetching GitHub project data - looking for repository `%s` projects.", repository_id) - projects: list[GithubProject] = self.safe_call(self.github_projects_instance.get_repository_projects)( + projects: list[GithubProject] = self.__safe_call(self.__github_projects_instance.get_repository_projects)( repository=repository, projects_title_filter=projects_title_filter ) @@ -247,9 +232,9 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: # Update every project with project issue related data for project in projects: logger.info("Fetching GitHub project data - fetching project data from `%s`.", project.title) - project_issues: list[ProjectIssue] = self.safe_call(self.github_projects_instance.get_project_issues)( - project=project - ) + project_issues: list[ProjectIssue] = self.__safe_call( + self.__github_projects_instance.get_project_issues + )(project=project) for project_issue in project_issues: key = make_issue_key( From 27a34c08e73a675aeb54a4b5cae84236540de679 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:48:30 +0200 Subject: [PATCH 14/60] Unittest using Pytest for utils folder. --- .../utils/decorators.py | 15 +- .../utils/logging_config.py | 8 ++ living_documentation_generator/utils/utils.py | 7 +- pyproject.toml | 20 ++- tests/conftest.py | 51 +++++++ tests/test_hello.py | 16 --- tests/utils/__init__.py | 15 ++ tests/utils/test_decorators.py | 69 +++++++++ tests/utils/test_github_project_queries.py | 68 +++++++++ tests/utils/test_github_rate_limiter.py | 64 +++++++++ tests/utils/test_logging_config.py | 71 ++++++++++ tests/utils/test_utils.py | 134 ++++++++++++++++++ 12 files changed, 516 insertions(+), 22 deletions(-) create mode 100644 tests/conftest.py delete mode 100644 tests/test_hello.py create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/test_decorators.py create mode 100644 tests/utils/test_github_project_queries.py create mode 100644 tests/utils/test_github_rate_limiter.py create mode 100644 tests/utils/test_logging_config.py create mode 100644 tests/utils/test_utils.py diff --git a/living_documentation_generator/utils/decorators.py b/living_documentation_generator/utils/decorators.py index 44939ce..a8fadaa 100644 --- a/living_documentation_generator/utils/decorators.py +++ b/living_documentation_generator/utils/decorators.py @@ -23,6 +23,8 @@ from typing import Callable, Optional, Any from functools import wraps +from github import GithubException +from requests.exceptions import Timeout, ConnectionError, RequestException from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter @@ -63,8 +65,17 @@ def decorator(method: Callable) -> Callable: def wrapped(*args, **kwargs) -> Optional[Any]: try: return method(*args, **kwargs) - except (ValueError, TypeError) as e: - logger.error("Error calling %s: %s", method.__name__, e, exc_info=True) + except (ConnectionError, Timeout) as e: + logger.error("Network error calling %s: %s.", method.__name__, e, exc_info=True) + return None + except GithubException as e: + logger.error("GitHub API error calling %s: %s.", method.__name__, e, exc_info=True) + return None + except RequestException as e: + logger.error("HTTP error calling %s: %s.", method.__name__, e, exc_info=True) + return None + except Exception as e: + logger.error("Unexpected error calling %s: %s.", method.__name__, e, exc_info=True) return None return wrapped diff --git a/living_documentation_generator/utils/logging_config.py b/living_documentation_generator/utils/logging_config.py index e066bea..bb2f9d1 100644 --- a/living_documentation_generator/utils/logging_config.py +++ b/living_documentation_generator/utils/logging_config.py @@ -42,3 +42,11 @@ def setup_logging() -> None: handlers=[logging.StreamHandler(sys.stdout)], ) sys.stdout.flush() + + # TODO + logging.info("Logging configuration set up.") + + if is_verbose_logging: + logging.debug("Verbose logging enabled.") + if is_debug_mode: + logging.debug("Debug mode enabled by CI runner.") diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 52cf5b2..f4f633e 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -22,6 +22,7 @@ import re import sys import logging +from typing import Optional logger = logging.getLogger(__name__) @@ -72,7 +73,7 @@ def make_absolute_path(path: str) -> str: # Github -def get_action_input(name: str, default: str = None) -> str: +def get_action_input(name: str, default: Optional[str] = None) -> str: """ Get the input value from the environment variables. @@ -80,7 +81,7 @@ def get_action_input(name: str, default: str = None) -> str: @param default: The default value to return if the environment variable is not set. @return: The value of the specified input parameter, or an empty string """ - return os.getenv(f"INPUT_{name}", default) + return os.getenv(f'INPUT_{name.replace("-", "_").upper()}', default=default) def set_action_output(name: str, value: str, default_output_path: str = "default_output.txt") -> None: @@ -107,5 +108,5 @@ def set_action_failed(message: str) -> None: @param message: The error message to display. @return: None """ - logger.error("::error:: %s", message) + print(f"::error::{message}") sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml index 68eb39c..34ce511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,22 @@ [tool.black] line-length = 120 target-version = ['py311'] -force-exclude = '''test''' +#force-exclude = '''test''' + +[tool.pytest.ini_options] +markers = [ + "safe_call_decorator: group of tests that test the safe call decorator", + "decorators: group of tests that test the decorators", + "debug_log_decorator: group of tests that test the debug log decorator", + "utils: the whole folder of tests that are utils modules", + "github_rate_limiter: group of tests that test the github rate limiter", + "github_project_queries: group of tests that test the github project queries", + "setup_logging: group of tests that test the logging config", + "make_issue_key: group of tests that test the make issue key function", + "sanitize_filename: group of tests that test the sanitize filename function", + "make_absolute_path: group of tests that test the make absolute path function", + "github_utils: group of tests that test the github utils", + "get_action_input: group of tests that test the get action input function", + "set_action_output: group of tests that test the set action output function", + "set_action_failed: group of tests that test the set action failed function", +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..189fe7b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,51 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import time + +import pytest +from github import Github +from github.Rate import Rate +from github.RateLimit import RateLimit + +from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter + + +@pytest.fixture +def rate_limiter(mocker, request): + mock_github_client = mocker.Mock(spec=Github) + mock_github_client.get_rate_limit.return_value = request.getfixturevalue("mock_rate_limiter") + return GithubRateLimiter(mock_github_client) + + +@pytest.fixture +def mock_rate_limiter(mocker): + mock_rate = mocker.Mock(spec=Rate) + mock_rate.timestamp = mocker.Mock(return_value=time.time() + 3600) + + mock_core = mocker.Mock(spec=RateLimit) + mock_core.reset = mock_rate + + mock = mocker.Mock(spec=GithubRateLimiter) + mock.core = mock_core + mock.core.remaining = 10 + + return mock + + +@pytest.fixture +def mock_logging_setup(mocker): + mock_log_config = mocker.patch("logging.basicConfig") + yield mock_log_config diff --git a/tests/test_hello.py b/tests/test_hello.py deleted file mode 100644 index a52abe8..0000000 --- a/tests/test_hello.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest -import sys - -sys.path.append("living_documentation_generator") # Adjust path to include the directory where hello.py is located - - -class TestHelloWorld(unittest.TestCase): - def test_hello_world(self): - """Test the output of the hello_world function.""" - expected = "Hello, World!" - result = "Hello, World!" - self.assertEqual(result, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..6f9c372 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py new file mode 100644 index 0000000..2ca332b --- /dev/null +++ b/tests/utils/test_decorators.py @@ -0,0 +1,69 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import pytest + +from living_documentation_generator.utils.decorators import debug_log_decorator, safe_call_decorator + +pytestmark = [pytest.mark.decorators, pytest.mark.utils] + + +# sample function to be decorated +def sample_function(x, y): + return x + y + + +@pytest.mark.debug_log_decorator +def test_debug_log_decorator(mocker): + # Mock logging + mock_log_debug = mocker.patch("living_documentation_generator.utils.decorators.logger.debug") + + decorated_function = debug_log_decorator(sample_function) + expected_call = [ + mocker.call("Calling method %s with args: %s and kwargs: %s.", "sample_function", (3, 4), {}), + mocker.call("Method %s returned %s.", "sample_function", 7), + ] + + actual = decorated_function(3, 4) + + assert actual == 7 + assert mock_log_debug.call_args_list == expected_call + + +@pytest.mark.safe_call_decorator +def test_safe_call_decorator_success(rate_limiter): + @safe_call_decorator(rate_limiter) + def sample_method(x, y): + return x + y + + actual = sample_method(2, 3) + + assert actual == 5 + + +@pytest.mark.safe_call_decorator +def test_safe_call_decorator_exception(rate_limiter, mocker): + mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") + + @safe_call_decorator(rate_limiter) + def sample_method(x, y): + return x / y + + actual = sample_method(2, 0) + + assert actual is None + mock_log_error.assert_called_once() + assert "Unexpected error calling %s:" in mock_log_error.call_args[0][0] + assert "sample_method" in mock_log_error.call_args[0][1] diff --git a/tests/utils/test_github_project_queries.py b/tests/utils/test_github_project_queries.py new file mode 100644 index 0000000..fa223d9 --- /dev/null +++ b/tests/utils/test_github_project_queries.py @@ -0,0 +1,68 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from living_documentation_generator.utils.constants import ( + PROJECTS_FROM_REPO_QUERY, + ISSUES_FROM_PROJECT_QUERY, + PROJECT_FIELD_OPTIONS_QUERY, + ISSUES_PER_PAGE_LIMIT, +) +from living_documentation_generator.utils.github_project_queries import ( + get_projects_from_repo_query, + get_issues_from_project_query, + get_project_field_options_query, +) + +pytestmark = [pytest.mark.github_project_queries, pytest.mark.utils] + + +def test_get_projects_from_repo_query(): + organization_name = "test_org" + repository_name = "test_repo" + expected_query = PROJECTS_FROM_REPO_QUERY.format( + organization_name=organization_name, repository_name=repository_name + ) + + actual_query = get_projects_from_repo_query(organization_name, repository_name) + + assert actual_query == expected_query + + +def test_get_issues_from_project_query(): + project_id = "test_project_id" + after_argument = "test_after_argument" + expected_query = ISSUES_FROM_PROJECT_QUERY.format( + project_id=project_id, issues_per_page=ISSUES_PER_PAGE_LIMIT, after_argument=after_argument + ) + + actual_query = get_issues_from_project_query(project_id, after_argument) + + assert actual_query == expected_query + + +def test_get_project_field_options_query(): + organization_name = "test_org" + repository_name = "test_repo" + project_number = 1 + expected_query = PROJECT_FIELD_OPTIONS_QUERY.format( + organization_name=organization_name, repository_name=repository_name, project_number=project_number + ) + + actual_query = get_project_field_options_query(organization_name, repository_name, project_number) + + assert actual_query == expected_query diff --git a/tests/utils/test_github_rate_limiter.py b/tests/utils/test_github_rate_limiter.py new file mode 100644 index 0000000..8330cff --- /dev/null +++ b/tests/utils/test_github_rate_limiter.py @@ -0,0 +1,64 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import time +import pytest + +pytestmark = [pytest.mark.github_rate_limiter, pytest.mark.utils] + + +def test_rate_limiter_extended_sleep_remaining_1(mocker, rate_limiter, mock_rate_limiter): + # Patch time.sleep to avoid actual delay and track call count + mock_sleep = mocker.patch("time.sleep", return_value=None) + mock_rate_limiter.core.remaining = 1 + + # Mock method to be wrapped + method_mock = mocker.Mock() + wrapped_method = rate_limiter(method_mock) + + wrapped_method() + + method_mock.assert_called_once() + mock_sleep.assert_called_once() + + +def test_rate_limiter_extended_sleep_remaining_10(mocker, rate_limiter): + # Patch time.sleep to avoid actual delay and track call count + mock_sleep = mocker.patch("time.sleep", return_value=None) + + # Mock method to be wrapped + method_mock = mocker.Mock() + wrapped_method = rate_limiter(method_mock) + + wrapped_method() + + method_mock.assert_called_once() + mock_sleep.assert_not_called() + + +def test_rate_limiter_extended_sleep_remaining_1_negative_reset_time(mocker, rate_limiter, mock_rate_limiter): + # Patch time.sleep to avoid actual delay and track call count + mock_sleep = mocker.patch("time.sleep", return_value=None) + mock_rate_limiter.core.remaining = 1 + mock_rate_limiter.core.reset.timestamp = mocker.Mock(return_value=time.time() - 1000) + + # Mock method to be wrapped + method_mock = mocker.Mock() + wrapped_method = rate_limiter(method_mock) + + wrapped_method() + + method_mock.assert_called_once() + mock_sleep.assert_called_once() diff --git a/tests/utils/test_logging_config.py b/tests/utils/test_logging_config.py new file mode 100644 index 0000000..33c5751 --- /dev/null +++ b/tests/utils/test_logging_config.py @@ -0,0 +1,71 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import logging +import os +import sys +from logging import StreamHandler + +from living_documentation_generator.utils.logging_config import setup_logging + +pytestmark = pytest.mark.setup_logging + + +def validate_logging_config(mock_logging_setup, caplog, expected_level, expected_message): + mock_logging_setup.assert_called_once() + + # Get the actual call arguments from the mock + call_args = mock_logging_setup.call_args[1] # Extract the kwargs from the call + + # Validate the logging level and format + assert call_args["level"] == expected_level + assert call_args["format"] == "%(asctime)s - %(levelname)s - %(message)s" + assert call_args["datefmt"] == "%Y-%m-%d %H:%M:%S" + + # Check that the handler is a StreamHandler and outputs to sys.stdout + handlers = call_args["handlers"] + assert len(handlers) == 1 # Only one handler is expected + assert isinstance(handlers[0], StreamHandler) # Handler should be StreamHandler + assert handlers[0].stream is sys.stdout # Stream should be sys.stdout + + # Check that the log message is present + assert expected_message in caplog.text + + +def test_setup_logging_default_logging_level(mock_logging_setup, caplog): + with caplog.at_level(logging.INFO): + setup_logging() + + validate_logging_config(mock_logging_setup, caplog, logging.INFO, "Logging configuration set up.") + + +def test_setup_logging_verbose_logging_enabled(mock_logging_setup, caplog): + os.environ["INPUT_VERBOSE_LOGGING"] = "true" + + with caplog.at_level(logging.DEBUG): + setup_logging() + + validate_logging_config(mock_logging_setup, caplog, logging.DEBUG, "Verbose logging enabled.") + + +def test_setup_logging_debug_mode_enabled_by_ci(mock_logging_setup, caplog): + os.environ["RUNNER_DEBUG"] = "1" + + with caplog.at_level(logging.DEBUG): + setup_logging() + + validate_logging_config(mock_logging_setup, caplog, logging.DEBUG, "Debug mode enabled by CI runner.") diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py new file mode 100644 index 0000000..3aa1827 --- /dev/null +++ b/tests/utils/test_utils.py @@ -0,0 +1,134 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import os + +from living_documentation_generator.utils.utils import ( + make_issue_key, + sanitize_filename, + make_absolute_path, + get_action_input, + set_action_output, + set_action_failed, +) + + +@pytest.mark.make_issue_key +def test_make_issue_key(): + organization_name = "org" + repository_name = "repo" + issue_number = 123 + + expected_key = "org/repo/123" + actual_key = make_issue_key(organization_name, repository_name, issue_number) + + assert actual_key == expected_key + + +@pytest.mark.sanitize_filename +@pytest.mark.parametrize( + "filename_example, expected_filename", + [ + ("in<>va::lid//.fi|le?na*me.txt", "invalid.filename.txt"), # Remove invalid characters for Windows filenames + ("another..invalid...filename.txt", "another.invalid.filename.txt"), # Reduce consecutive periods + ( + "filename with spaces.txt", + "filename_with_spaces.txt", + ), # Reduce consecutive spaces to a single space and replace spaces with '_' + ], +) +def test_sanitize_filename(filename_example, expected_filename): + actual_filename = sanitize_filename(filename_example) + assert actual_filename == expected_filename + + +@pytest.mark.make_absolute_path +def test_make_absolute_path_from_relative_path(): + relative_path = "relative/path" + expected_absolute_path = os.path.abspath(relative_path) + actual_absolute_path = make_absolute_path(relative_path) + + assert actual_absolute_path == expected_absolute_path + + +@pytest.mark.make_absolute_path +def test_make_absolute_path_from_absolute_path(): + absolute_path = "/absolute/path" + expected_absolute_path = absolute_path + actual_absolute_path = make_absolute_path(absolute_path) + + assert actual_absolute_path == expected_absolute_path + + +@pytest.mark.github_utils +@pytest.mark.get_action_input +def test_get_input_with_hyphen(mocker): + mock_getenv = mocker.patch("os.getenv", return_value="test_value") + + actual = get_action_input("test-input", default=None) + + mock_getenv.assert_called_with("INPUT_TEST_INPUT", default=None) + assert actual == "test_value" + + +@pytest.mark.github_utils +@pytest.mark.get_action_input +def test_get_input_without_hyphen(mocker): + mock_getenv = mocker.patch("os.getenv", return_value="another_test_value") + + actual = get_action_input("anotherinput", default=None) + + mock_getenv.assert_called_with("INPUT_ANOTHERINPUT", default=None) + assert actual == "another_test_value" + + +@pytest.mark.github_utils +@pytest.mark.set_action_output +def test_set_output_default(mocker): + mocker.patch("os.getenv", return_value="default_output.txt") + mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open) + + set_action_output("test-output", "test_value") + + mock_open.assert_called_with("default_output.txt", "a", encoding="utf-8") + handle = mock_open() + handle.write.assert_any_call("test-output=test_value\n") + + +@pytest.mark.github_utils +@pytest.mark.set_action_output +def test_set_output_custom_path(mocker): + mocker.patch("os.getenv", return_value="custom_output.txt") + mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open) + + set_action_output("custom-output", "custom_value", "default_output.txt") + + mock_open.assert_called_with("custom_output.txt", "a", encoding="utf-8") + handle = mock_open() + handle.write.assert_any_call("custom-output=custom_value\n") + + +@pytest.mark.github_utils +@pytest.mark.set_action_failed +def test_set_failed(mocker): + mock_print = mocker.patch("builtins.print", return_value=None) + mock_exit = mocker.patch("sys.exit", return_value=None) + + set_action_failed("failure message") + + mock_print.assert_called_with("::error::failure message") + mock_exit.assert_called_with(1) From 266e25f7dd3c42c251c0d35dd81e041556594575 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:19:24 +0200 Subject: [PATCH 15/60] New way how to write a pytest tests. --- .../utils/logging_config.py | 1 - living_documentation_generator/utils/utils.py | 2 +- pyproject.toml | 20 +---------- tests/__init__.py | 15 +++++++++ tests/conftest.py | 1 - tests/utils/test_decorators.py | 12 +++---- tests/utils/test_github_project_queries.py | 11 +++++-- tests/utils/test_github_rate_limiter.py | 4 +-- tests/utils/test_logging_config.py | 6 ++-- tests/utils/test_utils.py | 33 +++++++++++-------- 10 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 tests/__init__.py diff --git a/living_documentation_generator/utils/logging_config.py b/living_documentation_generator/utils/logging_config.py index bb2f9d1..936e595 100644 --- a/living_documentation_generator/utils/logging_config.py +++ b/living_documentation_generator/utils/logging_config.py @@ -43,7 +43,6 @@ def setup_logging() -> None: ) sys.stdout.flush() - # TODO logging.info("Logging configuration set up.") if is_verbose_logging: diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index f4f633e..1d5c0eb 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -72,7 +72,7 @@ def make_absolute_path(path: str) -> str: return os.path.abspath(path) -# Github +# GitHub action utils def get_action_input(name: str, default: Optional[str] = None) -> str: """ Get the input value from the environment variables. diff --git a/pyproject.toml b/pyproject.toml index 34ce511..68eb39c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,4 @@ [tool.black] line-length = 120 target-version = ['py311'] -#force-exclude = '''test''' - -[tool.pytest.ini_options] -markers = [ - "safe_call_decorator: group of tests that test the safe call decorator", - "decorators: group of tests that test the decorators", - "debug_log_decorator: group of tests that test the debug log decorator", - "utils: the whole folder of tests that are utils modules", - "github_rate_limiter: group of tests that test the github rate limiter", - "github_project_queries: group of tests that test the github project queries", - "setup_logging: group of tests that test the logging config", - "make_issue_key: group of tests that test the make issue key function", - "sanitize_filename: group of tests that test the sanitize filename function", - "make_absolute_path: group of tests that test the make absolute path function", - "github_utils: group of tests that test the github utils", - "get_action_input: group of tests that test the get action input function", - "set_action_output: group of tests that test the set action output function", - "set_action_failed: group of tests that test the set action failed function", -] \ No newline at end of file +force-exclude = '''test''' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6f9c372 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/conftest.py b/tests/conftest.py index 189fe7b..8d9636d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,6 @@ # limitations under the License. # import time - import pytest from github import Github from github.Rate import Rate diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index 2ca332b..21b9837 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -13,19 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest from living_documentation_generator.utils.decorators import debug_log_decorator, safe_call_decorator -pytestmark = [pytest.mark.decorators, pytest.mark.utils] - # sample function to be decorated def sample_function(x, y): return x + y -@pytest.mark.debug_log_decorator +# debug_log_decorator + + def test_debug_log_decorator(mocker): # Mock logging mock_log_debug = mocker.patch("living_documentation_generator.utils.decorators.logger.debug") @@ -42,7 +41,9 @@ def test_debug_log_decorator(mocker): assert mock_log_debug.call_args_list == expected_call -@pytest.mark.safe_call_decorator +# safe_call_decorator + + def test_safe_call_decorator_success(rate_limiter): @safe_call_decorator(rate_limiter) def sample_method(x, y): @@ -53,7 +54,6 @@ def sample_method(x, y): assert actual == 5 -@pytest.mark.safe_call_decorator def test_safe_call_decorator_exception(rate_limiter, mocker): mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") diff --git a/tests/utils/test_github_project_queries.py b/tests/utils/test_github_project_queries.py index fa223d9..4112998 100644 --- a/tests/utils/test_github_project_queries.py +++ b/tests/utils/test_github_project_queries.py @@ -14,8 +14,6 @@ # limitations under the License. # -import pytest - from living_documentation_generator.utils.constants import ( PROJECTS_FROM_REPO_QUERY, ISSUES_FROM_PROJECT_QUERY, @@ -28,7 +26,8 @@ get_project_field_options_query, ) -pytestmark = [pytest.mark.github_project_queries, pytest.mark.utils] + +# get_projects_from_repo_query def test_get_projects_from_repo_query(): @@ -43,6 +42,9 @@ def test_get_projects_from_repo_query(): assert actual_query == expected_query +# get_issues_from_project_query + + def test_get_issues_from_project_query(): project_id = "test_project_id" after_argument = "test_after_argument" @@ -55,6 +57,9 @@ def test_get_issues_from_project_query(): assert actual_query == expected_query +# get_project_field_options_query + + def test_get_project_field_options_query(): organization_name = "test_org" repository_name = "test_repo" diff --git a/tests/utils/test_github_rate_limiter.py b/tests/utils/test_github_rate_limiter.py index 8330cff..84687a7 100644 --- a/tests/utils/test_github_rate_limiter.py +++ b/tests/utils/test_github_rate_limiter.py @@ -14,9 +14,9 @@ # limitations under the License. # import time -import pytest -pytestmark = [pytest.mark.github_rate_limiter, pytest.mark.utils] + +# GithubRateLimiter __call__ method def test_rate_limiter_extended_sleep_remaining_1(mocker, rate_limiter, mock_rate_limiter): diff --git a/tests/utils/test_logging_config.py b/tests/utils/test_logging_config.py index 33c5751..6374ac8 100644 --- a/tests/utils/test_logging_config.py +++ b/tests/utils/test_logging_config.py @@ -14,7 +14,6 @@ # limitations under the License. # -import pytest import logging import os import sys @@ -22,8 +21,6 @@ from living_documentation_generator.utils.logging_config import setup_logging -pytestmark = pytest.mark.setup_logging - def validate_logging_config(mock_logging_setup, caplog, expected_level, expected_message): mock_logging_setup.assert_called_once() @@ -46,6 +43,9 @@ def validate_logging_config(mock_logging_setup, caplog, expected_level, expected assert expected_message in caplog.text +# setup_logging + + def test_setup_logging_default_logging_level(mock_logging_setup, caplog): with caplog.at_level(logging.INFO): setup_logging() diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 3aa1827..48eab21 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -27,7 +27,9 @@ ) -@pytest.mark.make_issue_key +# make_issue_key + + def test_make_issue_key(): organization_name = "org" repository_name = "repo" @@ -39,7 +41,9 @@ def test_make_issue_key(): assert actual_key == expected_key -@pytest.mark.sanitize_filename +# sanitize_filename + + @pytest.mark.parametrize( "filename_example, expected_filename", [ @@ -56,7 +60,9 @@ def test_sanitize_filename(filename_example, expected_filename): assert actual_filename == expected_filename -@pytest.mark.make_absolute_path +# make_absolute_path + + def test_make_absolute_path_from_relative_path(): relative_path = "relative/path" expected_absolute_path = os.path.abspath(relative_path) @@ -65,7 +71,6 @@ def test_make_absolute_path_from_relative_path(): assert actual_absolute_path == expected_absolute_path -@pytest.mark.make_absolute_path def test_make_absolute_path_from_absolute_path(): absolute_path = "/absolute/path" expected_absolute_path = absolute_path @@ -74,8 +79,10 @@ def test_make_absolute_path_from_absolute_path(): assert actual_absolute_path == expected_absolute_path -@pytest.mark.github_utils -@pytest.mark.get_action_input +# GitHub action utils +# get_action_input + + def test_get_input_with_hyphen(mocker): mock_getenv = mocker.patch("os.getenv", return_value="test_value") @@ -85,8 +92,6 @@ def test_get_input_with_hyphen(mocker): assert actual == "test_value" -@pytest.mark.github_utils -@pytest.mark.get_action_input def test_get_input_without_hyphen(mocker): mock_getenv = mocker.patch("os.getenv", return_value="another_test_value") @@ -96,8 +101,9 @@ def test_get_input_without_hyphen(mocker): assert actual == "another_test_value" -@pytest.mark.github_utils -@pytest.mark.set_action_output +# set_action_output + + def test_set_output_default(mocker): mocker.patch("os.getenv", return_value="default_output.txt") mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open) @@ -109,8 +115,6 @@ def test_set_output_default(mocker): handle.write.assert_any_call("test-output=test_value\n") -@pytest.mark.github_utils -@pytest.mark.set_action_output def test_set_output_custom_path(mocker): mocker.patch("os.getenv", return_value="custom_output.txt") mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open) @@ -122,8 +126,9 @@ def test_set_output_custom_path(mocker): handle.write.assert_any_call("custom-output=custom_value\n") -@pytest.mark.github_utils -@pytest.mark.set_action_failed +# set_action_failed + + def test_set_failed(mocker): mock_print = mocker.patch("builtins.print", return_value=None) mock_exit = mocker.patch("sys.exit", return_value=None) From 879ce94f7b974cdcd67f6932609f3f0e98c5359e Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:28:40 +0200 Subject: [PATCH 16/60] Updated README.md with unit test and test coverage section. --- README.md | 41 ++++++++++++++++++++--------------------- tests/model/__init__.py | 15 +++++++++++++++ 2 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 tests/model/__init__.py diff --git a/README.md b/README.md index 2188594..1b5187b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ - [Run Scripts Locally](#run-scripts-locally) - [Run Pylint Check Locally](#run-pylint-check-locally) - [Run Black Tool Locally](#run-black-tool-locally) -- [Run unit test](#run-unit-test) +- [Run Unit Test](#run-unit-test) +- [Code Coverage](#code-coverage) - [Deployment](#deployment) - [Features](#features) - [Data Mining from GitHub Repositories](#data-mining-from-github-repositories) @@ -438,35 +439,33 @@ All done! ✨ 🍰 ✨ ``` ## Run Unit Test -TODO - check this chapter and update by latest state -### Launch Unit Tests -``` -pytest -``` -### To Run Specific Tests or Get Verbose Output: -``` -pytest -v # Verbose mode -pytest path/to/test_file.py # Run specific test file +Unit tests are written using Pytest framework. To run alle the tests, use the following command: +```bash +pytest tests/ ``` -### To Check Test Coverage: -``` -pytest --cov=../scripts -``` +You can modify the directory to control the level of detail or granularity as per your needs. -### After running the tests +To run specific test, write the command following the pattern below: +```bash +pytest path/to/test_file.py::name_of_specific_test_method ``` -deactivate + +## Code Coverage + +This project uses [pytest-cov](https://pypi.org/project/pytest-cov/) plugin to generate test coverage reports. +The objective of the project is to achieve a minimal score of 80 %. + +To generate the coverage report, run the following command: +```bash +pytest --cov=living_documentation_generator --cov-fail-under=80 --cov-report=html ``` -### Commit Changes -After testing and ensuring that everything is functioning as expected, prepare your files for deployment: +See the coverage report on the path: ``` -git add action.yml dist/index.js # Adjust paths as needed -git commit -m "Prepare GitHub Action for deployment" -git push +htmlcov/index.html ``` ## Deployment diff --git a/tests/model/__init__.py b/tests/model/__init__.py new file mode 100644 index 0000000..6f9c372 --- /dev/null +++ b/tests/model/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From d6857fa79ee14ff53c1efcd3b7391b7f2c6e7373 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:57:24 +0200 Subject: [PATCH 17/60] Improved github test workflow check. --- .../workflows/static_analysis_and_tests.yml | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 9999f33..450b14e 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -66,6 +66,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4.1.5 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5.1.0 @@ -76,21 +78,9 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - pip install coverage pytest - - - name: Run tests - run: coverage run --source='similarity,column2Vec' -m pytest ./tests - -# - name: Show coverage -# run: coverage report -m -# - name: Create coverage file -# run: coverage xml + - name: Set PYTHONPATH environment variable + run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV -# - name: Get Cover -# uses: orgoro/coverage@v3.1 -# with: -# coverageFile: coverage.xml -# token: ${{ secrets.GITHUB_TOKEN }} -# thresholdAll: 0.7 -# thresholdNew: 0.9 + - name: Build and run unit tests + run: pytest --cov=living_documentation_generator --cov-report=html tests/ -vv --cov-fail-under=80 From 7606047ef4621c8482e1fe4296004bde69d668f8 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:59:39 +0200 Subject: [PATCH 18/60] Update static_analysis_and_tests.yml --- .github/workflows/static_analysis_and_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 450b14e..5c11363 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -83,4 +83,4 @@ jobs: run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - name: Build and run unit tests - run: pytest --cov=living_documentation_generator --cov-report=html tests/ -vv --cov-fail-under=80 + run: pytest -v --cov=living_documentation_generator tests/ --cov-fail-under=80 From d6b6e6b5e6f7720ae8abe4e50c8ad8762bf02ed9 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:02:46 +0200 Subject: [PATCH 19/60] Update static_analysis_and_tests.yml --- .github/workflows/static_analysis_and_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 5c11363..71ee6ce 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -83,4 +83,4 @@ jobs: run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - name: Build and run unit tests - run: pytest -v --cov=living_documentation_generator tests/ --cov-fail-under=80 + run: pytest --cov=living_documentation_generator tests/ --cov-fail-under=80 From 2d9f946dd48f04f68f7a5a065de812c3bff301f4 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:05:30 +0200 Subject: [PATCH 20/60] Update static_analysis_and_tests.yml --- .github/workflows/static_analysis_and_tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 71ee6ce..79d9d74 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -79,8 +79,5 @@ jobs: run: | pip install -r requirements.txt - - name: Set PYTHONPATH environment variable - run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - - name: Build and run unit tests - run: pytest --cov=living_documentation_generator tests/ --cov-fail-under=80 + run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 From 473f1ec073218ab976e1169d34aca0379f522c02 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:07:26 +0200 Subject: [PATCH 21/60] Update static_analysis_and_tests.yml --- .github/workflows/static_analysis_and_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 79d9d74..2b7f3ad 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -79,5 +79,8 @@ jobs: run: | pip install -r requirements.txt - - name: Build and run unit tests + - name: Set PYTHONPATH environment variable + run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV + + - name: Check project test coverage run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 From 802f2ab4514d56cdfcdeb7834abd7819703fdcf4 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:09:01 +0200 Subject: [PATCH 22/60] GH workflow updated. --- .github/workflows/static_analysis_and_tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 450b14e..fd276a3 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -78,9 +78,8 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - - name: Set PYTHONPATH environment variable run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - - name: Build and run unit tests - run: pytest --cov=living_documentation_generator --cov-report=html tests/ -vv --cov-fail-under=80 + - name: Check project test coverage + run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 From b5c0dec82039e5816a9032039f78e8820196d9b6 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:09:19 +0200 Subject: [PATCH 23/60] Update static_analysis_and_tests.yml --- .github/workflows/static_analysis_and_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 2b7f3ad..5b12d51 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -82,5 +82,5 @@ jobs: - name: Set PYTHONPATH environment variable run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - - name: Check project test coverage + - name: Check code coverage with Pytest run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 From 28539e1a95c2708f51588ce5250490febc0dd6d9 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:36:22 +0200 Subject: [PATCH 24/60] `Check code coverage with Pytest` GH workflow final touches. --- .github/workflows/static_analysis_and_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index fd276a3..7c76586 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -81,5 +81,5 @@ jobs: - name: Set PYTHONPATH environment variable run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - - name: Check project test coverage + - name: Check code coverage with Pytest run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 From cc075a30aca9aa34cb77c57cbfaeb7fd2e6b4491 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:41:23 +0200 Subject: [PATCH 25/60] Implementing the comments. --- .github/workflows/static_analysis_and_tests.yml | 2 +- .pylintrc | 2 ++ README.md | 4 ++-- pyproject.toml | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 7c76586..e892cfd 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -82,4 +82,4 @@ jobs: run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/living_documentation_generator/living_documentation_generator" >> $GITHUB_ENV - name: Check code coverage with Pytest - run: pytest --cov=living_documentation_generator -v tests/ --cov-fail-under=80 + run: pytest --cov=. -v tests/ --cov-fail-under=80 diff --git a/.pylintrc b/.pylintrc index 3a587bb..857c254 100644 --- a/.pylintrc +++ b/.pylintrc @@ -115,6 +115,8 @@ unsafe-load-any-extension=no # In verbose mode, extra non-checker-related info will be displayed. #verbose= +[MASTER] +ignore=tests [BASIC] diff --git a/README.md b/README.md index 1b5187b..cc08c31 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,7 @@ You can modify the directory to control the level of detail or granularity as pe To run specific test, write the command following the pattern below: ```bash -pytest path/to/test_file.py::name_of_specific_test_method +pytest tests/utils/test_utils.py::test_make_issue_key ``` ## Code Coverage @@ -459,7 +459,7 @@ The objective of the project is to achieve a minimal score of 80 %. To generate the coverage report, run the following command: ```bash -pytest --cov=living_documentation_generator --cov-fail-under=80 --cov-report=html +pytest --cov=. --cov-fail-under=80 --cov-report=html ``` See the coverage report on the path: diff --git a/pyproject.toml b/pyproject.toml index 68eb39c..daf4674 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,3 +2,6 @@ line-length = 120 target-version = ['py311'] force-exclude = '''test''' + +[tool.pytest.ini_options] +norecursedirs = "tests" From c4573e57ad125a692db47663925f2ca04c6e8ade Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:01:25 +0200 Subject: [PATCH 26/60] Implementing the comments about the utils tests. --- .../utils/decorators.py | 3 ++- tests/utils/test_decorators.py | 9 +++++++-- tests/utils/test_github_project_queries.py | 7 +++++++ tests/utils/test_utils.py | 19 ------------------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/living_documentation_generator/utils/decorators.py b/living_documentation_generator/utils/decorators.py index a8fadaa..d04d7f0 100644 --- a/living_documentation_generator/utils/decorators.py +++ b/living_documentation_generator/utils/decorators.py @@ -75,7 +75,8 @@ def wrapped(*args, **kwargs) -> Optional[Any]: logger.error("HTTP error calling %s: %s.", method.__name__, e, exc_info=True) return None except Exception as e: - logger.error("Unexpected error calling %s: %s.", method.__name__, e, exc_info=True) + exception_type = type(e).__name__ + logger.error("%s by calling %s: %s.", exception_type, method.__name__, e, exc_info=True) return None return wrapped diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index 21b9837..b1ec933 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -65,5 +65,10 @@ def sample_method(x, y): assert actual is None mock_log_error.assert_called_once() - assert "Unexpected error calling %s:" in mock_log_error.call_args[0][0] - assert "sample_method" in mock_log_error.call_args[0][1] + exception_message = mock_log_error.call_args[0][0] + assert "%s by calling %s: %s." in exception_message + exception_type = mock_log_error.call_args[0][1] + assert "ZeroDivisionError" in exception_type + method_name = mock_log_error.call_args[0][2] + assert "sample_method" in method_name + diff --git a/tests/utils/test_github_project_queries.py b/tests/utils/test_github_project_queries.py index 4112998..dc92091 100644 --- a/tests/utils/test_github_project_queries.py +++ b/tests/utils/test_github_project_queries.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import re from living_documentation_generator.utils.constants import ( PROJECTS_FROM_REPO_QUERY, @@ -40,6 +41,8 @@ def test_get_projects_from_repo_query(): actual_query = get_projects_from_repo_query(organization_name, repository_name) assert actual_query == expected_query + leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + assert not leftover_placeholders # get_issues_from_project_query @@ -55,6 +58,8 @@ def test_get_issues_from_project_query(): actual_query = get_issues_from_project_query(project_id, after_argument) assert actual_query == expected_query + leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + assert not leftover_placeholders # get_project_field_options_query @@ -71,3 +76,5 @@ def test_get_project_field_options_query(): actual_query = get_project_field_options_query(organization_name, repository_name, project_number) assert actual_query == expected_query + leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + assert not leftover_placeholders diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 48eab21..53cc822 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -60,25 +60,6 @@ def test_sanitize_filename(filename_example, expected_filename): assert actual_filename == expected_filename -# make_absolute_path - - -def test_make_absolute_path_from_relative_path(): - relative_path = "relative/path" - expected_absolute_path = os.path.abspath(relative_path) - actual_absolute_path = make_absolute_path(relative_path) - - assert actual_absolute_path == expected_absolute_path - - -def test_make_absolute_path_from_absolute_path(): - absolute_path = "/absolute/path" - expected_absolute_path = absolute_path - actual_absolute_path = make_absolute_path(absolute_path) - - assert actual_absolute_path == expected_absolute_path - - # GitHub action utils # get_action_input From 29f59b766498f5f19b2b212fa0ed542956f1a830 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:55:35 +0200 Subject: [PATCH 27/60] Comments changed. --- README.md | 8 +++++--- living_documentation_generator/utils/decorators.py | 3 +-- pyproject.toml | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cc08c31..322019d 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,8 @@ chmod +x run_script.sh ## Run Pylint Check Locally This project uses [Pylint](https://pypi.org/project/pylint/) tool for static code analysis. Pylint analyses your code without actually running it. -It checks for errors, enforces, coding standards, looks for code smells etc. +It checks for errors, enforces, coding standards, looks for code smells etc. +We do exclude the `tests/` file from the pylint check. Pylint displays a global evaluation score for the code, rated out of a maximum score of 10.0. We are aiming to keep our code quality high above the score 9.5. @@ -404,6 +405,7 @@ The coding style used can be viewed as a strict subset of PEP 8. The project root file `pyproject.toml` defines the Black tool configuration. In this project we are accepting the line length of 120 characters. +We also do exclude the `tests/` file from the black formatting. Follow these steps to format your code with Black locally: @@ -455,11 +457,11 @@ pytest tests/utils/test_utils.py::test_make_issue_key ## Code Coverage This project uses [pytest-cov](https://pypi.org/project/pytest-cov/) plugin to generate test coverage reports. -The objective of the project is to achieve a minimal score of 80 %. +The objective of the project is to achieve a minimal score of 80 %. We do exclude the `tests/` file from the coverage report. To generate the coverage report, run the following command: ```bash -pytest --cov=. --cov-fail-under=80 --cov-report=html +pytest --cov=. tests/ --cov-fail-under=80 --cov-report=html ``` See the coverage report on the path: diff --git a/living_documentation_generator/utils/decorators.py b/living_documentation_generator/utils/decorators.py index d04d7f0..e8481a6 100644 --- a/living_documentation_generator/utils/decorators.py +++ b/living_documentation_generator/utils/decorators.py @@ -75,8 +75,7 @@ def wrapped(*args, **kwargs) -> Optional[Any]: logger.error("HTTP error calling %s: %s.", method.__name__, e, exc_info=True) return None except Exception as e: - exception_type = type(e).__name__ - logger.error("%s by calling %s: %s.", exception_type, method.__name__, e, exc_info=True) + logger.error("%s by calling %s: %s.", type(e).__name__, method.__name__, e, exc_info=True) return None return wrapped diff --git a/pyproject.toml b/pyproject.toml index daf4674..7cf6438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,5 +3,5 @@ line-length = 120 target-version = ['py311'] force-exclude = '''test''' -[tool.pytest.ini_options] -norecursedirs = "tests" +[tool.coverage.run] +omit = ["tests/*"] From f7d38067a9630ce546606ac8209e2fda979c9456 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:12:49 +0200 Subject: [PATCH 28/60] New logic for testing correct query formatting. --- .../utils/github_project_queries.py | 4 ++ living_documentation_generator/utils/utils.py | 17 ++++++++ tests/utils/test_utils.py | 42 ++++++++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/living_documentation_generator/utils/github_project_queries.py b/living_documentation_generator/utils/github_project_queries.py index 3cd5ebc..69dc1e8 100644 --- a/living_documentation_generator/utils/github_project_queries.py +++ b/living_documentation_generator/utils/github_project_queries.py @@ -18,6 +18,7 @@ This module contains methods for formating the GitHub GraphQL queries. """ +from living_documentation_generator.utils.utils import validate_query_format from living_documentation_generator.utils.constants import ( PROJECTS_FROM_REPO_QUERY, ISSUES_FROM_PROJECT_QUERY, @@ -28,11 +29,13 @@ def get_projects_from_repo_query(organization_name: str, repository_name: str) -> str: """Update the placeholder values and formate the graphQL query""" + validate_query_format(PROJECTS_FROM_REPO_QUERY, {"organization_name", "repository_name"}) return PROJECTS_FROM_REPO_QUERY.format(organization_name=organization_name, repository_name=repository_name) def get_issues_from_project_query(project_id: str, after_argument: str) -> str: """Update the placeholder values and formate the graphQL query""" + validate_query_format(ISSUES_FROM_PROJECT_QUERY, {"project_id", "issues_per_page", "after_argument"}) return ISSUES_FROM_PROJECT_QUERY.format( project_id=project_id, issues_per_page=ISSUES_PER_PAGE_LIMIT, after_argument=after_argument ) @@ -40,6 +43,7 @@ def get_issues_from_project_query(project_id: str, after_argument: str) -> str: def get_project_field_options_query(organization_name: str, repository_name: str, project_number: int) -> str: """Update the placeholder values and formate the graphQL query""" + validate_query_format(PROJECT_FIELD_OPTIONS_QUERY, {"organization_name", "repository_name", "project_number"}) return PROJECT_FIELD_OPTIONS_QUERY.format( organization_name=organization_name, repository_name=repository_name, project_number=project_number ) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 1d5c0eb..b5b1fba 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -72,6 +72,23 @@ def make_absolute_path(path: str) -> str: return os.path.abspath(path) +def validate_query_format(query_string, expected_placeholders) -> None: + """ + Validate the placeholders in the query string. + Check if all the expected placeholders are present in the query and exit if not. + + @param query_string: The query string to validate. + @param expected_placeholders: The set of expected placeholders in the query. + @return: None + """ + actual_placeholders = set(re.findall(r'\{(\w+)\}', query_string)) + missing = expected_placeholders - actual_placeholders + extra = actual_placeholders - expected_placeholders + if missing or extra: + logger.error(f"Missing placeholders: {missing}, Extra placeholders: {extra}.\n For the query: {query_string}") + sys.exit(1) + + # GitHub action utils def get_action_input(name: str, default: Optional[str] = None) -> str: """ diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 53cc822..ad9aa28 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -15,12 +15,11 @@ # import pytest -import os from living_documentation_generator.utils.utils import ( make_issue_key, sanitize_filename, - make_absolute_path, + validate_query_format, get_action_input, set_action_output, set_action_failed, @@ -82,6 +81,45 @@ def test_get_input_without_hyphen(mocker): assert actual == "another_test_value" +# validate_query_format + + +def test_validate_query_format_missing_placeholder(mocker): + mock_exit = mocker.patch("sys.exit", return_value=None) + mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") + + # Test case where there are missing placeholders + query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" + expected_placeholders = {"placeholder1", "placeholder2", "placeholder3"} + validate_query_format(query_string, expected_placeholders) + mock_log_error.assert_called_with("Missing placeholders: {'placeholder3'}, Extra placeholders: set().\n For the query: This is a query with placeholders {placeholder1} and {placeholder2}") + mock_exit.assert_called_with(1) + + +def test_validate_query_format_extra_placeholder(mocker): + mock_exit = mocker.patch("sys.exit", return_value=None) + mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") + + # Test case where there are extra placeholders + query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" + expected_placeholders = {"placeholder1"} + validate_query_format(query_string, expected_placeholders) + mock_log_error.assert_called_with("Missing placeholders: set(), Extra placeholders: {'placeholder2'}.\n For the query: This is a query with placeholders {placeholder1} and {placeholder2}") + mock_exit.assert_called_with(1) + + +def test_validate_query_format_right_behaviour(mocker): + mock_exit = mocker.patch("sys.exit", return_value=None) + mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") + + # Test case where there are no missing or extra placeholders + query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" + expected_placeholders = {"placeholder1", "placeholder2"} + validate_query_format(query_string, expected_placeholders) + mock_log_error.assert_not_called() + mock_exit.assert_not_called() + + # set_action_output From 240f00d48d8c698a82da883b24f758f6b9fc7c38 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:35:03 +0200 Subject: [PATCH 29/60] Update logic for testing correct query formatting. --- living_documentation_generator/utils/decorators.py | 2 +- living_documentation_generator/utils/utils.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/living_documentation_generator/utils/decorators.py b/living_documentation_generator/utils/decorators.py index e8481a6..08fe737 100644 --- a/living_documentation_generator/utils/decorators.py +++ b/living_documentation_generator/utils/decorators.py @@ -24,7 +24,7 @@ from typing import Callable, Optional, Any from functools import wraps from github import GithubException -from requests.exceptions import Timeout, ConnectionError, RequestException +from requests.exceptions import Timeout, RequestException from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index b5b1fba..d232040 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -81,11 +81,13 @@ def validate_query_format(query_string, expected_placeholders) -> None: @param expected_placeholders: The set of expected placeholders in the query. @return: None """ - actual_placeholders = set(re.findall(r'\{(\w+)\}', query_string)) + actual_placeholders = set(re.findall(r"\{(\w+)\}", query_string)) missing = expected_placeholders - actual_placeholders extra = actual_placeholders - expected_placeholders if missing or extra: - logger.error(f"Missing placeholders: {missing}, Extra placeholders: {extra}.\n For the query: {query_string}") + logger.error( + "Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s", missing, extra, query_string + ) sys.exit(1) From a124d94b4ba1b1e7ea37434784f625e752e19f3f Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:00:22 +0200 Subject: [PATCH 30/60] Bug Fix. --- tests/utils/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index ad9aa28..10cbd4b 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -92,7 +92,7 @@ def test_validate_query_format_missing_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1", "placeholder2", "placeholder3"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with("Missing placeholders: {'placeholder3'}, Extra placeholders: set().\n For the query: This is a query with placeholders {placeholder1} and {placeholder2}") + mock_log_error.assert_called_with('Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s', {'placeholder3'}, set(), 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) @@ -104,7 +104,7 @@ def test_validate_query_format_extra_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with("Missing placeholders: set(), Extra placeholders: {'placeholder2'}.\n For the query: This is a query with placeholders {placeholder1} and {placeholder2}") + mock_log_error.assert_called_with('Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s', set(), {'placeholder2'}, 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) From d0ddc176769405918ad139fddb08e8a136754a7d Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:25:19 +0200 Subject: [PATCH 31/60] Adding tests for reaching over 90 % cov. --- tests/utils/test_decorators.py | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index b1ec933..65a6200 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from github import GithubException +from requests import RequestException from living_documentation_generator.utils.decorators import debug_log_decorator, safe_call_decorator @@ -54,6 +56,66 @@ def sample_method(x, y): assert actual == 5 +def test_safe_call_decorator_network_error(rate_limiter, mocker): + mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") + + @safe_call_decorator(rate_limiter) + def sample_method(): + raise ConnectionError("Test connection error") + + actual = sample_method() + + assert actual is None + assert mock_log_error.call_count == 1 + + args, kwargs = mock_log_error.call_args + assert args[0] == "Network error calling %s: %s." + assert args[1] == "sample_method" + assert isinstance(args[2], ConnectionError) + assert str(args[2]) == "Test connection error" + assert kwargs['exc_info'] + + +def test_safe_call_decorator_github_api_error(rate_limiter, mocker): + mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") + + @safe_call_decorator(rate_limiter) + def sample_method(): + raise GithubException(status=404) + + actual = sample_method() + + assert actual is None + assert mock_log_error.call_count == 1 + + args, kwargs = mock_log_error.call_args + assert args[0] == 'GitHub API error calling %s: %s.' + assert args[1] == "sample_method" + assert isinstance(args[2], GithubException) + assert str(args[2]) == "404" + assert kwargs['exc_info'] + + +def test_safe_call_decorator_http_error(mocker, rate_limiter): + mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") + + @safe_call_decorator(rate_limiter) + def sample_method(): + raise RequestException("Test HTTP error") + + actual = sample_method() + + assert actual is None + assert mock_log_error.call_count == 1 + + args, kwargs = mock_log_error.call_args + assert args[0] == "HTTP error calling %s: %s." + assert args[1] == "sample_method" + assert isinstance(args[2], RequestException) + assert str(args[2]) == "Test HTTP error" + assert kwargs['exc_info'] + + def test_safe_call_decorator_exception(rate_limiter, mocker): mock_log_error = mocker.patch("living_documentation_generator.utils.decorators.logger.error") @@ -72,3 +134,5 @@ def sample_method(x, y): method_name = mock_log_error.call_args[0][2] assert "sample_method" in method_name + + From 19f1accce5edcfca4499834dbe95ce3e6a48ac9f Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:53:36 +0200 Subject: [PATCH 32/60] Comments implemented. --- .../workflows/static_analysis_and_tests.yml | 4 ++-- .pylintrc | 3 --- README.md | 4 ++-- tests/utils/test_utils.py | 24 +++++++++---------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index e892cfd..674a4c7 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -26,7 +26,7 @@ jobs: - name: Analyze code with Pylint id: analyze-code run: | - pylint_score=$(pylint $(git ls-files '*.py')| grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) + pylint_score=$(pylint $(git ls-files '*.py'| grep -v '^tests/') | grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) echo "PYLINT_SCORE=$pylint_score" >> $GITHUB_ENV - name: Check Pylint score @@ -58,7 +58,7 @@ jobs: - name: Check code format with Black id: check-format run: | - black --check $(git ls-files '*.py') + black --check $(git ls-files '*.py'| grep -v '^tests/') python-tests: runs-on: ubuntu-latest diff --git a/.pylintrc b/.pylintrc index 857c254..301c32d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -115,9 +115,6 @@ unsafe-load-any-extension=no # In verbose mode, extra non-checker-related info will be displayed. #verbose= -[MASTER] -ignore=tests - [BASIC] # Naming style matching correct argument names. diff --git a/README.md b/README.md index 322019d..451036e 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ This command will also install a Pylint tool, since it is listed in the project ### Run Pylint Run Pylint on all files that are currently tracked by Git in the project. ```shell -pylint $(git ls-files '*.py') +pylint $(git ls-files '*.py' | grep -v '^tests/') ``` To run Pylint on a specific file, follow the pattern `pylint /.py`. @@ -423,7 +423,7 @@ This command will also install a Black tool, since it is listed in the project r ### Run Black Run Black on all files that are currently tracked by Git in the project. ```shell -black $(git ls-files '*.py') +black $(git ls-files '*.py' | grep -v '^tests/') ``` To run Black on a specific file, follow the pattern `black /.py`. diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 10cbd4b..db56a7c 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -84,6 +84,18 @@ def test_get_input_without_hyphen(mocker): # validate_query_format +def test_validate_query_format_right_behaviour(mocker): + mock_exit = mocker.patch("sys.exit", return_value=None) + mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") + + # Test case where there are no missing or extra placeholders + query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" + expected_placeholders = {"placeholder1", "placeholder2"} + validate_query_format(query_string, expected_placeholders) + mock_log_error.assert_not_called() + mock_exit.assert_not_called() + + def test_validate_query_format_missing_placeholder(mocker): mock_exit = mocker.patch("sys.exit", return_value=None) mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") @@ -108,18 +120,6 @@ def test_validate_query_format_extra_placeholder(mocker): mock_exit.assert_called_with(1) -def test_validate_query_format_right_behaviour(mocker): - mock_exit = mocker.patch("sys.exit", return_value=None) - mock_log_error = mocker.patch("living_documentation_generator.utils.utils.logger.error") - - # Test case where there are no missing or extra placeholders - query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" - expected_placeholders = {"placeholder1", "placeholder2"} - validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_not_called() - mock_exit.assert_not_called() - - # set_action_output From f7f24b2b204e551d81caca18e377731585b10046 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:56:35 +0200 Subject: [PATCH 33/60] Update of black tool. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7cf6438..4f5c62e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,6 @@ [tool.black] line-length = 120 target-version = ['py311'] -force-exclude = '''test''' [tool.coverage.run] omit = ["tests/*"] From 7bbab060e6823dc7a31e873bb91679eabf8c7909 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:37:01 +0200 Subject: [PATCH 34/60] Update of handeling the tools. --- .github/workflows/static_analysis_and_tests.yml | 2 +- .pylintrc | 3 +++ README.md | 4 ++-- pyproject.toml | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 674a4c7..1c9b335 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -58,7 +58,7 @@ jobs: - name: Check code format with Black id: check-format run: | - black --check $(git ls-files '*.py'| grep -v '^tests/') + black --check $(git ls-files '*.py') python-tests: runs-on: ubuntu-latest diff --git a/.pylintrc b/.pylintrc index 301c32d..49b5b45 100644 --- a/.pylintrc +++ b/.pylintrc @@ -115,6 +115,9 @@ unsafe-load-any-extension=no # In verbose mode, extra non-checker-related info will be displayed. #verbose= +[MASTER] +ignore-paths=tests + [BASIC] # Naming style matching correct argument names. diff --git a/README.md b/README.md index 451036e..322019d 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ This command will also install a Pylint tool, since it is listed in the project ### Run Pylint Run Pylint on all files that are currently tracked by Git in the project. ```shell -pylint $(git ls-files '*.py' | grep -v '^tests/') +pylint $(git ls-files '*.py') ``` To run Pylint on a specific file, follow the pattern `pylint /.py`. @@ -423,7 +423,7 @@ This command will also install a Black tool, since it is listed in the project r ### Run Black Run Black on all files that are currently tracked by Git in the project. ```shell -black $(git ls-files '*.py' | grep -v '^tests/') +black $(git ls-files '*.py') ``` To run Black on a specific file, follow the pattern `black /.py`. diff --git a/pyproject.toml b/pyproject.toml index 4f5c62e..7cf6438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.black] line-length = 120 target-version = ['py311'] +force-exclude = '''test''' [tool.coverage.run] omit = ["tests/*"] From 4f986217862d6adfe9dde9e9a6072b9c68b1ba1f Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:39:00 +0200 Subject: [PATCH 35/60] Bug fix. --- .github/workflows/static_analysis_and_tests.yml | 2 +- .pylintrc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static_analysis_and_tests.yml b/.github/workflows/static_analysis_and_tests.yml index 1c9b335..ffdab2e 100644 --- a/.github/workflows/static_analysis_and_tests.yml +++ b/.github/workflows/static_analysis_and_tests.yml @@ -26,7 +26,7 @@ jobs: - name: Analyze code with Pylint id: analyze-code run: | - pylint_score=$(pylint $(git ls-files '*.py'| grep -v '^tests/') | grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) + pylint_score=$(pylint $(git ls-files '*.py') | grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) echo "PYLINT_SCORE=$pylint_score" >> $GITHUB_ENV - name: Check Pylint score diff --git a/.pylintrc b/.pylintrc index 49b5b45..cdc470b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -116,6 +116,7 @@ unsafe-load-any-extension=no #verbose= [MASTER] + ignore-paths=tests [BASIC] From 83270948338e967b4bcc0951434e15482a420b48 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:52:08 +0200 Subject: [PATCH 36/60] Updating logging message for validate_query_format(). --- living_documentation_generator/utils/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index d232040..b80e5a2 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -85,8 +85,10 @@ def validate_query_format(query_string, expected_placeholders) -> None: missing = expected_placeholders - actual_placeholders extra = actual_placeholders - expected_placeholders if missing or extra: + missing_message = f"Missing placeholders: {missing}. " if missing else "" + extra_message = f"Extra placeholders: {extra}." if extra else "" logger.error( - "Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s", missing, extra, query_string + "%s%s\n For the query: %s", missing_message, extra_message, query_string ) sys.exit(1) From 133ca25731f7cfe8047f7cb61e0794920090275c Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 22:02:04 +0200 Subject: [PATCH 37/60] Test files updated with current logic. --- tests/utils/test_decorators.py | 15 ++++++++++++--- tests/utils/test_utils.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index 65a6200..c4039b0 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -81,7 +81,16 @@ def test_safe_call_decorator_github_api_error(rate_limiter, mocker): @safe_call_decorator(rate_limiter) def sample_method(): - raise GithubException(status=404) + status_code = 404 + error_data = { + "message": "Not Found", + "documentation_url": "https://developer.github.com/v3" + } + response_headers = { + "X-RateLimit-Limit": "60", + "X-RateLimit-Remaining": "0", + } + raise GithubException(status_code, error_data, response_headers) actual = sample_method() @@ -90,9 +99,9 @@ def sample_method(): args, kwargs = mock_log_error.call_args assert args[0] == 'GitHub API error calling %s: %s.' - assert args[1] == "sample_method" + assert args[1] == 'sample_method' assert isinstance(args[2], GithubException) - assert str(args[2]) == "404" + assert str(args[2]) == '404 {"message": "Not Found", "documentation_url": "https://developer.github.com/v3"}' assert kwargs['exc_info'] diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index db56a7c..62ae58c 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -104,7 +104,7 @@ def test_validate_query_format_missing_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1", "placeholder2", "placeholder3"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s', {'placeholder3'}, set(), 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with('%s%s\n For the query: %s', "Missing placeholders: {'placeholder3'}. ", '', 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) @@ -116,7 +116,7 @@ def test_validate_query_format_extra_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('Missing placeholders: %s, Extra placeholders: %s.\n For the query: %s', set(), {'placeholder2'}, 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with('%s%s\n For the query: %s', '', "Extra placeholders: {'placeholder2'}.", 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) From 0d6d50a297b23826197932c72c067f778347546b Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 7 Oct 2024 22:04:30 +0200 Subject: [PATCH 38/60] Black tool formating. --- living_documentation_generator/utils/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index b80e5a2..99a8432 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -87,9 +87,7 @@ def validate_query_format(query_string, expected_placeholders) -> None: if missing or extra: missing_message = f"Missing placeholders: {missing}. " if missing else "" extra_message = f"Extra placeholders: {extra}." if extra else "" - logger.error( - "%s%s\n For the query: %s", missing_message, extra_message, query_string - ) + logger.error("%s%s\n For the query: %s", missing_message, extra_message, query_string) sys.exit(1) From dfd74c68b18f2569dc037e92c10c682092dbb155 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:24:00 +0200 Subject: [PATCH 39/60] Logging message updated. --- living_documentation_generator/utils/utils.py | 2 +- tests/utils/test_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 99a8432..552b45b 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -87,7 +87,7 @@ def validate_query_format(query_string, expected_placeholders) -> None: if missing or extra: missing_message = f"Missing placeholders: {missing}. " if missing else "" extra_message = f"Extra placeholders: {extra}." if extra else "" - logger.error("%s%s\n For the query: %s", missing_message, extra_message, query_string) + logger.error("%s%s\nFor the query: %s", missing_message, extra_message, query_string) sys.exit(1) diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 62ae58c..537b900 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -104,7 +104,7 @@ def test_validate_query_format_missing_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1", "placeholder2", "placeholder3"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('%s%s\n For the query: %s', "Missing placeholders: {'placeholder3'}. ", '', 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with('%s%s\nFor the query: %s', "Missing placeholders: {'placeholder3'}. ", '', 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) @@ -116,7 +116,7 @@ def test_validate_query_format_extra_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('%s%s\n For the query: %s', '', "Extra placeholders: {'placeholder2'}.", 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with('%s%s\nFor the query: %s', '', "Extra placeholders: {'placeholder2'}.", 'This is a query with placeholders {placeholder1} and {placeholder2}') mock_exit.assert_called_with(1) From c2d36e23d9bfb735018de176b5241d680bfeef58 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:49:01 +0200 Subject: [PATCH 40/60] Unit tests using pytest for config_repository.py. --- .../action_inputs.py | 6 +- .../model/config_repository.py | 22 ++++-- tests/model/test_config_repository.py | 67 +++++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 tests/model/test_config_repository.py diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 7ada813..91f6a43 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -106,8 +106,10 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": for repository_json in repositories_json: config_repository = ConfigRepository() - config_repository.load_from_json(repository_json) - self.__repositories.append(config_repository) + if config_repository.load_from_json(repository_json): + self.__repositories.append(config_repository) + else: + logger.error("Failed to load repository from JSON: %s.", repository_json) return self diff --git a/living_documentation_generator/model/config_repository.py b/living_documentation_generator/model/config_repository.py index b0138fa..32073ca 100644 --- a/living_documentation_generator/model/config_repository.py +++ b/living_documentation_generator/model/config_repository.py @@ -18,8 +18,11 @@ This module contains a data container for Config Repository, which holds all the essential logic. """ +import logging from typing import Optional +logger = logging.getLogger(__name__) + class ConfigRepository: """ @@ -53,14 +56,21 @@ def projects_title_filter(self) -> list[str]: """Getter of the project title filter.""" return self.__projects_title_filter - def load_from_json(self, repository_json: dict) -> None: + def load_from_json(self, repository_json: dict) -> bool: """ Load the configuration from a JSON object. @param repository_json: The JSON object containing the repository configuration. - @return: None + @return: bool """ - self.__organization_name = repository_json["organization-name"] - self.__repository_name = repository_json["repository-name"] - self.__query_labels = repository_json["query-labels"] - self.__projects_title_filter = repository_json["projects-title-filter"] + try: + self.__organization_name = repository_json["organization-name"] + self.__repository_name = repository_json["repository-name"] + self.__query_labels = repository_json["query-labels"] + self.__projects_title_filter = repository_json["projects-title-filter"] + return True + except KeyError as e: + logger.error("The key is not found in the repository JSON input: %s.", e, exc_info=True) + except TypeError as e: + logger.error("The repository JSON input does not have a dictionary structure: %s.", e, exc_info=True) + return False diff --git a/tests/model/test_config_repository.py b/tests/model/test_config_repository.py new file mode 100644 index 0000000..ae1e026 --- /dev/null +++ b/tests/model/test_config_repository.py @@ -0,0 +1,67 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from living_documentation_generator.model.config_repository import ConfigRepository + + +def test_load_from_json_with_valid_input_loads_correctly(): + config_repository = ConfigRepository() + organization_name = "organizationABC" + repository_name = "repositoryABC" + query_labels = ["feature", "bug"] + projects_title_filter = ["project1"] + other_value = "other-value" + repository_json = { + "organization-name": organization_name, + "repository-name": repository_name, + "query-labels": query_labels, + "projects-title-filter": projects_title_filter, + "other-field": other_value, + } + + actual = config_repository.load_from_json(repository_json) + + assert actual + assert organization_name == config_repository.organization_name + assert repository_name == config_repository.repository_name + assert query_labels == config_repository.query_labels + assert projects_title_filter == config_repository.projects_title_filter + + +def test_load_from_json_with_missing_key_logs_error(mocker): + config_repository = ConfigRepository() + mock_log_error = mocker.patch("living_documentation_generator.model.config_repository.logger.error") + repository_json = {"non-existent-key": "value"} + + actual = config_repository.load_from_json(repository_json) + + assert actual is False + mock_log_error.assert_called_once_with( + "The key is not found in the repository JSON input: %s.", mocker.ANY, exc_info=True + ) + + +def test_load_from_json_with_wrong_structure_input_logs_error(mocker): + config_repository = ConfigRepository() + mock_log_error = mocker.patch("living_documentation_generator.model.config_repository.logger.error") + repository_json = "not a dictionary" + + actual = config_repository.load_from_json(repository_json) + + assert actual is False + mock_log_error.assert_called_once_with( + "The repository JSON input does not have a dictionary structure: %s.", mocker.ANY, exc_info=True + ) From e82a1f60f6f237404dd9f18b2182769a19cd5e61 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:11:35 +0200 Subject: [PATCH 41/60] Formatting for tests/utils/. --- tests/utils/test_decorators.py | 19 +++++++------------ tests/utils/test_github_project_queries.py | 7 ++++--- tests/utils/test_github_rate_limiter.py | 1 + tests/utils/test_utils.py | 14 ++++++++++++-- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index c4039b0..9eedd25 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + from github import GithubException from requests import RequestException @@ -73,7 +74,7 @@ def sample_method(): assert args[1] == "sample_method" assert isinstance(args[2], ConnectionError) assert str(args[2]) == "Test connection error" - assert kwargs['exc_info'] + assert kwargs["exc_info"] def test_safe_call_decorator_github_api_error(rate_limiter, mocker): @@ -82,10 +83,7 @@ def test_safe_call_decorator_github_api_error(rate_limiter, mocker): @safe_call_decorator(rate_limiter) def sample_method(): status_code = 404 - error_data = { - "message": "Not Found", - "documentation_url": "https://developer.github.com/v3" - } + error_data = {"message": "Not Found", "documentation_url": "https://developer.github.com/v3"} response_headers = { "X-RateLimit-Limit": "60", "X-RateLimit-Remaining": "0", @@ -98,11 +96,11 @@ def sample_method(): assert mock_log_error.call_count == 1 args, kwargs = mock_log_error.call_args - assert args[0] == 'GitHub API error calling %s: %s.' - assert args[1] == 'sample_method' + assert args[0] == "GitHub API error calling %s: %s." + assert args[1] == "sample_method" assert isinstance(args[2], GithubException) assert str(args[2]) == '404 {"message": "Not Found", "documentation_url": "https://developer.github.com/v3"}' - assert kwargs['exc_info'] + assert kwargs["exc_info"] def test_safe_call_decorator_http_error(mocker, rate_limiter): @@ -122,7 +120,7 @@ def sample_method(): assert args[1] == "sample_method" assert isinstance(args[2], RequestException) assert str(args[2]) == "Test HTTP error" - assert kwargs['exc_info'] + assert kwargs["exc_info"] def test_safe_call_decorator_exception(rate_limiter, mocker): @@ -142,6 +140,3 @@ def sample_method(x, y): assert "ZeroDivisionError" in exception_type method_name = mock_log_error.call_args[0][2] assert "sample_method" in method_name - - - diff --git a/tests/utils/test_github_project_queries.py b/tests/utils/test_github_project_queries.py index dc92091..5362613 100644 --- a/tests/utils/test_github_project_queries.py +++ b/tests/utils/test_github_project_queries.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import re from living_documentation_generator.utils.constants import ( @@ -41,7 +42,7 @@ def test_get_projects_from_repo_query(): actual_query = get_projects_from_repo_query(organization_name, repository_name) assert actual_query == expected_query - leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + leftover_placeholders = re.findall(r"\{\w+\}", actual_query) assert not leftover_placeholders @@ -58,7 +59,7 @@ def test_get_issues_from_project_query(): actual_query = get_issues_from_project_query(project_id, after_argument) assert actual_query == expected_query - leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + leftover_placeholders = re.findall(r"\{\w+\}", actual_query) assert not leftover_placeholders @@ -76,5 +77,5 @@ def test_get_project_field_options_query(): actual_query = get_project_field_options_query(organization_name, repository_name, project_number) assert actual_query == expected_query - leftover_placeholders = re.findall(r'\{\w+\}', actual_query) + leftover_placeholders = re.findall(r"\{\w+\}", actual_query) assert not leftover_placeholders diff --git a/tests/utils/test_github_rate_limiter.py b/tests/utils/test_github_rate_limiter.py index 84687a7..275ae60 100644 --- a/tests/utils/test_github_rate_limiter.py +++ b/tests/utils/test_github_rate_limiter.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import time diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 537b900..11519d0 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -104,7 +104,12 @@ def test_validate_query_format_missing_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1", "placeholder2", "placeholder3"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('%s%s\nFor the query: %s', "Missing placeholders: {'placeholder3'}. ", '', 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with( + "%s%s\nFor the query: %s", + "Missing placeholders: {'placeholder3'}. ", + "", + "This is a query with placeholders {placeholder1} and {placeholder2}", + ) mock_exit.assert_called_with(1) @@ -116,7 +121,12 @@ def test_validate_query_format_extra_placeholder(mocker): query_string = "This is a query with placeholders {placeholder1} and {placeholder2}" expected_placeholders = {"placeholder1"} validate_query_format(query_string, expected_placeholders) - mock_log_error.assert_called_with('%s%s\nFor the query: %s', '', "Extra placeholders: {'placeholder2'}.", 'This is a query with placeholders {placeholder1} and {placeholder2}') + mock_log_error.assert_called_with( + "%s%s\nFor the query: %s", + "", + "Extra placeholders: {'placeholder2'}.", + "This is a query with placeholders {placeholder1} and {placeholder2}", + ) mock_exit.assert_called_with(1) From ebf5cbec5faaca9e34fc1172b31b641f9403ee99 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:15:15 +0200 Subject: [PATCH 42/60] Unit tests using pytest for consolidated_issue.py. --- .../model/consolidated_issue.py | 18 +++++- tests/model/test_consolidated_issue.py | 62 +++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 tests/model/test_consolidated_issue.py diff --git a/living_documentation_generator/model/consolidated_issue.py b/living_documentation_generator/model/consolidated_issue.py index 8fd73f2..a4e099d 100644 --- a/living_documentation_generator/model/consolidated_issue.py +++ b/living_documentation_generator/model/consolidated_issue.py @@ -17,7 +17,7 @@ """ This module contains a data container for Consolidated Issue, which holds all the essential logic. """ - +import logging from typing import Optional from github.Issue import Issue @@ -25,6 +25,8 @@ from living_documentation_generator.utils.utils import sanitize_filename from living_documentation_generator.model.project_status import ProjectStatus +logger = logging.getLogger(__name__) + class ConsolidatedIssue: """ @@ -145,7 +147,17 @@ def generate_page_filename(self) -> str: @return: The generated page filename. """ - md_filename_base = f"{self.number}_{self.title.lower()}.md" - page_filename = sanitize_filename(md_filename_base) + try: + md_filename_base = f"{self.number}_{self.title.lower()}.md" + page_filename = sanitize_filename(md_filename_base) + except AttributeError: + logger.error( + "Issue page filename generation failed for Issue %s/%s (%s). Issue does not have a title.", + self.organization_name, + self.repository_name, + self.number, + exc_info=True, + ) + return f"{self.number}.md" return page_filename diff --git a/tests/model/test_consolidated_issue.py b/tests/model/test_consolidated_issue.py new file mode 100644 index 0000000..95b6b94 --- /dev/null +++ b/tests/model/test_consolidated_issue.py @@ -0,0 +1,62 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from github.Issue import Issue + +from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue +from living_documentation_generator.model.project_status import ProjectStatus + + +# update_with_project_data + + +# THIS MIGHT NOT BE NEEDED, SINCE WE TEST ASSIGN AND APPEND +def test_update_with_project_data_correct_behaviour(): + actual_consolidated_issue = ConsolidatedIssue("organization/repo") + project_status = ProjectStatus() + + actual_consolidated_issue.update_with_project_data(project_status) + + assert actual_consolidated_issue.linked_to_project + assert actual_consolidated_issue.project_issue_statuses == [project_status] + + +# generate_page_filename + + +def test_generate_page_filename_correct_behaviour(): + mock_issue = Issue(None, None, {"number": 1, "title": None}, completed=True) + consolidated_issue = ConsolidatedIssue("organization/repository", mock_issue) + + actual = consolidated_issue.generate_page_filename() + + assert actual == "1.md" + + +def test_generate_page_filename_with_none_title(mocker): + mock_log_error = mocker.patch("living_documentation_generator.model.consolidated_issue.logger.error") + mock_issue = Issue(None, None, {"number": 1, "title": None}, completed=True) + consolidated_issue = ConsolidatedIssue("organization/repository", mock_issue) + + actual = consolidated_issue.generate_page_filename() + + assert actual == "1.md" + mock_log_error.assert_called_once_with( + "Issue page filename generation failed for Issue %s/%s (%s). Issue does not have a title.", + "organization", + "repository", + 1, + exc_info=True, + ) From e760a988275b27be90f0b64acddf8118271c7bfa Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:41:04 +0200 Subject: [PATCH 43/60] Comment changes implemented. --- tests/model/test_consolidated_issue.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/tests/model/test_consolidated_issue.py b/tests/model/test_consolidated_issue.py index 95b6b94..fcbcdb0 100644 --- a/tests/model/test_consolidated_issue.py +++ b/tests/model/test_consolidated_issue.py @@ -16,33 +16,18 @@ from github.Issue import Issue from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue -from living_documentation_generator.model.project_status import ProjectStatus - - -# update_with_project_data - - -# THIS MIGHT NOT BE NEEDED, SINCE WE TEST ASSIGN AND APPEND -def test_update_with_project_data_correct_behaviour(): - actual_consolidated_issue = ConsolidatedIssue("organization/repo") - project_status = ProjectStatus() - - actual_consolidated_issue.update_with_project_data(project_status) - - assert actual_consolidated_issue.linked_to_project - assert actual_consolidated_issue.project_issue_statuses == [project_status] # generate_page_filename def test_generate_page_filename_correct_behaviour(): - mock_issue = Issue(None, None, {"number": 1, "title": None}, completed=True) + mock_issue = Issue(None, None, {"number": 1, "title": "Issue Title"}, completed=True) consolidated_issue = ConsolidatedIssue("organization/repository", mock_issue) actual = consolidated_issue.generate_page_filename() - assert actual == "1.md" + assert actual == "1_issue_title.md" def test_generate_page_filename_with_none_title(mocker): From 873c92b625cb565506e3200bdb50c0062512e1c2 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:21:14 +0200 Subject: [PATCH 44/60] Unit tests using pytest for github_project.py. --- .../model/github_project.py | 23 ++- tests/model/test_github_project.py | 136 ++++++++++++++++++ 2 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 tests/model/test_github_project.py diff --git a/living_documentation_generator/model/github_project.py b/living_documentation_generator/model/github_project.py index e58c16f..6985b51 100644 --- a/living_documentation_generator/model/github_project.py +++ b/living_documentation_generator/model/github_project.py @@ -72,17 +72,26 @@ def loads(self, project_json: dict, repository: Repository, field_option_respons @param field_option_response: The response containing the field options for the project. @return: The GithubProject object with the loaded data. """ - self.__id = project_json["id"] - self.__number = project_json["number"] - self.__title = project_json["title"] - self.__organization_name = repository.owner.login + try: + self.__id = project_json["id"] + self.__number = project_json["number"] + self.__title = project_json["title"] + self.__organization_name = repository.owner.login + + logger.debug("Updating field options for projects in repository `%s`.", repository.full_name) + except KeyError: + logger.error( + "There is no expected response structure for the project json: %s", + repository.full_name, + exc_info=True, + ) + return self - logger.debug("Updating field options for projects in repository `%s`.", repository.full_name) - self.__update_field_options(field_option_response) + self._update_field_options(field_option_response) return self - def __update_field_options(self, field_option_response: dict) -> None: + def _update_field_options(self, field_option_response: dict) -> None: """ Parse and update the field options of the project from a JSON response. diff --git a/tests/model/test_github_project.py b/tests/model/test_github_project.py new file mode 100644 index 0000000..006bce8 --- /dev/null +++ b/tests/model/test_github_project.py @@ -0,0 +1,136 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from github.Repository import Repository +from living_documentation_generator.model.github_project import GithubProject + + +# loads + + +def test_loads_with_valid_input_loads_correctly(mocker): + github_project = GithubProject() + project_json = { + "id": "123", + "number": 1, + "title": "Test Project" + } + repository = mocker.Mock(spec=Repository) + repository.owner.login = "organizationABC" + repository.full_name = "organizationABC/repoABC" + field_option_response = { + "repository": { + "projectV2": { + "fields": { + "nodes": [ + { + "name": "field", + "options": [{"name": "option1"}, {"name": "option2"}] + } + ] + } + } + } + } + + actual = github_project.loads(project_json, repository, field_option_response) + + assert actual.id == project_json["id"] + assert actual.number == project_json["number"] + assert actual.title == project_json["title"] + assert actual.organization_name == repository.owner.login + assert actual.field_options == {"field": ["option1", "option2"]} + + +def test_loads_with_missing_key(mocker): + github_project = GithubProject() + mock_log_error = mocker.patch("living_documentation_generator.model.github_project.logger.error") + project_json = { + "id": "123", + "title": "Test Project", + "unexpected_key": "unexpected_value" + } + repository = mocker.Mock(spec=Repository) + repository.owner.login = "organizationABC" + repository.full_name = "organizationABC/repoABC" + field_option_response = { + "repository": { + "projectV2": { + "fields": { + "nodes": [ + { + "name": "field1", + "options": [{"name": "option1"}, {"name": "option2"}] + } + ] + } + } + } + } + + github_project.loads(project_json, repository, field_option_response) + + mock_log_error.assert_called_once_with( + 'There is no expected response structure for the project json: %s', 'organizationABC/repoABC', exc_info=True) + + +# _update_field_options + + +def test_update_field_options_with_valid_input(): + github_project = GithubProject() + field_option_response = { + "repository": { + "projectV2": { + "fields": { + "nodes": [ + { + "name": "field1", + "options": [{"name": "option1"}, {"name": "option2"}] + }, + { + "wrong_name": "field2", + "wrong_options": [{"name": "option3"}, {"name": "option4"}] + } + ] + } + } + } + } + + github_project._update_field_options(field_option_response) + + assert github_project.field_options == {"field1": ["option1", "option2"]} + + +def test_update_field_options_with_no_expected_response_structure(mocker): + github_project = GithubProject() + mock_log_error = mocker.patch("living_documentation_generator.model.github_project.logger.error") + field_option_response = { + "unexpected_structure": { + "unexpected_key": "unexpected_value" + } + } + + github_project._update_field_options(field_option_response) + + assert github_project.field_options == {} + mock_log_error.assert_called_once_with( + "There is no expected response structure for field options fetched from project: %s", + github_project.title, + exc_info=True, + ) + From b2652d38cd94f8fc0dd7cc06438b0b00b885a353 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:43:50 +0200 Subject: [PATCH 45/60] Unit tests using pytest for project_issue.py. --- .../model/github_project.py | 10 ++ .../model/project_issue.py | 21 ++- tests/model/test_github_project.py | 50 ++----- tests/model/test_project_issue.py | 124 ++++++++++++++++++ 4 files changed, 158 insertions(+), 47 deletions(-) create mode 100644 tests/model/test_project_issue.py diff --git a/living_documentation_generator/model/github_project.py b/living_documentation_generator/model/github_project.py index 6985b51..befda68 100644 --- a/living_documentation_generator/model/github_project.py +++ b/living_documentation_generator/model/github_project.py @@ -53,6 +53,11 @@ def title(self) -> str: """Getter of the project title.""" return self.__title + @title.setter + def title(self, title: str) -> None: + """Setter of the project title.""" + self.__title = title + @property def organization_name(self) -> str: """Getter of the organization name.""" @@ -63,6 +68,11 @@ def field_options(self) -> dict[str, str]: """Getter of the project field options.""" return self.__field_options + @field_options.setter + def field_options(self, field_options: dict[str, str]) -> None: + """Setter of the project field options.""" + self.__field_options = field_options + def loads(self, project_json: dict, repository: Repository, field_option_response: dict) -> "GithubProject": """ Load the project data from several inputs. diff --git a/living_documentation_generator/model/project_issue.py b/living_documentation_generator/model/project_issue.py index fe2d260..578b51c 100644 --- a/living_documentation_generator/model/project_issue.py +++ b/living_documentation_generator/model/project_issue.py @@ -67,14 +67,23 @@ def loads(self, issue_json: dict, project: GithubProject) -> Optional["ProjectIs @param: project: The GithubProject object representing the project the issue belongs to. @return: The ProjectIssue object with the loaded data. """ - if not issue_json["content"]: - logger.debug("No issue data provided in received json.") - logger.debug(issue_json) + if "content" not in issue_json: + logger.debug("No issue data provided in received json: %s.", issue_json) return None - self.__number = issue_json["content"]["number"] - self.__organization_name = issue_json["content"]["repository"]["owner"]["login"] - self.__repository_name = issue_json["content"]["repository"]["name"] + try: + self.__number = issue_json["content"]["number"] + except KeyError: + logger.debug("Wrong project issue json structure for `number` value: %s.", issue_json) + try: + self.__repository_name = issue_json["content"]["repository"]["name"] + except KeyError: + logger.debug("Wrong project issue json structure for `repository_name` value: %s.", issue_json) + try: + self.__organization_name = issue_json["content"]["repository"]["owner"]["login"] + except KeyError: + logger.debug("Wrong project issue json structure for `organization_name` value: %s.", issue_json) + self.__project_status.project_title = project.title # Parse the field types from the response diff --git a/tests/model/test_github_project.py b/tests/model/test_github_project.py index 006bce8..7aaddba 100644 --- a/tests/model/test_github_project.py +++ b/tests/model/test_github_project.py @@ -23,25 +23,14 @@ def test_loads_with_valid_input_loads_correctly(mocker): github_project = GithubProject() - project_json = { - "id": "123", - "number": 1, - "title": "Test Project" - } + project_json = {"id": "123", "number": 1, "title": "Test Project"} repository = mocker.Mock(spec=Repository) repository.owner.login = "organizationABC" repository.full_name = "organizationABC/repoABC" field_option_response = { "repository": { "projectV2": { - "fields": { - "nodes": [ - { - "name": "field", - "options": [{"name": "option1"}, {"name": "option2"}] - } - ] - } + "fields": {"nodes": [{"name": "field", "options": [{"name": "option1"}, {"name": "option2"}]}]} } } } @@ -58,25 +47,14 @@ def test_loads_with_valid_input_loads_correctly(mocker): def test_loads_with_missing_key(mocker): github_project = GithubProject() mock_log_error = mocker.patch("living_documentation_generator.model.github_project.logger.error") - project_json = { - "id": "123", - "title": "Test Project", - "unexpected_key": "unexpected_value" - } + project_json = {"id": "123", "title": "Test Project", "unexpected_key": "unexpected_value"} repository = mocker.Mock(spec=Repository) repository.owner.login = "organizationABC" repository.full_name = "organizationABC/repoABC" field_option_response = { "repository": { "projectV2": { - "fields": { - "nodes": [ - { - "name": "field1", - "options": [{"name": "option1"}, {"name": "option2"}] - } - ] - } + "fields": {"nodes": [{"name": "field1", "options": [{"name": "option1"}, {"name": "option2"}]}]} } } } @@ -84,7 +62,8 @@ def test_loads_with_missing_key(mocker): github_project.loads(project_json, repository, field_option_response) mock_log_error.assert_called_once_with( - 'There is no expected response structure for the project json: %s', 'organizationABC/repoABC', exc_info=True) + "There is no expected response structure for the project json: %s", "organizationABC/repoABC", exc_info=True + ) # _update_field_options @@ -97,14 +76,8 @@ def test_update_field_options_with_valid_input(): "projectV2": { "fields": { "nodes": [ - { - "name": "field1", - "options": [{"name": "option1"}, {"name": "option2"}] - }, - { - "wrong_name": "field2", - "wrong_options": [{"name": "option3"}, {"name": "option4"}] - } + {"name": "field1", "options": [{"name": "option1"}, {"name": "option2"}]}, + {"wrong_name": "field2", "wrong_options": [{"name": "option3"}, {"name": "option4"}]}, ] } } @@ -119,11 +92,7 @@ def test_update_field_options_with_valid_input(): def test_update_field_options_with_no_expected_response_structure(mocker): github_project = GithubProject() mock_log_error = mocker.patch("living_documentation_generator.model.github_project.logger.error") - field_option_response = { - "unexpected_structure": { - "unexpected_key": "unexpected_value" - } - } + field_option_response = {"unexpected_structure": {"unexpected_key": "unexpected_value"}} github_project._update_field_options(field_option_response) @@ -133,4 +102,3 @@ def test_update_field_options_with_no_expected_response_structure(mocker): github_project.title, exc_info=True, ) - diff --git a/tests/model/test_project_issue.py b/tests/model/test_project_issue.py new file mode 100644 index 0000000..3d033fd --- /dev/null +++ b/tests/model/test_project_issue.py @@ -0,0 +1,124 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from living_documentation_generator.model.github_project import GithubProject +from living_documentation_generator.model.project_issue import ProjectIssue + + +# loads + + +def test_loads_with_valid_input(): + project_issue = ProjectIssue() + issue_json = { + "content": {"number": 1, "repository": {"owner": {"login": "organizationABC"}, "name": "repoABC"}}, + "fieldValues": { + "nodes": [ + {"__typename": "ProjectV2ItemFieldSingleSelectValue", "name": "Status1"}, + {"__typename": "ProjectV2ItemFieldSingleSelectValue", "name": "Priority1"}, + {"__typename": "ProjectV2ItemFieldSingleSelectValue", "name": "Size1"}, + {"__typename": "ProjectV2ItemFieldSingleSelectValue", "name": "MoSCoW1"}, + ] + }, + } + project = GithubProject() + project.title = "Test Project" + project.field_options = { + "Status": ["Status1", "Status2"], + "Priority": ["Priority1", "Priority2"], + "Size": ["Size1", "Size2"], + "MoSCoW": ["MoSCoW1", "MoSCoW2"], + } + + actual = project_issue.loads(issue_json, project) + + assert actual is not None + assert actual.number == issue_json["content"]["number"] + assert actual.organization_name == issue_json["content"]["repository"]["owner"]["login"] + assert actual.repository_name == issue_json["content"]["repository"]["name"] + assert actual.project_status.project_title == project.title + assert actual.project_status.status == "Status1" + assert actual.project_status.priority == "Priority1" + assert actual.project_status.size == "Size1" + assert actual.project_status.moscow == "MoSCoW1" + + +def test_loads_without_content_key_logs_debug(mocker): + project_issue = ProjectIssue() + mock_log = mocker.patch("living_documentation_generator.model.project_issue.logger") + issue_json = {} + project = GithubProject() + + actual = project_issue.loads(issue_json, project) + + assert actual is None + mock_log.debug.assert_called_once_with("No issue data provided in received json: %s.", {}) + + +def test_loads_with_incorrect_json_structure_for_number(mocker): + project_issue = ProjectIssue() + mock_log = mocker.patch("living_documentation_generator.model.project_issue.logger") + incorrect_json = {"content": {}} + project = GithubProject() + + actual = project_issue.loads(incorrect_json, project) + + assert actual.number == 0 + assert actual.organization_name == "" + assert actual.repository_name == "" + mock_log.assert_has_calls( + [ + mocker.call.debug("Wrong project issue json structure for `number` value: %s.", incorrect_json), + mocker.call.debug("Wrong project issue json structure for `repository_name` value: %s.", incorrect_json), + mocker.call.debug("Wrong project issue json structure for `organization_name` value: %s.", incorrect_json), + ] + ) + + +def test_loads_with_incorrect_json_structure_for_repository_name(mocker): + project_issue = ProjectIssue() + mock_log = mocker.patch("living_documentation_generator.model.project_issue.logger") + incorrect_json = {"content": {"number": 1}} + project = GithubProject() + + actual = project_issue.loads(incorrect_json, project) + + assert actual.number == 1 + assert actual.organization_name == "" + assert actual.repository_name == "" + mock_log.assert_has_calls( + [ + mocker.call.debug("Wrong project issue json structure for `repository_name` value: %s.", incorrect_json), + mocker.call.debug("Wrong project issue json structure for `organization_name` value: %s.", incorrect_json), + ] + ) + + +def test_loads_with_incorrect_json_structure_for_organization_name(mocker): + project_issue = ProjectIssue() + mock_log = mocker.patch("living_documentation_generator.model.project_issue.logger") + incorrect_json = { + "content": {"number": 1, "repository": {"owner": {"incorrect_key": "incorrect_name"}, "name": "repositoryABC"}} + } + project = GithubProject() + + actual = project_issue.loads(incorrect_json, project) + + assert actual.number == 1 + assert actual.repository_name == "repositoryABC" + assert actual.organization_name == "" + mock_log.debug.assert_called_once_with( + "Wrong project issue json structure for `organization_name` value: %s.", incorrect_json + ) From 08cda52dbb3453f0221fd5eb9d83a5c15f6be10c Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:55:43 +0200 Subject: [PATCH 46/60] Initial save commit. --- README.md | 4 +- .../action_inputs.py | 64 ++++++++++++++----- .../utils/constants.py | 1 + living_documentation_generator/utils/utils.py | 13 ++++ 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 322019d..653f742 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Configure the action by customizing the following parameters based on your needs ## Action Outputs The Living Documentation Generator action provides a key output that allows users to locate and access the generated documentation easily. This output can be utilized in various ways within your CI/CD pipeline to ensure the documentation is effectively distributed and accessible. +The output-path can not be an empty string. It can not aim to the root and other project directories as well. - **output-path** - **Description**: This output provides the path to the directory where the generated living documentation files are stored. @@ -316,7 +317,8 @@ Add the shebang line at the top of the sh script file. ### Set the Environment Variables Set the configuration environment variables in the shell script following the structure below. -Also make sure that the GITHUB_TOKEN is configured in your environment variables. +Also make sure that the INPUT_GITHUB_TOKEN is configured in your environment variables. +INPUT_OUTPUT_PATH can not be an empty string. It can not aim to the root and other project directories as well. ``` export INPUT_GITHUB_TOKEN=$(printenv GITHUB_TOKEN) export INPUT_REPOSITORIES='[ diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 91f6a43..c962e46 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -24,13 +24,13 @@ import sys from living_documentation_generator.model.config_repository import ConfigRepository -from living_documentation_generator.utils.utils import get_action_input, make_absolute_path +from living_documentation_generator.utils.utils import get_action_input, make_absolute_path, get_all_project_directories from living_documentation_generator.utils.constants import ( GITHUB_TOKEN, PROJECT_STATE_MINING, REPOSITORIES, OUTPUT_PATH, - STRUCTURED_OUTPUT, + STRUCTURED_OUTPUT, DEFAULT_OUTPUT_PATH, ) logger = logging.getLogger(__name__) @@ -74,6 +74,23 @@ def structured_output(self) -> bool: """Getter of the structured output switch.""" return self.__structured_output + @staticmethod + def validate_instance(input_value, expected_type: type, error_message: str, error_buffer: list) -> bool: + """ + Validates the input value against the expected type. + + @param input_value: The input value to validate. + @param expected_type: The expected type of the input value. + @param error_message: The error message to log if the validation fails. + @param error_buffer: The buffer to store the error messages. + @return: The boolean result of the validation. + """ + + if not isinstance(input_value, expected_type): + error_buffer.append(error_message) + return False + return True + def load_from_environment(self, validate: bool = True) -> "ActionInputs": """ Load the action inputs from the environment variables and validate them if needed. @@ -84,19 +101,19 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": self.__github_token = get_action_input(GITHUB_TOKEN) self.__is_project_state_mining_enabled = get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" self.__structured_output = get_action_input(STRUCTURED_OUTPUT, "false").lower() == "true" - out_path = get_action_input(OUTPUT_PATH, "./output") - self.__output_directory = make_absolute_path(out_path) repositories_json = get_action_input(REPOSITORIES, "") + out_path = get_action_input(OUTPUT_PATH, default=DEFAULT_OUTPUT_PATH) + self.__output_directory = make_absolute_path(out_path) + + # Validate inputs + if validate: + self.validate_inputs(repositories_json, out_path) logger.debug("Is project state mining allowed: %s.", self.is_project_state_mining_enabled) logger.debug("JSON repositories to fetch from: %s.", repositories_json) logger.debug("Output directory: %s.", self.output_directory) logger.debug("Is output directory structured: %s.", self.structured_output) - # Validate inputs - if validate: - self.validate_inputs(repositories_json) - # Parse repositories json string into json dictionary format try: repositories_json = json.loads(repositories_json) @@ -113,22 +130,39 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": return self - def validate_inputs(self, repositories_json: str) -> None: + def validate_inputs(self, repositories_json: str, out_path: str) -> None: """ Validate the input attributes of the action. @param repositories_json: The JSON string containing the repositories to fetch. + @param out_path: The output path to save the results to. @return: None """ + errors = [] - # Validate correct format of input repositories_json + # Validate INPUT_GITHUB_TOKEN + if not self.github_token: + errors.append("Input GitHub token could not be loaded from the environment.") + if not isinstance(self.github_token, str): + errors.append("Input GitHub token must be a string.") + + # Validate INPUT_REPOSITORIES and its correct JSON format try: json.loads(repositories_json) except json.JSONDecodeError: - logger.error("Input attr `repositories_json` is not a valid JSON string.", exc_info=True) - sys.exit(1) + errors.append("Input Repositories is not a valid JSON string.") + + # Validate INPUT_OUTPUT_PATH + if out_path == "": + errors.append("Input Output path can not be an empty string.") + + # Check that the INPUT_OUTPUT_PATH is not a directory in the project + # Note: That would cause a rewriting project files + project_directories = get_all_project_directories() + if out_path in project_directories: + errors.append("Output path can not be a directory in the project.") - # Validate GitHub token - if not self.__github_token: - logger.error("GitHub token could not be loaded from the environment.", exc_info=True) + if errors: + for error in errors: + logger.error(error, exc_info=True) sys.exit(1) diff --git a/living_documentation_generator/utils/constants.py b/living_documentation_generator/utils/constants.py index 4b8e865..fff60d6 100644 --- a/living_documentation_generator/utils/constants.py +++ b/living_documentation_generator/utils/constants.py @@ -23,6 +23,7 @@ PROJECT_STATE_MINING = "PROJECT_STATE_MINING" REPOSITORIES = "REPOSITORIES" OUTPUT_PATH = "OUTPUT_PATH" +DEFAULT_OUTPUT_PATH = "./output" STRUCTURED_OUTPUT = "STRUCTURED_OUTPUT" # GitHub API constants diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 552b45b..38e5cce 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -91,6 +91,19 @@ def validate_query_format(query_string, expected_placeholders) -> None: sys.exit(1) +def get_all_project_directories(path: str = '.') -> list[str]: + """ + Get all directories in the project starting from the specified path. + + @param path: The path to start searching for directories. + @return: A list of all directories in the project. + """ + directories = [] + for dir_path, dir_names, _ in os.walk(path): + directories.extend([os.path.join(dir_path, d) for d in dir_names]) + return directories + + # GitHub action utils def get_action_input(name: str, default: Optional[str] = None) -> str: """ From b1f6a13d1220d700f2c5cf897f99a48ce386ac69 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:11:46 +0200 Subject: [PATCH 47/60] Action Inputs validation logic added. --- living_documentation_generator/action_inputs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index c962e46..084acfa 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -142,27 +142,28 @@ def validate_inputs(self, repositories_json: str, out_path: str) -> None: # Validate INPUT_GITHUB_TOKEN if not self.github_token: - errors.append("Input GitHub token could not be loaded from the environment.") + errors.append("INPUT_GITHUB_TOKEN could not be loaded from the environment.") if not isinstance(self.github_token, str): - errors.append("Input GitHub token must be a string.") + errors.append("INPUT_GITHUB_TOKEN must be a string.") # Validate INPUT_REPOSITORIES and its correct JSON format try: json.loads(repositories_json) except json.JSONDecodeError: - errors.append("Input Repositories is not a valid JSON string.") + errors.append("INPUT_REPOSITORIES is not a valid JSON string.") # Validate INPUT_OUTPUT_PATH if out_path == "": - errors.append("Input Output path can not be an empty string.") + errors.append("INPUT_OUTPUT_PATH can not be an empty string.") # Check that the INPUT_OUTPUT_PATH is not a directory in the project # Note: That would cause a rewriting project files project_directories = get_all_project_directories() if out_path in project_directories: - errors.append("Output path can not be a directory in the project.") + errors.append("INPUT_OUTPUT_PATH can not be a project directory.") if errors: for error in errors: - logger.error(error, exc_info=True) + logger.error(error) + logger.info("GitHub Action is terminating as a cause of an input validation error.") sys.exit(1) From 6d08f650f3e772577f751fea8abd17d583f59897 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:14:51 +0200 Subject: [PATCH 48/60] Action Inputs bug fix. --- .../action_inputs.py | 20 ++----------------- tests/test_action_inputs.py | 0 2 files changed, 2 insertions(+), 18 deletions(-) create mode 100644 tests/test_action_inputs.py diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 084acfa..6092d1c 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -30,7 +30,8 @@ PROJECT_STATE_MINING, REPOSITORIES, OUTPUT_PATH, - STRUCTURED_OUTPUT, DEFAULT_OUTPUT_PATH, + STRUCTURED_OUTPUT, + DEFAULT_OUTPUT_PATH, ) logger = logging.getLogger(__name__) @@ -74,23 +75,6 @@ def structured_output(self) -> bool: """Getter of the structured output switch.""" return self.__structured_output - @staticmethod - def validate_instance(input_value, expected_type: type, error_message: str, error_buffer: list) -> bool: - """ - Validates the input value against the expected type. - - @param input_value: The input value to validate. - @param expected_type: The expected type of the input value. - @param error_message: The error message to log if the validation fails. - @param error_buffer: The buffer to store the error messages. - @return: The boolean result of the validation. - """ - - if not isinstance(input_value, expected_type): - error_buffer.append(error_message) - return False - return True - def load_from_environment(self, validate: bool = True) -> "ActionInputs": """ Load the action inputs from the environment variables and validate them if needed. diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py new file mode 100644 index 0000000..e69de29 From 910ee4913651dfc282890da1cd39239ca3033311 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:20:26 +0200 Subject: [PATCH 49/60] Black formatting. --- living_documentation_generator/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index 38e5cce..f7b5019 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -91,7 +91,7 @@ def validate_query_format(query_string, expected_placeholders) -> None: sys.exit(1) -def get_all_project_directories(path: str = '.') -> list[str]: +def get_all_project_directories(path: str = ".") -> list[str]: """ Get all directories in the project starting from the specified path. From f79f6c048fffb32081cdbe7e09476383c60c0e88 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:38:31 +0200 Subject: [PATCH 50/60] Implementation of new method `get_all_project_directories`. --- .../action_inputs.py | 6 ++++- tests/utils/test_utils.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 6092d1c..1f1971a 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -142,7 +142,11 @@ def validate_inputs(self, repositories_json: str, out_path: str) -> None: # Check that the INPUT_OUTPUT_PATH is not a directory in the project # Note: That would cause a rewriting project files - project_directories = get_all_project_directories() + # TODO: Do not know how to make sure, that we exclude folder that is made for output in the project + project_directories = get_all_project_directories(out_path) + if DEFAULT_OUTPUT_PATH in project_directories: + project_directories.remove(DEFAULT_OUTPUT_PATH) + if out_path in project_directories: errors.append("INPUT_OUTPUT_PATH can not be a project directory.") diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 11519d0..13cad79 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os import pytest @@ -23,6 +24,7 @@ get_action_input, set_action_output, set_action_failed, + get_all_project_directories, ) @@ -59,6 +61,29 @@ def test_sanitize_filename(filename_example, expected_filename): assert actual_filename == expected_filename +# get_all_project_directories + +def test_get_all_project_directories_correct_behaviour(): + base_path = "test_dir" + expected_directories = ["test_dir/sub_dir1", "test_dir/sub_dir2", "test_dir/sub_dir1/sub_sub_dir1"] + + os.makedirs(os.path.join(base_path, "sub_dir1", "sub_sub_dir1"), exist_ok=True) + os.makedirs(os.path.join(base_path, "sub_dir2"), exist_ok=True) + + actual_directories = get_all_project_directories(base_path) + + assert expected_directories == actual_directories + + +def test_get_all_project_directories_empty_dir(): + base_path = "empty_dir" + expected_directories = [] + + actual_directories = get_all_project_directories(base_path) + + assert expected_directories == actual_directories + + # GitHub action utils # get_action_input From 61323d53c3667bba950a78ee88138b79687f819c Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:15:18 +0200 Subject: [PATCH 51/60] New logic for ActionInputs. --- .../action_inputs.py | 118 +++++++----------- living_documentation_generator/generator.py | 70 +++++------ living_documentation_generator/utils/utils.py | 5 +- main.py | 15 ++- 4 files changed, 82 insertions(+), 126 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 1f1971a..19e7632 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -21,6 +21,7 @@ import json import logging +import os import sys from living_documentation_generator.model.config_repository import ConfigRepository @@ -43,60 +44,26 @@ class ActionInputs: and validating the inputs required for running the GH Action. """ - def __init__(self): - self.__github_token: str = "" - self.__is_project_state_mining_enabled: bool = False - self.__repositories: list[ConfigRepository] = [] - self.__output_directory: str = "" - self.__structured_output: bool = False - - @property - def github_token(self) -> str: + @staticmethod + def get_github_token() -> str: """Getter of the GitHub authorization token.""" - return self.__github_token + return get_action_input(GITHUB_TOKEN) - @property - def is_project_state_mining_enabled(self) -> bool: + @staticmethod + def get_is_project_state_mining_enabled() -> bool: """Getter of the project state mining switch.""" - return self.__is_project_state_mining_enabled - - @property - def repositories(self) -> list[ConfigRepository]: - """Getter of the list of repositories to fetch from.""" - return self.__repositories - - @property - def output_directory(self) -> str: - """Getter of the output directory.""" - return self.__output_directory + return get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" - @property - def structured_output(self) -> bool: + @staticmethod + def get_is_structured_output_enabled() -> bool: """Getter of the structured output switch.""" - return self.__structured_output + return get_action_input(STRUCTURED_OUTPUT, "false").lower() == "true" - def load_from_environment(self, validate: bool = True) -> "ActionInputs": - """ - Load the action inputs from the environment variables and validate them if needed. - - @param validate: Switch indicating if the inputs should be validated. - @return: The instance of the ActionInputs class. - """ - self.__github_token = get_action_input(GITHUB_TOKEN) - self.__is_project_state_mining_enabled = get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" - self.__structured_output = get_action_input(STRUCTURED_OUTPUT, "false").lower() == "true" + @staticmethod + def get_repositories() -> list[ConfigRepository]: + """Getter of the list of repositories to fetch from.""" + repositories = [] repositories_json = get_action_input(REPOSITORIES, "") - out_path = get_action_input(OUTPUT_PATH, default=DEFAULT_OUTPUT_PATH) - self.__output_directory = make_absolute_path(out_path) - - # Validate inputs - if validate: - self.validate_inputs(repositories_json, out_path) - - logger.debug("Is project state mining allowed: %s.", self.is_project_state_mining_enabled) - logger.debug("JSON repositories to fetch from: %s.", repositories_json) - logger.debug("Output directory: %s.", self.output_directory) - logger.debug("Is output directory structured: %s.", self.structured_output) # Parse repositories json string into json dictionary format try: @@ -108,50 +75,49 @@ def load_from_environment(self, validate: bool = True) -> "ActionInputs": for repository_json in repositories_json: config_repository = ConfigRepository() if config_repository.load_from_json(repository_json): - self.__repositories.append(config_repository) + repositories.append(config_repository) else: logger.error("Failed to load repository from JSON: %s.", repository_json) - return self + return repositories + + @staticmethod + def get_output_directory() -> str: + """Getter of the output directory.""" + out_path = get_action_input(OUTPUT_PATH, default=DEFAULT_OUTPUT_PATH) + return make_absolute_path(out_path) - def validate_inputs(self, repositories_json: str, out_path: str) -> None: + @staticmethod + def validate_inputs(out_path: str) -> None: """ - Validate the input attributes of the action. + Loads the inputs provided for the Living documentation generator. + Logs any validation errors and exits if any are found. - @param repositories_json: The JSON string containing the repositories to fetch. - @param out_path: The output path to save the results to. + @param out_path: The output path for the generated documentation. @return: None """ - errors = [] - # Validate INPUT_GITHUB_TOKEN - if not self.github_token: - errors.append("INPUT_GITHUB_TOKEN could not be loaded from the environment.") - if not isinstance(self.github_token, str): - errors.append("INPUT_GITHUB_TOKEN must be a string.") - - # Validate INPUT_REPOSITORIES and its correct JSON format - try: - json.loads(repositories_json) - except json.JSONDecodeError: - errors.append("INPUT_REPOSITORIES is not a valid JSON string.") + # Validate INPUT_REPOSITORIES + ActionInputs.get_repositories() # Validate INPUT_OUTPUT_PATH if out_path == "": - errors.append("INPUT_OUTPUT_PATH can not be an empty string.") + logger.error("INPUT_OUTPUT_PATH can not be an empty string.") + sys.exit(1) - # Check that the INPUT_OUTPUT_PATH is not a directory in the project + # Check that the INPUT_OUTPUT_PATH is not a project directory # Note: That would cause a rewriting project files - # TODO: Do not know how to make sure, that we exclude folder that is made for output in the project - project_directories = get_all_project_directories(out_path) + project_directories = get_all_project_directories() if DEFAULT_OUTPUT_PATH in project_directories: project_directories.remove(DEFAULT_OUTPUT_PATH) - if out_path in project_directories: - errors.append("INPUT_OUTPUT_PATH can not be a project directory.") + for project_directory in project_directories: + # Finds the common path between the absolute paths of out_path and project_directory + common_path = os.path.commonpath([os.path.abspath(out_path), os.path.abspath(project_directory)]) - if errors: - for error in errors: - logger.error(error) - logger.info("GitHub Action is terminating as a cause of an input validation error.") - sys.exit(1) + # Check if common path is equal to the absolute path of project_directory + if common_path == os.path.abspath(project_directory): + logger.error("INPUT_OUTPUT_PATH cannot be chosen as a part of any project folder.") + sys.exit(1) + + logger.debug("Action inputs validation successfully completed.") diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 3913d82..023e69c 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -31,7 +31,6 @@ from living_documentation_generator.action_inputs import ActionInputs from living_documentation_generator.github_projects import GithubProjects from living_documentation_generator.model.github_project import GithubProject -from living_documentation_generator.model.config_repository import ConfigRepository from living_documentation_generator.model.consolidated_issue import ConsolidatedIssue from living_documentation_generator.model.project_issue import ProjectIssue from living_documentation_generator.utils.decorators import safe_call_decorator @@ -61,35 +60,14 @@ class LivingDocumentationGenerator: ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "issue_detail_page_template.md") INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_template.md") - def __init__(self, action_inputs: ActionInputs): - self.__action_inputs = action_inputs + def __init__(self): + github_token = ActionInputs.get_github_token() - github_token = self.__action_inputs.github_token self.__github_instance: Github = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) self.__github_projects_instance: GithubProjects = GithubProjects(token=github_token) self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) self.__safe_call: Callable = safe_call_decorator(self.__rate_limiter) - @property - def repositories(self) -> list[ConfigRepository]: - """Getter of the list of config repository objects to fetch from.""" - return self.__action_inputs.repositories - - @property - def project_state_mining_enabled(self) -> bool: - """Getter of the project state mining switch.""" - return self.__action_inputs.is_project_state_mining_enabled - - @property - def structured_output(self) -> bool: - """Getter of the structured output switch.""" - return self.__action_inputs.structured_output - - @property - def output_path(self) -> str: - """Getter of the output directory.""" - return self.__action_inputs.output_directory - def generate(self) -> None: """ Generate the Living Documentation markdown pages output. @@ -123,15 +101,18 @@ def generate(self) -> None: self._generate_markdown_pages(consolidated_issues) logger.info("Markdown page generation - finished.") - def _clean_output_directory(self) -> None: + @staticmethod + def _clean_output_directory() -> None: """ Clean the output directory from the previous run. @return: None """ - if os.path.exists(self.output_path): - shutil.rmtree(self.output_path) - os.makedirs(self.output_path) + output_path = ActionInputs.get_output_directory() + + if os.path.exists(output_path): + shutil.rmtree(output_path) + os.makedirs(output_path) def _fetch_github_issues(self) -> dict[str, list[Issue]]: """ @@ -144,7 +125,7 @@ def _fetch_github_issues(self) -> dict[str, list[Issue]]: total_issues_number = 0 # Run the fetching logic for every config repository - for config_repository in self.repositories: + for config_repository in ActionInputs.get_repositories(): repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" repository = self.__safe_call(self.__github_instance.get_repo)(repository_id) @@ -194,7 +175,7 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: @return: A dictionary containing project issue objects with unique key. """ - if not self.project_state_mining_enabled: + if not ActionInputs.get_is_project_state_mining_enabled(): logger.info("Fetching GitHub project data - project mining is not allowed.") return {} @@ -203,7 +184,7 @@ def _fetch_github_project_issues(self) -> dict[str, list[ProjectIssue]]: # Mine project issues for every repository all_project_issues: dict[str, list[ProjectIssue]] = {} - for config_repository in self.repositories: + for config_repository in ActionInputs.get_repositories(): repository_id = f"{config_repository.organization_name}/{config_repository.repository_name}" projects_title_filter = config_repository.projects_title_filter logger.debug("Filtering projects: %s. If filter is empty, fetching all.", projects_title_filter) @@ -320,7 +301,7 @@ def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None logger.info("Markdown page generation - generated `%s` issue pages.", len(issues)) # Generate an index page with a summary table about all issues - if self.structured_output: + if ActionInputs.get_is_structured_output_enabled(): self._generate_structured_index_page(issue_index_page_template, issues) else: issues = list(issues.values()) @@ -402,7 +383,9 @@ def _generate_index_page( """ # Initializing the issue table header based on the project mining state issue_table = ( - TABLE_HEADER_WITH_PROJECT_DATA if self.project_state_mining_enabled else TABLE_HEADER_WITHOUT_PROJECT_DATA + TABLE_HEADER_WITH_PROJECT_DATA + if ActionInputs.get_is_project_state_mining_enabled() + else TABLE_HEADER_WITHOUT_PROJECT_DATA ) # Create an issue summary table for every issue @@ -426,7 +409,8 @@ def _generate_index_page( with open(os.path.join(index_directory_path, "_index.md"), "w", encoding="utf-8") as f: f.write(index_page) - def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: + @staticmethod + def _generate_markdown_line(consolidated_issue: ConsolidatedIssue) -> str: """ Generates a markdown summary line for a single issue. @@ -446,7 +430,7 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: status = ", ".join(status_list) if status_list else "---" # Change the bool values to more user-friendly characters - if self.project_state_mining_enabled: + if ActionInputs.get_is_project_state_mining_enabled(): if consolidated_issue.linked_to_project: linked_to_project = LINKED_TO_PROJECT_TRUE else: @@ -466,7 +450,8 @@ def _generate_markdown_line(self, consolidated_issue: ConsolidatedIssue) -> str: return md_issue_line - def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) -> str: + @staticmethod + def _generate_issue_summary_table(consolidated_issue: ConsolidatedIssue) -> str: """ Generates a string representation of feature info in a table format. @@ -508,7 +493,7 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - ] # Update the summary table, based on the project data mining situation - if self.project_state_mining_enabled: + if ActionInputs.get_is_project_state_mining_enabled(): project_statuses = consolidated_issue.project_issue_statuses if consolidated_issue.linked_to_project: @@ -546,18 +531,21 @@ def _generate_issue_summary_table(self, consolidated_issue: ConsolidatedIssue) - return issue_info - def _generate_directory_path(self, repository_id: Optional[str]) -> str: + @staticmethod + def _generate_directory_path(repository_id: Optional[str]) -> str: """ Generates a directory path based on if structured output is required. @param repository_id: The repository id. @return: The generated directory path. """ - if self.structured_output and repository_id: + output_path = ActionInputs.get_output_directory() + + if ActionInputs.get_is_structured_output_enabled() and repository_id: organization_name, repository_name = repository_id.split("/") - output_path = os.path.join(self.output_path, organization_name, repository_name) + output_path = os.path.join(output_path, organization_name, repository_name) else: - output_path = self.output_path + output_path = output_path os.makedirs(output_path, exist_ok=True) diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index f7b5019..dfc680e 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -98,10 +98,7 @@ def get_all_project_directories(path: str = ".") -> list[str]: @param path: The path to start searching for directories. @return: A list of all directories in the project. """ - directories = [] - for dir_path, dir_names, _ in os.walk(path): - directories.extend([os.path.join(dir_path, d) for d in dir_names]) - return directories + return [os.path.join(path, d) for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] # GitHub action utils diff --git a/main.py b/main.py index d958413..3023a7d 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,8 @@ from living_documentation_generator.action_inputs import ActionInputs from living_documentation_generator.generator import LivingDocumentationGenerator -from living_documentation_generator.utils.utils import set_action_output +from living_documentation_generator.utils.constants import OUTPUT_PATH, DEFAULT_OUTPUT_PATH +from living_documentation_generator.utils.utils import set_action_output, get_action_input from living_documentation_generator.utils.logging_config import setup_logging @@ -37,17 +38,21 @@ def run() -> None: logger = logging.getLogger(__name__) logger.info("Starting Living Documentation generation.") - action_inputs = ActionInputs().load_from_environment() + + # Validate the action inputs + out_path_from_config = get_action_input(OUTPUT_PATH, default=DEFAULT_OUTPUT_PATH) + ActionInputs.validate_inputs(out_path_from_config) # Create the Living Documentation Generator - generator = LivingDocumentationGenerator(action_inputs=action_inputs) + generator = LivingDocumentationGenerator() # Generate the Living Documentation generator.generate() # Set the output for the GitHub Action - set_action_output("output-path", generator.output_path) - logger.info("Living Documentation generation - output path set to `%s`.", generator.output_path) + output_path = ActionInputs.get_output_directory() + set_action_output("output-path", output_path) + logger.info("Living Documentation generation - output path set to `%s`.", output_path) logger.info("Living Documentation generation completed.") From 364e9d076a65ceb342c2ac9e5c8b8e2687782bae Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:22:21 +0200 Subject: [PATCH 52/60] Bug fix. --- tests/conftest.py | 2 +- tests/utils/test_utils.py | 25 ------------------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 93d0ffd..67bfeb1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import time import pytest from github import Github from github.Rate import Rate from github.RateLimit import RateLimit from github.Repository import Repository -from pytest_mock import mocker from living_documentation_generator.model.github_project import GithubProject from living_documentation_generator.utils.github_rate_limiter import GithubRateLimiter diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 39d8fb1..1d9cef0 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os import pytest @@ -24,7 +23,6 @@ get_action_input, set_action_output, set_action_failed, - get_all_project_directories, ) @@ -61,29 +59,6 @@ def test_sanitize_filename(filename_example, expected_filename): assert expected_filename == actual_filename -# get_all_project_directories - -def test_get_all_project_directories_correct_behaviour(): - base_path = "test_dir" - expected_directories = ["test_dir/sub_dir1", "test_dir/sub_dir2", "test_dir/sub_dir1/sub_sub_dir1"] - - os.makedirs(os.path.join(base_path, "sub_dir1", "sub_sub_dir1"), exist_ok=True) - os.makedirs(os.path.join(base_path, "sub_dir2"), exist_ok=True) - - actual_directories = get_all_project_directories(base_path) - - assert expected_directories == actual_directories - - -def test_get_all_project_directories_empty_dir(): - base_path = "empty_dir" - expected_directories = [] - - actual_directories = get_all_project_directories(base_path) - - assert expected_directories == actual_directories - - # GitHub action utils # get_action_input From a5029e7c2aa6b4b9ecfcfae365322d486106c469 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:23:48 +0200 Subject: [PATCH 53/60] Unit tests using Pytest for action_inputs.py. --- .../action_inputs.py | 49 ++++-- living_documentation_generator/generator.py | 2 - tests/model/test_project_issue.py | 4 +- tests/test_action_inputs.py | 159 ++++++++++++++++++ 4 files changed, 195 insertions(+), 19 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 19e7632..3e8c27f 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -46,38 +46,55 @@ class ActionInputs: @staticmethod def get_github_token() -> str: - """Getter of the GitHub authorization token.""" + """ + Getter of the GitHub authorization token. + @return: The GitHub authorization token. + """ return get_action_input(GITHUB_TOKEN) @staticmethod def get_is_project_state_mining_enabled() -> bool: - """Getter of the project state mining switch.""" + """ + Getter of the project state mining switch. + @return: True if project state mining is enabled, False otherwise. + """ return get_action_input(PROJECT_STATE_MINING, "false").lower() == "true" @staticmethod def get_is_structured_output_enabled() -> bool: - """Getter of the structured output switch.""" + """ + Getter of the structured output switch. + @return: True if structured output is enabled, False otherwise. + """ return get_action_input(STRUCTURED_OUTPUT, "false").lower() == "true" @staticmethod def get_repositories() -> list[ConfigRepository]: - """Getter of the list of repositories to fetch from.""" + """ + Getter and parser of the Config Repositories. + @return: A list of Config Repositories. + """ repositories = [] repositories_json = get_action_input(REPOSITORIES, "") - - # Parse repositories json string into json dictionary format try: - repositories_json = json.loads(repositories_json) - except json.JSONDecodeError as e: - logger.error("Error parsing JSON repositories: %s.", e, exc_info=True) - sys.exit(1) + # Parse repositories json string into json dictionary format + try: + repositories_json = json.loads(repositories_json) + except json.JSONDecodeError as e: + logger.error("Error parsing JSON repositories: %s.", e, exc_info=True) + sys.exit(1) + + # Load repositories into ConfigRepository object from JSON + for repository_json in repositories_json: + config_repository = ConfigRepository() + if config_repository.load_from_json(repository_json): + repositories.append(config_repository) + else: + logger.error("Failed to load repository from JSON: %s.", repository_json) - for repository_json in repositories_json: - config_repository = ConfigRepository() - if config_repository.load_from_json(repository_json): - repositories.append(config_repository) - else: - logger.error("Failed to load repository from JSON: %s.", repository_json) + except TypeError: + logger.error("Type error parsing input JSON repositories: `%s.`", repositories_json) + sys.exit(1) return repositories diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 023e69c..37d9100 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -544,8 +544,6 @@ def _generate_directory_path(repository_id: Optional[str]) -> str: if ActionInputs.get_is_structured_output_enabled() and repository_id: organization_name, repository_name = repository_id.split("/") output_path = os.path.join(output_path, organization_name, repository_name) - else: - output_path = output_path os.makedirs(output_path, exist_ok=True) diff --git a/tests/model/test_project_issue.py b/tests/model/test_project_issue.py index 268d3bd..d9070d0 100644 --- a/tests/model/test_project_issue.py +++ b/tests/model/test_project_issue.py @@ -77,4 +77,6 @@ def test_loads_with_incorrect_json_structure_for_repository_name(mocker): assert 1 == actual.number assert "" == actual.organization_name assert "" == actual.repository_name - mock_log.debug.assert_called_once_with("KeyError(%s) occurred while parsing issue json: %s.", "'repository'", incorrect_json) + mock_log.debug.assert_called_once_with( + "KeyError(%s) occurred while parsing issue json: %s.", "'repository'", incorrect_json + ) diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index e69de29..37e4af1 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -0,0 +1,159 @@ +# +# Copyright 2024 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json + +from living_documentation_generator.action_inputs import ActionInputs +from living_documentation_generator.model.config_repository import ConfigRepository + + +# get_repositories + + +def test_get_repositories_correct_behaviour(mocker): + repositories_json = [ + { + "organization-name": "organizationABC", + "repository-name": "repositoryABC", + "query-labels": ["feature"], + "projects-title-filter": [], + }, + { + "organization-name": "organizationXYZ", + "repository-name": "repositoryXYZ", + "query-labels": ["bug"], + "projects-title-filter": ["wanted_project"], + }, + ] + mocker.patch( + "living_documentation_generator.action_inputs.get_action_input", return_value=json.dumps(repositories_json) + ) + + actual = ActionInputs.get_repositories() + + assert 2 == len(actual) + assert isinstance(actual[0], ConfigRepository) + assert "organizationABC" == actual[0].organization_name + assert "repositoryABC" == actual[0].repository_name + assert ["feature"] == actual[0].query_labels + assert [] == actual[0].projects_title_filter + assert isinstance(actual[1], ConfigRepository) + assert "organizationXYZ" == actual[1].organization_name + assert "repositoryXYZ" == actual[1].repository_name + assert ["bug"] == actual[1].query_labels + assert ["wanted_project"] == actual[1].projects_title_filter + + +# FixMe: For some reason this test is called 11 times. Please help me to understand the reason. +# def test_get_repositories_error_parsing_json_from_json_string(mocker): +# mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") +# mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="not a JSON string") +# mock_exit = mocker.patch("sys.exit") +# +# ActionInputs.get_repositories() +# +# mock_log_error.assert_called_once_with("Error parsing input JSON repositories: `%s.`", mocker.ANY, exc_info=True) +# mock_exit.assert_called_once_with(1) + + +def test_get_repositories_type_error_parsing_json(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value=1) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.get_repositories() + + mock_log_error.assert_called_once_with("Type error parsing input JSON repositories: `%s.`", mocker.ANY) + mock_exit.assert_called_once_with(1) + + +def test_get_repositories_error_with_loading_repository_json(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="[{}]") + mocker.patch.object(ConfigRepository, "load_from_json", return_value=False) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.get_repositories() + + mock_log_error.assert_called_once_with("Failed to load repository from JSON: %s.", {}) + mock_exit.assert_not_called() + + +# validate_inputs + + +def test_validate_inputs_correct_behaviour(mocker): + mock_log_debug = mocker.patch("living_documentation_generator.action_inputs.logger.debug") + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + repositories_json = [ + { + "organization-name": "organizationABC", + "repository-name": "repositoryABC", + "query-labels": ["feature"], + "projects-title-filter": [], + } + ] + mocker.patch( + "living_documentation_generator.action_inputs.ActionInputs.get_repositories", return_value=repositories_json + ) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.validate_inputs("./output") + + mock_log_debug.assert_called_once_with("Action inputs validation successfully completed.") + mock_log_error.assert_not_called() + mock_exit.assert_not_called() + + +def test_validate_inputs_error_output_path_as_empty_string(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + repositories_json = [ + { + "organization-name": "organizationABC", + "repository-name": "repositoryABC", + "query-labels": ["feature"], + "projects-title-filter": [], + } + ] + mocker.patch( + "living_documentation_generator.action_inputs.ActionInputs.get_repositories", return_value=repositories_json + ) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.validate_inputs("") + + mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH can not be an empty string.") + mock_exit.assert_called_once_with(1) + + +def test_validate_inputs_error_output_path_as_project_directory(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + repositories_json = [ + { + "organization-name": "organizationABC", + "repository-name": "repositoryABC", + "query-labels": ["feature"], + "projects-title-filter": [], + } + ] + mocker.patch( + "living_documentation_generator.action_inputs.ActionInputs.get_repositories", return_value=repositories_json + ) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.validate_inputs("./templates/template_subfolder") + + mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH cannot be chosen as a part of any project folder.") + mock_exit.assert_called_once_with(1) From e9ed9b7c75e544212bf09f736f6be09367bf822c Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:38:58 +0100 Subject: [PATCH 54/60] Action inputs comments implemented. --- living_documentation_generator/action_inputs.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 3e8c27f..4e58dae 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -78,11 +78,7 @@ def get_repositories() -> list[ConfigRepository]: repositories_json = get_action_input(REPOSITORIES, "") try: # Parse repositories json string into json dictionary format - try: - repositories_json = json.loads(repositories_json) - except json.JSONDecodeError as e: - logger.error("Error parsing JSON repositories: %s.", e, exc_info=True) - sys.exit(1) + repositories_json = json.loads(repositories_json) # Load repositories into ConfigRepository object from JSON for repository_json in repositories_json: @@ -92,6 +88,10 @@ def get_repositories() -> list[ConfigRepository]: else: logger.error("Failed to load repository from JSON: %s.", repository_json) + except json.JSONDecodeError as e: + logger.error("Error parsing JSON repositories: %s.", e, exc_info=True) + sys.exit(1) + except TypeError: logger.error("Type error parsing input JSON repositories: `%s.`", repositories_json) sys.exit(1) @@ -125,8 +125,10 @@ def validate_inputs(out_path: str) -> None: # Check that the INPUT_OUTPUT_PATH is not a project directory # Note: That would cause a rewriting project files project_directories = get_all_project_directories() - if DEFAULT_OUTPUT_PATH in project_directories: - project_directories.remove(DEFAULT_OUTPUT_PATH) + default_output_abs_path = os.path.abspath(DEFAULT_OUTPUT_PATH) + + if default_output_abs_path in project_directories: + project_directories.remove(default_output_abs_path) for project_directory in project_directories: # Finds the common path between the absolute paths of out_path and project_directory From fa37ddd108b93ce56eb570b57c6a298599c74be4 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:12:01 +0100 Subject: [PATCH 55/60] Action inputs comments implemented. --- .../action_inputs.py | 8 +-- tests/test_action_inputs.py | 60 +++++++++++++++++-- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 4e58dae..6ef79f0 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -75,7 +75,7 @@ def get_repositories() -> list[ConfigRepository]: @return: A list of Config Repositories. """ repositories = [] - repositories_json = get_action_input(REPOSITORIES, "") + repositories_json = get_action_input(REPOSITORIES, "[]") try: # Parse repositories json string into json dictionary format repositories_json = json.loads(repositories_json) @@ -125,10 +125,8 @@ def validate_inputs(out_path: str) -> None: # Check that the INPUT_OUTPUT_PATH is not a project directory # Note: That would cause a rewriting project files project_directories = get_all_project_directories() - default_output_abs_path = os.path.abspath(DEFAULT_OUTPUT_PATH) - - if default_output_abs_path in project_directories: - project_directories.remove(default_output_abs_path) + if DEFAULT_OUTPUT_PATH in project_directories: + project_directories.remove(DEFAULT_OUTPUT_PATH) for project_directory in project_directories: # Finds the common path between the absolute paths of out_path and project_directory diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 37e4af1..67e68b7 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -68,15 +68,28 @@ def test_get_repositories_correct_behaviour(mocker): # mock_exit.assert_called_once_with(1) -def test_get_repositories_type_error_parsing_json(mocker): +def test_get_repositories_default_value_as_json(mocker): mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") - mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value=1) + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="[]") mock_exit = mocker.patch("sys.exit") - ActionInputs.get_repositories() + actual = ActionInputs.get_repositories() - mock_log_error.assert_called_once_with("Type error parsing input JSON repositories: `%s.`", mocker.ANY) - mock_exit.assert_called_once_with(1) + assert actual == [] + mock_exit.assert_not_called() + mock_log_error.assert_not_called() + + +def test_get_repositories_empty_object_as_input(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="{}") + mock_exit = mocker.patch("sys.exit") + + actual = ActionInputs.get_repositories() + + assert actual == [] + mock_exit.assert_not_called() + mock_log_error.assert_not_called() def test_get_repositories_error_with_loading_repository_json(mocker): @@ -87,8 +100,43 @@ def test_get_repositories_error_with_loading_repository_json(mocker): ActionInputs.get_repositories() - mock_log_error.assert_called_once_with("Failed to load repository from JSON: %s.", {}) mock_exit.assert_not_called() + mock_log_error.assert_called_once_with("Failed to load repository from JSON: %s.", {}) + + +def test_get_repositories_number_instead_of_json(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value=1) + mock_exit = mocker.patch("sys.exit") + + ActionInputs.get_repositories() + + mock_exit.assert_called_once_with(1) + mock_log_error.assert_called_once_with("Type error parsing input JSON repositories: `%s.`", mocker.ANY) + + +def test_get_repositories_empty_string_as_input(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="") + mock_exit = mocker.patch("sys.exit") + + actual = ActionInputs.get_repositories() + + assert actual == [] + mock_exit.assert_called_once() + mock_log_error.assert_called_once_with("Error parsing JSON repositories: %s.", mocker.ANY, exc_info=True) + + +def test_get_repositories_invalid_string_as_input(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mocker.patch("living_documentation_generator.action_inputs.get_action_input", return_value="string") + mock_exit = mocker.patch("sys.exit") + + actual = ActionInputs.get_repositories() + + assert actual == [] + mock_exit.assert_called_once() + mock_log_error.assert_called_once_with("Error parsing JSON repositories: %s.", mocker.ANY, exc_info=True) # validate_inputs From 585c4ad76f76973aa3a3fdbd6549a0a93e7407df Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:44:15 +0100 Subject: [PATCH 56/60] Action inputs comments implemented. --- .../action_inputs.py | 11 +++-- tests/model/test_consolidated_issue.py | 2 +- tests/test_action_inputs.py | 42 ++++++++----------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/living_documentation_generator/action_inputs.py b/living_documentation_generator/action_inputs.py index 6ef79f0..bfbd919 100644 --- a/living_documentation_generator/action_inputs.py +++ b/living_documentation_generator/action_inputs.py @@ -125,10 +125,15 @@ def validate_inputs(out_path: str) -> None: # Check that the INPUT_OUTPUT_PATH is not a project directory # Note: That would cause a rewriting project files project_directories = get_all_project_directories() - if DEFAULT_OUTPUT_PATH in project_directories: - project_directories.remove(DEFAULT_OUTPUT_PATH) + default_abspath_output_path = os.path.abspath(DEFAULT_OUTPUT_PATH) - for project_directory in project_directories: + # Ensure project directories are absolute paths + project_abspath_directories = [os.path.abspath(d) for d in project_directories] + + if default_abspath_output_path in project_abspath_directories: + project_abspath_directories.remove(default_abspath_output_path) + + for project_directory in project_abspath_directories: # Finds the common path between the absolute paths of out_path and project_directory common_path = os.path.commonpath([os.path.abspath(out_path), os.path.abspath(project_directory)]) diff --git a/tests/model/test_consolidated_issue.py b/tests/model/test_consolidated_issue.py index 625a9c8..2cd7724 100644 --- a/tests/model/test_consolidated_issue.py +++ b/tests/model/test_consolidated_issue.py @@ -36,7 +36,7 @@ def test_generate_page_filename_with_none_title(mocker): consolidated_issue = ConsolidatedIssue("organization/repository", mock_issue) actual = consolidated_issue.generate_page_filename() - + assert "1.md" == actual mock_log_error.assert_called_once_with( "Issue page filename generation failed for Issue %s/%s (%s). Issue does not have a title.", diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 67e68b7..2239a8e 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -160,48 +160,42 @@ def test_validate_inputs_correct_behaviour(mocker): ActionInputs.validate_inputs("./output") + mock_exit.assert_not_called() mock_log_debug.assert_called_once_with("Action inputs validation successfully completed.") mock_log_error.assert_not_called() - mock_exit.assert_not_called() def test_validate_inputs_error_output_path_as_empty_string(mocker): mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") - repositories_json = [ - { - "organization-name": "organizationABC", - "repository-name": "repositoryABC", - "query-labels": ["feature"], - "projects-title-filter": [], - } - ] - mocker.patch( - "living_documentation_generator.action_inputs.ActionInputs.get_repositories", return_value=repositories_json - ) mock_exit = mocker.patch("sys.exit") ActionInputs.validate_inputs("") - mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH can not be an empty string.") mock_exit.assert_called_once_with(1) + mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH can not be an empty string.") def test_validate_inputs_error_output_path_as_project_directory(mocker): mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") - repositories_json = [ - { - "organization-name": "organizationABC", - "repository-name": "repositoryABC", - "query-labels": ["feature"], - "projects-title-filter": [], - } - ] - mocker.patch( - "living_documentation_generator.action_inputs.ActionInputs.get_repositories", return_value=repositories_json - ) mock_exit = mocker.patch("sys.exit") ActionInputs.validate_inputs("./templates/template_subfolder") + mock_exit.assert_called_once_with(1) mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH cannot be chosen as a part of any project folder.") + + +def test_validate_inputs_absolute_output_path_with_relative_project_directories(mocker): + mock_log_error = mocker.patch("living_documentation_generator.action_inputs.logger.error") + mock_exit = mocker.patch("sys.exit") + mocker.patch( + "living_documentation_generator.action_inputs.get_all_project_directories", + return_value=["project/dir1", "project/dir2"], + ) + mocker.patch("os.path.abspath", side_effect=lambda path: f"/root/{path}" if not path.startswith("/") else path) + absolute_out_path = "/root/project/dir1/subfolder" + + ActionInputs.validate_inputs(absolute_out_path) + mock_exit.assert_called_once_with(1) + mock_log_error.assert_called_once_with("INPUT_OUTPUT_PATH cannot be chosen as a part of any project folder.") From 626a5fd65e5886462c945e8e4315c22d9fb046e3 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:11:08 +0100 Subject: [PATCH 57/60] Index output page compatible with html projects. --- living_documentation_generator/generator.py | 4 ++-- templates/_index_page_html_template.md | 20 +++++++++++++++++++ ...te.md => _index_page_markdown_template.md} | 12 +++++------ 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 templates/_index_page_html_template.md rename templates/{_index_page_template.md => _index_page_markdown_template.md} (59%) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 37d9100..28bf7a8 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -58,7 +58,7 @@ class LivingDocumentationGenerator: PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "issue_detail_page_template.md") - INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_template.md") + INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_html_template.md") def __init__(self): github_token = ActionInputs.get_github_token() @@ -395,7 +395,7 @@ def _generate_index_page( # Prepare issues replacement for the index page replacement = { "date": datetime.now().strftime("%Y-%m-%d"), - "issues": issue_table, + "issue-overview-table": issue_table, } # Replace the issue placeholders in the index template diff --git a/templates/_index_page_html_template.md b/templates/_index_page_html_template.md new file mode 100644 index 0000000..4f9bb59 --- /dev/null +++ b/templates/_index_page_html_template.md @@ -0,0 +1,20 @@ +--- +title: Issue Summary +toolbar_title: Issue Summary +description_title: Brief overview of all mined issues. +description: This is a comprehensive list and brief overview of all issues. +date: {date} +weight: 0 +--- + +

Issue Summary page

+ +Our project is designed with a myriad of issues to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these issues, their brief descriptions, and links to their detailed documentation. + +

Issue Overview

+ +
+ +{issue-overview-table} + +
diff --git a/templates/_index_page_template.md b/templates/_index_page_markdown_template.md similarity index 59% rename from templates/_index_page_template.md rename to templates/_index_page_markdown_template.md index dbecb16..fb0964c 100644 --- a/templates/_index_page_template.md +++ b/templates/_index_page_markdown_template.md @@ -1,9 +1,9 @@ --- -title: "Issue Summary" -toolbar_title: "Issue Summary" -description_title: "Brief overview of all mined issues." -description: "This is a comprehensive list and brief overview of all issues." -date: "{date}" +title: Issue Summary +toolbar_title: Issue Summary +description_title: Brief overview of all mined issues. +description: This is a comprehensive list and brief overview of all issues. +date: {date} weight: 0 --- @@ -13,4 +13,4 @@ Our project is designed with a myriad of issues to ensure seamless user experien ## Issue Overview -{issues} +{issue-overview-table} From 0dd7f7527f9be93438ce565a0e4348e127e56bab Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:37:45 +0100 Subject: [PATCH 58/60] Added logic for compatible mdoc output. --- living_documentation_generator/generator.py | 121 +++++++++++++++--- living_documentation_generator/utils/utils.py | 2 +- templates/_index_no_struct_page_template.md | 21 +++ templates/_index_org_level_page_template.md | 7 + templates/_index_page_html_template.md | 20 --- templates/_index_page_markdown_template.md | 16 --- templates/_index_repo_level_page_template.md | 17 +++ templates/_index_root_level_page_template.md | 9 ++ templates/issue_detail_page_template.md | 8 +- tests/utils/test_utils.py | 2 +- 10 files changed, 162 insertions(+), 61 deletions(-) create mode 100644 templates/_index_no_struct_page_template.md create mode 100644 templates/_index_org_level_page_template.md delete mode 100644 templates/_index_page_html_template.md delete mode 100644 templates/_index_page_markdown_template.md create mode 100644 templates/_index_repo_level_page_template.md create mode 100644 templates/_index_root_level_page_template.md diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index 28bf7a8..93f515a 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -58,7 +58,18 @@ class LivingDocumentationGenerator: PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) ISSUE_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "issue_detail_page_template.md") - INDEX_PAGE_TEMPLATE_FILE = os.path.join(PROJECT_ROOT, os.pardir, "templates", "_index_page_html_template.md") + INDEX_NO_STRUCT_TEMPLATE_FILE = os.path.join( + PROJECT_ROOT, os.pardir, "templates", "_index_no_struct_page_template.md" + ) + INDEX_ROOT_LEVEL_TEMPLATE_FILE = os.path.join( + PROJECT_ROOT, os.pardir, "templates", "_index_root_level_page_template.md" + ) + INDEX_ORG_LEVEL_TEMPLATE_FILE = os.path.join( + PROJECT_ROOT, os.pardir, "templates", "_index_org_level_page_template.md" + ) + INDEX_REPO_LEVEL_TEMPLATE_FILE = os.path.join( + PROJECT_ROOT, os.pardir, "templates", "_index_repo_level_page_template.md" + ) def __init__(self): github_token = ActionInputs.get_github_token() @@ -280,7 +291,10 @@ def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None @param issues: A dictionary containing all consolidated issues. """ issue_page_detail_template = None - issue_index_page_template = None + index_page_template = None + index_root_level_page = None + index_org_level_template = None + index_repo_level_template = None # Load the template files for generating the Markdown pages try: @@ -290,11 +304,35 @@ def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None logger.error("Issue page template file was not successfully loaded.", exc_info=True) try: - with open(LivingDocumentationGenerator.INDEX_PAGE_TEMPLATE_FILE, "r", encoding="utf-8") as f: - issue_index_page_template = f.read() + with open(LivingDocumentationGenerator.INDEX_NO_STRUCT_TEMPLATE_FILE, "r", encoding="utf-8") as f: + index_page_template = f.read() except IOError: logger.error("Index page template file was not successfully loaded.", exc_info=True) + try: + with open(LivingDocumentationGenerator.INDEX_ROOT_LEVEL_TEMPLATE_FILE, "r", encoding="utf-8") as f: + index_root_level_page = f.read() + except IOError: + logger.error( + "Structured index page template file for root level was not successfully loaded.", exc_info=True + ) + + try: + with open(LivingDocumentationGenerator.INDEX_ORG_LEVEL_TEMPLATE_FILE, "r", encoding="utf-8") as f: + index_org_level_template = f.read() + except IOError: + logger.error( + "Structured index page template file for organization level was not successfully loaded.", exc_info=True + ) + + try: + with open(LivingDocumentationGenerator.INDEX_REPO_LEVEL_TEMPLATE_FILE, "r", encoding="utf-8") as f: + index_repo_level_template = f.read() + except IOError: + logger.error( + "Structured index page template file for repository level was not successfully loaded.", exc_info=True + ) + # Generate a markdown page for every issue for consolidated_issue in issues.values(): self._generate_md_issue_page(issue_page_detail_template, consolidated_issue) @@ -302,10 +340,14 @@ def _generate_markdown_pages(self, issues: dict[str, ConsolidatedIssue]) -> None # Generate an index page with a summary table about all issues if ActionInputs.get_is_structured_output_enabled(): - self._generate_structured_index_page(issue_index_page_template, issues) + self._generate_structured_index_pages(index_repo_level_template, index_org_level_template, issues) + + output_path = ActionInputs.get_output_directory() + with open(os.path.join(output_path, "_index.md"), "w", encoding="utf-8") as f: + f.write(index_root_level_page) else: issues = list(issues.values()) - self._generate_index_page(issue_index_page_template, issues) + self._generate_index_page(index_page_template, issues) logger.info("Markdown page generation - generated `_index.md`") def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: ConsolidatedIssue) -> None: @@ -347,13 +389,17 @@ def _generate_md_issue_page(self, issue_page_template: str, consolidated_issue: logger.debug("Generated Markdown page: %s.", page_filename) - def _generate_structured_index_page( - self, issue_index_page_template: str, consolidated_issues: dict[str, ConsolidatedIssue] + def _generate_structured_index_pages( + self, + index_repo_level_template: str, + index_org_level_template: str, + consolidated_issues: dict[str, ConsolidatedIssue], ) -> None: """ - Generates a structured index page with a summary of one repository issues. + Generates a set of index pages due to a structured output feature. - @param issue_index_page_template: The template string for generating the index markdown page. + @param index_repo_level_template: The template string for generating the repository level index markdown page. + @param index_org_level_template: The template string for generating the organization level index markdown page. @param consolidated_issues: A dictionary containing all consolidated issues. @return: None """ @@ -367,8 +413,20 @@ def _generate_structured_index_page( # Generate an index page for each repository for repository_id, issues in issues_by_repository.items(): - self._generate_index_page(issue_index_page_template, issues, repository_id) - logger.info("Markdown page generation - generated `_index.md` for %s.", repository_id) + organization_name, _ = repository_id.split("/") + + self._generate_org_level_index_page(index_org_level_template, organization_name) + logger.debug( + "Generated organization level `_index.md` for %s as a cause of structured output feature.", + organization_name, + ) + + self._generate_index_page(index_repo_level_template, issues, repository_id) + logger.debug( + "Generated repository level `_index.md` for %s as a cause of structured output feature.", repository_id + ) + + logger.info("Markdown page generation - generated `_index.md` pages for %s.", repository_id) def _generate_index_page( self, issue_index_page_template: str, consolidated_issues: list[ConsolidatedIssue], repository_id: str = None @@ -398,6 +456,9 @@ def _generate_index_page( "issue-overview-table": issue_table, } + if ActionInputs.get_is_structured_output_enabled(): + replacement["repository_name"] = repository_id.split("/")[1] + # Replace the issue placeholders in the index template index_page = issue_index_page_template.format(**replacement) @@ -409,6 +470,29 @@ def _generate_index_page( with open(os.path.join(index_directory_path, "_index.md"), "w", encoding="utf-8") as f: f.write(index_page) + @staticmethod + def _generate_org_level_index_page(index_org_level_template: str, organization_name: str) -> None: + """ + Generates an organization level index page and save it. + + @param index_org_level_template: The template string for generating the organization level index markdown page. + @param organization_name: The name of the organization. + @return: None + """ + # Prepare issues replacement for the index page + replacement = { + "date": datetime.now().strftime("%Y-%m-%d"), + "organization_name": organization_name, + } + + # Replace the issue placeholders in the index template + org_level_index_page = index_org_level_template.format(**replacement) + + # Create a sub index page file + output_path = os.path.join(ActionInputs.get_output_directory(), organization_name) + with open(os.path.join(output_path, "_index.md"), "w", encoding="utf-8") as f: + f.write(org_level_index_page) + @staticmethod def _generate_markdown_line(consolidated_issue: ConsolidatedIssue) -> str: """ @@ -422,7 +506,8 @@ def _generate_markdown_line(consolidated_issue: ConsolidatedIssue) -> str: number = consolidated_issue.number title = consolidated_issue.title title = title.replace("|", " _ ") - page_filename = consolidated_issue.generate_page_filename() + issue_link_base = consolidated_issue.title.replace(" ", "-").lower() + issue_mdoc_link = f"features#{issue_link_base}" url = consolidated_issue.html_url state = consolidated_issue.state @@ -438,14 +523,14 @@ def _generate_markdown_line(consolidated_issue: ConsolidatedIssue) -> str: # Generate the Markdown issue line WITH extra project data md_issue_line = ( - f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" - f" {linked_to_project} | {status} |[GitHub link]({url}) |\n" + f"| {organization_name} | {repository_name} | [#{number} - {title}]({issue_mdoc_link}) |" + f" {linked_to_project} | {status} |GitHub link |\n" ) else: # Generate the Markdown issue line WITHOUT project data md_issue_line = ( - f"| {organization_name} | {repository_name} | [#{number} - {title}]({page_filename}) |" - f" {state} |[GitHub link]({url}) |\n" + f"| {organization_name} | {repository_name} | [#{number} - {title}]({issue_mdoc_link}) |" + f" {state} |GitHub link |\n" ) return md_issue_line @@ -464,7 +549,7 @@ def _generate_issue_summary_table(consolidated_issue: ConsolidatedIssue) -> str: # Format issue URL as a Markdown link issue_url = consolidated_issue.html_url - issue_url = f"[GitHub link]({issue_url})" if issue_url else None + issue_url = f"GitHub link " if issue_url else None # Define the header for the issue summary table headers = [ diff --git a/living_documentation_generator/utils/utils.py b/living_documentation_generator/utils/utils.py index dfc680e..bb78cf3 100644 --- a/living_documentation_generator/utils/utils.py +++ b/living_documentation_generator/utils/utils.py @@ -47,7 +47,7 @@ def sanitize_filename(filename: str) -> str: @return: The sanitized filename """ # Remove invalid characters for Windows filenames - sanitized_name = re.sub(r'[<>:"/|?*`]', "", filename) + sanitized_name = re.sub(r'[<>:"/|?*#{}()`]', "", filename) # Reduce consecutive periods sanitized_name = re.sub(r"\.{2,}", ".", sanitized_name) # Reduce consecutive spaces to a single space diff --git a/templates/_index_no_struct_page_template.md b/templates/_index_no_struct_page_template.md new file mode 100644 index 0000000..a0d2a70 --- /dev/null +++ b/templates/_index_no_struct_page_template.md @@ -0,0 +1,21 @@ +--- +title: Features +toolbar_title: Features +description_title: Living Documentation +description: > + This is a comprehensive list and brief overview of all mined features. +date: {date} +weight: 0 +--- + +

Feature Summary page

+ +Our project is designed with a myriad of features to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these features, their brief descriptions, and links to their detailed documentation. + +

Feature Overview

+ +
+ +{issue-overview-table} + +
diff --git a/templates/_index_org_level_page_template.md b/templates/_index_org_level_page_template.md new file mode 100644 index 0000000..35280c2 --- /dev/null +++ b/templates/_index_org_level_page_template.md @@ -0,0 +1,7 @@ +--- +title: "{organization_name}" +date: {date} +weight: 0 +--- + +This section displays the living documentation for all repositories within the organization: {organization_name} in a structured output. \ No newline at end of file diff --git a/templates/_index_page_html_template.md b/templates/_index_page_html_template.md deleted file mode 100644 index 4f9bb59..0000000 --- a/templates/_index_page_html_template.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Issue Summary -toolbar_title: Issue Summary -description_title: Brief overview of all mined issues. -description: This is a comprehensive list and brief overview of all issues. -date: {date} -weight: 0 ---- - -

Issue Summary page

- -Our project is designed with a myriad of issues to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these issues, their brief descriptions, and links to their detailed documentation. - -

Issue Overview

- -
- -{issue-overview-table} - -
diff --git a/templates/_index_page_markdown_template.md b/templates/_index_page_markdown_template.md deleted file mode 100644 index fb0964c..0000000 --- a/templates/_index_page_markdown_template.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: Issue Summary -toolbar_title: Issue Summary -description_title: Brief overview of all mined issues. -description: This is a comprehensive list and brief overview of all issues. -date: {date} -weight: 0 ---- - -# Issue Summary page - -Our project is designed with a myriad of issues to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these issues, their brief descriptions, and links to their detailed documentation. - -## Issue Overview - -{issue-overview-table} diff --git a/templates/_index_repo_level_page_template.md b/templates/_index_repo_level_page_template.md new file mode 100644 index 0000000..569c937 --- /dev/null +++ b/templates/_index_repo_level_page_template.md @@ -0,0 +1,17 @@ +--- +title: '{repository_name}' +date: {date} +weight: 0 +--- + +

Feature Summary page

+ +Our project is designed with a myriad of features to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these features, their brief descriptions, and links to their detailed documentation. + +

Feature Overview

+ +
+ +{issue-overview-table} + +
\ No newline at end of file diff --git a/templates/_index_root_level_page_template.md b/templates/_index_root_level_page_template.md new file mode 100644 index 0000000..11521bb --- /dev/null +++ b/templates/_index_root_level_page_template.md @@ -0,0 +1,9 @@ +--- +title: Liv Doc +toolbar_title: Features +description_title: Living Documentation +description: > + This is a comprehensive list and brief overview of all mined features. +date: {date} +weight: 0 +--- diff --git a/templates/issue_detail_page_template.md b/templates/issue_detail_page_template.md index 7693d16..62efc5f 100644 --- a/templates/issue_detail_page_template.md +++ b/templates/issue_detail_page_template.md @@ -1,13 +1,11 @@ --- title: "{title}" -date: "{date}" +date: {date} weight: 1 --- -# {page_issue_title} - {issue_summary_table} - -## Issue Content + +

Issue Content

{issue_content} diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 1d9cef0..e7864ed 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -46,7 +46,7 @@ def test_make_issue_key(): @pytest.mark.parametrize( "filename_example, expected_filename", [ - ("in<>va::lid//.fi|le?na*me.txt", "invalid.filename.txt"), # Remove invalid characters for Windows filenames + ("in<>va::l#(){}id//.fi|le?*.txt", "invalid.file.txt"), # Remove invalid characters for Windows filenames ("another..invalid...filename.txt", "another.invalid.filename.txt"), # Reduce consecutive periods ( "filename with spaces.txt", From 6d4c9c1abab6b0966b8dc76fd63b42a8fcea54fd Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:33:27 +0100 Subject: [PATCH 59/60] Bug fix. --- living_documentation_generator/generator.py | 2 +- templates/_index_repo_level_page_template.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index fef3cec..ff4b781 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -76,7 +76,7 @@ def __init__(self): def __init__(self): github_token = ActionInputs.get_github_token() - + self.__github_instance: Github = Github(auth=Auth.Token(token=github_token), per_page=ISSUES_PER_PAGE_LIMIT) self.__github_projects_instance: GithubProjects = GithubProjects(token=github_token) self.__rate_limiter: GithubRateLimiter = GithubRateLimiter(self.__github_instance) diff --git a/templates/_index_repo_level_page_template.md b/templates/_index_repo_level_page_template.md index 569c937..69e2d83 100644 --- a/templates/_index_repo_level_page_template.md +++ b/templates/_index_repo_level_page_template.md @@ -1,9 +1,11 @@ --- -title: '{repository_name}' +title: "{repository_name}" date: {date} weight: 0 --- +This section displays all the information about mined features for the repository: {repository_name}. +

Feature Summary page

Our project is designed with a myriad of features to ensure seamless user experience, top-tier functionality, and efficient operations. Here, you'll find a summarized list of all these features, their brief descriptions, and links to their detailed documentation. From a02c546faee44c4ef2b374be4566e477e74160b6 Mon Sep 17 00:00:00 2001 From: Tobias Mikula <72911271+MobiTikula@users.noreply.github.com> Date: Sun, 3 Nov 2024 12:11:04 +0100 Subject: [PATCH 60/60] Bug fix. --- living_documentation_generator/generator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/living_documentation_generator/generator.py b/living_documentation_generator/generator.py index ff4b781..93f515a 100644 --- a/living_documentation_generator/generator.py +++ b/living_documentation_generator/generator.py @@ -71,9 +71,6 @@ class LivingDocumentationGenerator: PROJECT_ROOT, os.pardir, "templates", "_index_repo_level_page_template.md" ) - def __init__(self): - github_token = ActionInputs.get_github_token() - def __init__(self): github_token = ActionInputs.get_github_token()