Skip to content

Commit

Permalink
Merge pull request #161 from tasmota/main_work
Browse files Browse the repository at this point in the history
Main work
  • Loading branch information
Jason2866 authored Oct 9, 2023
2 parents 0072a46 + 6cbc323 commit a9ec851
Show file tree
Hide file tree
Showing 632 changed files with 20,249 additions and 9,966 deletions.
2 changes: 0 additions & 2 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,6 @@ variables:
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
PYTHON_VER: 3.8.17

CLANG_TIDY_RUNNER_PROJ: 2107 # idf/clang-tidy-runner

# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"

Expand Down
15 changes: 4 additions & 11 deletions .gitlab/ci/static-code-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,16 @@ clang_tidy_check:
- .rules:patterns:clang_tidy
artifacts:
paths:
- $OUTPUT_DIR
- clang_tidy_reports/
when: always
expire_in: 1 day
variables:
CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
IDF_TOOLCHAIN: clang
script:
- internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
- export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
- dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
- run_cmd idf_clang ${dirs}
--output-path ${OUTPUT_DIR}
--limit-file ${RULES_FILE}
- run_cmd idf_clang_tidy $(cat tools/ci/clang_tidy_dirs.txt | xargs)
--output-path clang_tidy_reports
--limit-file tools/ci/static-analysis-rules.yml
--xtensa-include-dir
--run-clang-tidy-py run-clang-tidy

check_pylint:
extends:
Expand Down
56 changes: 56 additions & 0 deletions .gitlab/dangerjs/configParameters.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
let outputStatuses = [];

/**
* Logs the status of a rule with padded formatting and stores it in the `outputStatuses` array.
* If the rule already exists in the array, its status is updated.
* @param message The name of the rule
* @param status The output (exit) status of the rule
*/
function recordRuleExitStatus(message, status) {
// Check if the rule already exists in the array
const existingRecord = outputStatuses.find(
(rule) => rule.message === message
);

if (existingRecord) {
// Update the status of the existing rule
existingRecord.status = status;
} else {
// If the rule doesn't exist, add it to the array
outputStatuses.push({ message, status });
}
}

/**
* Displays all the rule output statuses stored in the `outputStatuses` array.
* Filters out any empty lines, sorts them alphabetically, and prints the statuses
* with a header and separator.
* These statuses are later displayed in CI job tracelog.
*/
function displayAllOutputStatuses() {
const lineLength = 100;
const sortedStatuses = outputStatuses.sort((a, b) =>
a.message.localeCompare(b.message)
);

const formattedLines = sortedStatuses.map((statusObj) => {
const paddingLength =
lineLength - statusObj.message.length - statusObj.status.length;
const paddedMessage = statusObj.message.padEnd(
statusObj.message.length + paddingLength,
"."
);
return `${paddedMessage} ${statusObj.status}`;
});

console.log(
"DangerJS checks (rules) output states:\n" + "=".repeat(lineLength + 2)
);
console.log(formattedLines.join("\n"));
console.log("=".repeat(lineLength + 2));
}

module.exports = {
displayAllOutputStatuses,
recordRuleExitStatus,
};
8 changes: 6 additions & 2 deletions .gitlab/dangerjs/dangerfile.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
const { displayAllOutputStatuses } = require("./configParameters.js");

/*
* Modules with checks are stored in ".gitlab/dangerjs/<module_name>". To import them, use path relative to "dangerfile.js"
*/

async function runChecks() {
// Checks for merge request title
require("./mrTitleNoDraftOrWip.js")();
Expand All @@ -28,13 +29,16 @@ async function runChecks() {
// Checks for Source branch name
require("./mrSourceBranchName.js")();

// Show DangerJS individual checks statuses - visible in CI job tracelog
displayAllOutputStatuses();

// Add success log if no issues
if (
results.fails.length === 0 &&
results.warnings.length === 0 &&
results.messages.length === 0
) {
return message("Good Job! All checks are passing!");
return message("🎉 Good Job! All checks are passing!");
}
}

Expand Down
9 changes: 8 additions & 1 deletion .gitlab/dangerjs/mrAreaLabels.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if MR has area labels (light blue labels)
*
* @dangerjs WARN
*/
module.exports = async function () {
const ruleName = "Merge request area labels";
const projectId = 103; // ESP-IDF
const areaLabelColor = /^#d2ebfa$/i; // match color code (case-insensitive)
const projectLabels = await danger.gitlab.api.Labels.all(projectId); // Get all project labels
Expand All @@ -13,8 +16,12 @@ module.exports = async function () {
const mrLabels = danger.gitlab.mr.labels; // Get MR labels

if (!mrLabels.some((label) => areaLabels.includes(label))) {
warn(
recordRuleExitStatus(ruleName, "Failed");
return warn(
`Please add some [area labels](${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/labels) to this MR.`
);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, "Passed");
};
8 changes: 7 additions & 1 deletion .gitlab/dangerjs/mrCommitsCommitMessage.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ const {
maximumBodyLineChars,
allowedTypes,
} = require("./mrCommitsConstants.js");
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check that commit messages are based on the Espressif ESP-IDF project's rules for git commit messages.
*
* @dangerjs WARN
*/
module.exports = async function () {
const ruleName = "Commit messages style";
const mrCommits = danger.gitlab.commits;
const lint = require("@commitlint/lint").default;

Expand Down Expand Up @@ -154,6 +156,10 @@ module.exports = async function () {
dangerMessage += AImessageSuggestion;
}

warn(dangerMessage);
recordRuleExitStatus(ruleName, "Failed");
return warn(dangerMessage);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, "Passed");
};
7 changes: 7 additions & 0 deletions .gitlab/dangerjs/mrCommitsEmail.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if the author is accidentally making a commit using a personal email
*
* @dangerjs INFO
*/
module.exports = function () {
const ruleName = 'Commits from outside Espressif';
const mrCommitAuthorEmails = danger.gitlab.commits.map(commit => commit.author_email);
const mrCommitCommitterEmails = danger.gitlab.commits.map(commit => commit.committer_email);
const emailPattern = /.*@espressif\.com/;
const filteredEmails = [...mrCommitAuthorEmails, ...mrCommitCommitterEmails].filter((email) => !emailPattern.test(email));
if (filteredEmails.length) {
recordRuleExitStatus(ruleName, "Failed");
return message(
`Some of the commits were authored or committed by developers outside Espressif: ${filteredEmails.join(', ')}. Please check if this is expected.`
);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, 'Passed');
};
7 changes: 7 additions & 0 deletions .gitlab/dangerjs/mrCommitsTooManyCommits.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if MR has not an excessive numbers of commits (if squashed)
*
* @dangerjs INFO
*/
module.exports = function () {
const ruleName = 'Number of commits in merge request';
const tooManyCommitThreshold = 2; // above this number of commits, squash commits is suggested
const mrCommits = danger.gitlab.commits;

if (mrCommits.length > tooManyCommitThreshold) {
recordRuleExitStatus(ruleName, "Passed (with suggestions)");
return message(
`You might consider squashing your ${mrCommits.length} commits (simplifying branch history).`
);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, 'Passed');
};
10 changes: 9 additions & 1 deletion .gitlab/dangerjs/mrDescriptionJiraLinks.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/** Check that there are valid JIRA links in MR description.
*
* This check extracts the "Related" section from the MR description and
Expand All @@ -10,6 +12,7 @@
*
*/
module.exports = async function () {
const ruleName = 'Jira ticket references';
const axios = require("axios");
const mrDescription = danger.gitlab.mr.description;
const mrCommitMessages = danger.gitlab.commits.map(
Expand All @@ -26,6 +29,7 @@ module.exports = async function () {
!sectionRelated.header || // No section Related in MR description or ...
!jiraTicketRegex.test(sectionRelated.content) // no Jira links in section Related
) {
recordRuleExitStatus(ruleName, 'Passed (with suggestions)');
return message(
"Please consider adding references to JIRA issues in the `Related` section of the MR description."
);
Expand Down Expand Up @@ -88,6 +92,9 @@ module.exports = async function () {
createReport();
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, 'Passed');

// ---------------------------------------------------------------

/**
Expand Down Expand Up @@ -225,6 +232,7 @@ module.exports = async function () {
let dangerMessage = `Some issues found for the related JIRA tickets in this MR:\n${partMessages.join(
"\n"
)}`;
warn(dangerMessage);
recordRuleExitStatus(ruleName, "Failed");
return warn(dangerMessage);
}
};
7 changes: 7 additions & 0 deletions .gitlab/dangerjs/mrDescriptionLongEnough.js
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if MR Description has accurate description".
*
* @dangerjs WARN
*/
module.exports = function () {
const ruleName = "Merge request sufficient description";
const mrDescription = danger.gitlab.mr.description;
const descriptionChunk = mrDescription.match(/^([^#]*)/)[1].trim(); // Extract all text before the first section header (i.e., the text before the "## Release notes")

const shortMrDescriptionThreshold = 50; // Description is considered too short below this number of characters

if (descriptionChunk.length < shortMrDescriptionThreshold) {
recordRuleExitStatus(ruleName, "Failed");
return warn(
"The MR description looks very brief, please check if more details can be added."
);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, "Passed");
};
14 changes: 10 additions & 4 deletions .gitlab/dangerjs/mrDescriptionReleaseNotes.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if MR Description contains mandatory section "Release notes"
*
Expand All @@ -6,6 +8,7 @@
* @dangerjs WARN (if section missing, is empty or wrong markdown format)
*/
module.exports = function () {
const ruleName = 'Merge request Release Notes section';
const mrDescription = danger.gitlab.mr.description;
const wiki_link = `${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/wikis/rfc/How-to-write-release-notes-properly`;

Expand All @@ -15,8 +18,8 @@ module.exports = function () {

const sectionReleaseNotes = mrDescription.match(regexSectionReleaseNotes);
if (!sectionReleaseNotes) {
warn(`The \`Release Notes\` section seems to be missing. Please check if the section header in MR description is present and in the correct markdown format ("## Release Notes").\n\nSee [Release Notes Format Rules](${wiki_link}).`);
return null;
recordRuleExitStatus(ruleName, "Failed");
return warn(`The \`Release Notes\` section seems to be missing. Please check if the section header in MR description is present and in the correct markdown format ("## Release Notes").\n\nSee [Release Notes Format Rules](${wiki_link}).`);
}

const releaseNotesLines = sectionReleaseNotes[1].replace(/<!--[\s\S]*?-->/g, '')
Expand Down Expand Up @@ -55,9 +58,12 @@ module.exports = function () {
if (error_output.length > 0) {
// Paragraphs joined by double `\n`s.
error_output = [...error_output, `See [Release Notes Format Guide](${wiki_link}).`].join('\n\n');
warn(error_output);
recordRuleExitStatus(ruleName, "Failed");
return warn(error_output);
}
return null;

// At this point, the rule has passed
recordRuleExitStatus(ruleName, 'Passed');
};

function check_entry(entry) {
Expand Down
9 changes: 9 additions & 0 deletions .gitlab/dangerjs/mrDocsTranslation.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check the documentation files in this MR.
*
Expand All @@ -24,6 +26,7 @@
*
*/
module.exports = async function () {
const ruleName = 'Documentation translation';
let partMessages = []; // Create a blank field for future records of individual issues
const pathProject = "espressif/esp-idf";
const regexIncludeLink = /\.\.\sinclude::\s((\.\.\/)+)en\//;
Expand Down Expand Up @@ -90,6 +93,9 @@ module.exports = async function () {
// Create a report with found issues with documents in MR
createReport();

// At this point, the rule has passed
recordRuleExitStatus(ruleName, 'Passed');

/**
* Generates an object that represents the relationships between files in two different languages found in this MR.
*
Expand Down Expand Up @@ -245,6 +251,7 @@ module.exports = async function () {

// No docs issues found in MR, but translation labels have been added anyway
if (!partMessages.length && translationLabelsPresent) {
recordRuleExitStatus(ruleName, "Failed");
return warn(
`Please remove the \`needs translation: XX\` labels. For documents that need to translate from scratch, Doc team will translate them in the future. For the current stage, we only focus on updating exiting EN and CN translation to make them in sync.`
);
Expand All @@ -261,9 +268,11 @@ module.exports = async function () {
dangerMessage += `
\nWhen synchronizing the EN and CN versions, please follow the [Documentation Code](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/contribute/documenting-code.html#standardize-document-format). The total number of lines of EN and CN should be same.\n
\nIf you have difficulty in providing translation, you can contact Documentation team by adding <kbd>needs translation: CN</kbd> or <kbd>needs translation: EN</kbd> labels into this MR and retrying Danger CI job. The documentation team will be automatically notified and will help you with the translations before the merge.\n`;
recordRuleExitStatus(ruleName, "Failed");
return warn(dangerMessage); // no "needs translation: XX" labels in MR; report issues as warn
} else {
dangerMessage += `\nTranslation labels <kbd>needs translation: CN</kbd> or <kbd>needs translation: EN</kbd> were added - this will automatically notify the Documentation team to help you with translation issues.`;
recordRuleExitStatus(ruleName, 'Passed (with suggestions)');
return message(dangerMessage); // "needs translation: XX" labels were found in MR and Docs team was notified; report issues as info
}
}
Expand Down
7 changes: 7 additions & 0 deletions .gitlab/dangerjs/mrSizeTooLarge.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
const { recordRuleExitStatus } = require("./configParameters.js");

/**
* Check if MR is too large (more than 1000 lines of changes)
*
* @dangerjs INFO
*/
module.exports = async function () {
const ruleName = "Merge request size (number of changed lines)";
const bigMrLinesOfCodeThreshold = 1000;
const totalLines = await danger.git.linesOfCode();

if (totalLines > bigMrLinesOfCodeThreshold) {
recordRuleExitStatus(ruleName, "Passed (with suggestions)");
return message(
`This MR seems to be quite large (total lines of code: ${totalLines}), you might consider splitting it into smaller MRs`
);
}

// At this point, the rule has passed
recordRuleExitStatus(ruleName, "Passed");
};
Loading

0 comments on commit a9ec851

Please sign in to comment.