From 1f646bb978ed94c25f6224e801779c929096c0ae Mon Sep 17 00:00:00 2001 From: Alexandru Croitor Date: Fri, 6 Sep 2024 13:54:24 +0200 Subject: [PATCH] coin: Build qdoc as part of repo instructions for CI doc building Backstory. The current approach for building documentation in the CI is that we build a static qdoc in a Jenkins job, manually provision it in qt5.git, and then build qt documentation using that somewhat old qdoc version. Updating the provisioned qdoc is a very slow process, because changes first need to be merged into qttools, then a Jenkins job needs to be manually run, then provisioning needs to be updated, and sometimes it also depends on a full qt5.git submodule update or has to wait for other unrelated changes to qt5.git. The new approach. This change adds CI instructions to build a static qdoc (and other relevant tools necessary for documentation generation) using the latest qttools/dev/HEAD sha1 in a specific host platform's qtbase build instructions, and then uses those tools for documentation generation and doc warning checks in each repo's TestDocs CI test phase. To enable these new instructions, the host platform and the doc checking platform need to be tagged with the 'DocsGenerationV2' feature. The built tools are cached in a separate archive using a new Coin feature, which is then extracted during each repo's TestDocs phase. Pros and cons. The benefit of this new approach is that we will always use a mostly up-to-date qdoc from qttools/dev/HEAD when generating documentation, making the documentation team's life easier. Specifically, once the doc tools are built in qtbase with the latest qttools/dev/HEAD sha1 that was available during that integration, the resulting built tools are then used for doc checks in dependent repos. A newer qttools/dev/HEAD sha1 will only be used for a particular repo once the qtbase dependencies.yaml sha1 is updated, or when doing qt5.git changes that cause a rebuild of qtbase. Note that while new tools are built due to dependency updates, the used qttools sha1 is not recorded in any way in any of the dependencies.yaml files. qtbase doc checks will always use the latest qttools/dev/HEAD sha1 at the integration time, because the tools will always be built from scratch. The downside of the new approach is that, by default, the doc generation will not use a pinned version of qdoc anymore, but rather follow the flow described above. This has a chance of introducing failures which are unrelated to integrating changes. This should happen rarely because the doc team usually tries to fix doc warnings before changing qdoc's code. If temporary pinning of qttools or qt5.git is required when testing the docs of a specific repo, it can be achieved as follows: - set the QT_CI_BUILD_REPO_DOC_TOOLS env var to "1" in the repo coin/module_config.yaml file build instructions, to ensure the doc tools are built in that repo, rather than reuse the ones built in qtbase - set the QT_CI_DOC_TOOLS_PIN_GIT_REF env var to the desired qttools sha1 or other git ref, in the same build instructions phase - optionally set the QT_CI_DOC_TOOLS_TOP_LEVEL_PIN_GIT_REF env var to the desired qt5.git sha1 or other git ref, in the same build instructions phase, if a different version of qt5.git is required - optionally set the QT_CI_DOC_TOOLS_USE_CI_TOP_LEVEL_BRANCH env var to ON, in case if the module branch that the CI job computes should be used as the qt5.git branch - set the QT_CI_FETCH_REPO_DOC_TOOLS env var to "1" in the repo test instructions, to ensure the just built doc tools are fetched instead of the qtbase-built ones Test running this new approach will allow us to collect some feedback on how often breakages happen, and how much easier it makes the documentation team's work, or how much harder it makes the life of regular integrations. The abundant amount of optional pinning options should be enough to avoid any permanent integration dead locks. Pick-to: 6.8 Task-number: QTBUG-128730 Change-Id: I8606cb3076036a4a0ec652d0fa74d270e8f5dfdf Reviewed-by: Alexey Edelev Reviewed-by: Paul Wicking --- coin/instructions/coin_build_doc_tools.yaml | 47 +++++++++++++++++++ .../coin_build_doc_tools_checked.yaml | 25 ++++++++++ coin/instructions/coin_fetch_doc_tools.yaml | 40 ++++++++++++++++ .../coin_module_build_template_v2.yaml | 8 ++++ .../coin_qtbase_build_template_v2.yaml | 6 +++ coin/instructions/coin_test_docs_common.yaml | 6 +++ 6 files changed, 132 insertions(+) create mode 100644 coin/instructions/coin_build_doc_tools.yaml create mode 100644 coin/instructions/coin_build_doc_tools_checked.yaml create mode 100644 coin/instructions/coin_fetch_doc_tools.yaml diff --git a/coin/instructions/coin_build_doc_tools.yaml b/coin/instructions/coin_build_doc_tools.yaml new file mode 100644 index 00000000000..b1a5c51a3ed --- /dev/null +++ b/coin/instructions/coin_build_doc_tools.yaml @@ -0,0 +1,47 @@ +type: Group +instructions: + - type: SetBuildDirectory + directory: "{{.SourceDir}}" + - type: ChangeDirectory + directory: "{{.BuildDir}}" + + - !include "{{qt/qtbase}}/prepare_building_env.yaml" + + # This tells the script where to install the doc tools, and what files the archive will contain. + - type: EnvironmentVariable + variableName: QT_CI_DOC_TOOLS_INSTALL_DIR + variableValue: "{{.BuildDir}}/doc_tools_build/install" + + - type: ExecuteCommand + command: "{{.Env.ENV_PREFIX}} cmake -P {{.SourceDir}}/cmake/QtBuildStaticDocToolsScript.cmake" + executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution + maxTimeInSeconds: 6000 + maxTimeBetweenOutput: 1200 + userMessageOnFailure: > + Failed to build doc tools. + enable_if: + condition: runtime + env_var: TESTED_MODULE_COIN + contains_value: qtbase + + # For non-qtbase repos, call the build script from the installed folder. + - type: ExecuteCommand + command: "{{.Env.ENV_PREFIX}} cmake -P {{.Env.COIN_CMAKE_INSTALL_DIR}}/lib/cmake/Qt6/QtBuildStaticDocToolsScript.cmake" + executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution + maxTimeInSeconds: 6000 + maxTimeBetweenOutput: 1200 + userMessageOnFailure: > + Failed to build doc tools. + disable_if: + condition: runtime + env_var: TESTED_MODULE_COIN + contains_value: qtbase + + - type: UploadArtifact + archiveDirectory: "{{.Env.QT_CI_DOC_TOOLS_INSTALL_DIR}}" + transferType: GenericArtifactWithName + filename: "static_doc_tools_{{.Env.TESTED_MODULE_COIN}}.tar.gz" + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + userMessageOnFailure: > + Failed to upload built doc tools. diff --git a/coin/instructions/coin_build_doc_tools_checked.yaml b/coin/instructions/coin_build_doc_tools_checked.yaml new file mode 100644 index 00000000000..b2bba88b57e --- /dev/null +++ b/coin/instructions/coin_build_doc_tools_checked.yaml @@ -0,0 +1,25 @@ +type: Group +instructions: + - type: Group + instructions: + - !include "{{qt/qtbase}}/coin_build_doc_tools.yaml" + enable_if: + condition: or + conditions: + # This is set by qtbase instructions. + - condition: runtime + env_var: QT_CI_BUILD_DOC_TOOLS + equals_value: "1" + # This could potentially bet set by non-qtbase instructions. + - condition: runtime + env_var: QT_CI_BUILD_REPO_DOC_TOOLS + equals_value: "1" +enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_property: target.os + - condition: property + property: features + contains_value: DocsGenerationV2 diff --git a/coin/instructions/coin_fetch_doc_tools.yaml b/coin/instructions/coin_fetch_doc_tools.yaml new file mode 100644 index 00000000000..a29f6f6d370 --- /dev/null +++ b/coin/instructions/coin_fetch_doc_tools.yaml @@ -0,0 +1,40 @@ +type: Group +instructions: + # This tells the script where to install the soon to be downloaded doc tools + - type: EnvironmentVariable + variableName: QT_CI_DOC_TOOLS_INSTALL_DIR + variableValue: "{{.BuildDir}}/doc_tools_build/install" + + # Define the doc tools archive name when it is built by qtbase. + - type: EnvironmentVariable + variableName: QT_CI_DOC_TOOLS_ARCHIVE_NAME + variableValue: "static_doc_tools_qtbase.tar.gz" + enable_if: + condition: runtime + env_var: QT_CI_FETCH_REPO_DOC_TOOLS + not_equals_value: "1" + + # Define the doc tools archive name when it is built by a repo other than qtbase. + - type: EnvironmentVariable + variableName: QT_CI_DOC_TOOLS_ARCHIVE_NAME + variableValue: "static_doc_tools_{{.Env.TESTED_MODULE_COIN}}.tar.gz" + enable_if: + condition: runtime + env_var: QT_CI_FETCH_REPO_DOC_TOOLS + equals_value: "1" + + # Downloads the doc tools archive and extracts it to the install directory + - type: InstallBinaryArchive + directory: "{{.Env.QT_CI_DOC_TOOLS_INSTALL_DIR}}" + regex: "{{.Env.QT_CI_DOC_TOOLS_ARCHIVE_NAME}}" + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + + - type: AppendToEnvironmentVariable + variableName: COIN_TEST_DOCS_CONFIGURE_EXTRA_CMAKE_ARGS + variableValue: " -DQT_OPTIONAL_TOOLS_PATH={{.Env.QT_CI_DOC_TOOLS_INSTALL_DIR}}" + +enable_if: + condition: property + property: features + contains_value: DocsGenerationV2 diff --git a/coin/instructions/coin_module_build_template_v2.yaml b/coin/instructions/coin_module_build_template_v2.yaml index a07119b8d50..07c31f111b8 100644 --- a/coin/instructions/coin_module_build_template_v2.yaml +++ b/coin/instructions/coin_module_build_template_v2.yaml @@ -87,3 +87,11 @@ instructions: condition: property property: features contains_value: "DebianPackaging" + + # Normally the doc tools are only built in qtbase, but a repository can choose + # to opt into building the doc tools during the repo's build phase, instead of using + # the qtbase tools, perhaps to use a different qttools sha1. + # In that case, the repository must set the QT_CI_BUILD_REPO_DOC_TOOLS=1 environment variable + # in its module CI build phase instructions, and QT_CI_FETCH_REPO_DOC_TOOLS=1 in its test phase + # instructions. + - !include "{{qt/qtbase}}/coin_build_doc_tools_checked.yaml" diff --git a/coin/instructions/coin_qtbase_build_template_v2.yaml b/coin/instructions/coin_qtbase_build_template_v2.yaml index e7a8329ffc4..f792dbf3af3 100644 --- a/coin/instructions/coin_qtbase_build_template_v2.yaml +++ b/coin/instructions/coin_qtbase_build_template_v2.yaml @@ -78,3 +78,9 @@ instructions: condition: property property: features contains_value: "DebianPackaging" + + # Signal that qtbase should always build the doc tools (if all other requirements are met). + - type: EnvironmentVariable + variableName: QT_CI_BUILD_DOC_TOOLS + variableValue: "1" + - !include "{{qt/qtbase}}/coin_build_doc_tools_checked.yaml" diff --git a/coin/instructions/coin_test_docs_common.yaml b/coin/instructions/coin_test_docs_common.yaml index f3b7b5b2da8..429ecbe57fa 100644 --- a/coin/instructions/coin_test_docs_common.yaml +++ b/coin/instructions/coin_test_docs_common.yaml @@ -14,6 +14,12 @@ instructions: command: "cp -rfs /opt/qt-doctools/. {{.InstallDir}}" userMessageOnFailure: > Failed to create links to provisioned binaries. + enable_if: + condition: property + property: features + not_contains_value: "DocsGenerationV2" + + - !include "{{qt/qtbase}}/coin_fetch_doc_tools.yaml" - !include "{{qt/qtbase}}/prepare_building_env.yaml"