Plumb 'build_environment_variables' into doc jobs (#1079)

This buildfarm configuration variable has always been included for doc
job configurations, but has been left unused. With this change, those
variables are now used in 'doc' and 'rosdoc2' jobs in the same way that
they're used for devel, CI, and release jobs.

ros_buildfarm/doc_job.py

@@ -269,6 +269,12 @@ def _get_doc_job_config(
     repository_args, script_generating_key_files = \
         get_repositories_and_script_generating_key_files(build_file=build_file)
 
+    build_environment_variables = []
+    if build_file.build_environment_variables:
+        build_environment_variables = [
+            '%s=%s' % (var, value)
+            for var, value in sorted(build_file.build_environment_variables.items())]
+
     maintainer_emails = set([])
     if build_file.notify_maintainers and dist_cache and repo_name and \
             repo_name in dist_cache.distribution_file.repositories:
@@ -311,6 +317,7 @@ def _get_doc_job_config(
         'arch': arch,
         'build_tool': build_file.build_tool,
         'repository_args': repository_args,
+        'build_environment_variables': build_environment_variables,
 
         'upload_user': build_file.upload_user,
         'upload_host': build_file.upload_host,

ros_buildfarm/scripts/doc/create_doc_task_generator.py

@@ -31,6 +31,7 @@
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
 from ros_buildfarm.argument import add_argument_dockerfile_dir
+from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.argument import add_argument_force
 from ros_buildfarm.argument import add_argument_output_dir
 from ros_buildfarm.argument import add_argument_repository_name
@@ -108,6 +109,7 @@ def main(argv=sys.argv[1:]):
     add_argument_vcs_information(parser)
     add_argument_distribution_repository_urls(parser)
     add_argument_distribution_repository_key_files(parser)
+    add_argument_env_vars(parser)
     add_argument_force(parser)
     add_argument_output_dir(parser, required=True)
     add_argument_dockerfile_dir(parser)
@@ -578,6 +580,11 @@ def main(argv=sys.argv[1:]):
                 debian_pkg_names.remove(debian_pkg_name)
             print('# END SUBSECTION')
 
+        env_vars = {
+            **args.env_vars,
+            'ROS_PYTHON_VERSION': condition_context['ROS_PYTHON_VERSION'],
+        }
+
         # generate Dockerfile
         data = {
             'os_name': args.os_name,
@@ -590,8 +597,7 @@ def main(argv=sys.argv[1:]):
                 args.distribution_repository_urls,
                 args.distribution_repository_key_files),
 
-            'environment_variables': [
-                'ROS_PYTHON_VERSION={}'.format(condition_context['ROS_PYTHON_VERSION'])],
+            'environment_variables': ['%s=%s' % key_value for key_value in env_vars.items()],
 
             'rosdistro_name': args.rosdistro_name,
 

ros_buildfarm/scripts/doc/create_rosdoc2_task_generator.py

@@ -19,6 +19,7 @@
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
 from ros_buildfarm.argument import add_argument_dockerfile_dir
+from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.common import get_distribution_repository_keys
 from ros_buildfarm.common import get_user_id
 from ros_buildfarm.templates import create_dockerfile
@@ -45,6 +46,7 @@ def main(argv=sys.argv[1:]):
         help="The architecture (e.g. 'amd64')")
     add_argument_distribution_repository_urls(parser)
     add_argument_distribution_repository_key_files(parser)
+    add_argument_env_vars(parser)
     add_argument_dockerfile_dir(parser)
     args = parser.parse_args(argv)
 
@@ -60,6 +62,8 @@ def main(argv=sys.argv[1:]):
             args.distribution_repository_urls,
             args.distribution_repository_key_files),
 
+        'environment_variables': ['%s=%s' % key_value for key_value in args.env_vars.items()],
+
         'uid': get_user_id(),
     }
     create_dockerfile('doc/rosdoc2_task.Dockerfile.em', data, args.dockerfile_dir)

ros_buildfarm/scripts/doc/run_doc_job.py

@@ -26,6 +26,7 @@
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
 from ros_buildfarm.argument import add_argument_dockerfile_dir
+from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.argument import add_argument_force
 from ros_buildfarm.argument import add_argument_os_code_name
 from ros_buildfarm.argument import add_argument_os_name
@@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]):
     add_argument_custom_rosdep_urls(parser)
     add_argument_custom_rosdep_update_options(parser)
     add_argument_force(parser)
+    add_argument_env_vars(parser)
     add_argument_dockerfile_dir(parser)
     args = parser.parse_args(argv)
 

ros_buildfarm/scripts/doc/run_rosdoc2_job.py

@@ -21,6 +21,7 @@
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
 from ros_buildfarm.argument import add_argument_dockerfile_dir
+from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.argument import add_argument_os_code_name
 from ros_buildfarm.argument import add_argument_os_name
 from ros_buildfarm.common import get_distribution_repository_keys
@@ -36,6 +37,7 @@ def main(argv=sys.argv[1:]):
     add_argument_arch(parser)
     add_argument_distribution_repository_urls(parser)
     add_argument_distribution_repository_key_files(parser)
+    add_argument_env_vars(parser)
     add_argument_dockerfile_dir(parser)
     args = parser.parse_args(argv)
 

ros_buildfarm/templates/doc/doc_create_task.Dockerfile.em

@@ -76,6 +76,7 @@ cmds = [
     ' --vcs-info "%s"' % vcs_info + \
     ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \
     ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \
+    ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items()]) + \
     (' --force' if force else '') + \
     ' --output-dir /tmp/generated_documentation' + \
     ' --dockerfile-dir /tmp/docker_doc',

ros_buildfarm/templates/doc/doc_job.xml.em

@@ -168,6 +168,7 @@ else:
         ' ' + os_code_name +
         ' ' + arch +
         ' --build-tool ' + build_tool +
+        ' --env-vars ' + ' '.join([v.replace('$', '\\$',) for v in build_environment_variables]) +
         ' --vcs-info "%s %s %s"' % (doc_repo_spec.type, doc_repo_spec.version if doc_repo_spec.version is not None else '', doc_repo_spec.url) +
         ' ' + ' '.join(repository_args) +
         ' $FORCE_FLAG' +

ros_buildfarm/templates/doc/rosdoc2_create_task.Dockerfile.em

@@ -58,6 +58,7 @@ cmds = [
     ' --arch ' + arch + \
     ' --distribution-repository-urls ' + ' '.join(distribution_repository_urls) + \
     ' --distribution-repository-key-files ' + ' ' .join(['/tmp/keys/%d.key' % i for i in range(len(distribution_repository_keys))]) + \
+    ' --env-vars ' + ' ' .join(['%s=%s' % key_value for key_value in env_vars.items()]) + \
     ' --dockerfile-dir /tmp/docker_doc',
 ]
 }@

ros_buildfarm/templates/doc/rosdoc2_job.xml.em

@@ -129,6 +129,7 @@ else:
         ' ' + os_code_name +
         ' ' + arch +
         ' ' + ' '.join(repository_args) +
+        ' --env-vars ' + ' '.join([v.replace('$', '\\$',) for v in build_environment_variables]) +
         ' --dockerfile-dir $WORKSPACE/docker_generating_docker',
         'echo "# END SECTION"',
         '',

ros_buildfarm/templates/doc/rosdoc2_task.Dockerfile.em

@@ -21,6 +21,11 @@ ENV DEBIAN_FRONTEND noninteractive
     uid=uid,
 ))@
 
+@(TEMPLATE(
+    'snippet/set_environment_variables.Dockerfile.em',
+    environment_variables=environment_variables,
+))@
+
 @(TEMPLATE(
     'snippet/add_distribution_repositories.Dockerfile.em',
     distribution_repository_keys=distribution_repository_keys,