diff --git a/src/builder.py b/src/builder.py index adffa5a..0ed36f4 100644 --- a/src/builder.py +++ b/src/builder.py @@ -1,11 +1,12 @@ +import logging import os -import sys import shutil +import sys from collections import OrderedDict + +from src import common_utils from src import config from src import config_parser -from src import common_utils -import logging logger = logging.getLogger("builder") @@ -28,7 +29,7 @@ def build_deb(version, src_archive_name, downloaded_modules, common_utils.extract_archive(src_archive_name, config.SRC_PATH) common_utils.extract_archive(scripts_archive_name, config.SRC_PATH) - source_dir = os.path.join(config.SRC_PATH, "nginx-{}".format(version)) + source_dir = os.path.join(config.SRC_PATH, f"nginx-{version}") modules_dir = os.path.join(config.SRC_PATH, "modules") scripts_dir = os.path.join(config.SRC_PATH, "debian") shutil.move(modules_dir, scripts_dir) @@ -45,10 +46,8 @@ def build_deb(version, src_archive_name, downloaded_modules, change_control(scripts_dir, control_file_params) logger.info("Running 'dh_make'...") - dh_make_command = "dh_make --copyright {} -e {} --createorig -s -y -p nginx_{}".format( - config.LICENSE_TYPE, - config.EMAIL_CREATOR, - version + dh_make_command = ( + f"dh_make --copyright {config.LICENSE_TYPE} -e {config.EMAIL_CREATOR} --createorig -s -y -p nginx_{version}" ) common_utils.execute_command(dh_make_command, source_dir) @@ -60,10 +59,10 @@ def build_deb(version, src_archive_name, downloaded_modules, package_name = None for file in os.listdir(config.SRC_PATH): - if file.startswith("nginx_{}".format(version)) and file.endswith(".deb"): + if file.startswith(f"nginx_{version}") and file.endswith(".deb"): package_name = file - return [ os.path.join(config.SRC_PATH, package_name) ] + return [os.path.join(config.SRC_PATH, package_name)] def build_rpm(version, downloaded_modules, revision, configure_params, patches): @@ -94,7 +93,7 @@ def build_rpm(version, downloaded_modules, revision, configure_params, patches): for patch in patches: patch_file = patch.replace("/", "_") shutil.move(os.path.join(modules_dir, patch), os.path.join(scripts_dir, patch_file)) - outfile.write("Patch{}: {}\n".format(i, patch_file)) + outfile.write(f"Patch{i}: {patch_file}\n") i += 1 with open(spec_path) as infile: @@ -109,15 +108,18 @@ def build_rpm(version, downloaded_modules, revision, configure_params, patches): modules_dir = os.path.join(scripts_dir, "modules") prepare_rules_rpm(specs_dir, downloaded_modules, modules_dir, revision, configure_params) - common_utils.execute_command("rpmbuild -bb {}".format(spec_file), specs_dir) + common_utils.execute_command(f"rpmbuild -bb {spec_file}", specs_dir) package_name = None package_debuginfo_name = None for file in os.listdir(os.path.join(rpms_dir, config.PLATFORM_ARCH)): - if file.startswith("nginx-{}".format(version)) and file.endswith(".rpm"): + if file.startswith(f"nginx-{version}") and file.endswith(".rpm"): package_name = file - elif file.startswith("nginx-debuginfo-{}".format(version)) and file.endswith(".rpm"): + elif file.startswith(f"nginx-debuginfo-{version}") and file.endswith(".rpm"): package_debuginfo_name = file - return [ os.path.join(rpms_dir, config.PLATFORM_ARCH, package_name), os.path.join(rpms_dir, config.PLATFORM_ARCH, package_debuginfo_name) ] + return [ + os.path.join(rpms_dir, config.PLATFORM_ARCH, package_name), + os.path.join(rpms_dir, config.PLATFORM_ARCH, package_debuginfo_name), + ] def prepare_changelog(source_dir, version, revision): @@ -128,13 +130,13 @@ def prepare_changelog(source_dir, version, revision): :param revision: :return: """ - with open('{}/changelog'.format(source_dir), 'r') as input_file: + with open(f'{source_dir}/changelog', 'r') as input_file: content_file = input_file.readlines() - with open('{}/changelog'.format(source_dir), 'w') as output_file: - replace_line = "nginx ({}-1~".format(version) + with open(f'{source_dir}/changelog', 'w') as output_file: + replace_line = f"nginx ({version}-1~" for line in content_file: if replace_line in line: - line = "nginx ({}-{}) {}; urgency=low\n".format(version, revision, config.OS_RELEASE) + line = f"nginx ({version}-{revision}) {config.OS_RELEASE}; urgency=low\n" output_file.write(line) @@ -150,15 +152,15 @@ def prepare_rules(source_dir, downloaded_modules, configure_params): for configure_param in configure_params: configure_command.append(configure_param) for module in downloaded_modules: - configure_command.append("--add-module=$(MODULESDIR)/{}".format(module)) + configure_command.append(f"--add-module=$(MODULESDIR)/{module}") configure_command = " ".join(configure_command) - with open('{}/rules'.format(source_dir), 'r') as input_file: + with open(f'{source_dir}/rules', 'r') as input_file: content_file = input_file.readlines() - with open('{}/rules'.format(source_dir), 'w') as output_file: + with open(f'{source_dir}/rules', 'w') as output_file: for line in content_file: if "CFLAGS=" in line: - line = 'CFLAGS="" {}'.format(configure_command + "\n") + line = f'CFLAGS="" {configure_command}\n' if "default.conf" in line: continue output_file.write(line) @@ -180,17 +182,17 @@ def prepare_rules_rpm(source_dir, downloaded_modules, modules_dir, revision, con for configure_param in configure_params: configure_command.append(configure_param) for module in downloaded_modules: - configure_command.append("--add-module={}/{}".format(modules_dir, module)) + configure_command.append(f"--add-module={modules_dir}/{module}") configure_command = " ".join(configure_command) - with open('{}/nginx.spec'.format(source_dir), 'r') as input_file: + with open(f'{source_dir}/nginx.spec', 'r') as input_file: content_file = input_file.readlines() - with open('{}/nginx.spec'.format(source_dir), 'w') as output_file: + with open(f'{source_dir}/nginx.spec', 'w') as output_file: for line in content_file: if "./configure" in line: line = configure_command if "%define main_release" in line: - line = "%define main_release {}.ngx".format(revision) + line = f"%define main_release {revision}.ngx" output_file.write(line) @@ -200,9 +202,9 @@ def prepare_nginx_dirs(source_dir): :param source_dir: :return: """ - with open('{}/nginx.dirs'.format(source_dir), 'r') as input_file: + with open(f'{source_dir}/nginx.dirs', 'r') as input_file: content_file = input_file.readlines() - with open('{}/nginx.dirs'.format(source_dir), 'w') as output_file: + with open(f'{source_dir}/nginx.dirs', 'w') as output_file: for line in content_file: output_file.write(line) if "/var/log/nginx" in line: @@ -218,14 +220,14 @@ def change_control(source_dir, control_changes): """ if not control_changes: return False - parsed_control_file = config_parser.parse_control_file('{}/control'.format(source_dir)) + parsed_control_file = config_parser.parse_control_file(f'{source_dir}/control') repaired_keys = repair_keys(control_changes) merged_dict = merge_dicts(parsed_control_file, repaired_keys) - with open('{}/control'.format(source_dir), 'w') as output_file: + with open(f'{source_dir}/control', 'w') as output_file: for key in merged_dict: for k in merged_dict[key]: - output_file.write("{}: {}\n".format(k, merged_dict[key][k])) + output_file.write(f"{k}: {merged_dict[key][k]}\n") output_file.write("\n") return True @@ -243,7 +245,7 @@ def repair_keys(block): z = ''.join(x for x in key.title() if not x.isspace()).replace('_', '-') repaired_keys[z] = block_part[key] index = list(repaired_keys)[0] - index_q = '{}_{}'.format(index, repaired_keys[index]) + index_q = f'{index}_{repaired_keys[index]}' repaired_keys_dict[index_q] = repaired_keys return repaired_keys_dict @@ -260,7 +262,7 @@ def merge_dicts(control_file_dict, control_changes_dict): state = control_changes_dict[key]['State'] changes = control_changes_dict[key] index = list(changes)[0] - index_q = '{}_{}'.format(index, changes[index]) + index_q = f'{index}_{changes[index]}' if state == 'present': del changes['State'] control_file_dict[index_q] = changes @@ -269,7 +271,7 @@ def merge_dicts(control_file_dict, control_changes_dict): if k not in ['State', 'Source', 'Package'] and control_file_dict[index_q][k]: control_file_dict[index_q][k] = str.join(', ', (control_file_dict[index_q][k], changes[k])) except: - logger.error('Не указан параметр state для {}'.format(key)) + logger.error(f'Не указан параметр state для {key}') sys.exit(1) return control_file_dict @@ -283,6 +285,6 @@ def apply_patch(modules_dir, source_dir, patches): :return: """ for patch in patches: - logger.info("Apply patch {}".format(patch)) - patch_command = "patch -p1 < {}".format(os.path.join(modules_dir, patch)) + logger.info(f"Apply patch {patch}") + patch_command = f"patch -p1 < {os.path.join(modules_dir, patch)}" common_utils.execute_command(patch_command, source_dir) diff --git a/src/common_utils.py b/src/common_utils.py index 78298ed..fba15c4 100644 --- a/src/common_utils.py +++ b/src/common_utils.py @@ -1,10 +1,10 @@ -import tarfile -import zipfile -import subprocess -import sys -import os import getpass import logging +import os +import subprocess +import sys +import tarfile +import zipfile logger = logging.getLogger("builder") @@ -59,7 +59,7 @@ def extract_archive(file_name, dest_path): extracted_file = zip.namelist()[0] zip.close() else: - logger.error("Archive format {} is not valid".format(file_name)) + logger.error(f"Archive format {file_name} is not valid") return extracted_file.rstrip('/') diff --git a/src/config.py b/src/config.py index 0b99df7..1902b24 100644 --- a/src/config.py +++ b/src/config.py @@ -1,6 +1,6 @@ -import distro import platform +import distro # Параметры системы OS_RELEASE = distro.codename().split(' ')[0].lower() @@ -10,12 +10,12 @@ # Адрес загрузки исходного кода nginx NGINX_URL = "http://nginx.org/download" -NGINX_SRPM_URL_MAINLINE = "http://nginx.org/packages/mainline/centos/{}/SRPMS".format(OS_VERSION) -NGINX_SRPM_URL_STABLE = "http://nginx.org/packages/centos/{}/SRPMS".format(OS_VERSION) +NGINX_SRPM_URL_MAINLINE = f"http://nginx.org/packages/mainline/centos/{OS_VERSION}/SRPMS" +NGINX_SRPM_URL_STABLE = f"http://nginx.org/packages/centos/{OS_VERSION}/SRPMS" # Архив со скриптами для создания пакета -DEB_PACKAGE_SCRIPTS_URL_MAINLINE = "http://nginx.org/packages/mainline/{}/pool/nginx/n/nginx".format(OS_DISTRIBUTION) -DEB_PACKAGE_SCRIPTS_URL_STABLE = "http://nginx.org/packages/{}/pool/nginx/n/nginx".format(OS_DISTRIBUTION) +DEB_PACKAGE_SCRIPTS_URL_MAINLINE = f"http://nginx.org/packages/mainline/{OS_DISTRIBUTION}/pool/nginx/n/nginx" +DEB_PACKAGE_SCRIPTS_URL_STABLE = f"http://nginx.org/packages/{OS_DISTRIBUTION}/pool/nginx/n/nginx" # Путь до директории сборки пакета SRC_PATH = "/usr/src/nginx" diff --git a/src/config_parser.py b/src/config_parser.py index bd63783..c8d4c0c 100644 --- a/src/config_parser.py +++ b/src/config_parser.py @@ -1,7 +1,8 @@ -import yaml +import logging import sys from collections import OrderedDict -import logging + +import yaml logger = logging.getLogger("builder") @@ -14,11 +15,14 @@ def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): :param object_pairs_hook: :return: """ + class OrderedLoader(Loader): pass + def construct_mapping(loader, node): loader.flatten_mapping(node) return object_pairs_hook(loader.construct_pairs(node)) + OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) @@ -31,7 +35,7 @@ def parse_yaml(file_name): :param file_name: :return config: """ - logger.info("Parse yaml file: {}".format(file_name)) + logger.info(f"Parse yaml file: {file_name}") with open(file_name, 'r') as stream: try: config = ordered_load(stream, yaml.SafeLoader) @@ -67,6 +71,5 @@ def parse_control_file(control_file): except yaml.YAMLError as exc: logger.error(exc) index = list(parsed_paragraph)[0] - index_q = '{}_{}'.format(index, parsed_paragraph[index]) - parsed_paragraph_dict[index_q] = parsed_paragraph + parsed_paragraph_dict[f'{index}_{parsed_paragraph[index]}'] = parsed_paragraph return parsed_paragraph_dict diff --git a/src/downloader.py b/src/downloader.py index 317feb3..f241c75 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -1,14 +1,16 @@ -from requests import get -from packaging import version -from src import config -from src import common_utils -import git +import logging import os -import sys import shutil -import logging +import sys + +import git import requests from bs4 import BeautifulSoup +from packaging import version +from requests import get + +from src import common_utils +from src import config logger = logging.getLogger("builder") @@ -21,9 +23,9 @@ def download_source(src_version): :return file_name: """ logger.info("Downloading nginx src...") - file_name = "nginx-{}.tar.gz".format(src_version) - url = "{}/{}".format(config.NGINX_URL, file_name) - logger.info("--> {}".format(url)) + file_name = f"nginx-{src_version}.tar.gz" + url = f"{config.NGINX_URL}/{file_name}" + logger.info(f"--> {url}") with open(os.path.join(config.SRC_PATH, file_name), "wb") as file: response = get(url) file.write(response.content) @@ -42,20 +44,20 @@ def get_src_rpm_filename(url, src_version): response_text = response.text soup = BeautifulSoup(response_text, 'html.parser') for node in soup.find_all('a'): - if node.get('href').endswith('rpm') and "nginx-{}-".format(src_version) in node.get('href'): + if node.get('href').endswith('rpm') and f"nginx-{src_version}-" in node.get('href'): file_name = node.get('href') elif 400 <= response.status_code < 500: - logger.error(u"{} Client Error: {} for url: {}".format(response.status_code, response.reason, url)) + logger.error(f"{response.status_code} Client Error: {response.reason} for url: {url}") sys.exit(1) elif 500 <= response.status_code < 600: - logger.error(u"{} Server Error: {} for url: {}".format(response.status_code, response.reason, url)) + logger.error(f"{response.status_code} Server Error: {response.reason} for url: {url}") sys.exit(1) if 'file_name' in locals(): return file_name else: - logger.error("Cannot find nginx source rpm(SRPM) with version {} in url {}".format(src_version, url)) + logger.error(f"Cannot find nginx source rpm(SRPM) with version {src_version} in url {url}") sys.exit(1) @@ -71,10 +73,7 @@ def download_source_rpm(src_version): nginx_srpm_url = config.NGINX_SRPM_URL_STABLE file_name = get_src_rpm_filename(nginx_srpm_url, src_version) - common_utils.execute_command("rpm --upgrade --verbose --hash {}/{}".format( - nginx_srpm_url, - file_name - ), os.getcwd()) + common_utils.execute_command(f"rpm --upgrade --verbose --hash {nginx_srpm_url}/{file_name}", os.getcwd()) def download_modules(modules): @@ -121,22 +120,22 @@ def download_module_from_git(module): git_tag = module.get('git_tag') if git_tag: - logger.info("Module {} will download by tag".format(module_name)) + logger.info(f"Module {module_name} will download by tag") downloaded_git_branchortag = git_tag elif git_branch: - logger.info("Module {} will download by branch".format(module_name)) + logger.info(f"Module {module_name} will download by branch") downloaded_git_branchortag = git_branch module_dir = os.path.join(config.SRC_PATH, "modules", module_name) r = repo.clone_from(git_url, module_dir, branch=downloaded_git_branchortag) - logger.info("-- Done: {}".format(module_name)) + logger.info(f"-- Done: {module_name}") if r.submodules: - logger.info("-- Checking for {}/submodules...".format(module_name)) + logger.info(f"-- Checking for {module_name}/submodules...") for submodule in r.submodules: - logger.info("-- Downloading: {}...".format(submodule)) + logger.info(f"-- Downloading: {submodule}...") submodule.update(init=True) - logger.info("---- Done: {}/{}".format(module_name, submodule)) + logger.info(f"---- Done: {module_name}/{submodule}") return module_name @@ -153,7 +152,7 @@ def download_module_from_web(module): module_name = set_module_name(module.get('name'), web_url) file_name = web_url[web_url.rfind("/") + 1:] - logger.info("Module {} will downloading".format(module_name)) + logger.info(f"Module {module_name} will downloading") with open(os.path.join(config.SRC_PATH, "modules", file_name), "wb") as file: response = get(web_url) file.write(response.content) @@ -186,7 +185,7 @@ def download_module_embedded(module): :return: """ if module.get('name') is not None: - config.DEFAULT_CONFIGURE_PARAMS.append("--with-{}".format(module.get('name'))) + config.DEFAULT_CONFIGURE_PARAMS.append(f"--with-{module.get('name')}") def download_package_scripts_deb(src_version): @@ -195,28 +194,19 @@ def download_package_scripts_deb(src_version): :return file_name: """ common_utils.ensure_directory(config.SRC_PATH) - deb_package_scripts_filename = "nginx_{}-1~{}.debian.tar.xz".format( - src_version, - config.OS_RELEASE - ) - deb_package_scripts_url = "{}/{}".format( - config.DEB_PACKAGE_SCRIPTS_URL_MAINLINE, - deb_package_scripts_filename - ) + deb_package_scripts_filename = f"nginx_{src_version}-1~{config.OS_RELEASE}.debian.tar.xz" + deb_package_scripts_url = f"{config.DEB_PACKAGE_SCRIPTS_URL_MAINLINE}/{deb_package_scripts_filename}" if version.parse(src_version).release[1] % 2 == 0: - deb_package_scripts_url = "{}/{}".format( - config.DEB_PACKAGE_SCRIPTS_URL_STABLE, - deb_package_scripts_filename - ) + deb_package_scripts_url = f"{config.DEB_PACKAGE_SCRIPTS_URL_STABLE}/{deb_package_scripts_filename}" logger.info("Download scripts for build deb package") with open(os.path.join(config.SRC_PATH, deb_package_scripts_filename), "wb") as file: response = get(deb_package_scripts_url) if 400 <= response.status_code < 500: - logger.error(u"{} Client Error: {} for url: {}".format(response.status_code, response.reason, deb_package_scripts_url)) + logger.error(f"{response.status_code} Client Error: {response.reason} for url: {deb_package_scripts_url}") sys.exit(1) elif 500 <= response.status_code < 600: - logger.error(u"{} Server Error: {} for url: {}".format(response.status_code, response.reason, deb_package_scripts_url)) + logger.error(f"{response.status_code} Server Error: {response.reason} for url: {deb_package_scripts_url}") sys.exit(1) file.write(response.content) @@ -251,7 +241,7 @@ def install_deb_packages(all_deps): for dependency in all_deps: pkg = cache[dependency] if pkg.is_installed: - logger.warning("'{}' already installed".format(dependency)) + logger.warning(f"'{dependency}' already installed") continue pkg.mark_install()