diff --git a/conf/deployment.yaml b/conf/deployment.yaml new file mode 100644 index 00000000..95dc0b30 --- /dev/null +++ b/conf/deployment.yaml @@ -0,0 +1,50 @@ +#Copyright (C) 2022 TomTom NV. All rights reserved. + +restrictedRepos: + # You can exclude certain repos from safe-settings processing + # If no file is specified, then the following repositories - 'admin', '.github', 'safe-settings' are exempted by default + exclude: + - "^.github$" + - "^fork(ed)*-.*$" + - "^hackathon-.*$" + - "^personal-.*$" + - "^safe-settings$" + - "^ttlab-.*$" + - ".*-fork(ed)*$" + - ".*-personal$" + - ".*-test$" + - ".*-trial$" + # Alternatively you can only include certain repos + #include: ['test'] + +configvalidators: + - plugin: collaborators + error: | + `Admin cannot be assigned to collaborators` + script: | + console.log(`validator.collaborators: baseConfig ${JSON.stringify(baseconfig)}`) + if (baseconfig) { + return baseconfig.permission != 'admin' + } + return true +overridevalidators: + - plugin: branches + error: | + `Branch protection required_approving_review_count cannot be set to zero` + script: | + console.log(`overridevalidators.branches: baseConfig ${JSON.stringify(baseconfig)}`) + console.log(`overridevalidators.branches: overrideConfig ${JSON.stringify(overrideconfig)}`) + if (overrideconfig?.protection?.required_pull_request_reviews?.required_approving_review_count) { + return overrideconfig.protection.required_pull_request_reviews.required_approving_review_count >= 1 + } + return true + - plugin: repository + error: | + `Repository visibility cannot be overriden to public` + script: | + console.log(`validator.repository: baseConfig ${JSON.stringify(baseconfig)}`) + console.log(`validator.repository: overrideConfig ${JSON.stringify(overrideconfig)}`) + if (overrideconfig?.visibility) { + return overrideconfig.visibility != 'public' + } + return true diff --git a/docker-compose.yml b/docker-compose.yml index bd857557..784d3a01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,158 @@ -version: '3.4' +version: "3.4" -services: +x-logging: &default-logging + driver: "json-file" + options: + max-size: "200m" + max-file: "10" + +x-labels: &default-labels + application: "github-safe-settings-probot" + autoheal: true + env: ${ENV} + gh_org: ${GH_ORG} + +x-probot-enviornment: &probot-environment + NODE_ENV: production + LOG_LEVEL: "${LOG_LEVEL}" + LOG_LEVEL_IN_STRING: "true" + +x-probot-healthcheck: &probot-healthcheck + interval: "10s" + timeout: "10s" + +x-probot-labels: &probot-labels + <<: *default-labels + com.scalyr.config.log.attributes.parser: json + traefik.enable: true + +networks: probot: + external: false + +services: + autoheal: + image: willfarrell/autoheal:1.2.0 + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - probot + logging: *default-logging + labels: *default-labels + + probot_cron: image: safe-settings build: . environment: - NODE_ENV: production - APP_ID: ${APP_ID} + <<: *probot-environment + CRON: "${CRON}" GH_ORG: ${GH_ORG} - WEBHOOK_PROXY_URL: ${WEBHOOK_PROXY_URL} - PRIVATE_KEY: ${PRIVATE_KEY} + APP_ID: ${PROBOT_CRON_APP_ID} + PRIVATE_KEY: ${PROBOT_CRON_PRIVATE_KEY} WEBHOOK_SECRET: ${WEBHOOK_SECRET} + WEBHOOK_PROXY_URL: ${WEBHOOK_PROXY_URL} GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID} GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET} + NODE_OPTIONS: "--max-old-space-size=8192" + expose: + - 3000 ports: - 3000:3000 + volumes: + - ${PWD}/conf/deployment.yml:/opt/safe-settings/deployment-settings.yml + labels: + <<: *probot-labels + traefik.port: 3000 + traefik.http.routers.probot_cron.entrypoints: web + traefik.http.routers.probot_cron.rule: "PathPrefix(`/`) && !HeadersRegexp(`X-GitHub-Event`, `.*`)" + healthcheck: + <<: *probot-healthcheck + test: "wget --no-verbose --tries=1 --spider http://probot_cron:3000/ || exit 1" + networks: + - probot + logging: *default-logging + restart: always + profiles: + - cron + + probot_event: + image: safe-settings + build: . + environment: + <<: *probot-environment + CRON: "${CRON}" + GH_ORG: ${GH_ORG} + APP_ID: ${PROBOT_EVENT_APP_ID} + PRIVATE_KEY: ${PROBOT_EVENT_PRIVATE_KEY} + WEBHOOK_SECRET: ${WEBHOOK_SECRET} + WEBHOOK_PROXY_URL: ${WEBHOOK_PROXY_URL} + GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID} + GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET} + expose: + - 3000 + ports: + - 4000:3000 + volumes: + - ${PWD}/conf/deployment.yml:/opt/safe-settings/deployment-settings.yml + labels: + <<: *probot-labels + traefik.port: 4000 + traefik.http.routers.probot_event.entrypoints: web + traefik.http.routers.probot_event.rule: "PathPrefix(`/`)" + healthcheck: + <<: *probot-healthcheck + test: "wget --no-verbose --tries=1 --spider http://probot_event:3000/ || exit 1" + networks: + - probot + logging: *default-logging + restart: always + + proxy: + image: traefik:v3.0 + command: + - "--log.level=DEBUG" + - "--api.insecure=true" + - "--api.dashboard=true" + - "--api.debug=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--providers.docker.network=safe-settings_probot" + - "--entrypoints.web.address=:80" + - "--ping=true" + ports: + - 80:80 + - 8080:8080 + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + labels: *default-labels + healthcheck: + test: "traefik healthcheck --ping || exit 1" + interval: "10s" + timeout: "5s" + networks: + - probot + logging: *default-logging + restart: always + + scalyr: + image: scalyr/scalyr-agent-docker-json:2.2.4-alpine + depends_on: + - proxy + environment: + SCALYR_API_KEY: "${SCALYR_API_KEY}" + SCALYR_SERVER: "https://upload.eu.scalyr.com" + SCALYR_LABELS_AS_ATTRIBUTES: "true" + volumes: + - /var/run/docker.sock:/var/scalyr/docker.sock + - /var/lib/docker/containers:/var/lib/docker/containers + labels: *default-labels + healthcheck: + test: "/usr/sbin/scalyr-agent-2 status || exit 1" + interval: "60s" + timeout: "30s" + networks: + - probot + logging: *default-logging + restart: always diff --git a/index.js b/index.js index 7e4ef032..32424eaf 100644 --- a/index.js +++ b/index.js @@ -14,13 +14,14 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => let appName = 'safe-settings' let appSlug = 'safe-settings' async function syncAllSettings (nop, context, repo = context.repo(), ref) { + const log = robot.log.child({ context: 'index', repository: repo.repo }) try { deploymentConfig = await loadYamlFileSystem() robot.log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) const configManager = new ConfigManager(context, ref) const runtimeConfig = await configManager.loadGlobalSettingsYaml() const config = Object.assign({}, deploymentConfig, runtimeConfig) - robot.log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) + log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) if (ref) { return Settings.syncAll(nop, context, repo, config, ref) } else { @@ -34,7 +35,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => deploymentConfig = {} } const nopcommand = new NopCommand(filename, repo, null, e, 'ERROR') - robot.log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) + log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) Settings.handleError(nop, context, repo, deploymentConfig, ref, nopcommand) } else { throw e @@ -43,13 +44,14 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } async function syncSubOrgSettings (nop, context, suborg, repo = context.repo(), ref) { + const log = robot.log.child({ context: 'index', suborg, repository: repo.repo }) try { deploymentConfig = await loadYamlFileSystem() - robot.log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) + log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) const configManager = new ConfigManager(context, ref) const runtimeConfig = await configManager.loadGlobalSettingsYaml() const config = Object.assign({}, deploymentConfig, runtimeConfig) - robot.log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) + log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) return Settings.syncSubOrgs(nop, context, suborg, repo, config, ref) } catch (e) { if (nop) { @@ -59,7 +61,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => deploymentConfig = {} } const nopcommand = new NopCommand(filename, repo, null, e, 'ERROR') - robot.log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) + log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) Settings.handleError(nop, context, repo, deploymentConfig, ref, nopcommand) } else { throw e @@ -68,13 +70,14 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } async function syncSettings (nop, context, repo = context.repo(), ref) { + const log = robot.log.child({ context: 'index', repository: repo.repo }) try { deploymentConfig = await loadYamlFileSystem() - robot.log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) + log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) const configManager = new ConfigManager(context, ref) const runtimeConfig = await configManager.loadGlobalSettingsYaml() const config = Object.assign({}, deploymentConfig, runtimeConfig) - robot.log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) + log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) return Settings.sync(nop, context, repo, config, ref) } catch (e) { if (nop) { @@ -84,7 +87,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => deploymentConfig = {} } const nopcommand = new NopCommand(filename, repo, null, e, 'ERROR') - robot.log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) + log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) Settings.handleError(nop, context, repo, deploymentConfig, ref, nopcommand) } else { throw e @@ -93,14 +96,15 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } async function renameSync (nop, context, repo = context.repo(), rename, ref) { + const log = robot.log.child({ context: 'index', repository: repo.repo }) try { deploymentConfig = await loadYamlFileSystem() - robot.log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) + log.debug(`deploymentConfig is ${JSON.stringify(deploymentConfig)}`) const configManager = new ConfigManager(context, ref) const runtimeConfig = await configManager.loadGlobalSettingsYaml() const config = Object.assign({}, deploymentConfig, runtimeConfig) const renameConfig = Object.assign({}, config, rename) - robot.log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) + log.debug(`config for ref ${ref} is ${JSON.stringify(config)}`) return Settings.sync(nop, context, repo, renameConfig, ref ) } catch (e) { if (nop) { @@ -110,7 +114,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => deploymentConfig = {} } const nopcommand = new NopCommand(filename, repo, null, e, 'ERROR') - robot.log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) + log.error(`NOPCOMMAND ${JSON.stringify(nopcommand)}`) Settings.handleError(nop, context, repo, deploymentConfig, ref, nopcommand) } else { throw e @@ -136,55 +140,58 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } function getAllChangedSubOrgConfigs (payload) { + const log = robot.log.child({ context: 'index' }) const settingPattern = new Glob(`${env.CONFIG_PATH}/suborgs/*.yml`) // Changes will be an array of files that were added const added = payload.commits.map(c => { return (c.added.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(settingPattern) >= 0) })) }).flat(2) const modified = payload.commits.map(c => { return (c.modified.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(settingPattern) >= 0) })) }).flat(2) const changes = added.concat(modified) const configs = changes.map(file => { const matches = file.match(settingPattern) - robot.log.debug(`${JSON.stringify(file)} \n ${matches[1]}`) + log.debug(`${JSON.stringify(file)} \n ${matches[1]}`) return { name: matches[1] + '.yml', path: file } }) return configs } function getAllChangedRepoConfigs (payload, owner) { + const log = robot.log.child({ context: 'index' }) const settingPattern = new Glob(`${env.CONFIG_PATH}/repos/*.yml`) // Changes will be an array of files that were added const added = payload.commits.map(c => { return (c.added.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(settingPattern) >= 0) })) }).flat(2) const modified = payload.commits.map(c => { return (c.modified.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(settingPattern) >= 0) })) }).flat(2) const changes = added.concat(modified) const configs = changes.map(file => { - robot.log.debug(`${JSON.stringify(file)}`) + log.debug(`${JSON.stringify(file)}`) return { repo: file.match(settingPattern)[1], owner } }) return configs } function getChangedRepoConfigName (glob, files, owner) { + const log = robot.log.child({ context: 'index' }) const modifiedFiles = files.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(glob) >= 0) }) @@ -194,18 +201,20 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } function getChangedSubOrgConfigName (glob, files) { + const log = robot.log.child({ context: 'index', repository: repo.repo }) const modifiedFiles = files.filter(s => { - robot.log.debug(JSON.stringify(s)) + log.debug(JSON.stringify(s)) return (s.search(glob) >= 0) }) return modifiedFiles.map(modifiedFile => { - robot.log.debug(`${JSON.stringify(modifiedFile)}`) + log.debug(`${JSON.stringify(modifiedFile)}`) return { name: modifiedFile.match(glob)[1] + '.yml', path: modifiedFile } }) } async function createCheckRun (context, pull_request, head_sha, head_branch) { + const log = robot.log.child({ context: 'index' }) const { payload } = context // robot.log.debug(`Check suite was requested! for ${context.repo()} ${pull_request.number} ${head_sha} ${head_branch}`) const res = await context.octokit.checks.create({ @@ -214,28 +223,30 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => name: 'Safe-setting validator', head_sha }) - robot.log.debug(JSON.stringify(res, null)) + log.debug(JSON.stringify(res, null)) } async function info() { + const log = robot.log.child({ context: 'index', repository: repo.repo }) const github = await robot.auth() const installations = await github.paginate( github.apps.listInstallations.endpoint.merge({ per_page: 100 }) ) - robot.log.debug(`installations: ${JSON.stringify(installations)}`) + log.debug(`installations: ${JSON.stringify(installations)}`) if (installations.length > 0) { const installation = installations[0] const github = await robot.auth(installation.id) const app = await github.apps.getAuthenticated() appName = app.data.name appSlug = app.data.slug - robot.log.debug(`Validated the app is configured properly = \n${JSON.stringify(app.data, null, 2)}`) + log.debug(`Validated the app is configured properly = \n${JSON.stringify(app.data, null, 2)}`) } } async function syncInstallation () { - robot.log.trace('Fetching installations') + const log = robot.log.child({ context: 'index' }) + log.trace('Fetching installations') const github = await robot.auth() const installations = await github.paginate( @@ -261,6 +272,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.on('push', async context => { const { payload } = context const { repository } = payload + const log = robot.log.child({ context: 'index', event: 'push', repository: repository.name }) const adminRepo = repository.name === env.ADMIN_REPO if (!adminRepo) { @@ -269,7 +281,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => const defaultBranch = payload.ref === 'refs/heads/' + repository.default_branch if (!defaultBranch) { - robot.log.debug('Not working on the default branch, returning...') + log.debug('Not working on the default branch, returning...') return } @@ -278,7 +290,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => commit.modified.includes(Settings.FILE_NAME) }) if (settingsModified) { - robot.log.debug(`Changes in '${Settings.FILE_NAME}' detected, doing a full synch...`) + log.debug(`Changes in '${Settings.FILE_NAME}' detected, doing a full synch...`) return syncAllSettings(false, context) } @@ -296,20 +308,21 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => })) } - robot.log.debug(`No changes in '${Settings.FILE_NAME}' detected, returning...`) + log.debug(`No changes in '${Settings.FILE_NAME}' detected, returning...`) }) robot.on('create', async context => { + const log = robot.log.child({ context: 'index', event: 'create' }) const { payload } = context const { sender } = payload - robot.log.debug('Branch Creation by ', JSON.stringify(sender)) + log.debug('Branch Creation by ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Branch Creation by Bot') + log.debug('Branch Creation by Bot') return } - robot.log.debug('Branch Creation by a Human') + log.debug('Branch Creation by a Human') if (payload.repository.default_branch !== payload.ref) { - robot.log.debug('Not default Branch') + log.debug('Not default Branch') return } @@ -318,38 +331,41 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.on('branch_protection_rule', async context => { const { payload } = context - const { sender } = payload - robot.log.debug('Branch Protection edited by ', JSON.stringify(sender)) + const { sender, repository } = payload + const log = robot.log.child({ context: 'index', event: 'branch_protection_rule', repository: repository.name }) + log.debug('Branch Protection edited by ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Branch Protection edited by Bot') + log.debug('Branch Protection edited by Bot') return } - robot.log.debug('Branch Protection edited by a Human') + log.debug('Branch Protection edited by a Human') return syncSettings(false, context) }) robot.on('custom_property_values', async context => { const { payload } = context - const { sender } = payload - robot.log.debug('Custom Property Value Updated for a repo by ', JSON.stringify(sender)) + const { sender, repository } = payload + const log = robot.log.child({ context: 'index', event: 'custom_property_values', repository: repository.name }) + log.debug('Custom Property Value Updated for a repo by ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Custom Property Value edited by Bot') + log.debug('Custom Property Value edited by Bot') return } - robot.log.debug('Custom Property Value edited by a Human') + log.debug('Custom Property Value edited by a Human') return syncSettings(false, context) }) robot.on('repository_ruleset', async context => { const { payload } = context - const { sender } = payload - robot.log.debug('Repository Ruleset edited by ', JSON.stringify(sender)) + const { sender, repository } = payload + const log = robot.log.child({ context: 'index', event: 'repository_ruleset', repository: repository.name }) + log.debug('Repository Ruleset edited by ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Repository Ruleset edited by Bot') + log.debug('Repository Ruleset edited by Bot') return } - robot.log.debug('Repository Repository edited by a Human') + log.debug('Repository Repository edited by a Human') if (payload.repository_ruleset.source_type === 'Organization') { // For org-level events, we need to update the context since context.repo() won't work const updatedContext = Object.assign({}, context, { @@ -371,25 +387,27 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.on(member_change_events, async context => { const { payload } = context const { sender } = payload - robot.log.debug('Repository member edited by ', JSON.stringify(sender)) + const log = robot.log.child({ context: 'index', event: 'member_change_events', repository: repository.name }) + log.debug('Repository member edited by ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Repository member edited by Bot') + log.debug('Repository member edited by Bot') return } - robot.log.debug('Repository member edited by a Human') + log.debug('Repository member edited by a Human') return syncSettings(false, context) }) robot.on('repository.edited', async context => { const { payload } = context - const { sender } = payload - robot.log.debug('repository.edited payload from ', JSON.stringify(sender)) + const { sender, repository } = payload + const log = robot.log.child({ context: 'index', event: 'repository.edited', repository: repository.name }) + log.debug('repository.edited payload from ', JSON.stringify(sender)) if (sender.type === 'Bot') { - robot.log.debug('Repository Edited by a Bot') + log.debug('Repository Edited by a Bot') return } - robot.log.debug('Repository Edited by a Human') + log.debug('Repository Edited by a Human') return syncSettings(false, context) }) @@ -401,18 +419,19 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } const { payload } = context const { sender } = payload + const log = robot.log.child({ context: 'index', event: 'repository.renamed', repository: repository.name }) - robot.log.debug(`repository renamed from ${payload.changes.repository.name.from} to ${payload.repository.name} by ', ${sender.login}`) + log.debug(`repository renamed from ${payload.changes.repository.name.from} to ${payload.repository.name} by ', ${sender.login}`) if (sender.type === 'Bot') { - robot.log.debug('Repository Edited by a Bot') + log.debug('Repository Edited by a Bot') if (sender.login === `${appSlug}[bot]`) { - robot.log.debug('Renamed by safe-settings app') + log.debug('Renamed by safe-settings app') return } const oldPath = `.github/repos/${payload.changes.repository.name.from}.yml` const newPath = `.github/repos/${payload.repository.name}.yml` - robot.log.debug(oldPath) + log.debug(oldPath) try { const repofile = await context.octokit.request('GET /repos/{owner}/{repo}/contents/{path}', { owner: payload.repository.owner.login, @@ -423,7 +442,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } }) let content = Buffer.from(repofile.data.content, 'base64').toString() - robot.log.debug(content) + log.debug(content) content = `# Repo Renamed and safe-settings renamed the file from ${payload.changes.repository.name.from} to ${payload.repository.name}\n# change the repo name in the config for consistency\n\n${content}` content = Buffer.from(content).toString('base64') try { @@ -451,22 +470,22 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => 'X-GitHub-Api-Version': '2022-11-28' } }) - robot.log.debug(`Created a new setting file ${newPath}`) + log.debug(`Created a new setting file ${newPath}`) } else { - robot.log.error(error) + log.error(error) } - } + } } catch (error) { if (error.status === 404) { //nop - } else { - robot.log.error(error) + } else { + log.error(error) } - } + } return } else { - robot.log.debug('Repository Edited by a Human') + log.debug('Repository Edited by a Human') // Create a repository config to reset the name back to the previous name const rename = {repository: { name: payload.changes.repository.name.from, oldname: payload.repository.name}} const repo = {repo: payload.changes.repository.name.from, owner: payload.repository.owner.login} @@ -478,15 +497,16 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.on('check_suite.requested', async context => { const { payload } = context const { repository } = payload + const log = robot.log.child({ context: 'index', event: 'check_suite.requested', repository: repository.name }) const adminRepo = repository.name === env.ADMIN_REPO - robot.log.debug(`Is Admin repo event ${adminRepo}`) + log.debug(`Is Admin repo event ${adminRepo}`) if (!adminRepo) { - robot.log.debug('Not working on the Admin repo, returning...') + log.debug('Not working on the Admin repo, returning...') return } const defaultBranch = payload.check_suite.head_branch === repository.default_branch if (defaultBranch) { - robot.log.debug(' Working on the default branch, returning...') + log.debug(' Working on the default branch, returning...') return } const { @@ -496,7 +516,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => } = context.payload.check_suite if (!Array.isArray(pullRequests) || !pullRequests[0]) { - robot.log.debug('Not working on a PR, returning...') + log.debug('Not working on a PR, returning...') return } const pull_request = payload.check_suite.pull_requests[0] @@ -507,15 +527,16 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.log.debug('Pull_request opened !') const { payload } = context const { repository } = payload + const log = robot.log.child({ context: 'index', event: 'pull_request.opened', repository: repository.name }) const adminRepo = repository.name === env.ADMIN_REPO - robot.log.debug(`Is Admin repo event ${adminRepo}`) + log.debug(`Is Admin repo event ${adminRepo}`) if (!adminRepo) { - robot.log.debug('Not working on the Admin repo, returning...') + log.debug('Not working on the Admin repo, returning...') return } const defaultBranch = payload.pull_request.head_branch === repository.default_branch if (defaultBranch) { - robot.log.debug(' Working on the default branch, returning...') + log.debug(' Working on the default branch, returning...') return } const pull_request = payload.pull_request @@ -526,18 +547,19 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.log.debug('Pull_request REopened !') const { payload } = context const { repository } = payload + const log = robot.log.child({ context: 'index', event: 'pull_request.reopened', repository: repository.name }) const pull_request = payload.pull_request const adminRepo = repository.name === env.ADMIN_REPO - robot.log.debug(`Is Admin repo event ${adminRepo}`) + log.debug(`Is Admin repo event ${adminRepo}`) if (!adminRepo) { - robot.log.debug('Not working on the Admin repo, returning...') + log.debug('Not working on the Admin repo, returning...') return } const defaultBranch = payload.pull_request.head_branch === repository.default_branch if (defaultBranch) { - robot.log.debug(' Working on the default branch, returning...') + log.debug(' Working on the default branch, returning...') return } return createCheckRun(context, pull_request, payload.pull_request.head.sha, payload.pull_request.head.ref) @@ -557,29 +579,30 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => robot.log.debug('Check run was created!') const { payload } = context const { repository } = payload + const log = robot.log.child({ context: 'index', event: 'check_run.created', repository: repository.name }) const { check_run } = payload const { check_suite } = check_run const pull_request = check_suite.pull_requests[0] const source = payload.check_run.name === 'Safe-setting validator' if (!source) { - robot.log.debug(' Not triggered by Safe-settings...') + log.debug(' Not triggered by Safe-settings...') return } if (check_run.status === 'completed') { - robot.log.debug(' Checkrun created as completed, returning') + log.debug(' Checkrun created as completed, returning') return } const adminRepo = repository.name === env.ADMIN_REPO - robot.log.debug(`Is Admin repo event ${adminRepo}`) + log.debug(`Is Admin repo event ${adminRepo}`) if (!adminRepo) { - robot.log.debug('Not working on the Admin repo, returning...') + log.debug('Not working on the Admin repo, returning...') return } if (!pull_request) { - robot.log.debug('Not working on a PR, returning...') + log.debug('Not working on a PR, returning...') return } @@ -591,7 +614,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => started_at: new Date().toISOString(), output: { title: 'Starting NOP', summary: 'initiating...' } } - robot.log.debug(`Updating check run ${JSON.stringify(params)}`) + log.debug(`Updating check run ${JSON.stringify(params)}`) await context.octokit.checks.update(params) // guarding against null value from upstream libary that is @@ -607,7 +630,7 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => const settingsModified = files.includes(Settings.FILE_NAME) if (settingsModified) { - robot.log.debug(`Changes in '${Settings.FILE_NAME}' detected, doing a full synch...`) + log.debug(`Changes in '${Settings.FILE_NAME}' detected, doing a full synch...`) return syncAllSettings(true, context, context.repo(), pull_request.head.ref) } @@ -635,14 +658,15 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => conclusion: 'success', output: { title: 'No Safe-settings changes detected', summary: 'No changes detected' } } - robot.log.debug(`Completing check run ${JSON.stringify(params)}`) + log.debug(`Completing check run ${JSON.stringify(params)}`) await context.octokit.checks.update(params) }) robot.on('repository.created', async context => { const { payload } = context - const { sender } = payload - robot.log.debug('repository.created payload from ', JSON.stringify(sender)) + const { sender, repository } = payload + const log = robot.log.child({ context: 'index', event: 'repository.created', repository: repository.name }) + log.debug('repository.created payload from ', JSON.stringify(sender)) return syncSettings(false, context) }) @@ -659,11 +683,12 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) => # * * * * * * */ cron.schedule(process.env.CRON, () => { - robot.log.debug('running a task every minute') + const log = robot.log.child({ event: 'cron' }) + log.debug('running a task every minute') syncInstallation() }) } - + // Get info about the app info() diff --git a/lib/configManager.js b/lib/configManager.js index 58f5bb43..dd7b4977 100644 --- a/lib/configManager.js +++ b/lib/configManager.js @@ -6,7 +6,7 @@ module.exports = class ConfigManager { constructor (context, ref) { this.context = context this.ref = ref - this.log = context.log + this.log = context.log.child({ context: 'ConfigManager' }) } /** diff --git a/lib/mergeDeep.js b/lib/mergeDeep.js index d47fd5ce..a43a5db7 100644 --- a/lib/mergeDeep.js +++ b/lib/mergeDeep.js @@ -7,7 +7,7 @@ const GET_NAME_USERNAME_PROPERTY = item => { if (NAME_USERNAME_PROPERTY(item)) r class MergeDeep { constructor (log, github, ignorableFields = [], configvalidators = {}, overridevalidators = {}) { - this.log = log + this.log = log.child({ context: 'MergeDeep' }) this.github = github this.ignorableFields = ignorableFields this.configvalidators = DeploymentConfig.configvalidators @@ -67,6 +67,9 @@ class MergeDeep { const target = firstInvocation && Array.isArray(t) ? Object.assign({}, { __array: t }) : t const source = firstInvocation && Array.isArray(s) ? Object.assign({}, { __array: s }) : s + this.log.debug('compareDeep begin BaseConfig %o', target) + this.log.debug('compareDeep begin OverrideConfig %o', source) + // Also initialize the additions, modifications, and deletions for the first invocation if (firstInvocation) { if (Array.isArray(source)) { @@ -83,6 +86,8 @@ class MergeDeep { // If the target is empty, then all the source is added to additions if (t === undefined || t === null || (this.isEmpty(t) && !this.isEmpty(s))) { additions = Object.assign(additions, s) + this.log.debug('compareDeep end BaseConfig %o', target) + this.log.debug('compareDeep end OverrideConfig %o', source) return ({ additions, modifications, hasChanges: true }) } diff --git a/lib/plugins/branches.js b/lib/plugins/branches.js index cac0aadf..397879ee 100644 --- a/lib/plugins/branches.js +++ b/lib/plugins/branches.js @@ -6,7 +6,7 @@ const previewHeaders = { accept: 'application/vnd.github.hellcat-preview+json,ap module.exports = class Branches extends ErrorStash { constructor (nop, github, repo, settings, log, errors) { - super(errors) + super(errors, log, repo.repo) this.github = github this.repo = repo this.branches = settings @@ -21,12 +21,13 @@ module.exports = class Branches extends ErrorStash { this.branches .filter(branch => branch.protection !== undefined) .map(branch => { + this.log.debug(`Processing branch ${JSON.stringify(branch)}`) // If branch protection is empty if (this.isEmpty(branch.protection)) { let p = Object.assign(this.repo, { branch: branch.name }) if (branch.name === 'default') { p = Object.assign(this.repo, { branch: currentRepo.data.default_branch }) - this.log(`Deleting default branch protection for branch ${currentRepo.data.default_branch}`) + this.log.info(`Deleting default branch protection for branch ${currentRepo.data.default_branch}`) } // Hack to handle closures and keep params from changing const params = Object.assign({}, p) @@ -43,7 +44,7 @@ module.exports = class Branches extends ErrorStash { let p = Object.assign(this.repo, { branch: branch.name }) if (branch.name === 'default') { p = Object.assign(this.repo, { branch: currentRepo.data.default_branch }) - // this.log(`Setting default branch protection for branch ${currentRepo.data.default_branch}`) + // this.log.info(`Setting default branch protection for branch ${currentRepo.data.default_branch}`) } // Hack to handle closures and keep params from changing const params = Object.assign({}, p) @@ -72,6 +73,7 @@ module.exports = class Branches extends ErrorStash { resArray.push(new NopCommand(this.constructor.name, this.repo, this.github.repos.updateBranchProtection.endpoint(params), 'Add Branch Protection')) return Promise.resolve(resArray) } + this.useCurrentBranchProtectionIfNotOverriddenOrDisable(this.reformatAndReturnBranchProtection(result.data), params) this.log.debug(`Adding branch protection ${JSON.stringify(params)}`) return this.github.repos.updateBranchProtection(params).then(res => this.log(`Branch protection applied successfully ${JSON.stringify(res.url)}`)).catch(e => { this.logError(`Error applying branch protection ${JSON.stringify(e)}`); return [] }) }).catch((e) => { @@ -82,7 +84,7 @@ module.exports = class Branches extends ErrorStash { return Promise.resolve(resArray) } this.log.debug(`Adding branch protection ${JSON.stringify(params)}`) - return this.github.repos.updateBranchProtection(params).then(res => this.log(`Branch protection applied successfully ${JSON.stringify(res.url)}`)).catch(e => { this.logError(`Error applying branch protection ${JSON.stringify(e)}`); return [] }) + return this.github.repos.updateBranchProtection(params).then(res => this.log.info(`Branch protection applied successfully ${JSON.stringify(res.url)}`)).catch(e => { this.logError(`Error applying branch protection ${JSON.stringify(e)}`); return [] }) } else { this.logError(e) if (this.nop) { @@ -122,4 +124,54 @@ module.exports = class Branches extends ErrorStash { } return protection } + + /** + * Method applies branch protection settings coming from GitHub in case user did not specify them in safe-settings configurations. + * + * @param {*} currentBranchProtection + * @param {*} overriddenBranchProtection + */ + useCurrentBranchProtectionIfNotOverriddenOrDisable (currentBranchProtection, overriddenBranchProtection) { + this.log.debug('Branch protection current: %o', currentBranchProtection) + this.log.debug('Branch protection overridden: %o', overriddenBranchProtection) + + if (overriddenBranchProtection?.required_status_checks === undefined) { + this.log.debug('required_status_checks not defined in settings files') + if (currentBranchProtection.required_status_checks) { + const { strict, checks } = currentBranchProtection.required_status_checks + overriddenBranchProtection.required_status_checks = { strict, checks } + this.log.debug('Enabling required_status_checks: %o', overriddenBranchProtection.required_status_checks) + } + else { + this.log.debug('Disabling required_status_checks') + overriddenBranchProtection.required_status_checks = null + } + } + const supportedAttributes = [{"name": "enforce_admins", "required": true}, + {"name": "required_linear_history", "required": false}, + {"name": "restrictions", "required": true} + ] + supportedAttributes.forEach((e) => { + this.checkAttribute(currentBranchProtection, overriddenBranchProtection, e.name, e.required) + }) + + this.log.debug('Branch protection updated: %o', overriddenBranchProtection) + } + + checkAttribute(currentBranchProtection, overriddenBranchProtection, key, required = false) { + this.log.debug('Checking whether %s is defined in UI', key) + if (overriddenBranchProtection[key] === undefined) { + this.log.debug('%s not defined in settings files', key) + if (currentBranchProtection.hasOwnProperty(key)) { + overriddenBranchProtection[key] = currentBranchProtection[key] + this.log.debug('Setting %s to %o from UI', key, overriddenBranchProtection[key]) + } + else { + if (required) { + this.log.debug('Setting required attribute %s to null', key) + overriddenBranchProtection[key] = null + } + } + } + } } diff --git a/lib/plugins/diffable.js b/lib/plugins/diffable.js index 74871691..02dfb129 100644 --- a/lib/plugins/diffable.js +++ b/lib/plugins/diffable.js @@ -25,7 +25,7 @@ const NopCommand = require('../nopcommand') const ignorableFields = ['id', 'node_id', 'default', 'url'] module.exports = class Diffable extends ErrorStash { constructor (nop, github, repo, entries, log, errors) { - super(errors) + super(errors, log, repo.repo) this.github = github this.repo = repo this.entries = entries diff --git a/lib/plugins/errorStash.js b/lib/plugins/errorStash.js index 4b625559..b53fe05e 100644 --- a/lib/plugins/errorStash.js +++ b/lib/plugins/errorStash.js @@ -1,7 +1,8 @@ // Base class to make it easy to log errors as issue in the `admin` repo module.exports = class ErrorStash { - constructor (errors) { + constructor (errors, log, repoName) { this.errors = errors + this.log = log.child({ plugin: this.constructor.name, repository: repoName }) } logError (msg) { diff --git a/lib/plugins/repository.js b/lib/plugins/repository.js index 42bbd28e..f10dada2 100644 --- a/lib/plugins/repository.js +++ b/lib/plugins/repository.js @@ -41,7 +41,7 @@ const ignorableFields = [ module.exports = class Repository extends ErrorStash { constructor (nop, github, repo, settings, installationId, log, errors) { - super(errors) + super(errors, log, repo.repo) this.installationId = installationId this.github = github this.settings = Object.assign({ mediaType: { previews: ['nebula-preview'] } }, settings, repo) @@ -91,7 +91,7 @@ module.exports = class Repository extends ErrorStash { const promises = [] if (topicChanges.hasChanges) { promises.push(this.updatetopics(resp.data, resArray)) - } + } if (changes.hasChanges) { this.log.debug('There are repo changes') let updateDefaultBranchPromise = Promise.resolve() @@ -150,7 +150,7 @@ module.exports = class Repository extends ErrorStash { } } } else { - this.logError(` Error ${JSON.stringify(e)}`) + this.logError(`Error while syncing repository: ${JSON.stringify(e)}`) } }) } @@ -184,7 +184,7 @@ module.exports = class Repository extends ErrorStash { if (e.status === 404) { return this.renameBranch(oldname, newname, resArray) } else { - this.logError(`Error ${JSON.stringify(e)}`) + this.logError(`Error while updating default branch: ${JSON.stringify(e)}`) } }) } diff --git a/lib/plugins/validator.js b/lib/plugins/validator.js index 45f04d60..4d105311 100644 --- a/lib/plugins/validator.js +++ b/lib/plugins/validator.js @@ -1,6 +1,7 @@ const NopCommand = require('../nopcommand') -module.exports = class Validator { +module.exports = class Validator extends ErrorStash { constructor (nop, github, repo, settings, log) { + super([], log, repo.repo) this.github = github this.pattern = settings.pattern // this.regex = /[a-zA-Z0-9_-]+_\w[a-zA-Z0-9_-]+.*/gm @@ -20,7 +21,7 @@ module.exports = class Validator { } }).then(res => { if (this.repo.repo.search(this.regex) >= 0) { - this.log(`Repo ${this.repo.repo} Passed Validation for pattern ${this.pattern}`) + this.log.debug(`Repo ${this.repo.repo} Passed Validation for pattern ${this.pattern}`) if (this.nop) { return Promise.resolve([ new NopCommand(this.constructor.name, this.repo, null, `Passed Validation for pattern ${this.pattern}`) @@ -38,7 +39,7 @@ module.exports = class Validator { }) } } else { - this.log(`Repo ${this.repo.repo} Failed Validation for pattern ${this.pattern}`) + this.log.warn(`Repo ${this.repo.repo} Failed Validation for pattern ${this.pattern}`) if (this.nop) { return Promise.resolve([ new NopCommand(this.constructor.name, this.repo, null, `Failed Validation for pattern ${this.pattern}`, 'ERROR') @@ -59,7 +60,7 @@ module.exports = class Validator { }) .catch(() => {}) } catch (error) { - this.log(`Error in Validation checking ${error}`) + this.log.error(`Error in Validation checking ${error}`) } } } diff --git a/lib/settings.js b/lib/settings.js index 21cac18b..098e82c5 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -68,7 +68,7 @@ class Settings { if (suborg) { this.subOrgConfigMap = [suborg] } - this.log = context.log + this.log = context.log.child({ context: 'Settings', repository: this.repo.repo }) this.results = [] this.errors = [] this.configvalidators = {} @@ -393,10 +393,10 @@ ${this.results.reduce((x, y) => { if (Array.isArray(baseConfig) && Array.isArray(config)) { for (const baseEntry of baseConfig) { const newEntry = config.find(e => e.name === baseEntry.name) - this.validate(section, baseEntry, newEntry) + this.validate(repoName, section, baseEntry, newEntry) } } else { - this.validate(section, baseConfig, config) + this.validate(repoName, section, baseConfig, config) } if (section !== 'repositories' && section !== 'repository') { // Ignore any config that is not a plugin @@ -410,7 +410,7 @@ ${this.results.reduce((x, y) => { return childPlugins } - validate (section, baseConfig, overrideConfig) { + validate (repoName, section, baseConfig, overrideConfig) { const configValidator = this.configvalidators[section] if (configValidator) { this.log.debug(`Calling configvalidator for key ${section} `)