From 8fc437e8739eb12f8d74bce9897f0db92f09bded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 25 Mar 2021 17:52:03 +0100 Subject: [PATCH] [ci-app][CIAPP-936] Add More Git Metadata to Tests (#1254) --- .eslintignore | 1 + LICENSE-3rdparty.csv | 1 + .../src/plugins/util/ci-app-spec.json | 8 +- packages/dd-trace/src/plugins/util/git.js | 76 +++- packages/dd-trace/src/plugins/util/test.js | 7 +- .../dd-trace/test/plugins/util/git.spec.js | 88 ++++- vendor/git-repo-info.js | 356 ++++++++++++++++++ 7 files changed, 516 insertions(+), 21 deletions(-) create mode 100644 vendor/git-repo-info.js diff --git a/.eslintignore b/.eslintignore index 6aaf4fcb132..dbe1a934094 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,3 +6,4 @@ node_modules protobuf versions acmeair-nodejs +vendor diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 20d53d1cb4a..b4ee6b845a8 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -60,3 +60,4 @@ dev,tape,MIT,Copyright James Halliday dev,wait-on,MIT,Copyright 2015 Jeff Barczewski file,profile.proto,Apache license 2.0,Copyright 2016 Google Inc. file,TDigest.h,Apache license 2.0,Copyright Derrick R. Burns +file,git-repo-info.js,MIT,Copyright Robert Jackson and Contributors diff --git a/packages/dd-trace/src/plugins/util/ci-app-spec.json b/packages/dd-trace/src/plugins/util/ci-app-spec.json index 62ac8e96d48..e5f8236bab5 100644 --- a/packages/dd-trace/src/plugins/util/ci-app-spec.json +++ b/packages/dd-trace/src/plugins/util/ci-app-spec.json @@ -4,7 +4,6 @@ "test.framework", "test.suite", "test.name", - "resource.name", "ci.pipeline.id", "ci.pipeline.name", "ci.pipeline.number", @@ -18,6 +17,13 @@ "git.commit.sha", "git.branch", "git.tag", + "git.commit.message", + "git.commit.committer.date", + "git.commit.committer.email", + "git.commit.committer.name", + "git.commit.author.date", + "git.commit.author.email", + "git.commit.author.name", "os.platform", "os.version", "os.architecture", diff --git a/packages/dd-trace/src/plugins/util/git.js b/packages/dd-trace/src/plugins/util/git.js index 1686a509f52..8c5269690ee 100644 --- a/packages/dd-trace/src/plugins/util/git.js +++ b/packages/dd-trace/src/plugins/util/git.js @@ -1,22 +1,80 @@ +const coalesce = require('koalas') +const getRepoInfo = require('../../../../../vendor/git-repo-info') + const { sanitizedExec } = require('./exec') const GIT_COMMIT_SHA = 'git.commit.sha' const GIT_BRANCH = 'git.branch' const GIT_REPOSITORY_URL = 'git.repository_url' const GIT_TAG = 'git.tag' +const GIT_COMMIT_MESSAGE = 'git.commit.message' +const GIT_COMMIT_COMMITTER_DATE = 'git.commit.committer.date' +const GIT_COMMIT_COMMITTER_EMAIL = 'git.commit.committer.email' +const GIT_COMMIT_COMMITTER_NAME = 'git.commit.committer.name' +const GIT_COMMIT_AUTHOR_DATE = 'git.commit.author.date' +const GIT_COMMIT_AUTHOR_EMAIL = 'git.commit.author.email' +const GIT_COMMIT_AUTHOR_NAME = 'git.commit.author.name' + +// Receives a string with the form 'John Doe ' +// and returns { name: 'John Doe', email: 'john.doe@gmail.com' } +function parseUser (user) { + if (!user) { + return { name: '', email: '' } + } + let email = '' + const matchEmail = user.match(/[^@<\s]+@[^@\s>]+/g) + if (matchEmail) { + email = matchEmail[0] + } + return { name: user.replace(`<${email}>`, '').trim(), email } +} // If there is ciMetadata, it takes precedence. function getGitMetadata (ciMetadata) { - const { commitSHA, branch, repositoryUrl } = ciMetadata - // With stdio: 'pipe', errors in this command will not be output to the parent process, - // so if `git` is not present in the env, we'll just fallback to the default - // and not show a warning to the user. - const execOptions = { stdio: 'pipe' } + const { commitSHA, branch, repositoryUrl, tag } = ciMetadata + + const { + author, + committer, + authorDate, + committerDate, + commitMessage, + branch: gitBranch, + tag: gitTag, + sha: gitCommitSHA + } = getRepoInfo(process.cwd()) + + const { name: authorName, email: authorEmail } = parseUser(author) + const { name: committerName, email: committerEmail } = parseUser(committer) + return { - [GIT_REPOSITORY_URL]: repositoryUrl || sanitizedExec('git ls-remote --get-url', execOptions), - [GIT_BRANCH]: branch || sanitizedExec('git rev-parse --abbrev-ref HEAD', execOptions), - [GIT_COMMIT_SHA]: commitSHA || sanitizedExec('git rev-parse HEAD', execOptions) + // With stdio: 'pipe', errors in this command will not be output to the parent process, + // so if `git` is not present in the env, we won't show a warning to the user. + [GIT_REPOSITORY_URL]: repositoryUrl || sanitizedExec('git ls-remote --get-url', { stdio: 'pipe' }), + [GIT_BRANCH]: coalesce(branch, gitBranch), + [GIT_COMMIT_SHA]: coalesce(commitSHA, gitCommitSHA), + [GIT_TAG]: coalesce(tag, gitTag), + [GIT_COMMIT_MESSAGE]: commitMessage, + [GIT_COMMIT_AUTHOR_DATE]: authorDate, + [GIT_COMMIT_AUTHOR_NAME]: authorName, + [GIT_COMMIT_AUTHOR_EMAIL]: authorEmail, + [GIT_COMMIT_COMMITTER_DATE]: committerDate, + [GIT_COMMIT_COMMITTER_NAME]: committerName, + [GIT_COMMIT_COMMITTER_EMAIL]: committerEmail } } -module.exports = { getGitMetadata, GIT_COMMIT_SHA, GIT_BRANCH, GIT_REPOSITORY_URL, GIT_TAG } +module.exports = { + getGitMetadata, + GIT_COMMIT_SHA, + GIT_BRANCH, + GIT_REPOSITORY_URL, + GIT_TAG, + GIT_COMMIT_MESSAGE, + GIT_COMMIT_COMMITTER_DATE, + GIT_COMMIT_COMMITTER_EMAIL, + GIT_COMMIT_COMMITTER_NAME, + GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_AUTHOR_EMAIL, + GIT_COMMIT_AUTHOR_NAME +} diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 14741b68b0e..8ac0dd62d97 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -1,4 +1,4 @@ -const { getGitMetadata, GIT_BRANCH, GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('./git') +const { getGitMetadata, GIT_BRANCH, GIT_COMMIT_SHA, GIT_REPOSITORY_URL, GIT_TAG } = require('./git') const { getCIMetadata } = require('./ci') const { getRuntimeAndOSMetadata } = require('./env') @@ -23,10 +23,11 @@ function getTestEnvironmentMetadata (testFramework) { const { [GIT_COMMIT_SHA]: commitSHA, [GIT_BRANCH]: branch, - [GIT_REPOSITORY_URL]: repositoryUrl + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_TAG]: tag } = ciMetadata - const gitMetadata = getGitMetadata({ commitSHA, branch, repositoryUrl }) + const gitMetadata = getGitMetadata({ commitSHA, branch, repositoryUrl, tag }) const runtimeAndOSMetadata = getRuntimeAndOSMetadata() diff --git a/packages/dd-trace/test/plugins/util/git.spec.js b/packages/dd-trace/test/plugins/util/git.spec.js index 24f9c9b8bd2..e83c156c1a8 100644 --- a/packages/dd-trace/test/plugins/util/git.spec.js +++ b/packages/dd-trace/test/plugins/util/git.spec.js @@ -1,39 +1,111 @@ const proxyquire = require('proxyquire') const sanitizedExecStub = sinon.stub() +const gitRepoInfoStub = sinon.stub().returns({ + author: 'author ', + committer: 'committer ', + authorDate: '1970', + committerDate: '1971', + commitMessage: 'commit message', + branch: 'gitBranch', + tag: 'gitTag', + sha: 'gitSha' +}) + const { getGitMetadata, GIT_COMMIT_SHA, GIT_BRANCH, - GIT_REPOSITORY_URL -} = proxyquire('../../../src/plugins/util/git', { './exec': { - 'sanitizedExec': sanitizedExecStub -} }) + GIT_TAG, + GIT_REPOSITORY_URL, + GIT_COMMIT_MESSAGE, + GIT_COMMIT_COMMITTER_DATE, + GIT_COMMIT_COMMITTER_EMAIL, + GIT_COMMIT_COMMITTER_NAME, + GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_AUTHOR_EMAIL, + GIT_COMMIT_AUTHOR_NAME +} = proxyquire('../../../src/plugins/util/git', + { + './exec': { + 'sanitizedExec': sanitizedExecStub + }, + '../../../../../vendor/git-repo-info': gitRepoInfoStub + } +) describe('git', () => { afterEach(() => { sanitizedExecStub.reset() }) + const commonGitMetadata = { + [GIT_COMMIT_MESSAGE]: 'commit message', + [GIT_COMMIT_COMMITTER_DATE]: '1971', + [GIT_COMMIT_COMMITTER_EMAIL]: 'committer@commit.com', + [GIT_COMMIT_COMMITTER_NAME]: 'committer', + [GIT_COMMIT_AUTHOR_DATE]: '1970', + [GIT_COMMIT_AUTHOR_EMAIL]: 'author@commit.com', + [GIT_COMMIT_AUTHOR_NAME]: 'author', + [GIT_TAG]: 'gitTag', + [GIT_BRANCH]: 'gitBranch' + } it('calls git when some ci metadata is not present', () => { - const ciMetadata = { commitSHA: 'ciSHA', branch: 'ciBranch' } + const ciMetadata = { commitSHA: 'ciSHA' } const metadata = getGitMetadata(ciMetadata) expect(metadata).to.include( { [GIT_COMMIT_SHA]: 'ciSHA', - [GIT_BRANCH]: 'ciBranch' + ...commonGitMetadata } ) expect(metadata[GIT_REPOSITORY_URL]).not.to.equal('ciRepositoryUrl') expect(sanitizedExecStub).to.have.been.calledWith('git ls-remote --get-url', { stdio: 'pipe' }) + expect(gitRepoInfoStub).to.have.been.called }) it('returns ci metadata if present and does not call git', () => { - const ciMetadata = { commitSHA: 'ciSHA', branch: 'ciBranch', repositoryUrl: 'ciRepositoryUrl' } + const ciMetadata = { commitSHA: 'ciSHA', branch: 'ciBranch', repositoryUrl: 'ciRepositoryUrl', tag: 'tag' } const metadata = getGitMetadata(ciMetadata) expect(metadata).to.eql( - { [GIT_COMMIT_SHA]: 'ciSHA', [GIT_BRANCH]: 'ciBranch', [GIT_REPOSITORY_URL]: 'ciRepositoryUrl' } + { + ...commonGitMetadata, + [GIT_COMMIT_SHA]: 'ciSHA', + [GIT_BRANCH]: 'ciBranch', + [GIT_REPOSITORY_URL]: 'ciRepositoryUrl', + [GIT_TAG]: 'tag' + } ) expect(sanitizedExecStub).not.to.have.been.called }) + it('does not crash with empty or badly shapen author or committer', () => { + gitRepoInfoStub.returns({ + author: 'author <>', + committer: undefined, + authorDate: '1970', + committerDate: '1971', + commitMessage: 'commit message', + branch: 'gitBranch', + tag: 'gitTag', + sha: 'gitSha' + }) + const ciMetadata = { repositoryUrl: 'ciRepositoryUrl' } + const metadata = getGitMetadata(ciMetadata) + + expect(metadata).to.eql( + { + [GIT_COMMIT_MESSAGE]: 'commit message', + [GIT_COMMIT_COMMITTER_DATE]: '1971', + [GIT_COMMIT_COMMITTER_EMAIL]: '', + [GIT_COMMIT_COMMITTER_NAME]: '', + [GIT_COMMIT_AUTHOR_DATE]: '1970', + [GIT_COMMIT_AUTHOR_EMAIL]: '', + [GIT_COMMIT_AUTHOR_NAME]: 'author', + [GIT_TAG]: 'gitTag', + [GIT_BRANCH]: 'gitBranch', + [GIT_COMMIT_SHA]: 'gitSha', + [GIT_REPOSITORY_URL]: 'ciRepositoryUrl' + } + ) + }) }) diff --git a/vendor/git-repo-info.js b/vendor/git-repo-info.js new file mode 100644 index 00000000000..ca973779461 --- /dev/null +++ b/vendor/git-repo-info.js @@ -0,0 +1,356 @@ +// From https://github.com/rwjblue/git-repo-info/blob/d3ab418ef8b392eabbe911a37871708b15201b70/index.js +// License MIT, Copyright Robert Jackson and Contributors +'use strict'; + +var fs = require('fs'); +var path = require('path'); +var zlib = require('zlib'); + +var GIT_DIR = '.git'; + +function changeGitDir(newDirName) { + GIT_DIR = newDirName; +} + +function findRepoHandleLinkedWorktree(gitPath) { + var stat = fs.statSync(gitPath); + var root = path.dirname(path.resolve(gitPath)); + if (stat.isDirectory()) { + return { + // for the base (non-linked) dir, there is no distinction between where we + // find the HEAD file and where we find the rest of .git + worktreeGitDir: gitPath, + commonGitDir: gitPath, + root: root, + }; + } else { + // We have a file that tells us where to find the worktree git dir. Once we + // look there we'll know how to find the common git dir, depending on + // whether it's a linked worktree git dir, or a submodule dir + + var linkedGitDir = fs.readFileSync(gitPath).toString(); + var absolutePath=path.resolve(path.dirname(gitPath)); + var worktreeGitDirUnresolved = /gitdir: (.*)/.exec(linkedGitDir)[1]; + var worktreeGitDir = path.resolve(absolutePath,worktreeGitDirUnresolved); + var commonDirPath = path.join(worktreeGitDir, 'commondir'); + if (fs.existsSync(commonDirPath)) { + // this directory contains a `commondir` file; we're in a linked worktree + + var commonDirRelative = fs.readFileSync(commonDirPath).toString().replace(/\r?\n$/, ''); + var commonDir = path.resolve(path.join(worktreeGitDir, commonDirRelative)); + + return { + worktreeGitDir: worktreeGitDir, + commonGitDir: commonDir, + root: path.dirname(commonDir), + }; + } else { + // there is no `commondir` file; we're in a submodule + return { + worktreeGitDir: worktreeGitDir, + commonGitDir: worktreeGitDir, + root: root, + }; + } + } +} + +function findRepo(startingPath) { + var gitPath, lastPath; + var currentPath = startingPath; + + if (!currentPath) { currentPath = process.cwd(); } + + do { + gitPath = path.join(currentPath, GIT_DIR); + + if (fs.existsSync(gitPath)) { + return findRepoHandleLinkedWorktree(gitPath); + } + + lastPath = currentPath; + currentPath = path.resolve(currentPath, '..'); + } while (lastPath !== currentPath); + + return null; +} + +function findPackedTags(gitPath, refPath) { + return getPackedRefsForType(gitPath, refPath, 'tag'); +} + +function findPackedCommit(gitPath, refPath) { + return getPackedRefsForType(gitPath, refPath, 'commit')[0]; +} + +function getPackedRefsForType(gitPath, refPath, type) { + var packedRefsFile = getPackedRefsFile(gitPath); + if (packedRefsFile) { + return getLinesForRefPath(packedRefsFile, type, refPath).map(function(shaLine) { + return getShaBasedOnType(type, shaLine); + }); + } + return []; +} + +function getPackedRefsFile(gitPath) { + var packedRefsFilePath = path.join(gitPath, 'packed-refs'); + return fs.existsSync(packedRefsFilePath) ? fs.readFileSync(packedRefsFilePath, { encoding: 'utf8' }) : false; +} + +function getLinesForRefPath(packedRefsFile, type, refPath) { + return packedRefsFile.split(/\r?\n/).reduce(function(acc, line, idx, arr) { + var targetLine = line.indexOf('^') > -1 ? arr[idx-1] : line; + return doesLineMatchRefPath(type, line, refPath) ? acc.concat(targetLine) : acc; + }, []); +} + +function doesLineMatchRefPath(type, line, refPath) { + var refPrefix = type === 'tag' ? 'refs/tags' : 'refs/heads'; + return (line.indexOf(refPrefix) > -1 || line.indexOf('^') > -1) && line.indexOf(refPath) > -1; +} + +function getShaBasedOnType(type, shaLine) { + var shaResult = ''; + if (type === 'tag') { + shaResult = shaLine.split('tags/')[1]; + } else if (type === 'commit') { + shaResult = shaLine.split(' ')[0]; + } + + return shaResult; +} + +function commitForTag(gitPath, tag) { + var tagPath = path.join(gitPath, 'refs', 'tags', tag); + var taggedObject = fs.readFileSync(tagPath, { encoding: 'utf8' }).trim(); + var objectPath = path.join(gitPath, 'objects', taggedObject.slice(0, 2), taggedObject.slice(2)); + + if (!zlib.inflateSync || !fs.existsSync(objectPath)) { + // we cannot support annotated tags on node v0.10 because + // zlib does not allow sync access + return taggedObject; + } + + var objectContents = zlib.inflateSync(fs.readFileSync(objectPath)).toString(); + + // 'tag 172\u0000object c1ee41c325d54f410b133e0018c7a6b1316f6cda\ntype commit\ntag awesome-tag\ntagger Robert Jackson + // 1429100021 -0400\n\nI am making an annotated tag.\n' + if (objectContents.slice(0,3) === 'tag') { + var sections = objectContents.split(/\0|\r?\n/); + var sha = sections[1].slice(7); + + return sha; + } else { + // this will return the tag for lightweight tags + return taggedObject; + } +} + +function findTag(gitPath, sha) { + var tags = findPackedTags(gitPath, sha) + .concat(findUnpackedTags(gitPath, sha)); + tags.sort(); + return tags.length ? tags[0] : false; +} + +var LAST_TAG_CACHE = {}; + +function findLastTagCached(gitPath, sha) { + if(!LAST_TAG_CACHE[gitPath]) { + LAST_TAG_CACHE[gitPath] = {}; + } + + if(!LAST_TAG_CACHE[gitPath][sha]) { + LAST_TAG_CACHE[gitPath][sha] = findLastTag(gitPath, sha); + } + + return LAST_TAG_CACHE[gitPath][sha]; +} + +function findLastTag(gitPath, sha) { + var queue = [{ sha: sha, depth: 0 }]; + var seenCommits = new Set(); + while (queue.length) { + var element = queue.shift(); + if (seenCommits.has(element.sha)) { + continue; + } + seenCommits.add(element.sha); + var tag = findTag(gitPath, element.sha); + if (tag) { + return { + tag: tag, + commitsSinceLastTag: element.depth + }; + } + var commitData = getCommitData(gitPath, sha); + if (commitData && commitData.parents) { + for (var i = 0; i < commitData.parents.length; i++) { + queue.push({ sha: commitData.parents[i], depth: element.depth + 1 }); + } + } + } + return { tag: null, commitsSinceLastTag: Infinity }; +} + +function findUnpackedTags(gitPath, sha) { + var unpackedTags = []; + var tags = findLooseRefsForType(gitPath, 'tags'); + for (var i = 0, l = tags.length; i < l; i++) { + var commitAtTag = commitForTag(gitPath, tags[i]); + if (commitAtTag === sha) { + unpackedTags.push(tags[i]); + } + } + return unpackedTags; +} + +function findLooseRefsForType(gitPath, type) { + var refsPath = path.join(gitPath, 'refs', type); + return fs.existsSync(refsPath) ? fs.readdirSync(refsPath) : []; +} + +module.exports = function(gitPath) { + var gitPathInfo = findRepo(gitPath); + + var result = { + sha: null, + abbreviatedSha: null, + branch: null, + tag: null, + committer: null, + committerDate: null, + author: null, + authorDate: null, + commitMessage: null, + root: null, + commonGitDir: null, + worktreeGitDir: null, + lastTag: null, + commitsSinceLastTag: 0, + }; + + if (!gitPathInfo) { return result; } + + try { + result.root = gitPathInfo.root; + result.commonGitDir = gitPathInfo.commonGitDir; + result.worktreeGitDir = gitPathInfo.worktreeGitDir; + + var headFilePath = path.join(gitPathInfo.worktreeGitDir, 'HEAD'); + + if (fs.existsSync(headFilePath)) { + var headFile = fs.readFileSync(headFilePath, {encoding: 'utf8'}); + var branchName = headFile.split('/').slice(2).join('/').trim(); + if (!branchName) { + branchName = headFile.split('/').slice(-1)[0].trim(); + } + var refPath = headFile.split(' ')[1]; + + // Find branch and SHA + if (refPath) { + refPath = refPath.trim(); + var branchPath = path.join(gitPathInfo.commonGitDir, refPath); + + result.branch = branchName; + if (fs.existsSync(branchPath)) { + result.sha = fs.readFileSync(branchPath, { encoding: 'utf8' }).trim(); + } else { + result.sha = findPackedCommit(gitPathInfo.commonGitDir, refPath); + } + } else { + result.sha = branchName; + } + + result.abbreviatedSha = result.sha.slice(0,10); + + // Find commit data + var commitData = getCommitData(gitPathInfo.commonGitDir, result.sha); + if (commitData) { + result = Object.keys(commitData).reduce(function(r, key) { + result[key] = commitData[key]; + return result; + }, result); + } + + // Find tag + var tag = findTag(gitPathInfo.commonGitDir, result.sha); + if (tag) { + result.tag = tag; + } + + var lastTagInfo = findLastTagCached(gitPathInfo.commonGitDir, result.sha); + result.lastTag = lastTagInfo.tag; + result.commitsSinceLastTag = lastTagInfo.commitsSinceLastTag; + } + } catch (e) { + if (!module.exports._suppressErrors) { + throw e; // helps with testing and scenarios where we do not expect errors + } else { + // eat the error + } + } + + return result; +}; + +module.exports._suppressErrors = true; +module.exports._findRepo = findRepo; +module.exports._changeGitDir = changeGitDir; + +function getCommitData(gitPath, sha) { + var objectPath = path.join(gitPath, 'objects', sha.slice(0, 2), sha.slice(2)); + + if (zlib.inflateSync && fs.existsSync(objectPath)) { + var objectContents = zlib.inflateSync(fs.readFileSync(objectPath)).toString(); + + return objectContents.split(/\0|\r?\n/) + .filter(function(item) { + return !!item; + }) + .reduce(function(data, section) { + var part = section.slice(0, section.indexOf(' ')).trim(); + + switch(part) { + case 'commit': + case 'tag': + case 'object': + case 'type': + case 'tree': + //ignore these for now + break; + case 'author': + case 'committer': + var parts = section.match(/^(?:author|committer)\s(.+)\s(\d+\s(?:\+|\-)\d{4})$/); + + if (parts) { + data[part] = parts[1]; + data[part + 'Date'] = parseDate(parts[2]); + } + break; + case 'parent': + if (!data.parents) { + data.parents = []; + } + data.parents.push(section.split(' ')[1]); + break; + default: + // Added compatibility with multiline commit message + // https://github.com/rwjblue/git-repo-info/pull/60 + if (!data.commitMessage) { + data.commitMessage = section; + } else { + data.commitMessage = `${data.commitMessage}\n${section}`; + } + } + + return data; + }, {}); + } +} + +function parseDate(d) { + var epoch = d.split(' ')[0]; + return new Date(epoch * 1000).toISOString(); +}