Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
leordev committed Jul 31, 2024
1 parent 44654bc commit d3ecefb
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 11 deletions.
1 change: 1 addition & 0 deletions metrics-collector/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"dotenv": "^16.4.5",
"express": "^4.19.2",
"faker": "^6.6.6",
"json2csv": "6.0.0-alpha.2",
"moment": "^2.30.1",
"pg": "^8.11.5",
"ts-node": "^10.9.2",
Expand Down
30 changes: 30 additions & 0 deletions metrics-collector/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

87 changes: 87 additions & 0 deletions metrics-collector/src/gh-contributions-to-csv.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
const fs = require('fs');
const { Parser } = require('json2csv');

const filePath = "./gh_contributions_metrics.json";

// Read JSON data from the file
function readJsonFile(filePath) {
if (fs.existsSync(filePath)) {
const data = fs.readFileSync(filePath, "utf8");
return JSON.parse(data);
}
return {};
}

const data = readJsonFile(filePath);

function extractData(data) {
const projects = Object.keys(data);
const allMonths = new Set();

// Collect all months from all projects
projects.forEach(project => {
data[project].forEach(entry => {
const { data } = entry;
Object.values(data).forEach(item => {
Object.keys(item).forEach(month => {
allMonths.add(month);
});
});
});
});

const monthsArray = Array.from(allMonths).sort();

const csvData = [];

projects.forEach(project => {
const projectRows = {
issues: { project: project, type: 'issues' },
internalMemberIssues: { project: project, type: 'internalMemberIssues' },
prs: { project: project, type: 'prs' },
internalMemberPrs: { project: project, type: 'internalMemberPrs' },
comments: { project: project, type: 'comments' },
internalMemberComments: { project: project, type: 'internalMemberComments' }
};

monthsArray.forEach(month => {
projectRows.issues[month] = 0;
projectRows.internalMemberIssues[month] = 0;
projectRows.prs[month] = 0;
projectRows.internalMemberPrs[month] = 0;
projectRows.comments[month] = 0;
projectRows.internalMemberComments[month] = 0;
});

data[project].forEach(entry => {
const { data } = entry;
Object.entries(data).forEach(([key, values]) => {
Object.entries(values).forEach(([month, count]) => {
projectRows[key][month] = (projectRows[key][month] || 0) + count;
});
});
});

csvData.push(projectRows.issues);
csvData.push(projectRows.internalMemberIssues);
csvData.push(projectRows.prs);
csvData.push(projectRows.internalMemberPrs);
csvData.push(projectRows.comments);
csvData.push(projectRows.internalMemberComments);
});

return { csvData, monthsArray };
}

function buildCSV(data) {
const { csvData, monthsArray } = extractData(data);
const fields = ['project', 'type', ...monthsArray];
const opts = { fields };
const parser = new Parser(opts);
const csv = parser.parse(csvData);

fs.writeFileSync('gh_contributions_metrics.csv', csv);
console.log('CSV file created successfully.');
}

buildCSV(data);
34 changes: 23 additions & 11 deletions metrics-collector/src/gh-contributions.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,18 @@ let octokit;

const orgName = "TBD54566975";
const repos = [
"tbdex",
"tbdex-js",
"tbdex-kt",
"tbdex-swift",
"tbdex-rs",
"web5-spec",
"web5-js",
"web5-kt",
"web5-swift",
"web5-rs",
// "dwn-sdk-js",
"dwn-sdk-js",
"dwn-server",
];

const KNOWN_PAST_MEMBERS = ["amika-sq"];
Expand All @@ -26,7 +29,7 @@ const KNOWN_BOTS = ['codecov-commenter', 'dependabot[bot]', 'renovate[bot]'];

// Cache members to avoid rate limiting
const membersCache = new Map(
KNOWN_PAST_MEMBERS.map((kpm) => [kpm, true])
KNOWN_PAST_MEMBERS.map((kpm) => [kpm, 'org'])
);

async function fetchIssues(owner, repo) {
Expand Down Expand Up @@ -93,8 +96,8 @@ async function isMember(org, user) {

if (user.type === 'Bot' || KNOWN_BOTS.includes(user.login)) {
console.info("Skipping bot", user.login);
membersCache.set(username, true);
return true;
membersCache.set(username, 'bot');
return 'bot';
}

try {
Expand All @@ -105,9 +108,9 @@ async function isMember(org, user) {
if (res.status === 302) {
throw new Error("Forbidden to check membership!");
} else if (res.status === 204) {
membersCache.set(username, true);
membersCache.set(username, 'org');
console.info("member found in org", username);
return true;
return 'org';
} else {
console.info("member not found in org", username);
membersCache.set(username, false);
Expand All @@ -134,12 +137,15 @@ async function aggregateData(owner, repo) {
console.info(`Fetched ${issues.length} issues, ${prs.length} PRs, and ${comments.length} comments for ${owner}/${repo}`);

const now = moment();
const beginningTime = now.clone().subtract(3, 'months');
const beginningTime = now.clone().subtract(4, 'months');

const monthlyData = {
issues: {},
internalMemberIssues: {},
prs: {},
comments: {}
internalMemberPrs: {},
comments: {},
internalMemberComments: {},
};

function addToMonthlyData(type, date) {
Expand All @@ -151,7 +157,9 @@ async function aggregateData(owner, repo) {
console.info("Computing issues numbers...");
for (const issue of issues) {
const member = await isMember(owner, issue.user);
if (!member && moment(issue.created_at).isAfter(beginningTime)) {
if (member === 'org') {
addToMonthlyData('internalMemberIssues', issue.created_at);
} else if (member !== 'bot') {
addToMonthlyData('issues', issue.created_at);
// print issue details with link
console.info(`[${issue.user.login}]: ${issue.title} (${issue.html_url})`);
Expand All @@ -161,7 +169,9 @@ async function aggregateData(owner, repo) {
console.info("Computing PRs numbers...");
for (const pr of prs) {
const member = await isMember(owner, pr.user);
if (!member && moment(pr.created_at).isAfter(beginningTime)) {
if (member === 'org') {
addToMonthlyData('internalMemberPrs', pr.created_at);
} else if (member !== 'bot') {
addToMonthlyData('prs', pr.created_at);
// print PR details with link
console.info(`[${pr.user.login}]: ${pr.title} (${pr.html_url})`);
Expand All @@ -171,7 +181,9 @@ async function aggregateData(owner, repo) {
console.info("Computing comments numbers...");
for (const comment of comments) {
const member = await isMember(owner, comment.user);
if (!member && moment(comment.created_at).isAfter(beginningTime)) {
if (member === 'org') {
addToMonthlyData('internalMemberComments', comment.created_at);
} else if (member !== 'bot') {
addToMonthlyData('comments', comment.created_at);
// print comment details with link
console.info(`[${comment.user.login}]: ${comment.body.substring(0, 48)}... (${comment.html_url})`);
Expand Down

0 comments on commit d3ecefb

Please sign in to comment.