Skip to content

Commit 4ffab6b

Browse files
stragerstrager
committed
feat(website-analytics): import VS Code marketplace stats
Pull VS Code extension download statistics from the Visual Studio Marketplace and throw it onto the download analytics charts. This information is privileged so authentication is required.
1 parent bcf3a18 commit 4ffab6b

File tree

8 files changed

+481
-2
lines changed

8 files changed

+481
-2
lines changed

‎infrastructure/quick-lint-js-web-2/roles/analytics/files/quick-lint-js-website-analytics.service

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ Type = oneshot
99

1010
ExecStart = node src/import-apache-logs.mjs
1111
ExecStart = node src/import-matomo-logs.mjs
12+
ExecStart = node src/import-vscode-stats.mjs
1213
ExecStart = node src/make-charts.mjs
1314

1415
WorkingDirectory = /home/qljs-analytics/quick-lint-js-website-analytics/

‎infrastructure/quick-lint-js-web-2/roles/analytics/templates/quick-lint-js-website-analytics-config.json.j2

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,7 @@
88
"matomo_analytics.db_socket": null,
99
"matomo_analytics.db_user": "matomo_analytics",
1010
"matomo_analytics.db_password": "{{ matomo_db_password }}",
11-
"matomo_analytics.db_database": "matomo_analytics"
11+
"matomo_analytics.db_database": "matomo_analytics",
12+
13+
"vscode.marketplace-personal-access-token": "{{ vscode_marketplace_personal_access_token }}"
1214
}

‎infrastructure/quick-lint-js-web-2/vault-dev.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ matomo_db_password: hunter12
88
# Access token for Discord. Used to sync messages between Discord and IRC.
99
discord_token: ""
1010

11+
# OAuth token for the Visual Studio Marketplace.
12+
# https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token
13+
vscode_marketplace_personal_access_token: ""
14+
1115
# quick-lint-js finds bugs in JavaScript programs.
1216
# Copyright (C) 2020 Matthew Glazar
1317
#

‎website/analytics/config.example.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,7 @@
88
"matomo_analytics.db_socket": null,
99
"matomo_analytics.db_user": "root",
1010
"matomo_analytics.db_password": "",
11-
"matomo_analytics.db_database": "matomo_analytics"
11+
"matomo_analytics.db_database": "matomo_analytics",
12+
13+
"vscode.marketplace-personal-access-token": ""
1214
}

‎website/analytics/src/analytics-db.mjs

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,111 @@ export class AnalyticsDB {
6464
)
6565
.run();
6666

67+
this.#sqlite3DB
68+
.prepare(
69+
sql`
70+
CREATE TABLE IF NOT EXISTS vscode_stats (
71+
id INTEGER PRIMARY KEY NOT NULL,
72+
73+
-- UNIX timestamp in seconds.
74+
timestamp NOT NULL,
75+
-- Cache: Start of the day containing 'timestamp'.
76+
timestamp_day NOT NULL,
77+
-- Cache: Start of the week containing 'timestamp'.
78+
timestamp_week NOT NULL,
79+
80+
-- Version of quick-lint-js, e.g. "2.16.0".
81+
version NOT NULL,
82+
83+
average_rating NOT NULL,
84+
install_count NOT NULL,
85+
uninstall_count NOT NULL,
86+
web_download_count NOT NULL,
87+
web_page_views NOT NULL
88+
)
89+
`
90+
)
91+
.run();
92+
this.#sqlite3DB
93+
.prepare(
94+
sql`
95+
CREATE UNIQUE INDEX IF NOT EXISTS vscode_stats_uniqueness ON vscode_stats (
96+
timestamp,
97+
version
98+
)
99+
`
100+
)
101+
.run();
102+
103+
this.#sqlite3DB
104+
.prepare(
105+
sql`
106+
CREATE TABLE IF NOT EXISTS vscode_stats_archive (
107+
id INTEGER PRIMARY KEY NOT NULL,
108+
109+
archive_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
110+
111+
-- See vscode_stats.
112+
timestamp,
113+
timestamp_day,
114+
timestamp_week,
115+
version,
116+
average_rating,
117+
install_count,
118+
uninstall_count,
119+
web_download_count,
120+
web_page_views
121+
)
122+
`
123+
)
124+
.run();
125+
this.#sqlite3DB
126+
.prepare(
127+
sql`
128+
CREATE TRIGGER IF NOT EXISTS vscode_stats_archive_trigger
129+
BEFORE UPDATE OF
130+
average_rating,
131+
install_count,
132+
uninstall_count,
133+
web_download_count,
134+
web_page_views
135+
ON vscode_stats
136+
WHEN
137+
old.average_rating != new.average_rating
138+
OR old.install_count != new.install_count
139+
OR old.uninstall_count != new.uninstall_count
140+
OR old.web_download_count != new.web_download_count
141+
OR old.web_page_views != new.web_page_views
142+
BEGIN
143+
INSERT INTO vscode_stats_archive (
144+
timestamp,
145+
timestamp_day,
146+
timestamp_week,
147+
version,
148+
149+
average_rating,
150+
install_count,
151+
uninstall_count,
152+
web_download_count,
153+
web_page_views
154+
)
155+
VALUES (
156+
old.timestamp,
157+
old.timestamp_day,
158+
old.timestamp_week,
159+
old.version,
160+
161+
old.average_rating,
162+
old.install_count,
163+
old.uninstall_count,
164+
old.web_download_count,
165+
old.web_page_views
166+
);
167+
END
168+
`
169+
)
170+
.run();
171+
67172
this.#checkDownloadConflictQuery = this.#sqlite3DB.prepare(
68173
sql`
69174
SELECT
@@ -159,6 +264,7 @@ export class AnalyticsDB {
159264
)
160265
VALUES (
161266
@timestamp,
267+
-- TODO(strager): Deduplicate.
162268
STRFTIME('%s', DATETIME(@timestamp, 'unixepoch'), 'start of day'),
163269
STRFTIME('%s', DATETIME(@timestamp, 'unixepoch'), 'start of day', '-6 days', 'weekday 1'),
164270
@downloader_ip,
@@ -292,6 +398,103 @@ export class AnalyticsDB {
292398
}
293399
return { dates, counts };
294400
}
401+
402+
addVSCodeDownloadStats(vscodeDownloadStats) {
403+
for (let statsForOneDay of vscodeDownloadStats) {
404+
this.#sqlite3DB
405+
.prepare(
406+
sql`
407+
INSERT INTO vscode_stats (
408+
timestamp,
409+
timestamp_day,
410+
timestamp_week,
411+
version,
412+
average_rating,
413+
install_count,
414+
uninstall_count,
415+
web_download_count,
416+
web_page_views
417+
)
418+
VALUES (
419+
STRFTIME('%s', @timestamp),
420+
-- TODO(strager): Deduplicate.
421+
STRFTIME('%s', @timestamp, 'start of day'),
422+
STRFTIME('%s', @timestamp, 'start of day', '-6 days', 'weekday 1'),
423+
@version,
424+
@average_rating,
425+
@install_count,
426+
@uninstall_count,
427+
@web_download_count,
428+
@web_page_views
429+
)
430+
ON CONFLICT (timestamp, version)
431+
DO UPDATE SET
432+
average_rating = @average_rating,
433+
install_count = @install_count,
434+
uninstall_count = @uninstall_count,
435+
web_download_count = @web_download_count,
436+
web_page_views = @web_page_views
437+
`
438+
)
439+
.run({
440+
timestamp: statsForOneDay.statisticDate,
441+
version: statsForOneDay.version,
442+
average_rating: statsForOneDay.counts.averageRating ?? -1,
443+
install_count: statsForOneDay.counts.installCount ?? 0,
444+
uninstall_count: statsForOneDay.counts.uninstallCount ?? 0,
445+
web_download_count: statsForOneDay.counts.webDownloadCount ?? 0,
446+
web_page_views: statsForOneDay.counts.webPageViews ?? 0,
447+
});
448+
}
449+
}
450+
451+
countDailyVSCodeDownloads() {
452+
let rows = this.#sqlite3DB
453+
.prepare(
454+
sql`
455+
SELECT
456+
vscode_stats.timestamp_day AS timestamp_day,
457+
SUM(install_count) + SUM(web_download_count) AS count
458+
FROM vscode_stats
459+
GROUP BY vscode_stats.timestamp_day
460+
`
461+
)
462+
.raw()
463+
.all();
464+
let dates = [];
465+
let counts = [];
466+
for (let [timestamp, count] of rows) {
467+
dates.push(timestamp * 1000);
468+
counts.push(count);
469+
}
470+
return { dates, counts };
471+
}
472+
473+
countWeeklyVSCodeDownloads() {
474+
let rows = this.#sqlite3DB
475+
.prepare(
476+
sql`
477+
SELECT
478+
vscode_stats.timestamp_week AS timestamp_week,
479+
SUM(install_count) + SUM(web_download_count) AS count
480+
FROM vscode_stats
481+
GROUP BY vscode_stats.timestamp_week
482+
`
483+
)
484+
.raw()
485+
.all();
486+
let dates = [];
487+
let counts = [];
488+
for (let [timestamp, count] of rows) {
489+
dates.push(timestamp * 1000);
490+
counts.push(count);
491+
}
492+
return { dates, counts };
493+
}
494+
495+
_querySQLForTesting(sqlQuery) {
496+
return this.#sqlite3DB.prepare(sqlQuery).all();
497+
}
295498
}
296499

297500
function timestampMSToS(timestamp) {

‎website/analytics/src/import-vscode-stats.mjs

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env node
2+
// Copyright (C) 2020 Matthew "strager" Glazar
3+
// See end of file for extended copyright information.
4+
5+
import glob from "glob";
6+
import isbot from "isbot";
7+
import util from "node:util";
8+
import { AnalyticsDB } from "./analytics-db.mjs";
9+
import { loadConfigAsync } from "./config.mjs";
10+
import { parseLogFileAsync } from "./parse-log-file.mjs";
11+
12+
let globAsync = util.promisify(glob);
13+
14+
async function mainAsync() {
15+
let config = await loadConfigAsync();
16+
let db = AnalyticsDB.fromFile(config["db.file"]);
17+
18+
let personalAccessToken = config["vscode.marketplace-personal-access-token"];
19+
let response = await fetch(
20+
"https://marketplace.visualstudio.com/_apis/gallery/publishers/quick-lint/extensions/quick-lint-js/stats?aggregate=0",
21+
{
22+
headers: {
23+
Authorization: `Basic ${btoa(`OAuth:${personalAccessToken}`)}`,
24+
},
25+
}
26+
);
27+
if (!response.ok) {
28+
throw new Error(
29+
`fetching VS Code stats failed with status ${response.status} ${response.statusText}`
30+
);
31+
}
32+
let statsData = await response.json();
33+
db.addVSCodeDownloadStats(statsData.dailyStats);
34+
35+
console.log(
36+
`imported ${statsData.dailyStats.length} stats from the Visual Studio Marketplace`
37+
);
38+
39+
db.close();
40+
}
41+
42+
mainAsync().catch((e) => {
43+
console.error(e?.stack || e);
44+
process.exit(1);
45+
});
46+
47+
// quick-lint-js finds bugs in JavaScript programs.
48+
// Copyright (C) 2020 Matthew "strager" Glazar
49+
//
50+
// This file is part of quick-lint-js.
51+
//
52+
// quick-lint-js is free software: you can redistribute it and/or modify
53+
// it under the terms of the GNU General Public License as published by
54+
// the Free Software Foundation, either version 3 of the License, or
55+
// (at your option) any later version.
56+
//
57+
// quick-lint-js is distributed in the hope that it will be useful,
58+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
59+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
60+
// GNU General Public License for more details.
61+
//
62+
// You should have received a copy of the GNU General Public License
63+
// along with quick-lint-js. If not, see <https://www.gnu.org/licenses/>.

‎website/analytics/src/make-charts.mjs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,16 @@ async function mainAsync() {
7878
}
7979
}
8080

81+
let dailyVSCodeDownloads = db.countDailyVSCodeDownloads();
82+
dailyWebDownloadersLabels.push("VS Code");
83+
dailyWebDownloadersKeys.push(dailyVSCodeDownloads.dates);
84+
dailyWebDownloadersValues.push(dailyVSCodeDownloads.counts);
85+
86+
let weeklyVSCodeDownloads = db.countWeeklyVSCodeDownloads();
87+
weeklyWebDownloadersLabels.push("VS Code");
88+
weeklyWebDownloadersKeys.push(weeklyVSCodeDownloads.dates);
89+
weeklyWebDownloadersValues.push(weeklyVSCodeDownloads.counts);
90+
8191
let data = {
8292
dailyWebDownloaders: {
8393
labels: dailyWebDownloadersLabels,

0 commit comments

Comments
 (0)