From a5abc67220a43ae6cecbbdf49fe38ec455105878 Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Wed, 12 Apr 2023 05:18:02 +0530 Subject: [PATCH 01/10] first commit upgrading to manifest v3 --- webextension/manifest.json | 40 +++++++++++------- webextension/rules_1.json | 15 +++++++ webextension/scripts/background.js | 65 +++++++++++++++++------------- webextension/scripts/utils.js | 7 +--- 4 files changed, 78 insertions(+), 49 deletions(-) create mode 100644 webextension/rules_1.json diff --git a/webextension/manifest.json b/webextension/manifest.json index 0663a963..8ca30187 100644 --- a/webextension/manifest.json +++ b/webextension/manifest.json @@ -1,7 +1,7 @@ { - "manifest_version": 2, + "manifest_version": 3, "name": "Wayback Machine", - "version": "3.2", + "version": "3.2.0.1", "description": "The Official Wayback Machine Extension - by the Internet Archive.", "icons": { "16": "images/app-icon/mini-icon16.png", @@ -15,7 +15,7 @@ "512": "images/app-icon/app-icon512.png", "1024": "images/app-icon/app-icon1024.png" }, - "browser_action": { + "action": { "default_icon": { "16": "images/toolbar/toolbar-icon-archive16.png", "24": "images/toolbar/toolbar-icon-archive24.png", @@ -30,20 +30,23 @@ "contextMenus", "notifications", "storage", + "scripting", "webRequest", - "webRequestBlocking", - "https://archive.org/*", - "https://*.archive.org/*", - "https://hypothes.is/*", - "" + "declarativeNetRequestWithHostAccess", + "declarativeNetRequest", + "declarativeNetRequestFeedback" + ], + "host_permissions":[ + "https://archive.org/*", + "https://*.archive.org/*", + "https://hypothes.is/*", + "" ], "optional_permissions": [ "bookmarks" ], "background": { - "scripts": ["scripts/utils.js", - "scripts/background.js"], - "persistent": true + "service_worker": "scripts/background.js" }, "content_scripts": [ { @@ -52,8 +55,15 @@ "css": ["css/wikipedia.css"] } ], - "web_accessible_resources": [ - "css/archive.css", - "images/*" - ] + "web_accessible_resources": [{ + "resources":[ "/css/archive.css","/images/*"], + "matches": [""] + }], + "declarative_net_request" : { + "rule_resources" : [{ + "id": "ruleset_1", + "enabled": true, + "path": "rules_1.json" + }] + } } diff --git a/webextension/rules_1.json b/webextension/rules_1.json new file mode 100644 index 00000000..2bc5f0be --- /dev/null +++ b/webextension/rules_1.json @@ -0,0 +1,15 @@ +[ + { + "id": 1, + "priority": 1, + "action": { + "type": "modifyHeaders", + "requestHeaders": [ + { "header": "Wayback_Browser_Extension", "operation": "set", "value": "ok" } + ] + }, + "condition": { + "urlFilter": "*.archive.org" + } + } +] diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index baffb564..fc4aa19b 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -3,6 +3,7 @@ // License: AGPL-3 // Copyright 2016-2020, Internet Archive +importScripts('utils.js') // from 'utils.js' /* global isNotExcludedUrl, getCleanUrl, isArchiveUrl, isValidUrl, notifyMsg, openByWindowSetting, sleep, wmAvailabilityCheck, hostURL */ /* global initDefaultOptions, badgeCountText, getWaybackCount, newshosts, dateToTimestamp, fixedEncodeURIComponent, checkLastError */ @@ -21,18 +22,19 @@ const API_RETRY = 1000 const SPN_RETRY = 6000 let tabIdPromise -// updates User-Agent header in Chrome & Firefox, but not in Safari -function rewriteUserAgentHeader(e) { - for (let header of e.requestHeaders) { - if (header.name.toLowerCase() === 'user-agent') { - const customUA = gCustomUserAgent - const statusUA = 'Status-code/' + gStatusCode - // add customUA only if not yet present in user-agent - header.value += ((header.value.indexOf(customUA) === -1) ? ' ' + customUA : '') + (gStatusCode ? ' ' + statusUA : '') - } - } - return { requestHeaders: e.requestHeaders } -} +// not required because in manifest v3, a new header is set using declarativeNetRequest rules +// // updates User-Agent header in Chrome & Firefox, but not in Safari +// function rewriteUserAgentHeader(e) { +// for (let header of e.requestHeaders) { +// if (header.name.toLowerCase() === 'user-agent') { +// const customUA = gCustomUserAgent +// const statusUA = 'Status-code/' + gStatusCode +// // add customUA only if not yet present in user-agent +// header.value += ((header.value.indexOf(customUA) === -1) ? ' ' + customUA : '') + (gStatusCode ? ' ' + statusUA : '') +// } +// } +// return { requestHeaders: e.requestHeaders } +// } /* * * API Calls * * */ @@ -394,7 +396,7 @@ function getCachedFactCheck(url, onSuccess, onFail) { // Setup toolbar button action. chrome.storage.local.get({ agreement: false }, (settings) => { if (settings && settings.agreement) { - chrome.browserAction.setPopup({ popup: chrome.runtime.getURL('index.html') }, checkLastError) + chrome.action.setPopup({ popup: chrome.runtime.getURL('index.html') }, checkLastError) setupContextMenus(true) } else { setupContextMenus(false) @@ -415,7 +417,7 @@ chrome.runtime.onInstalled.addListener((details) => { }) // Opens Welcome page on toolbar click if terms not yet accepted. -chrome.browserAction.onClicked.addListener((tab) => { +chrome.action.onClicked.addListener((tab) => { chrome.storage.local.get({ agreement: false }, (settings) => { if (settings && (settings.agreement === false)) { openByWindowSetting(chrome.runtime.getURL('welcome.html'), 'tab') @@ -423,11 +425,16 @@ chrome.browserAction.onClicked.addListener((tab) => { }) }) -chrome.webRequest.onBeforeSendHeaders.addListener( - rewriteUserAgentHeader, - { urls: [hostURL + '*'] }, - ['blocking', 'requestHeaders'] // FIXME: not supported in Safari -) +// chrome.webRequest.onBeforeSendHeaders is not supported in manifest v3 and currently we are not looking to rewrite the user-agent +// as we already use diffent host-names for different endpoints + +// chrome.webRequest.onBeforeSendHeaders.addListener( +// rewriteUserAgentHeader, +// { urls: [hostURL + '*'] }, +// ['blocking', 'requestHeaders'] // FIXME: not supported in Safari +// ) + +// chrome.webRequest.onErrorOccurred is not supported in manifest v3. // Checks for error in page loading such as when a domain doesn't exist. // @@ -466,7 +473,6 @@ chrome.webRequest.onCompleted.addListener((details) => { }) } } - }, { urls: [''], types: ['main_frame'] }) // Check for 404 Not Found and other status errors. @@ -495,7 +501,10 @@ function checkNotFound(details) { function checkWM(tab, details2, bannerFlag) { wmAvailabilityCheck(details2.url, (wayback_url, url) => { if (bannerFlag && ('id' in tab)) { - chrome.tabs.executeScript(tab.id, { file: '/scripts/archive.js' }, () => { + chrome.scripting.executeScript({ + target: { tabId: tab.id }, + files: ['/scripts/archive.js'] + }).then(() => { update(tab, url, wayback_url, details2.statusCode, bannerFlag) }) } else { @@ -1004,18 +1013,18 @@ function updateWaybackCountBadge(atab, url) { if ((values.total >= 0) && !getToolbarState(atab).has('S')) { // display badge let text = badgeCountText(values.total) - chrome.browserAction.setBadgeBackgroundColor({ color: '#9A3B38' }, checkLastError) // red - chrome.browserAction.setBadgeText({ tabId: atab.id, text: text }, checkLastError) + chrome.action.setBadgeBackgroundColor({ color: '#9A3B38' }, checkLastError) // red + chrome.action.setBadgeText({ tabId: atab.id, text: text }, checkLastError) } else { - chrome.browserAction.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) + chrome.action.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) } }, (error) => { console.log('wayback count error: ' + error) - chrome.browserAction.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) + chrome.action.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) }) } else { - chrome.browserAction.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) + chrome.action.setBadgeText({ tabId: atab.id, text: '' }, checkLastError) } }) } @@ -1031,7 +1040,7 @@ function updateWaybackCountBadge(atab, url) { function setToolbarIcon(name, tabId = null) { const validToolbarIcons = new Set(['R', 'S', 'F', 'V', 'check', 'archive']) const checkBadgePos = new Set(['R', 'F', 'V']) - const path = 'images/toolbar/' + const path = '../images/toolbar/' const n = validToolbarIcons.has(name) ? name : 'archive' const ab = checkBadgePos.has(name) ? (isBadgeOnTop() ? 'b' : 'a') : '' const prefix = isDevVersion() ? 'dev-icon-' : 'toolbar-icon-' @@ -1042,7 +1051,7 @@ function setToolbarIcon(name, tabId = null) { '64': (path + prefix + n + ab + '64.png') } let details = (tabId) ? { path: allPaths, tabId: tabId } : { path: allPaths } - chrome.browserAction.setIcon(details, checkLastError) + chrome.action.setIcon(details, checkLastError) } // Add state to the state set for given Tab, and update toolbar. diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 1d973ebf..11a1d4d9 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -777,11 +777,6 @@ function setupContextMenus(enabled) { 'contexts': ['page', 'frame', 'link'], 'documentUrlPatterns': ['*://*/*', 'ftp://*/*'] }, checkLastError) - chrome.contextMenus.create({ - 'type': 'separator', - 'contexts': ['page', 'frame', 'link'], - 'documentUrlPatterns': ['*://*/*', 'ftp://*/*'] - }) chrome.contextMenus.create({ 'id': 'first', 'title': 'Oldest Version', @@ -845,7 +840,7 @@ function afterAcceptTerms () { private_mode_setting: false, not_found_setting: true }) - chrome.browserAction.setPopup({ popup: chrome.runtime.getURL('index.html') }, checkLastError) + chrome.action.setPopup({ popup: chrome.runtime.getURL('index.html') }, checkLastError) setupContextMenus(true) } From c6d3a6695cf8eab0a407c37fec1fff45dc06bfd3 Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Wed, 12 Apr 2023 06:03:08 +0530 Subject: [PATCH 02/10] use chrome.windows.getCurrent get height and width --- webextension/scripts/utils.js | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 11a1d4d9..9cfc62c1 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -668,20 +668,25 @@ function opener(url, option, callback) { } }) } else { - let w = window.screen.availWidth, h = window.screen.availHeight - if (w > h) { - // landscape screen - const maxW = 1200 - w = Math.floor(((w > maxW) ? maxW : w) * 0.666) - h = Math.floor(w * 0.75) - } else { // option === 'window' - // portrait screen (likely mobile) - w = Math.floor(w * 0.9) - h = Math.floor(h * 0.9) - } - chrome.windows.create({ url: url, width: w, height: h, type: 'popup' }, (window) => { - if (callback) { callback(window.tabs[0].id) } - }) + + chrome.windows.getCurrent({populate: true}, (window) => { + // Access window properties + let h = window.width; + let w = window.height; + if (w > h) { + // landscape screen + const maxW = 1200 + w = Math.floor(((w > maxW) ? maxW : w) * 0.666) + h = Math.floor(w * 0.75) + } else { // option === 'window' + // portrait screen (likely mobile) + w = Math.floor(w * 0.9) + h = Math.floor(h * 0.9) + } + chrome.windows.create({ url: url, width: w, height: h, type: 'popup' }, (window) => { + if (callback) { callback(window.tabs[0].id) } + }) + }); } } From 9c7fd9cdcbc813bf77c5eccd9d773e18fbbc68e1 Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Wed, 12 Apr 2023 06:17:58 +0530 Subject: [PATCH 03/10] update the request header in the rules file --- webextension/rules_1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webextension/rules_1.json b/webextension/rules_1.json index 2bc5f0be..c9f4d097 100644 --- a/webextension/rules_1.json +++ b/webextension/rules_1.json @@ -5,7 +5,7 @@ "action": { "type": "modifyHeaders", "requestHeaders": [ - { "header": "Wayback_Browser_Extension", "operation": "set", "value": "ok" } + { "header": "Wayback-Browser-Extension", "operation": "set", "value": "ok" } ] }, "condition": { From c809c014789f8e6f4089940910cb9627ca5ffabd Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Thu, 11 Apr 2024 05:01:43 +0530 Subject: [PATCH 04/10] updated the version and move some background.js variable to chrome.storage.local --- .../Wayback Machine.xcodeproj/project.pbxproj | 8 +- webextension/manifest.json | 2 +- webextension/scripts/background.js | 166 +++++++++++++----- webextension/scripts/utils.js | 8 + 4 files changed, 139 insertions(+), 45 deletions(-) diff --git a/safari/Wayback Machine.xcodeproj/project.pbxproj b/safari/Wayback Machine.xcodeproj/project.pbxproj index 764ded4f..3beec898 100644 --- a/safari/Wayback Machine.xcodeproj/project.pbxproj +++ b/safari/Wayback Machine.xcodeproj/project.pbxproj @@ -465,7 +465,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.14; - MARKETING_VERSION = 3.2; + MARKETING_VERSION = 3.4; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac.extension; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -490,7 +490,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.14; - MARKETING_VERSION = 3.2; + MARKETING_VERSION = 3.4; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac.extension; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -518,7 +518,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.14; - MARKETING_VERSION = 3.2; + MARKETING_VERSION = 3.4; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; @@ -545,7 +545,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 10.14; - MARKETING_VERSION = 3.2; + MARKETING_VERSION = 3.4; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; diff --git a/webextension/manifest.json b/webextension/manifest.json index 8ca30187..33f62748 100644 --- a/webextension/manifest.json +++ b/webextension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Wayback Machine", - "version": "3.2.0.1", + "version": "3.4", "description": "The Official Wayback Machine Extension - by the Internet Archive.", "icons": { "16": "images/app-icon/mini-icon16.png", diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index fc4aa19b..57b9d457 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -11,15 +11,15 @@ importScripts('utils.js') /* global isDevVersion, checkAuthentication, setupContextMenus, cropPrefix, alertMsg */ // Used to store the statuscode of the if it is a httpFailCodes -let gStatusCode = 0 +//let gStatusCode = 0 let gToolbarStates = {} -let waybackCountCache = {} +// let waybackCountCache = {} let globalAPICache = new Map() -const API_CACHE_SIZE = 5 -const API_LOADING = 'LOADING' -const API_TIMEOUT = 10000 -const API_RETRY = 1000 -const SPN_RETRY = 6000 +// const API_CACHE_SIZE = 5 +// const API_LOADING = 'LOADING' +// const API_TIMEOUT = 10000 +// const API_RETRY = 1000 +// const SPN_RETRY = 6000 let tabIdPromise // not required because in manifest v3, a new header is set using declarativeNetRequest rules @@ -56,8 +56,15 @@ function savePageNowChecked(atab, pageUrl, silent, options) { * @param silent {bool}: if false, include notify popup if supported by browser/OS, and open Resource List window if setting is on. * @param options {Object}: key/value pairs to send in POST data. See SPN API spec. */ -function savePageNow(atab, pageUrl, silent = false, options = {}, loggedInFlag = true) { +async function savePageNow(atab, pageUrl, silent = false, options = {}, loggedInFlag = true) { + let { API_TIMEOUT, SPN_RETRY } = await new Promise((resolve) => { + chrome.storage.local.get(['API_TIMEOUT', 'SPN_RETRY'], function(result) { + resolve(result); + }); + }); + + console.log(API_TIMEOUT,SPN_RETRY) if (!(isValidUrl(pageUrl) && isNotExcludedUrl(pageUrl))) { console.log('savePageNow URL excluded') return @@ -146,13 +153,17 @@ function extractJobIdFromHTML(html) { * @param silent {bool}: to pass to statusSuccess() or statusFailed(). * @param jobId {string}: job_id returned by SPN response, passed to Status API. */ -function savePageStatus(atab, pageUrl, silent = false, jobId) { - +async function savePageStatus(atab, pageUrl, silent = false, jobId) { + let { API_TIMEOUT, SPN_RETRY } = await new Promise((resolve) => { + chrome.storage.local.get(['API_TIMEOUT', 'SPN_RETRY'], function(result) { + resolve(result); + }); + }); // setup api // Accept header required when logged-out, even though response is in JSON. let headers = new Headers(hostHeaders) headers.set('Accept', 'text/html,application/xhtml+xml,application/xml') - + // call status after SPN response const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, API_TIMEOUT) fetch(hostURL + 'save/status/' + jobId, { @@ -162,7 +173,6 @@ function savePageStatus(atab, pageUrl, silent = false, jobId) { }) .then(resolve, reject) }) - // call api let retryAfter = SPN_RETRY timeoutPromise @@ -274,7 +284,12 @@ function statusFailed(atab, pageUrl, silent, data, err) { * @param timestamp {string}: Wayback timestamp as "yyyyMMddHHmmss" in UTC. * @return Promise: which should return this JSON on success: { "success": true } */ -function saveToMyWebArchive(url, timestamp) { +async function saveToMyWebArchive(url, timestamp) { + let { API_TIMEOUT } = await new Promise((resolve) => { + chrome.storage.local.get(['API_TIMEOUT'], function(result) { + resolve(result); + }); + }); const postData = { 'url': url, 'snapshot': timestamp, 'tags': [] } const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, API_TIMEOUT) @@ -298,7 +313,12 @@ function saveToMyWebArchive(url, timestamp) { * @param postData {object}: if present, uses POST instead of GET and sends postData object converted to json. * @return Promise */ -function fetchAPI(url, onSuccess, onFail, postData = null) { +async function fetchAPI(url, onSuccess, onFail, postData = null) { + let { API_TIMEOUT } = await new Promise((resolve) => { + chrome.storage.local.get(['API_TIMEOUT'], function(result) { + resolve(result); + }); + }); const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, API_TIMEOUT) let headers = new Headers(hostHeaders) @@ -333,7 +353,12 @@ function fetchAPI(url, onSuccess, onFail, postData = null) { * @param postData {object}: uses POST if present. * @return Promise if calls API, json data if in cache, null if loading in progress. */ -function fetchCachedAPI(url, onSuccess, onFail, postData = null) { +async function fetchCachedAPI(url, onSuccess, onFail, postData = null) { + let { API_CACHE_SIZE, API_LOADING, API_RETRY} = await new Promise((resolve) => { + chrome.storage.local.get(['API_CACHE_SIZE', 'API_LOADING', 'API_RETRY'], function(result) { + resolve(result); + }); + }); let data = globalAPICache.get(url) if (data === API_LOADING) { // re-call after delay if previous fetch hadn't returned yet @@ -445,7 +470,7 @@ chrome.webRequest.onErrorOccurred.addListener((details) => { const url = details.url if (isNotExcludedUrl(url) && isValidUrl(url)) { chrome.tabs.get(details.tabId, (tab) => { - gStatusCode = 999 + chrome.storage.local.set({ gStatusCode : 999 }); saveTabData(tab, { 'statusCode': 999, 'statusUrl': url }) }) } @@ -485,7 +510,7 @@ function checkNotFound(details) { function update(tab, statusUrl, waybackUrl, statusCode, bannerFlag) { checkLastError() addToolbarState(tab, 'V') - gStatusCode = statusCode + chrome.storage.local.set({ gStatusCode : statusCode }); // need the following to store statusWaybackUrl, other keys are overwritten with the same values. saveTabData(tab, { 'statusCode': statusCode, 'statusUrl': statusUrl, 'statusWaybackUrl': waybackUrl }) if (bannerFlag && ('id' in tab)) { @@ -501,6 +526,7 @@ function checkNotFound(details) { function checkWM(tab, details2, bannerFlag) { wmAvailabilityCheck(details2.url, (wayback_url, url) => { if (bannerFlag && ('id' in tab)) { + // scripting permission is required as executeScript() is moved from the tabs API to the scripting API chrome.scripting.executeScript({ target: { tabId: tab.id }, files: ['/scripts/archive.js'] @@ -967,21 +993,57 @@ function factCheck(atab, url) { } /* * * Wayback Count * * */ - function getCachedWaybackCount(url, onSuccess, onFail) { - let cacheValues = waybackCountCache[url] - if (cacheValues) { - onSuccess(cacheValues) - } else { - getWaybackCount(url, (values) => { - waybackCountCache[url] = values - onSuccess(values) - }, onFail) - } + // Retrieve the waybackCountCache object from chrome.storage + chrome.storage.local.get(['waybackCountCache'], function(result) { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + onFail(); // Call onFail callback in case of error + return; + } + + // Get the waybackCountCache object from the result or initialize it if it doesn't exist + let waybackCountCache = result.waybackCountCache || {}; + + // Check if cacheValues for the specified URL exist in the waybackCountCache + let cacheValues = waybackCountCache[url]; + + if (cacheValues) { + // If cacheValues exist, call onSuccess callback with cacheValues + onSuccess(cacheValues); + } else { + // If cacheValues don't exist, fetch them using getWaybackCount + getWaybackCount(url, function(values) { + // Update waybackCountCache with the fetched values + waybackCountCache[url] = values; + + // Store the updated waybackCountCache back into chrome.storage + chrome.storage.local.set({ waybackCountCache: waybackCountCache }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + onFail(); // Call onFail callback in case of error while storing + } else { + // Call onSuccess callback with the fetched values + onSuccess(values); + } + }); + }, onFail); + } + }); } + + function clearCountCache() { - waybackCountCache = {} + let waybackCountCache = {}; + // Store the updated waybackCountCache into chrome.storage + chrome.storage.local.set({ waybackCountCache: waybackCountCache }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + } else { + console.log('waybackCountCache reset and stored successfully'); + } + }); } /** @@ -990,21 +1052,45 @@ function clearCountCache() { * Doesn't update if cached "total" value was < 0. * @param url {string} */ -function incrementCount(url) { - let cacheValues = waybackCountCache[url] - let timestamp = dateToTimestamp(new Date()) - if (cacheValues && cacheValues.total) { - if (cacheValues.total > 0) { - cacheValues.total += 1 - cacheValues.last_ts = timestamp - waybackCountCache[url] = cacheValues + function incrementCount(url) { + // Retrieve the waybackCountCache object from chrome.storage + chrome.storage.local.get(['waybackCountCache'], function(result) { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + return; // Exit if there's an error } - // else don't update if total is a special value < 0 - } else { - waybackCountCache[url] = { total: 1, last_ts: timestamp } - } + + // Get the waybackCountCache object from the result or initialize it if it doesn't exist + let waybackCountCache = result.waybackCountCache || {}; + + // Get the current timestamp + let timestamp = dateToTimestamp(new Date()); + + // Get the cacheValues for the specified URL + let cacheValues = waybackCountCache[url] || {}; + + // Increment count and update last_ts if cacheValues and cacheValues.total exist and is greater than 0 + if (cacheValues.total && cacheValues.total > 0) { + cacheValues.total += 1; + cacheValues.last_ts = timestamp; + waybackCountCache[url] = cacheValues; + } else { + // Set count to 1 and update last_ts if cacheValues.total doesn't exist or is not greater than 0 + waybackCountCache[url] = { total: 1, last_ts: timestamp }; + } + + // Store the updated waybackCountCache back into chrome.storage + chrome.storage.local.set({ waybackCountCache: waybackCountCache }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + } else { + console.log('waybackCountCache updated and stored successfully'); + } + }); + }); } + function updateWaybackCountBadge(atab, url) { if (!atab) { return } chrome.storage.local.get(['wm_count_setting'], (settings) => { diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 9cfc62c1..9c21dcfb 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -813,11 +813,19 @@ function setupContextMenus(enabled) { // Default Settings prior to accepting terms. function initDefaultOptions () { + let globalAPICache = new Map(); chrome.storage.local.set({ agreement: false, // needed for firefox spn_outlinks: false, spn_screenshot: false, selectedFeature: null, + gStatusCode: 0, + waybackCountCache: {}, + API_CACHE_SIZE: 5, + API_LOADING: 'LOADING', + API_TIMEOUT: 10000, + API_RETRY: 1000, + SPN_RETRY: 6000, /* Features */ private_mode_setting: true, not_found_setting: false, From 22b3e2de157593b0c9348d0aa97baae29eb6e21f Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Thu, 11 Apr 2024 05:36:48 +0530 Subject: [PATCH 05/10] move globalAPICache to chrome.storage --- webextension/scripts/background.js | 73 +++++++++++++++++++----------- webextension/scripts/utils.js | 1 + 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index 57b9d457..840d6a61 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -14,7 +14,7 @@ importScripts('utils.js') //let gStatusCode = 0 let gToolbarStates = {} // let waybackCountCache = {} -let globalAPICache = new Map() +// let globalAPICache = new Map() // const API_CACHE_SIZE = 5 // const API_LOADING = 'LOADING' // const API_TIMEOUT = 10000 @@ -353,38 +353,59 @@ async function fetchAPI(url, onSuccess, onFail, postData = null) { * @param postData {object}: uses POST if present. * @return Promise if calls API, json data if in cache, null if loading in progress. */ -async function fetchCachedAPI(url, onSuccess, onFail, postData = null) { - let { API_CACHE_SIZE, API_LOADING, API_RETRY} = await new Promise((resolve) => { + async function fetchCachedAPI(url, onSuccess, onFail, postData = null) { + let { API_CACHE_SIZE, API_LOADING, API_RETRY } = await new Promise((resolve) => { chrome.storage.local.get(['API_CACHE_SIZE', 'API_LOADING', 'API_RETRY'], function(result) { resolve(result); }); }); - let data = globalAPICache.get(url) - if (data === API_LOADING) { - // re-call after delay if previous fetch hadn't returned yet - setTimeout(() => { - fetchCachedAPI(url, onSuccess, onFail, postData) - }, API_RETRY) - return null - } else if (data !== undefined) { - onSuccess(data) - return data - } else { - // if cache full, remove first object which is the oldest from the cache - if (globalAPICache.size >= API_CACHE_SIZE) { - globalAPICache.delete(globalAPICache.keys().next().value) + + chrome.storage.local.get(['globalAPICache'], async function(result) { + let globalAPICache = result.globalAPICache ? new Map(Object.entries(result.globalAPICache)) : new Map(); + let data = globalAPICache.get(url); + + if (data === API_LOADING) { + setTimeout(() => { + fetchCachedAPI(url, onSuccess, onFail, postData) + }, API_RETRY); + return null; + } else if (data !== undefined) { + onSuccess(data); + return data; + } else { + // if cache full, remove first object which is the oldest from the cache + if (globalAPICache.size >= API_CACHE_SIZE) { + globalAPICache.delete(globalAPICache.keys().next().value); + } + globalAPICache.set(url, API_LOADING); + + // Convert Map back to object before storing it in chrome.storage + let globalAPICacheObject = Object.fromEntries(globalAPICache); + + chrome.storage.local.set({ globalAPICache: globalAPICacheObject }, function() { + fetchAPI(url, (json) => { + globalAPICache.set(url, json); + + // Convert Map back to object before storing it in chrome.storage + let globalAPICacheObject = Object.fromEntries(globalAPICache); + chrome.storage.local.set({ globalAPICache: globalAPICacheObject }); + + onSuccess(json); + }, (error) => { + globalAPICache.delete(url); + + // Convert Map back to object before storing it in chrome.storage + let globalAPICacheObject = Object.fromEntries(globalAPICache); + chrome.storage.local.set({ globalAPICache: globalAPICacheObject }); + + onFail(error); + }, postData); + }); } - globalAPICache.set(url, API_LOADING) - return fetchAPI(url, (json) => { - globalAPICache.set(url, json) - onSuccess(json) - }, (error) => { - globalAPICache.delete(url) - onFail(error) - }, postData) - } + }); } + /** * The books API uses both GET, and POST with isbns array as the body. * @param url {string}: Must include '?url=' as entire url is used as cache key. diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 9c21dcfb..77ad682d 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -821,6 +821,7 @@ function initDefaultOptions () { selectedFeature: null, gStatusCode: 0, waybackCountCache: {}, + globalAPICache: {}, API_CACHE_SIZE: 5, API_LOADING: 'LOADING', API_TIMEOUT: 10000, From dd711496c1b5a78849004a5ac279d6c19928820b Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Thu, 11 Apr 2024 10:21:12 +0530 Subject: [PATCH 06/10] move gToolbarStates to chrome.storage --- webextension/scripts/background.js | 285 +++++++++++++++++++++-------- webextension/scripts/utils.js | 1 + 2 files changed, 209 insertions(+), 77 deletions(-) diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index 840d6a61..02c89aa7 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -12,7 +12,7 @@ importScripts('utils.js') // Used to store the statuscode of the if it is a httpFailCodes //let gStatusCode = 0 -let gToolbarStates = {} +// let gToolbarStates = {} // let waybackCountCache = {} // let globalAPICache = new Map() // const API_CACHE_SIZE = 5 @@ -23,7 +23,7 @@ let gToolbarStates = {} let tabIdPromise // not required because in manifest v3, a new header is set using declarativeNetRequest rules -// // updates User-Agent header in Chrome & Firefox, but not in Safari +// updates User-Agent header in Chrome & Firefox, but not in Safari // function rewriteUserAgentHeader(e) { // for (let header of e.requestHeaders) { // if (header.name.toLowerCase() === 'user-agent') { @@ -64,7 +64,6 @@ async function savePageNow(atab, pageUrl, silent = false, options = {}, loggedIn }); }); - console.log(API_TIMEOUT,SPN_RETRY) if (!(isValidUrl(pageUrl) && isNotExcludedUrl(pageUrl))) { console.log('savePageNow URL excluded') return @@ -206,11 +205,11 @@ async function savePageStatus(atab, pageUrl, silent = false, jobId) { * @param silent {bool}: set false to include notify popup. * @param data {Object}: Response data from Status API. */ -function statusSuccess(atab, pageUrl, silent, data) { +async function statusSuccess(atab, pageUrl, silent, data) { // update UI - removeToolbarState(atab, 'S') - addToolbarState(atab, 'check') + await removeToolbarState(atab, 'S') + await addToolbarState(atab, 'check') incrementCount(data.original_url) updateWaybackCountBadge(atab, data.original_url) chrome.runtime.sendMessage({ @@ -247,9 +246,9 @@ function statusSuccess(atab, pageUrl, silent, data) { * @param data {Object}: Response data from Status API. (optional) * @param err {Error}: Error from a catch block. (optional) */ -function statusFailed(atab, pageUrl, silent, data, err) { +async function statusFailed(atab, pageUrl, silent, data, err) { - removeToolbarState(atab, 'S') + await removeToolbarState(atab, 'S') if (err) { chrome.runtime.sendMessage({ message: 'resource_list_show_error', @@ -528,9 +527,9 @@ function checkNotFound(details) { if (!details) { return } // save status, display 'V' toolbar icon and banner - function update(tab, statusUrl, waybackUrl, statusCode, bannerFlag) { + async function update(tab, statusUrl, waybackUrl, statusCode, bannerFlag) { checkLastError() - addToolbarState(tab, 'V') + await addToolbarState(tab, 'V') chrome.storage.local.set({ gStatusCode : statusCode }); // need the following to store statusWaybackUrl, other keys are overwritten with the same values. saveTabData(tab, { 'statusCode': statusCode, 'statusUrl': statusUrl, 'statusWaybackUrl': waybackUrl }) @@ -650,8 +649,8 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { }) } else if (message.message === 'getToolbarState') { // retrieve the toolbar state set & custom tab data - let state = getToolbarState(message.atab) - readTabData(message.atab, (data) => { + readTabData(message.atab, async (data) => { + let state = await getToolbarState(message.atab) sendResponse({ stateArray: Array.from(state), customData: data }) }) return true @@ -659,10 +658,10 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { // add one or more states to the toolbar // States will fail to show if tab is loading but not in focus! // Content scripts cannot use tabs.query and send the tab, so it must be called here. - chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { + chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => { if (tabs && tabs[0] && (tabs[0].url === message.url)) { for (let state of message.states) { - addToolbarState(tabs[0], state) + await addToolbarState(tabs[0], state) } } }) @@ -682,7 +681,7 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { } }) } else if (message.message === 'clearResource') { - chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { + chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => { if (tabs && tabs[0]) { if (message.settings) { // clear 'R' state if wiki, amazon, or tvnews settings have been cleared @@ -690,19 +689,19 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { if (((message.settings.wiki_setting === false) && tabs[0].url.match(/^https?:\/\/[\w.]*wikipedia.org/)) || ((message.settings.amazon_setting === false) && tabs[0].url.includes('www.amazon')) || ((message.settings.tvnews_setting === false) && newshosts.has(news_host))) { - removeToolbarState(tabs[0], 'R') + await removeToolbarState(tabs[0], 'R') } } else { // clear 'R' if settings not provided - removeToolbarState(tabs[0], 'R') + await removeToolbarState(tabs[0], 'R') } } }) } else if (message.message === 'clearFactCheck') { // fact check settings unchecked - chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { + chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => { if (tabs && tabs[0]) { - removeToolbarState(tabs[0], 'F') + await removeToolbarState(tabs[0], 'F') } }) } else if (message.message === 'getCachedWaybackCount') { @@ -750,13 +749,13 @@ chrome.tabs.onUpdated.addListener((tabId, info, tab) => { // 404 not found if (settings && settings.not_found_setting) { - readTabData(tab, (data) => { + readTabData(tab, async (data) => { // cropPrefix used because Wayback's URL may not match exactly (e.g. "http" != "https") if (data && ('statusCode' in data) && (cropPrefix(data.statusUrl) === cropPrefix(url))) { if (data.statusCode >= 400) { checkNotFound({ 'tabId': tabId, 'statusCode': data.statusCode, 'url': data.statusUrl }) } else { - removeToolbarState(tab, 'V') + await removeToolbarState(tab, 'V') } } }) @@ -771,10 +770,10 @@ chrome.tabs.onUpdated.addListener((tabId, info, tab) => { if (settings && settings.wiki_setting && url.match(/^https?:\/\/[\w.]*wikipedia.org/)) { // if the papers API were to be updated similar to books API, then this would move to wikipedia.js getCachedPapers(url, - (data) => { + async (data) => { if (data && (data.status !== 'error')) { - addToolbarState(tab, 'R') - addToolbarState(tab, 'papers') + await addToolbarState(tab, 'R') + await addToolbarState(tab, 'papers') } }, () => {} ) @@ -801,9 +800,9 @@ chrome.tabs.onUpdated.addListener((tabId, info, tab) => { headers: headers }) .then(resp => resp.json()) - .then(resp => { + .then(async (resp) => { if (('metadata' in resp && 'identifier' in resp['metadata']) || 'ocaid' in resp) { - addToolbarState(tab, 'R') + await addToolbarState(tab, 'R') // Storing the tab url as well as the fetched archive url for future use chrome.storage.local.set({ 'tab_url': url, 'detail_url': resp['metadata']['identifier-access'] }, () => {}) } @@ -815,9 +814,9 @@ chrome.tabs.onUpdated.addListener((tabId, info, tab) => { const news_host = new URL(url).hostname // FIXME if (settings && settings.tvnews_setting && newshosts.has(news_host)) { getCachedTvNews(url, - (clips) => { + async (clips) => { if (clips && (clips.status !== 'error')) { - addToolbarState(tab, 'R') + await addToolbarState(tab, 'R') } }, () => {} ) @@ -832,33 +831,34 @@ chrome.tabs.onUpdated.addListener((tabId, info, tab) => { chrome.tabs.onActivated.addListener((info) => { chrome.storage.local.get(['fact_check_setting', 'wiki_setting', 'amazon_setting', 'tvnews_setting'], (settings) => { checkLastError() - chrome.tabs.get(info.tabId, (tab) => { + chrome.tabs.get(info.tabId, async (tab) => { checkLastError() if (typeof tab === 'undefined') { return } + toolbarState = await getToolbarState(tab) // fact check settings unchecked - if (settings && (settings.fact_check_setting === false) && getToolbarState(tab).has('F')) { - removeToolbarState(tab, 'F') + if (settings && (settings.fact_check_setting === false) && toolbarState.has('F')) { + await removeToolbarState(tab, 'F') } // wiki_setting settings unchecked - if (settings && (settings.wiki_setting === false) && getToolbarState(tab).has('R')) { - if (tab.url.match(/^https?:\/\/[\w.]*wikipedia.org/)) { removeToolbarState(tab, 'R') } + if (settings && (settings.wiki_setting === false) && toolbarState.has('R')) { + if (tab.url.match(/^https?:\/\/[\w.]*wikipedia.org/)) { await removeToolbarState(tab, 'R') } } // amazon_setting settings unchecked - if (settings && (settings.amazon_setting === false) && getToolbarState(tab).has('R')) { - if (tab.url.includes('www.amazon')) { removeToolbarState(tab, 'R') } + if (settings && (settings.amazon_setting === false) && toolbarState.has('R')) { + if (tab.url.includes('www.amazon')) { await removeToolbarState(tab, 'R') } } // tvnews_setting settings unchecked - if (settings && (settings.tvnews_setting === false) && getToolbarState(tab).has('R')) { + if (settings && (settings.tvnews_setting === false) && toolbarState.has('R')) { const news_host = new URL(tab.url).hostname - if (newshosts.has(news_host)) { removeToolbarState(tab, 'R') } + if (newshosts.has(news_host)) { await removeToolbarState(tab, 'R') } } // clear '404 not found' dot if tab URL doesn't match stored URL - readTabData(tab, (data) => { + readTabData(tab, async (data) => { if (data && ('statusUrl' in data) && (cropPrefix(data.statusUrl) !== cropPrefix(tab.url))) { - removeToolbarState(tab, 'V') + await removeToolbarState(tab, 'V') } }) - updateToolbar(tab) + await updateToolbar(tab) // update or clear count badge updateWaybackCountBadge(tab, tab.url) }) @@ -875,8 +875,9 @@ chrome.tabs.onActivated.addListener((info) => { * @param beforeDate {Date}: Date that will be checked only if url previously saved in WM. * Leave empty to always save. Set to null to save only if hadn't been previously saved. */ -function autoSave(atab, url, beforeDate = new Date()) { - if (isValidUrl(url) && isNotExcludedUrl(url) && !getToolbarState(atab).has('S')) { +async function autoSave(atab, url, beforeDate = new Date()) { + toolbarState = getToolbarState(atab) + if (isValidUrl(url) && isNotExcludedUrl(url) && !toolbarState.has('S')) { chrome.storage.local.get(['auto_exclude_list'], (items) => { if (!('auto_exclude_list' in items) || (('auto_exclude_list' in items) && items.auto_exclude_list && !isUrlInList(url, items.auto_exclude_list))) { @@ -963,7 +964,7 @@ function factCheck(atab, url) { if (isValidUrl(url) && isNotExcludedUrl(url)) { // retrieve fact check results getCachedFactCheck(url, - (json) => { + async (json) => { // json is an object containing: // "notices": [ { "notice": "...", "where": { "timestamp": [ "-yyyymmdd123456" ] } } ], // "status": "success" @@ -1000,7 +1001,7 @@ function factCheck(atab, url) { if (contextUrl !== url) { // only show context button if URL different than current URL in address bar saveTabData(atab, { 'contextUrl': contextUrl }) - addToolbarState(atab, 'F') + await addToolbarState(atab, 'F') } } } @@ -1116,8 +1117,9 @@ function updateWaybackCountBadge(atab, url) { if (!atab) { return } chrome.storage.local.get(['wm_count_setting'], (settings) => { if (settings && settings.wm_count_setting && isValidUrl(url) && isNotExcludedUrl(url) && !isArchiveUrl(url)) { - getCachedWaybackCount(url, (values) => { - if ((values.total >= 0) && !getToolbarState(atab).has('S')) { + getCachedWaybackCount(url, async(values) => { + let toolbarstate = await getToolbarState(atab) + if ((values.total >= 0) && !toolbarstate.has('S')) { // display badge let text = badgeCountText(values.total) chrome.action.setBadgeBackgroundColor({ color: '#9A3B38' }, checkLastError) // red @@ -1144,7 +1146,7 @@ function updateWaybackCountBadge(atab, url) { * @param name {string} = one of 'archive', 'check', 'R', or 'S' * @param tabId {int} (optional) = tab id, else sets current or global icon. */ -function setToolbarIcon(name, tabId = null) { +async function setToolbarIcon(name, tabId = null) { const validToolbarIcons = new Set(['R', 'S', 'F', 'V', 'check', 'archive']) const checkBadgePos = new Set(['R', 'F', 'V']) const path = '../images/toolbar/' @@ -1164,74 +1166,203 @@ function setToolbarIcon(name, tabId = null) { // Add state to the state set for given Tab, and update toolbar. // state is 'S', 'R', or 'check' // Add 'books' or 'papers' to display popup buttons for wikipedia resources. -function addToolbarState(atab, state) { +async function addToolbarState(atab, state) { if (!atab) { return } - const tabKey = getTabKey(atab) + const tabKey = getTabKey(atab); + // Retrieve gToolbarStates from chrome.storage + const gToolbarStatesSerialized = await new Promise((resolve) => { + chrome.storage.local.get(['gToolbarStates'], function(result) { + resolve(result.gToolbarStates || {}); + }); + }); + // Convert arrays back to Set objects + const gToolbarStates = {}; + for (const [key, value] of Object.entries(gToolbarStatesSerialized)) { + gToolbarStates[key] = new Set(value); + } + // Initialize the Set for the tabKey if it doesn't exist if (!gToolbarStates[tabKey]) { - gToolbarStates[tabKey] = new Set() + gToolbarStates[tabKey] = new Set(); } - gToolbarStates[tabKey].add(state) - updateToolbar(atab) + // Add the state to the Set for the tabKey + gToolbarStates[tabKey].add(state); + // Convert Set objects to arrays before storing in chrome.storage + const gToolbarStatesToStore = {}; + for (const [key, value] of Object.entries(gToolbarStates)) { + gToolbarStatesToStore[key] = Array.from(value); + } + // Store the updated gToolbarStates back into chrome.storage + chrome.storage.local.set({ gToolbarStates: gToolbarStatesToStore }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + } else { + console.log('gToolbarStates updated and stored successfully'); + } + }); + + // Update the toolbar + await updateToolbar(atab); } + // Remove state from the state set for given Tab, and update toolbar. -function removeToolbarState(atab, state) { +async function removeToolbarState(atab, state) { if (!atab) { return } - const tabKey = getTabKey(atab) + + const tabKey = getTabKey(atab); + // Retrieve gToolbarStates from chrome.storage + const gToolbarStatesSerialized = await new Promise((resolve) => { + chrome.storage.local.get(['gToolbarStates'], function(result) { + resolve(result.gToolbarStates || {}); + }); + }); + + // Convert arrays back to Set objects + const gToolbarStates = {}; + for (const [key, value] of Object.entries(gToolbarStatesSerialized)) { + gToolbarStates[key] = new Set(value); + } + + // Check if the state set for the tabKey exists if (gToolbarStates[tabKey]) { - gToolbarStates[tabKey].delete(state) + // Remove state from the state set for the given Tab + gToolbarStates[tabKey].delete(state); } - updateToolbar(atab) + + // Convert Set objects to arrays before storing in chrome.storage + const gToolbarStatesToStore = {}; + for (const [key, value] of Object.entries(gToolbarStates)) { + gToolbarStatesToStore[key] = Array.from(value); + } + + // Store the updated gToolbarStates back into chrome.storage + chrome.storage.local.set({ gToolbarStates: gToolbarStatesToStore }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + } else { + console.log('gToolbarStates updated and stored successfully'); + } + }); + + // Update the toolbar + await updateToolbar(atab); } + // Returns a Set of toolbar states, or an empty set. -function getToolbarState(atab) { +async function getToolbarState(atab) { if (!atab) { return new Set() } - const tabKey = getTabKey(atab) - return (gToolbarStates[tabKey]) ? gToolbarStates[tabKey] : new Set() + + const tabKey = getTabKey(atab); + + // Retrieve gToolbarStates from chrome.storage + const gToolbarStatesSerialized = await new Promise((resolve) => { + chrome.storage.local.get(['gToolbarStates'], function(result) { + resolve(result.gToolbarStates || {}); + }); + }); + + // Convert arrays back to Set objects + const gToolbarStates = {}; + for (const [key, value] of Object.entries(gToolbarStatesSerialized)) { + gToolbarStates[key] = new Set(value); + } + + // Return the state set for the tabKey or an empty Set if it doesn't exist + return (gToolbarStates[tabKey]) ? gToolbarStates[tabKey] : new Set(); } // Clears state for given Tab and update toolbar icon. -function clearToolbarState(atab) { +async function clearToolbarState(atab) { if (!atab) { return } - const tabKey = getTabKey(atab) + + const tabKey = getTabKey(atab); + + // Retrieve gToolbarStates from chrome.storage + const gToolbarStatesSerialized = await new Promise((resolve) => { + chrome.storage.local.get(['gToolbarStates'], function(result) { + resolve(result.gToolbarStates || {}); + }); + }); + + // Convert arrays back to Set objects + const gToolbarStates = {}; + for (const [key, value] of Object.entries(gToolbarStatesSerialized)) { + gToolbarStates[key] = new Set(value); + } + + // Clear the state set for the tabKey and delete the tabKey entry if (gToolbarStates[tabKey]) { - gToolbarStates[tabKey].clear() - delete gToolbarStates[tabKey] + gToolbarStates[tabKey].clear(); + delete gToolbarStates[tabKey]; } - updateToolbar(atab) + + // Convert Set objects to arrays before storing in chrome.storage + const gToolbarStatesToStore = {}; + for (const [key, value] of Object.entries(gToolbarStates)) { + gToolbarStatesToStore[key] = Array.from(value); + } + + // Store the updated gToolbarStates back into chrome.storage + chrome.storage.local.set({ gToolbarStates: gToolbarStatesToStore }, function() { + if (chrome.runtime.lastError) { + console.error(chrome.runtime.lastError); + } else { + console.log('gToolbarStates updated and stored successfully'); + } + }); + + // Update the toolbar + await updateToolbar(atab); } + /** * Updates the toolbar icon using the state set stored in gToolbarStates. * Only updates icon if atab is the currently active tab, else does nothing. * @param atab {Tab} */ -function updateToolbar(atab) { +async function updateToolbar(atab) { if (!atab) { return } - const tabKey = getTabKey(atab) - // type 'normal' prevents updation of toolbar icon when it's a popup window - chrome.tabs.query({ active: true, windowId: atab.windowId, windowType: 'normal' }, (tabs) => { + + const tabKey = getTabKey(atab); + + // Retrieve gToolbarStates from chrome.storage + const gToolbarStatesSerialized = await new Promise((resolve) => { + chrome.storage.local.get(['gToolbarStates'], function(result) { + resolve(result.gToolbarStates || {}); + }); + }); + + // Convert arrays back to Set objects + const gToolbarStates = {}; + for (const [key, value] of Object.entries(gToolbarStatesSerialized)) { + gToolbarStates[key] = new Set(value); + } + + // Query active tab to update toolbar icon + chrome.tabs.query({ active: true, windowId: atab.windowId, windowType: 'normal' }, async (tabs) => { if (tabs && tabs[0] && (tabs[0].id === atab.id) && (tabs[0].windowId === atab.windowId)) { - let state = gToolbarStates[tabKey] - // this order defines the priority of what icon to display + let state = gToolbarStates[tabKey]; + // Determine which icon to display based on state if (state && state.has('S')) { - setToolbarIcon('S', atab.id) + await setToolbarIcon('S', atab.id); } else if (state && state.has('V')) { - setToolbarIcon('V', atab.id) + await setToolbarIcon('V', atab.id); } else if (state && state.has('F')) { - setToolbarIcon('F', atab.id) + await setToolbarIcon('F', atab.id); } else if (state && state.has('R')) { - setToolbarIcon('R', atab.id) + await setToolbarIcon('R', atab.id); } else if (state && state.has('check')) { - setToolbarIcon('check', atab.id) + await setToolbarIcon('check', atab.id); } else { - setToolbarIcon('archive', atab.id) + await setToolbarIcon('archive', atab.id); } } - }) + }); } + /* * * Right-click Menu * * */ // Right-click context menu "Wayback Machine" inside the page. diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 77ad682d..0c23e3b4 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -821,6 +821,7 @@ function initDefaultOptions () { selectedFeature: null, gStatusCode: 0, waybackCountCache: {}, + gToolbarStates: {}, globalAPICache: {}, API_CACHE_SIZE: 5, API_LOADING: 'LOADING', From 6a395ccbf9fd3f81afee666d01dd9c1e39a3901d Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Wed, 22 May 2024 21:59:57 +0530 Subject: [PATCH 07/10] remove declarativeNetRequest request from manifest --- webextension/manifest.json | 14 ++------------ webextension/rules_1.json | 15 --------------- webextension/scripts/background.js | 4 ++-- 3 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 webextension/rules_1.json diff --git a/webextension/manifest.json b/webextension/manifest.json index 33f62748..8590f9c3 100644 --- a/webextension/manifest.json +++ b/webextension/manifest.json @@ -31,10 +31,7 @@ "notifications", "storage", "scripting", - "webRequest", - "declarativeNetRequestWithHostAccess", - "declarativeNetRequest", - "declarativeNetRequestFeedback" + "webRequest" ], "host_permissions":[ "https://archive.org/*", @@ -58,12 +55,5 @@ "web_accessible_resources": [{ "resources":[ "/css/archive.css","/images/*"], "matches": [""] - }], - "declarative_net_request" : { - "rule_resources" : [{ - "id": "ruleset_1", - "enabled": true, - "path": "rules_1.json" - }] - } + }] } diff --git a/webextension/rules_1.json b/webextension/rules_1.json deleted file mode 100644 index c9f4d097..00000000 --- a/webextension/rules_1.json +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "id": 1, - "priority": 1, - "action": { - "type": "modifyHeaders", - "requestHeaders": [ - { "header": "Wayback-Browser-Extension", "operation": "set", "value": "ok" } - ] - }, - "condition": { - "urlFilter": "*.archive.org" - } - } -] diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index da4916bf..1ac9f4bc 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -22,7 +22,7 @@ importScripts('utils.js') // const SPN_RETRY = 6000 let tabIdPromise -// not required because in manifest v3, a new header is set using declarativeNetRequest rules +// not required because in manifest v3, we use different subdomain of hostURLs for different browsers // updates User-Agent header in Chrome & Firefox, but not in Safari // function rewriteUserAgentHeader(e) { // for (let header of e.requestHeaders) { @@ -471,7 +471,7 @@ chrome.action.onClicked.addListener((tab) => { }) // chrome.webRequest.onBeforeSendHeaders is not supported in manifest v3 and currently we are not looking to rewrite the user-agent -// as we already use diffent host-names for different endpoints +// as we already use diffent host-names for different browsers // chrome.webRequest.onBeforeSendHeaders.addListener( // rewriteUserAgentHeader, From 6f204eabe71af5047974d5775d1463bea44d9f9e Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Wed, 31 Jul 2024 22:28:32 +0530 Subject: [PATCH 08/10] use the constant variable and fix missing async await --- .../Wayback Machine.xcodeproj/project.pbxproj | 8 +-- webextension/manifest.json | 2 +- webextension/scripts/background.js | 66 ++++++------------- webextension/scripts/utils.js | 5 -- 4 files changed, 24 insertions(+), 57 deletions(-) diff --git a/safari/Wayback Machine.xcodeproj/project.pbxproj b/safari/Wayback Machine.xcodeproj/project.pbxproj index f7eacf58..928c6e8a 100644 --- a/safari/Wayback Machine.xcodeproj/project.pbxproj +++ b/safari/Wayback Machine.xcodeproj/project.pbxproj @@ -468,7 +468,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = "$(RECOMMENDED_MACOSX_DEPLOYMENT_TARGET)"; - MARKETING_VERSION = 3.4; + MARKETING_VERSION = 3.5; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac.extension; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -494,7 +494,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = "$(RECOMMENDED_MACOSX_DEPLOYMENT_TARGET)"; - MARKETING_VERSION = 3.4; + MARKETING_VERSION = 3.5; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac.extension; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -522,7 +522,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = "$(RECOMMENDED_MACOSX_DEPLOYMENT_TARGET)"; - MARKETING_VERSION = 3.4; + MARKETING_VERSION = 3.5; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; @@ -549,7 +549,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = "$(RECOMMENDED_MACOSX_DEPLOYMENT_TARGET)"; - MARKETING_VERSION = 3.4; + MARKETING_VERSION = 3.5; PRODUCT_BUNDLE_IDENTIFIER = archive.org.waybackmachine.mac; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; diff --git a/webextension/manifest.json b/webextension/manifest.json index 8590f9c3..9ee78e42 100644 --- a/webextension/manifest.json +++ b/webextension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Wayback Machine", - "version": "3.4", + "version": "3.5", "description": "The Official Wayback Machine Extension - by the Internet Archive.", "icons": { "16": "images/app-icon/mini-icon16.png", diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index 1ac9f4bc..be0ea544 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -15,11 +15,11 @@ importScripts('utils.js') // let gToolbarStates = {} // let waybackCountCache = {} // let globalAPICache = new Map() -// const API_CACHE_SIZE = 5 -// const API_LOADING = 'LOADING' -// const API_TIMEOUT = 10000 -// const API_RETRY = 1000 -// const SPN_RETRY = 6000 +const API_CACHE_SIZE = 5 +const API_LOADING = 'LOADING' +const API_TIMEOUT = 10000 +const API_RETRY = 1000 +const SPN_RETRY = 6000 let tabIdPromise // not required because in manifest v3, we use different subdomain of hostURLs for different browsers @@ -58,12 +58,6 @@ function savePageNowChecked(atab, pageUrl, silent, options) { */ async function savePageNow(atab, pageUrl, silent = false, options = {}, loggedInFlag = true) { - let { API_TIMEOUT, SPN_RETRY } = await new Promise((resolve) => { - chrome.storage.local.get(['API_TIMEOUT', 'SPN_RETRY'], function(result) { - resolve(result); - }); - }); - if (!(isValidUrl(pageUrl) && isNotExcludedUrl(pageUrl))) { console.log('savePageNow URL excluded') return @@ -153,11 +147,6 @@ function extractJobIdFromHTML(html) { * @param jobId {string}: job_id returned by SPN response, passed to Status API. */ async function savePageStatus(atab, pageUrl, silent = false, jobId) { - let { API_TIMEOUT, SPN_RETRY } = await new Promise((resolve) => { - chrome.storage.local.get(['API_TIMEOUT', 'SPN_RETRY'], function(result) { - resolve(result); - }); - }); // setup api // Accept header required when logged-out, even though response is in JSON. let headers = new Headers(hostHeaders) @@ -283,25 +272,19 @@ async function statusFailed(atab, pageUrl, silent, data, err) { * @param timestamp {string}: Wayback timestamp as "yyyyMMddHHmmss" in UTC. * @return Promise: which should return this JSON on success: { "success": true } */ -async function saveToMyWebArchive(url, timestamp) { - let { API_TIMEOUT } = await new Promise((resolve) => { - chrome.storage.local.get(['API_TIMEOUT'], function(result) { - resolve(result); - }); - }); - const postData = { 'url': url, 'snapshot': timestamp, 'tags': [] } + async function saveToMyWebArchive(url, timestamp) { + const postData = { 'url': url, 'snapshot': timestamp, 'tags': [] }; const timeoutPromise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, API_TIMEOUT) - let headers = new Headers(hostHeaders) - headers.set('Content-Type', 'application/json') - fetch(hostURL + '__wb/web-archive', { - method: 'POST', - body: JSON.stringify(postData), - headers: headers - }) - .then(resolve, reject) - }) - return timeoutPromise + setTimeout(() => reject(new Error('timeout')), API_TIMEOUT); + fetch(hostURL + '__wb/web-archive', { + method: 'POST', + body: JSON.stringify(postData), + headers: new Headers({ 'Content-Type': 'application/json', ...hostHeaders }) + }) + .then(resolve) + .catch(reject); + }); + return timeoutPromise; } /** @@ -313,11 +296,6 @@ async function saveToMyWebArchive(url, timestamp) { * @return Promise */ async function fetchAPI(url, onSuccess, onFail, postData = null) { - let { API_TIMEOUT } = await new Promise((resolve) => { - chrome.storage.local.get(['API_TIMEOUT'], function(result) { - resolve(result); - }); - }); const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, API_TIMEOUT) let headers = new Headers(hostHeaders) @@ -353,12 +331,6 @@ async function fetchAPI(url, onSuccess, onFail, postData = null) { * @return Promise if calls API, json data if in cache, null if loading in progress. */ async function fetchCachedAPI(url, onSuccess, onFail, postData = null) { - let { API_CACHE_SIZE, API_LOADING, API_RETRY } = await new Promise((resolve) => { - chrome.storage.local.get(['API_CACHE_SIZE', 'API_LOADING', 'API_RETRY'], function(result) { - resolve(result); - }); - }); - chrome.storage.local.get(['globalAPICache'], async function(result) { let globalAPICache = result.globalAPICache ? new Map(Object.entries(result.globalAPICache)) : new Map(); let data = globalAPICache.get(url); @@ -725,7 +697,7 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { return false }) -chrome.tabs.onUpdated.addListener((tabId, info, tab) => { +chrome.tabs.onUpdated.addListener( async (tabId, info, tab) => { const url = tab.url if (!(isNotExcludedUrl(url) && isValidUrl(url)) || isArchiveUrl(url)) { return } @@ -876,7 +848,7 @@ chrome.tabs.onActivated.addListener((info) => { * Leave empty to always save. Set to null to save only if hadn't been previously saved. */ async function autoSave(atab, url, beforeDate = new Date()) { - toolbarState = getToolbarState(atab) + toolbarState = await getToolbarState(atab) if (isValidUrl(url) && isNotExcludedUrl(url) && !toolbarState.has('S')) { chrome.storage.local.get(['auto_exclude_list'], (items) => { if (!('auto_exclude_list' in items) || diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 5db2914b..06124f8e 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -824,11 +824,6 @@ function initDefaultOptions () { waybackCountCache: {}, gToolbarStates: {}, globalAPICache: {}, - API_CACHE_SIZE: 5, - API_LOADING: 'LOADING', - API_TIMEOUT: 10000, - API_RETRY: 1000, - SPN_RETRY: 6000, /* Features */ private_mode_setting: true, not_found_setting: false, From 83701dfa3a4cde29692e29335bc32fa313a15ba0 Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Thu, 1 Aug 2024 06:31:40 +0530 Subject: [PATCH 09/10] use aync/await in checkSaveToMyWebArchive --- webextension/scripts/background.js | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index be0ea544..9710f431 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -211,7 +211,7 @@ async function statusSuccess(atab, pageUrl, silent, data) { // notify if (!silent && data && ('timestamp' in data)) { // since not silent, saves to My Web Archive only if SPN explicitly clicked and turned on - checkSaveToMyWebArchive(pageUrl, data.timestamp) + await checkSaveToMyWebArchive(pageUrl, data.timestamp) // replace message if present in result let msg = 'Successfully saved! Click to view snapshot.' if (('message' in data) && (data.message.length > 0)) { @@ -546,21 +546,22 @@ function checkNotFound(details) { // Calls saveToMyWebArchive() if setting is set, and outputs errors to console. // -function checkSaveToMyWebArchive(url, timestamp) { - chrome.storage.local.get(['my_archive_setting'], (settings) => { - if (settings && settings.my_archive_setting) { - saveToMyWebArchive(url, timestamp) - .then(response => response.json()) - .then(data => { - if (!(data && (data['success'] === true))) { - console.log('Save to My Web Archive FAILED: ', data.error) - } - }) - .catch(error => { - console.log('Save to My Web Archive FAILED: ', error) - }) - } - }) +async function checkSaveToMyWebArchive(url, timestamp) { + const settings = await new Promise((resolve) => { + chrome.storage.local.get(['my_archive_setting'], resolve); + }); + if (settings && settings.my_archive_setting) { + await saveToMyWebArchive(url, timestamp) + .then(response => response.json()) + .then(data => { + if (!(data && (data['success'] === true))) { + console.log('Save to My Web Archive FAILED: ', data.error) + } + }) + .catch(error => { + console.log('Save to My Web Archive FAILED: ', error) + }) + } } // Listens for messages to call background functions from other scripts. From 18e54e8564de9d953fc3dda2868ac54d8f010d4c Mon Sep 17 00:00:00 2001 From: Anish Sarangi Date: Thu, 1 Aug 2024 06:36:41 +0530 Subject: [PATCH 10/10] store globalAPICache locally --- webextension/scripts/background.js | 70 +++++++++++------------------- webextension/scripts/utils.js | 1 - 2 files changed, 26 insertions(+), 45 deletions(-) diff --git a/webextension/scripts/background.js b/webextension/scripts/background.js index 9710f431..c7bfb546 100644 --- a/webextension/scripts/background.js +++ b/webextension/scripts/background.js @@ -14,7 +14,7 @@ importScripts('utils.js') //let gStatusCode = 0 // let gToolbarStates = {} // let waybackCountCache = {} -// let globalAPICache = new Map() +let globalAPICache = new Map() const API_CACHE_SIZE = 5 const API_LOADING = 'LOADING' const API_TIMEOUT = 10000 @@ -330,50 +330,32 @@ async function fetchAPI(url, onSuccess, onFail, postData = null) { * @param postData {object}: uses POST if present. * @return Promise if calls API, json data if in cache, null if loading in progress. */ - async function fetchCachedAPI(url, onSuccess, onFail, postData = null) { - chrome.storage.local.get(['globalAPICache'], async function(result) { - let globalAPICache = result.globalAPICache ? new Map(Object.entries(result.globalAPICache)) : new Map(); - let data = globalAPICache.get(url); - - if (data === API_LOADING) { - setTimeout(() => { - fetchCachedAPI(url, onSuccess, onFail, postData) - }, API_RETRY); - return null; - } else if (data !== undefined) { - onSuccess(data); - return data; - } else { - // if cache full, remove first object which is the oldest from the cache - if (globalAPICache.size >= API_CACHE_SIZE) { - globalAPICache.delete(globalAPICache.keys().next().value); - } - globalAPICache.set(url, API_LOADING); - - // Convert Map back to object before storing it in chrome.storage - let globalAPICacheObject = Object.fromEntries(globalAPICache); - - chrome.storage.local.set({ globalAPICache: globalAPICacheObject }, function() { - fetchAPI(url, (json) => { - globalAPICache.set(url, json); - - // Convert Map back to object before storing it in chrome.storage - let globalAPICacheObject = Object.fromEntries(globalAPICache); - chrome.storage.local.set({ globalAPICache: globalAPICacheObject }); - - onSuccess(json); - }, (error) => { - globalAPICache.delete(url); - - // Convert Map back to object before storing it in chrome.storage - let globalAPICacheObject = Object.fromEntries(globalAPICache); - chrome.storage.local.set({ globalAPICache: globalAPICacheObject }); - - onFail(error); - }, postData); - }); + function fetchCachedAPI(url, onSuccess, onFail, postData = null) { + if (typeof globalAPICache === 'undefined') { globalAPICache = new Map() } + let data = globalAPICache.get(url) + if (data === API_LOADING) { + // re-call after delay if previous fetch hadn't returned yet + setTimeout(() => { + fetchCachedAPI(url, onSuccess, onFail, postData) + }, API_RETRY) + return null + } else if (data !== undefined) { + onSuccess(data) + return data + } else { + // if cache full, remove first object which is the oldest from the cache + if (globalAPICache.size >= API_CACHE_SIZE) { + globalAPICache.delete(globalAPICache.keys().next().value) } - }); + globalAPICache.set(url, API_LOADING) + return fetchAPI(url, (json) => { + globalAPICache.set(url, json) + onSuccess(json) + }, (error) => { + globalAPICache.delete(url) + onFail(error) + }, postData) + } } diff --git a/webextension/scripts/utils.js b/webextension/scripts/utils.js index 06124f8e..b9144421 100644 --- a/webextension/scripts/utils.js +++ b/webextension/scripts/utils.js @@ -823,7 +823,6 @@ function initDefaultOptions () { gStatusCode: 0, waybackCountCache: {}, gToolbarStates: {}, - globalAPICache: {}, /* Features */ private_mode_setting: true, not_found_setting: false,