From 175031b5dab2e2764453ac61c780705770829c07 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 4 Dec 2024 11:47:01 +0100 Subject: [PATCH 1/5] Update engine --- package-lock.json | 40 +++++++++++++++++++++++----------------- package.json | 2 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 983244ec..546bb1a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "license": "EUPL-1.2", "dependencies": { - "@opentermsarchive/engine": "~2.5.0", + "@opentermsarchive/engine": "~4.0.0", "isomorphic-fetch": "^3.0.0" } }, @@ -2330,9 +2330,9 @@ "integrity": "sha512-AanzbulOHljrku1NGfafxdpTCfw2ENaWzH01N2vqQM+cUFbk868Cgh0xylz0JIM9BoKbfI++bdD6EYX0Q/UTEw==" }, "node_modules/@opentermsarchive/engine": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@opentermsarchive/engine/-/engine-2.5.0.tgz", - "integrity": "sha512-ziZ+inihix3Y3cnqdhHsZQa+ZclLolGkIxz+ZrFYweDISZVA3ioe2LhBuTj2EPSYXhNS0jlK898lH1viiRowaQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@opentermsarchive/engine/-/engine-4.0.0.tgz", + "integrity": "sha512-5PxDUTh2KG7cEVid0niU+uPMk6+5xVfCSLXaNQtwy7snvV9W7OJq5+VgKmFuO823kl2LsqjXELexJ0YhWB0n2A==", "dependencies": { "@accordproject/markdown-cicero": "^0.15.2", "@accordproject/markdown-pdf": "^0.15.2", @@ -2368,6 +2368,7 @@ "https-proxy-agent": "^5.0.0", "iconv-lite": "^0.6.3", "joplin-turndown-plugin-gfm": "^1.0.12", + "js-yaml": "^4.1.0", "jsdom": "^18.1.0", "json-source-map": "^0.6.1", "lodash": "^4.17.21", @@ -2382,7 +2383,7 @@ "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "sib-api-v3-sdk": "^8.2.1", - "simple-git": "^3.8.0", + "simple-git": "^3.27.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "winston": "^3.9.0", @@ -2395,7 +2396,7 @@ "node": ">=16.0.0" }, "peerDependencies": { - "@opentermsarchive/terms-types": "^1.4.0" + "@opentermsarchive/terms-types": "^2.0.0" } }, "node_modules/@opentermsarchive/fetch-charset-detection": { @@ -2413,9 +2414,9 @@ } }, "node_modules/@opentermsarchive/terms-types": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@opentermsarchive/terms-types/-/terms-types-1.4.0.tgz", - "integrity": "sha512-LXfCGfjiv7JS6xxSPIgpBeS65WdIadF+gn2CagVtZM6OYmQIoNJ9sir7QZl6scrt0ZmA5CT7QOBzT7nMDhnUKA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@opentermsarchive/terms-types/-/terms-types-2.0.0.tgz", + "integrity": "sha512-q6ck1vzVsxoLxaK1gA5DjgUB1YhBq4PbodchreCsC7bEcrxtKFrS5UhJZpRJpiu5e+orNJaFsDeIlU6QO5NaoA==", "peer": true }, "node_modules/@opentermsarchive/turndown": { @@ -9900,13 +9901,13 @@ } }, "node_modules/simple-git": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.21.0.tgz", - "integrity": "sha512-oTzw9248AF5bDTMk9MrxsRzEzivMlY+DWH0yWS4VYpMhNLhDWnN06pCtaUyPnqv/FpsdeNmRqmZugMABHRPdDA==", + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.27.0.tgz", + "integrity": "sha512-ivHoFS9Yi9GY49ogc6/YAi3Fl9ROnF4VyubNylgCkA+RVqLaKWnDSzXOVzya8csELIaWaYNutsEuAhZrtOjozA==", "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", - "debug": "^4.3.4" + "debug": "^4.3.5" }, "funding": { "type": "github", @@ -9914,11 +9915,11 @@ } }, "node_modules/simple-git/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -9929,6 +9930,11 @@ } } }, + "node_modules/simple-git/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", diff --git a/package.json b/package.json index c42d009c..a165ee8e 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "dataset:schedule": "npm run dataset -- --publish --remove-local-copy --schedule" }, "dependencies": { - "@opentermsarchive/engine": "~2.5.0", + "@opentermsarchive/engine": "~4.0.0", "isomorphic-fetch": "^3.0.0" } } From 018436ec8feae0abd309d776b317c2308a023333 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 4 Dec 2024 11:47:26 +0100 Subject: [PATCH 2/5] Delegate timestamps in logs to process manager --- config/production.json | 3 +++ deployment/pm2.config.cjs | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/production.json b/config/production.json index 889b4163..eba1dd30 100644 --- a/config/production.json +++ b/config/production.json @@ -44,6 +44,9 @@ "title": "contrib", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/contrib-versions" }, + "logger": { + "timestampPrefix": false + }, "collection-api": { "port": 3000, "basePath": "/collection-api" diff --git a/deployment/pm2.config.cjs b/deployment/pm2.config.cjs index b6d3d678..f4d2055d 100644 --- a/deployment/pm2.config.cjs +++ b/deployment/pm2.config.cjs @@ -7,6 +7,7 @@ module.exports = { max_restarts: 2, min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. restart_delay: 3 * 60 * 60 * 1000, // likely related to a connectivity problem that will take some time to be fixed + log_date_format: "YYYY-MM-DDTHH:mm:ssZ" }, { name: 'ota-api', @@ -15,7 +16,8 @@ module.exports = { min_uptime: '10s', max_restarts: 10, restart_delay: 1000, - exponential_backoff_restart_delay: true + exponential_backoff_restart_delay: true, + log_date_format: "YYYY-MM-DDTHH:mm:ssZ" }, { name: 'ota-release', @@ -24,7 +26,8 @@ module.exports = { min_uptime: '10s', max_restarts: 10, restart_delay: 60 * 60 * 1000, // likely related to a GitHub availability problem that will take some time to be fixed - exponential_backoff_restart_delay: true + exponential_backoff_restart_delay: true, + log_date_format: "YYYY-MM-DDTHH:mm:ssZ" } ], }; From f1885f9aed94c6e79fc12cd28967e95181909b70 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 4 Dec 2024 11:47:31 +0100 Subject: [PATCH 3/5] Add metadata --- metadata.yml | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 metadata.yml diff --git a/metadata.yml b/metadata.yml new file mode 100644 index 00000000..d7974315 --- /dev/null +++ b/metadata.yml @@ -0,0 +1,57 @@ +id: contrib +name: Contrib +tagline: Documents added by volunteer contributors and historically imported from TOSBack.org +description: | + The Contrib collection is maintained collaboratively by volunteer contributors. It contains all document versions that are not maintained by a dedicated actor. Data from the Contrib collection has been reused in a number of projects, ranging from research on the impact of social media terms of use on human rights to improving document readability. + + The documents in this collection fall under European Union and U.S. jurisdictions and are in French and English. This collection tracks documents from over 300 services. +dataset: https://github.com/OpenTermsArchive/contrib-versions/releases +declarations: https://github.com/OpenTermsArchive/contrib-declarations +versions: https://github.com/OpenTermsArchive/contrib-versions +snapshots: https://github.com/OpenTermsArchive/contrib-snapshots +donations: https://opencollective.com/opentermsarchive +logo: https://opentermsarchive.org/images/collections/contrib.png +languages: [en, fr] +jurisdictions: [EU, US] +trackingPeriods: + startDate: 2021-11-14 + schedule: "0 6,18 * * *" + serverLocation: London, GB +governance: + hosts: + - name: Ministry for Europe and Foreign Affairs + url: https://www.diplomatie.gouv.fr/en/ + logo: https://opentermsarchive.org/images/contributors/meae.png + administrators: + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png + curators: + - name: Volunteer contributors + maintainers: + - name: Volunteer contributors + sponsors: + - name: Ministry for Europe and Foreign Affairs + url: https://www.diplomatie.gouv.fr/en/ + logo: https://opentermsarchive.org/images/contributors/meae.png + +i18n: + fr: + tagline: Documents ajoutés par des contributeurs bénévoles et importés historiquement de TOSBack.org + description: | + La collection Contrib est maintenue de manière collaborative par des contributeurs bénévoles. Elle contient toutes les versions de documents qui ne sont pas gérées par un acteur dédié. Les données de la collection Contrib ont été réutilisées dans de nombreux projets, allant de la recherche sur l’impact des conditions d’utilisation des réseaux sociaux sur les droits humains à l’amélioration de la lisibilité des documents. + + Les documents de cette collection relèvent des juridictions de l’Union européenne et des États-Unis et sont rédigés en français et en anglais. Cette collection suit les documents de plus de 300 services. + governance: + hosts: + - name: Ministère de l'Europe et des Affaires étrangères + url: https://www.diplomatie.gouv.fr + logo: https://opentermsarchive.org/images/contributors/meae.png + curators: + - name: Contributeurs bénévoles + maintainers: + - name: Contributeurs bénévoles + sponsors: + - name: Ministère de l'Europe et des Affaires étrangères + url: https://www.diplomatie.gouv.fr + logo: https://opentermsarchive.org/images/contributors/meae.png From ce90bf1a3210bb979d2a3711155b8cf455cce6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Wed, 4 Dec 2024 15:52:57 +0100 Subject: [PATCH 4/5] Improve description --- metadata.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/metadata.yml b/metadata.yml index d7974315..a0612861 100644 --- a/metadata.yml +++ b/metadata.yml @@ -2,9 +2,11 @@ id: contrib name: Contrib tagline: Documents added by volunteer contributors and historically imported from TOSBack.org description: | - The Contrib collection is maintained collaboratively by volunteer contributors. It contains all document versions that are not maintained by a dedicated actor. Data from the Contrib collection has been reused in a number of projects, ranging from research on the impact of social media terms of use on human rights to improving document readability. + The Contrib collection contains terms written in English that may concern any industry and any jurisdiction. It was created thanks to a collaboration with [Tosback](https://tosback.org/), which enabled the import of its data history. + + It's one of the federation's largest collections, tracking over 300 services maintained by volunteer contributors. It is also a good entry point into the Open Terms Archive ecosystem to became familiar with monitoring and change analysis. - The documents in this collection fall under European Union and U.S. jurisdictions and are in French and English. This collection tracks documents from over 300 services. + Data from the this collection has been reused in a number of projects, ranging from research on the impact of social media terms of use on human rights to improving document readability. dataset: https://github.com/OpenTermsArchive/contrib-versions/releases declarations: https://github.com/OpenTermsArchive/contrib-declarations versions: https://github.com/OpenTermsArchive/contrib-versions @@ -39,9 +41,11 @@ i18n: fr: tagline: Documents ajoutés par des contributeurs bénévoles et importés historiquement de TOSBack.org description: | - La collection Contrib est maintenue de manière collaborative par des contributeurs bénévoles. Elle contient toutes les versions de documents qui ne sont pas gérées par un acteur dédié. Les données de la collection Contrib ont été réutilisées dans de nombreux projets, allant de la recherche sur l’impact des conditions d’utilisation des réseaux sociaux sur les droits humains à l’amélioration de la lisibilité des documents. - - Les documents de cette collection relèvent des juridictions de l’Union européenne et des États-Unis et sont rédigés en français et en anglais. Cette collection suit les documents de plus de 300 services. + La collection Contrib contient des documents rédigés en anglais qui peuvent concerner n'importe quelle industries et juridictions. Elle a été créée grâce à une collaboration avec [Tosback](https://tosback.org/), qui a permis d'importer son historique de données. + + C'est l'une des plus grandes collections de la fédération, avec plus de 300 services maintenus par des contributeurs bénévoles. C'est également un bon point d'entrée dans l'écosystème d'Open Terms Archive pour se familiariser avec le suivi et l'analyse du changement. + + Les données de cette collection ont été réutilisées dans un certain nombre de projets, allant de la recherche sur l'impact des conditions d'utilisation des médias sociaux sur les droits de l'homme à l'amélioration de la lisibilité des documents. governance: hosts: - name: Ministère de l'Europe et des Affaires étrangères From 0dba6d6921f82ac83ef324573559a5eb3d79d67e Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Fri, 6 Dec 2024 11:38:14 +0100 Subject: [PATCH 5/5] Update metadata --- metadata.yml | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/metadata.yml b/metadata.yml index a0612861..6bbd2dd6 100644 --- a/metadata.yml +++ b/metadata.yml @@ -1,12 +1,14 @@ id: contrib name: Contrib -tagline: Documents added by volunteer contributors and historically imported from TOSBack.org +tagline: Collection open to all contributions description: | - The Contrib collection contains terms written in English that may concern any industry and any jurisdiction. It was created thanks to a collaboration with [Tosback](https://tosback.org/), which enabled the import of its data history. - - It's one of the federation's largest collections, tracking over 300 services maintained by volunteer contributors. It is also a good entry point into the Open Terms Archive ecosystem to became familiar with monitoring and change analysis. + The **Contrib** collection gathers terms spanning a wide variety of industries and jurisdictions. - Data from the this collection has been reused in a number of projects, ranging from research on the impact of social media terms of use on human rights to improving document readability. + It was initially created thanks to a collaboration with [Tosback](https://tosback.org), which enabled the import of its data history. + + It’s one of the federation’s largest collections, tracking over 300 services maintained by volunteer contributors. It is also a good entry point into the Open Terms Archive ecosystem to became familiar with monitoring and change analysis. + + The data in this collection has been reused in a variety of projects, from research into the impact of social media usage conditions on human rights, to improving the readability of documents. dataset: https://github.com/OpenTermsArchive/contrib-versions/releases declarations: https://github.com/OpenTermsArchive/contrib-declarations versions: https://github.com/OpenTermsArchive/contrib-versions @@ -30,8 +32,14 @@ governance: logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png curators: - name: Volunteer contributors + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png maintainers: - name: Volunteer contributors + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png sponsors: - name: Ministry for Europe and Foreign Affairs url: https://www.diplomatie.gouv.fr/en/ @@ -39,13 +47,15 @@ governance: i18n: fr: - tagline: Documents ajoutés par des contributeurs bénévoles et importés historiquement de TOSBack.org + tagline: Collection ouverte à toutes contributions description: | - La collection Contrib contient des documents rédigés en anglais qui peuvent concerner n'importe quelle industries et juridictions. Elle a été créée grâce à une collaboration avec [Tosback](https://tosback.org/), qui a permis d'importer son historique de données. - - C'est l'une des plus grandes collections de la fédération, avec plus de 300 services maintenus par des contributeurs bénévoles. C'est également un bon point d'entrée dans l'écosystème d'Open Terms Archive pour se familiariser avec le suivi et l'analyse du changement. - - Les données de cette collection ont été réutilisées dans un certain nombre de projets, allant de la recherche sur l'impact des conditions d'utilisation des médias sociaux sur les droits de l'homme à l'amélioration de la lisibilité des documents. + La collection **Contrib** rassemble des conditions d’utilisation couvrant une large diversité d’industries et de juridictions. + + Elle a été initialement créée en collaboration avec [Tosback](https://tosback.org), qui a permis d’importer son historique de données. + + C’est l’une des plus grandes collections de la fédération, avec plus de 300 services maintenus par des contributeurs bénévoles. C’est également un bon point d’entrée dans l’écosystème d’Open Terms Archive pour se familiariser avec le suivi et l’analyse du changement. + + Les données de cette collection ont été réutilisées dans divers projets, allant de la recherche sur l’impact des conditions d’utilisation des médias sociaux sur les droits de l’homme à l’amélioration de la lisibilité des documents. governance: hosts: - name: Ministère de l'Europe et des Affaires étrangères @@ -53,8 +63,14 @@ i18n: logo: https://opentermsarchive.org/images/contributors/meae.png curators: - name: Contributeurs bénévoles + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png maintainers: - name: Contributeurs bénévoles + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png sponsors: - name: Ministère de l'Europe et des Affaires étrangères url: https://www.diplomatie.gouv.fr