From e46ed146bb51eb45883a7e7fa4c5d10b29d4baae Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Thu, 18 Apr 2024 15:37:53 -0500 Subject: [PATCH 1/4] Add removePodcastsByIds script --- package.json | 2 ++ src/controllers/podcast.ts | 27 +++++++++++++++++++-- src/scripts/podcasts/removePodcastsByIds.ts | 25 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 src/scripts/podcasts/removePodcastsByIds.ts diff --git a/package.json b/package.json index c674674a..f932ec9c 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "dev:scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts", "dev:scripts:removeDeadChapters": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs/removeDeadChapters.ts", "dev:scripts:removeDeadEpisodes": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts", + "dev:scripts:removePodcastsByIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts", "dev:scripts:syncWithFeedUrlsCSVDump": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts", "dev:scripts:updateRecentEpisodesTables": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts", "dev:seeds:categories": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/seeds/categories.ts", @@ -76,6 +77,7 @@ "scripts:refreshEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/refreshEpisodesMostRecentMaterializedView.ts", "scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts", "scripts:removeDeadEpisodes": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts", + "scripts:removePodcastsByIds": "ts-node -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts", "scripts:syncWithFeedUrlsCSVDump": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts", "scripts:updateRecentEpisodesTables": "ts-node -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts", "seeds:categories": "ts-node -r tsconfig-paths/register ./src/seeds/categories.ts", diff --git a/src/controllers/podcast.ts b/src/controllers/podcast.ts index 20d378e9..7ded994a 100644 --- a/src/controllers/podcast.ts +++ b/src/controllers/podcast.ts @@ -2,7 +2,14 @@ import { getRepository, In } from 'typeorm' import { config } from '~/config' import { getUserSubscribedPodcastIds } from '~/controllers/user' import { FeedUrl, Podcast, User } from '~/entities' -import { addOrderByToQuery, getManticoreOrderByColumnName, removeAllSpaces } from '~/lib/utility' +import { + _logEnd, + _logStart, + addOrderByToQuery, + getManticoreOrderByColumnName, + logPerformance, + removeAllSpaces +} from '~/lib/utility' import { validateSearchQueryString } from '~/lib/utility/validation' import { manticoreWildcardSpecialCharacters, searchApi } from '~/services/manticore' import { deleteNotification } from './notification' @@ -506,6 +513,21 @@ const getPodcastWebUrl = async ({ podcastGuid, podcastIndexId }: GetPodcastWebUr } } +const removePodcasts = async (ids: string[]) => { + logPerformance('removePodcasts', _logStart) + const repository = getRepository(Podcast) + for (const id of ids) { + try { + logPerformance(`removePodcast ${id}`, _logStart) + await repository.delete(id) + logPerformance(`removePodcast ${id}`, _logEnd) + } catch (error) { + console.log(`error removing podcast ${id}`, error) + } + } + logPerformance('removePodcasts', _logEnd) +} + export { findPodcastsByFeedUrls, getPodcast, @@ -519,5 +541,6 @@ export { subscribeToPodcast, toggleSubscribeToPodcast, updateHasPodcastIndexValueTags, - getPodcastWebUrl + getPodcastWebUrl, + removePodcasts } diff --git a/src/scripts/podcasts/removePodcastsByIds.ts b/src/scripts/podcasts/removePodcastsByIds.ts new file mode 100644 index 00000000..67b9323c --- /dev/null +++ b/src/scripts/podcasts/removePodcastsByIds.ts @@ -0,0 +1,25 @@ +import { removePodcasts } from '~/controllers/podcast' +import { connectToDb } from '~/lib/db' +;(async function () { + const removePodcastsByIds = async () => { + try { + const podcastIdsString = process.argv.length > 2 ? process.argv[2] : '' + + if (!podcastIdsString) { + console.log('Provide a comma delimited string of podcastIds as a parameter.') + return + } + + const podcastIds = podcastIdsString.split(',') + const connection = await connectToDb() + + if (connection && connection.isConnected) { + await removePodcasts(podcastIds) + } + } catch (error) { + console.log(error) + } + } + + removePodcastsByIds() +})() From ed645c1ac1342491d926a7961be6920d4c4e3e02 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Thu, 18 Apr 2024 16:05:06 -0500 Subject: [PATCH 2/4] Add generateDeadRowCSV script --- package.json | 3 +++ src/controllers/devtools/db.ts | 20 +++++++++++++++++++ .../devtools/db/generateDeadRowsCSV.ts | 17 ++++++++++++++++ yarn.lock | 18 +++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 src/controllers/devtools/db.ts create mode 100644 src/scripts/devtools/db/generateDeadRowsCSV.ts diff --git a/package.json b/package.json index f932ec9c..336566c3 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dev:scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts", "dev:scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts", "dev:scripts:backupDatabase": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts", + "dev:scripts:devtools:generateDeadRowsCSV": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts", "dev:scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts", "dev:scripts:hideDeadPodcasts": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts", "dev:scripts:parseFeedUrlsByPodcastIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts", @@ -63,6 +64,7 @@ "scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts", "scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts", "scripts:backupDatabase": "ts-node -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts", + "scripts:devtools:generateDeadRowsCSV": "ts-node -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts", "scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts", "scripts:hideDeadPodcasts": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts", "scripts:parseFeedUrlsByPodcastIds": "ts-node -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts", @@ -179,6 +181,7 @@ "isomorphic-fetch": "^3.0.0", "jest": "26.1.0", "joi": "14.3.1", + "json-2-csv": "^5.5.1", "koa": "2.11.0", "koa-bodyparser": "4.2.1", "koa-helmet": "5.2.0", diff --git a/src/controllers/devtools/db.ts b/src/controllers/devtools/db.ts new file mode 100644 index 00000000..1fa58096 --- /dev/null +++ b/src/controllers/devtools/db.ts @@ -0,0 +1,20 @@ +import { getConnection } from 'typeorm' +import { _logEnd, _logStart, logPerformance } from '~/lib/utility' +const json2csv = require('json-2-csv') + +const generateDeadRowsCSV = async () => { + logPerformance('generateDeadRowCSV', _logStart) + const em = await getConnection().createEntityManager() + const results = await em.query(` + SELECT n_live_tup, n_dead_tup, relname + FROM pg_stat_all_tables + ORDER BY n_dead_tup DESC; + `) + + const csv = json2csv.json2csv(results) + console.log('csv', csv) + + logPerformance('generateDeadRowCSV', _logEnd) +} + +export { generateDeadRowsCSV } diff --git a/src/scripts/devtools/db/generateDeadRowsCSV.ts b/src/scripts/devtools/db/generateDeadRowsCSV.ts new file mode 100644 index 00000000..44236a38 --- /dev/null +++ b/src/scripts/devtools/db/generateDeadRowsCSV.ts @@ -0,0 +1,17 @@ +import { generateDeadRowsCSV as generateDeadRowsCSVCont } from '~/controllers/devtools/db' +import { connectToDb } from '~/lib/db' +;(async function () { + const generateDeadRowCSV = async () => { + try { + const connection = await connectToDb() + + if (connection && connection.isConnected) { + await generateDeadRowsCSVCont() + } + } catch (error) { + console.log(error) + } + } + + generateDeadRowCSV() +})() diff --git a/yarn.lock b/yarn.lock index 6900e409..55f06d83 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3227,6 +3227,11 @@ dedent@^0.7.0: resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== +deeks@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/deeks/-/deeks-3.1.0.tgz#ecc47c600bcc6dbda656c6d86bbc96cbe126fd41" + integrity sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A== + deep-eql@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" @@ -3404,6 +3409,11 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" +doc-path@4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-4.1.1.tgz#5be8c1671877f6b719af5a6077c41904be8913ac" + integrity sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ== + docker-cli-js@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/docker-cli-js/-/docker-cli-js-2.9.0.tgz#1584461134cbb6beb548fed2c33de8f5593280bf" @@ -6276,6 +6286,14 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +json-2-csv@^5.5.1: + version "5.5.1" + resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-5.5.1.tgz#2ad4e69d4b2bb4590203948a36f11347a4d7aed9" + integrity sha512-KgAtAXTQopRwe90gh8SgjRSxgt9bUWbGAPMo9W0TZLA8SqiQH7khtagFfeEUjG3NBPwJu/+9uX5pMvunKaPvrQ== + dependencies: + deeks "3.1.0" + doc-path "4.1.1" + json-bigint@^0.3.0: version "0.3.1" resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-0.3.1.tgz#0c1729d679f580d550899d6a2226c228564afe60" From 626e0bd9e17c09d34b3fd8117d5de19bcf4e0b61 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 19 Apr 2024 00:06:44 -0500 Subject: [PATCH 3/4] Add followRedirect true to findHTTPS endpoint --- src/routes/tools.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/tools.ts b/src/routes/tools.ts index 95bfc927..a2cd8ca1 100644 --- a/src/routes/tools.ts +++ b/src/routes/tools.ts @@ -25,7 +25,7 @@ router.post('/findHTTPS', async (ctx) => { const extractHTTPSFromURL = async (url, tries) => { try { - const res = await request(url, { followRedirect: false, method: 'head' }) + const res = await request(url, { followRedirect: true, method: 'head' }) if (tries > 5) { throw new createError.NotFound('Secure URL for ' + url + ' was not found. Too many redirects') } else if (!res.location) { @@ -35,7 +35,7 @@ const extractHTTPSFromURL = async (url, tries) => { try { const attemptHttpsUrl = url.replace('http://', 'https://') // If no error is thrown,then assume it is a valid url. - await request(attemptHttpsUrl, { followRedirect: false, method: 'head' }) + await request(attemptHttpsUrl, { followRedirect: true, method: 'head' }) return attemptHttpsUrl } catch (error) { throw new createError.NotFound('Secure URL for ' + url + ' was not found.') From bd14fd41e112553d9f40ab80f687d2102383a374 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 19 Apr 2024 00:06:54 -0500 Subject: [PATCH 4/4] Bump to version 4.16.9 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 336566c3..1e3fdce8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "podverse-api", - "version": "4.16.8", + "version": "4.16.9", "description": "Data API, database migration scripts, and backend services for all Podverse models.", "contributors": [ "Mitch Downey"