diff --git a/migrations/0048_seasons.sql b/migrations/0048_seasons.sql new file mode 100644 index 00000000..dd1bc4de --- /dev/null +++ b/migrations/0048_seasons.sql @@ -0,0 +1,14 @@ +ALTER TABLE ONLY public."episodes" + ADD COLUMN "itunesEpisode" integer; + +ALTER TABLE ONLY public."episodes" + ADD COLUMN "itunesEpisodeType" varchar; + +ALTER TABLE ONLY public."episodes" + ADD COLUMN "itunesSeason" integer; + +CREATE INDEX CONCURRENTLY "episodes_itunesEpisode" ON "episodes" ("itunesEpisode"); + +CREATE INDEX CONCURRENTLY "episodes_itunesEpisodeType" ON "episodes" ("itunesEpisodeType"); + +CREATE INDEX CONCURRENTLY "episodes_itunesSeason" ON "episodes" ("itunesSeason"); diff --git a/migrations/0049_podcast_itunesFeedType_hasSeasons.sql b/migrations/0049_podcast_itunesFeedType_hasSeasons.sql new file mode 100644 index 00000000..452d952d --- /dev/null +++ b/migrations/0049_podcast_itunesFeedType_hasSeasons.sql @@ -0,0 +1,7 @@ +ALTER TABLE ONLY public."podcasts" + ADD COLUMN "itunesFeedType" varchar; + +CREATE INDEX CONCURRENTLY "podcasts_itunesFeedType" ON "podcasts" ("itunesFeedType"); + +ALTER TABLE ONLY public."podcasts" + ADD COLUMN "hasSeasons" boolean DEFAULT false NOT NULL; diff --git a/package.json b/package.json index 7b0f57e6..ba9476bc 100644 --- a/package.json +++ b/package.json @@ -196,7 +196,7 @@ "paypal-rest-sdk": "2.0.0-rc.2", "pg": "8.7.3", "podcast-partytime": "4.6.2", - "podverse-shared": "^4.12.10", + "podverse-shared": "^4.13.14", "reflect-metadata": "0.1.13", "request": "^2.88.2", "request-promise-native": "1.0.8", diff --git a/src/controllers/episode.ts b/src/controllers/episode.ts index 2956ffd8..74ecb937 100644 --- a/src/controllers/episode.ts +++ b/src/controllers/episode.ts @@ -7,6 +7,7 @@ import { validateSearchQueryString } from '~/lib/utility/validation' import { manticoreWildcardSpecialCharacters, searchApi } from '~/services/manticore' import { liveItemStatuses } from './liveItem' import { createMediaRef, updateMediaRef } from './mediaRef' +import { getPodcast } from './podcast' const createError = require('http-errors') const SqlString = require('sqlstring') const { superUserId } = config @@ -21,6 +22,8 @@ const relations = [ 'podcast.categories' ] +const maxResultsEpisodes = 5000 + const getEpisode = async (id) => { const repository = getRepository(Episode) const episode = await repository.findOne( @@ -118,6 +121,9 @@ const addSelectsToQueryBuilder = (qb) => { .addSelect('episode.imageUrl') .addSelect('episode.isExplicit') .addSelect('episode.isPublic') + .addSelect('episode.itunesEpisode') + .addSelect('episode.itunesEpisodeType') + .addSelect('episode.itunesSeason') .addSelect('episode.linkUrl') .addSelect('episode.mediaFilesize') .addSelect('episode.mediaType') @@ -317,6 +323,47 @@ const getEpisodesByPodcastId = async (query, qb, podcastIds) => { return handleGetEpisodesWithOrdering({ maxResults, qb, query, skip, sort, take }, allowRandom, shouldLimitCount) } +// When a podcast has seasons, we always return all the episodes, +// and in the order the podcaster intended. +// If the podcast has serial type, then return in chronological order +// instead of the default most-recent order. +const getEpisodesByPodcastIdWithSeasons = async ({ + searchTitle, + sincePubDate, + hasVideo, + itunesFeedType, + podcastId +}) => { + const includePodcast = false + const shouldUseEpisodesMostRecent = false + const liveItemStatus = null + const qb = generateEpisodeSelects( + includePodcast, + searchTitle, + sincePubDate, + hasVideo, + shouldUseEpisodesMostRecent, + liveItemStatus + ) + + const isSerial = itunesFeedType === 'serial' + const sort = isSerial ? 'oldest' : 'most-recent' + const seasonsQuery = { + podcastId, + maxResults: maxResultsEpisodes, + searchTitle, + sort + } + + const resultsArray = await getEpisodesByPodcastId(seasonsQuery, qb, [podcastId]) + resultsArray.push({ + hasSeasons: true, + isSerial + }) + + return resultsArray +} + const getEpisodesByPodcastIds = async (query) => { const { hasVideo, @@ -343,7 +390,19 @@ const getEpisodesByPodcastIds = async (query) => { ) if (podcastIds.length === 1) { - return getEpisodesByPodcastId(query, qb, podcastIds) + const id = podcastIds[0] + const podcast = await getPodcast(id) + if (podcast?.hasSeasons) { + return getEpisodesByPodcastIdWithSeasons({ + searchTitle, + sincePubDate, + hasVideo, + itunesFeedType: podcast.itunesFeedType, + podcastId + }) + } else { + return getEpisodesByPodcastId(query, qb, podcastIds) + } } qb.andWhere('episode.podcastId IN(:...podcastIds)', { podcastIds }) @@ -363,7 +422,7 @@ const handleGetEpisodesWithOrdering = async ( totalOverride? ) => { const { maxResults, skip, sort, take } = obj - const finalTake = maxResults ? 5000 : take + const finalTake = maxResults ? maxResultsEpisodes : take let { qb } = obj qb.offset(skip) diff --git a/src/controllers/podcast.ts b/src/controllers/podcast.ts index ce6e3f7c..38279fd6 100644 --- a/src/controllers/podcast.ts +++ b/src/controllers/podcast.ts @@ -200,10 +200,12 @@ const getPodcasts = async (query, countOverride?, isFromManticoreSearch?) => { .addSelect('podcast.feedLastUpdated') .addSelect('podcast.funding') .addSelect('podcast.hasLiveItem') + .addSelect('podcast.hasSeasons') .addSelect('podcast.hasVideo') .addSelect('podcast.hideDynamicAdsWarning') .addSelect('podcast.imageUrl') .addSelect('podcast.isExplicit') + .addSelect('podcast.itunesFeedType') .addSelect('podcast.lastEpisodePubDate') .addSelect('podcast.lastEpisodeTitle') .addSelect('podcast.lastFoundInPodcastIndex') diff --git a/src/controllers/userHistoryItem.ts b/src/controllers/userHistoryItem.ts index 1ce58e9b..ecb31e65 100644 --- a/src/controllers/userHistoryItem.ts +++ b/src/controllers/userHistoryItem.ts @@ -118,6 +118,9 @@ export const generateGetUserItemsQuery = (table, tableName, loggedInUserId) => { .addSelect('episode.funding', 'episodeFunding') .addSelect('episode.guid', 'episodeGuid') .addSelect('episode.imageUrl', 'episodeImageUrl') + .addSelect('episode.itunesEpisode', 'episodeItunesEpisode') + .addSelect('episode.itunesEpisodeType', 'episodeItunesEpisodeType') + .addSelect('episode.itunesSeason', 'episodeItunesSeason') .addSelect('episode.mediaType', 'episodeMediaType') .addSelect('episode.mediaUrl', 'episodeMediaUrl') .addSelect('episode.pubDate', 'episodePubDate') @@ -127,10 +130,12 @@ export const generateGetUserItemsQuery = (table, tableName, loggedInUserId) => { .addSelect('episode.transcript', 'episodeTranscript') .addSelect('episode.value', 'episodeValue') .addSelect('podcast.funding', 'podcastFunding') + .addSelect('podcast.hasSeasons', 'podcastHasSeasons') .addSelect('podcast.id', 'podcastId') .addSelect('podcast.imageUrl', 'podcastImageUrl') .addSelect('podcast.podcastIndexId', 'podcastPodcastIndexId') .addSelect('podcast.podcastGuid', 'podcastGuid') + .addSelect('podcast.itunesFeedType', 'podcastItunesFeedType') .addSelect('podcast.shrunkImageUrl', 'podcastShrunkImageUrl') .addSelect('podcast.title', 'podcastTitle') .addSelect('podcast.value', 'podcastValue') @@ -143,6 +148,9 @@ export const generateGetUserItemsQuery = (table, tableName, loggedInUserId) => { .addSelect('clipEpisode.funding', 'clipEpisodeFunding') .addSelect('clipEpisode.guid', 'clipEpisodeGuid') .addSelect('clipEpisode.imageUrl', 'clipEpisodeImageUrl') + .addSelect('clipEpisode.itunesEpisode', 'clipEpisodeItunesEpisode') + .addSelect('clipEpisode.itunesEpisodeType', 'clipEpisodeItunesEpisodeType') + .addSelect('clipEpisode.itunesSeason', 'clipEpisodeItunesSeason') .addSelect('clipEpisode.mediaType', 'clipEpisodeMediaType') .addSelect('clipEpisode.mediaUrl', 'clipEpisodeMediaUrl') .addSelect('clipEpisode.pubDate', 'clipEpisodePubDate') @@ -153,9 +161,11 @@ export const generateGetUserItemsQuery = (table, tableName, loggedInUserId) => { .addSelect('clipEpisode.value', 'clipEpisodeValue') .addSelect('clipPodcast.id', 'clipPodcastId') .addSelect('clipPodcast.funding', 'clipPodcastFunding') + .addSelect('clipPodcast.hasSeasons', 'clipPodcastHasSeasons') .addSelect('clipPodcast.imageUrl', 'clipPodcastImageUrl') - .addSelect('clipPodcast.podcastIndexId', 'clipPodcastIndexId') + .addSelect('clipPodcast.itunesFeedType', 'clipPodcastItunesFeedType') .addSelect('clipPodcast.podcastGuid', 'clipPodcastGuid') + .addSelect('clipPodcast.podcastIndexId', 'clipPodcastIndexId') .addSelect('clipPodcast.shrunkImageUrl', 'clipPodcastShrunkImageUrl') .addSelect('clipPodcast.title', 'clipPodcastTitle') .addSelect('clipPodcast.value', 'clipPodcastValue') diff --git a/src/entities/episode.ts b/src/entities/episode.ts index 4a91a92d..a244dfb1 100644 --- a/src/entities/episode.ts +++ b/src/entities/episode.ts @@ -107,6 +107,18 @@ export class Episode { @Column({ default: false }) isPublic: boolean + @Index() + @Column({ nullable: true }) + itunesEpisode?: number + + @Index() + @Column({ nullable: true }) + itunesEpisodeType?: string + + @Index() + @Column({ nullable: true }) + itunesSeason?: number + @ValidateIf((a) => a.linkUrl != null) @IsUrl() @Column({ nullable: true }) diff --git a/src/entities/episodes_most_recent.ts b/src/entities/episodes_most_recent.ts index 6efaaeda..e6d635ee 100644 --- a/src/entities/episodes_most_recent.ts +++ b/src/entities/episodes_most_recent.ts @@ -37,7 +37,13 @@ export class EpisodeMostRecent { credentialsRequired?: boolean @ViewColumn() - description?: string + itunesEpisode?: number + + @ViewColumn() + itunesEpisodeType?: string + + @ViewColumn() + itunesSeason?: number @ViewColumn() duration?: number diff --git a/src/entities/podcast.ts b/src/entities/podcast.ts index 0f68d678..0fc58022 100644 --- a/src/entities/podcast.ts +++ b/src/entities/podcast.ts @@ -43,6 +43,8 @@ type ValueRecipient = { type: string } +export const podcastItunesTypeDefaultValue = 'episodic' + @Index(['hasVideo', 'pastAllTimeTotalUniquePageviews']) @Index(['hasVideo', 'pastHourTotalUniquePageviews']) @Index(['hasVideo', 'pastDayTotalUniquePageviews']) @@ -110,6 +112,9 @@ export class Podcast { @Column({ default: false }) hasPodcastIndexValueTag?: boolean + @Column({ default: false }) + hasSeasons: boolean + @Column({ default: false }) hasVideo: boolean @@ -128,6 +133,9 @@ export class Podcast { @Column({ default: false }) isPublic: boolean + @Column({ default: podcastItunesTypeDefaultValue }) + itunesFeedType: string + @Column({ nullable: true }) language?: string diff --git a/src/services/parser.ts b/src/services/parser.ts index 08d2bfea..1c71369f 100644 --- a/src/services/parser.ts +++ b/src/services/parser.ts @@ -13,6 +13,7 @@ import { config } from '~/config' import { updateSoundBites } from '~/controllers/mediaRef' import { getPodcast } from '~/controllers/podcast' import { Author, Category, Episode, FeedUrl, LiveItem, Podcast } from '~/entities' +import { podcastItunesTypeDefaultValue } from '~/entities/podcast' import type { Value } from '~/entities/podcast' import { _logEnd, @@ -193,6 +194,7 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = generator: feed.generator, guid: feed.guid, imageURL: feed.itunesImage || feed.image?.url, + itunesType: feed.itunesType || podcastItunesTypeDefaultValue, language: feed.language, lastBuildDate: feed.lastBuildDate, link: feed.link, @@ -222,6 +224,9 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = // funding: Array.isArray(episode.podcastFunding) ? episode.podcastFunding?.map((f) => fundingCompat(f)) : [], guid: episode.guid, imageURL: episode.image, + itunesEpisode: episode.itunesEpisode, + itunesEpisodeType: episode.itunesEpisodeType, + itunesSeason: episode.itunesSeason, link: episode.link, pubDate: episode.pubDate, socialInteraction: episode.podcastSocialInteraction ?? [], @@ -372,6 +377,7 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = podcast.isExplicit = meta.explicit podcast.isPublic = true + podcast.itunesFeedType = meta.itunesType podcast.language = meta.language /* @@ -415,6 +421,7 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = logPerformance('findOrGenerateParsedLiveItems', _logEnd) podcast.hasLiveItem = hasLiveItem + podcast.hasSeasons = episodesResults.hasSeasons || liveItemsResults.hasSeasons podcast.hasVideo = episodesResults.hasVideo || liveItemsResults.hasVideo newEpisodes = episodesResults.newEpisodes @@ -942,6 +949,11 @@ const assignParsedEpisodeData = async ( episode.guid = parsedEpisode.guid || parsedEpisode.enclosure.url episode.imageUrl = parsedEpisode.imageURL episode.isExplicit = parsedEpisode.explicit + + episode.itunesEpisode = parsedEpisode.itunesEpisode + episode.itunesEpisodeType = parsedEpisode.itunesEpisodeType + episode.itunesSeason = parsedEpisode.itunesSeason + episode.linkUrl = parsedEpisode.link episode.mediaType = parsedEpisode.enclosure.type @@ -1043,6 +1055,7 @@ const findOrGenerateParsedLiveItems = async (parsedLiveItems, podcast, pvEpisode /* If a feed has more video episodes than audio episodes, mark it as a hasVideo podcast. */ let videoCount = 0 let audioCount = 0 + let hasSeasons = false // If episode is already saved, then merge the matching episode found in // the parsed object with what is already saved. @@ -1058,6 +1071,10 @@ const findOrGenerateParsedLiveItems = async (parsedLiveItems, podcast, pvEpisode pvEpisodesValueTagsByGuid ) + if (parsedLiveItem.itunesSeason) { + hasSeasons = true + } + if (parsedLiveItem.mediaType && checkIfVideoMediaType(parsedLiveItem.mediaType)) { videoCount++ } else { @@ -1075,6 +1092,10 @@ const findOrGenerateParsedLiveItems = async (parsedLiveItems, podcast, pvEpisode let episode = new Episode() as ExtendedEpisode episode = await assignParsedEpisodeData(episode, newParsedLiveItem, podcast, pvEpisodesValueTagsByGuid) + if (newParsedLiveItem.itunesSeason) { + hasSeasons = true + } + if (newParsedLiveItem.mediaType && checkIfVideoMediaType(newParsedLiveItem.mediaType)) { videoCount++ } else { @@ -1116,6 +1137,7 @@ const findOrGenerateParsedLiveItems = async (parsedLiveItems, podcast, pvEpisode return { newLiveItems, updatedSavedLiveItems, + hasSeasons, hasVideo: videoCount > audioCount, liveItemNotificationsData } @@ -1186,6 +1208,7 @@ const findOrGenerateParsedEpisodes = async (parsedEpisodes, podcast, pvEpisodesV /* If a feed has more video episodes than audio episodes, mark it as a hasVideo podcast. */ let videoCount = 0 let audioCount = 0 + let hasSeasons = false // If episode is already saved, then merge the matching episode found in // the parsed object with what is already saved. @@ -1193,6 +1216,10 @@ const findOrGenerateParsedEpisodes = async (parsedEpisodes, podcast, pvEpisodesV const parsedEpisode = validParsedEpisodes.find((x) => x.guid === existingEpisode.guid) existingEpisode = await assignParsedEpisodeData(existingEpisode, parsedEpisode, podcast, pvEpisodesValueTagsByGuid) + if (existingEpisode.itunesSeason) { + hasSeasons = true + } + if (existingEpisode.mediaType && checkIfVideoMediaType(existingEpisode.mediaType)) { videoCount++ } else { @@ -1210,6 +1237,10 @@ const findOrGenerateParsedEpisodes = async (parsedEpisodes, podcast, pvEpisodesV let episode = new Episode() as ExtendedEpisode episode = await assignParsedEpisodeData(episode, newParsedEpisode, podcast, pvEpisodesValueTagsByGuid) + if (newParsedEpisode.itunesSeason) { + hasSeasons = true + } + if (newParsedEpisode.mediaType && checkIfVideoMediaType(newParsedEpisode.mediaType)) { videoCount++ } else { @@ -1224,6 +1255,7 @@ const findOrGenerateParsedEpisodes = async (parsedEpisodes, podcast, pvEpisodesV return { newEpisodes, updatedSavedEpisodes, + hasSeasons, hasVideo: videoCount > audioCount } } diff --git a/yarn.lock b/yarn.lock index 818a62d8..9d39246c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8079,10 +8079,10 @@ podcast-partytime@4.6.2: ramda "^0.27.1" tiny-invariant "^1.2.0" -podverse-shared@^4.12.10: - version "4.12.10" - resolved "https://registry.yarnpkg.com/podverse-shared/-/podverse-shared-4.12.10.tgz#53c36b9d432125ffac5bdf8028ff14208c193151" - integrity sha512-JeVqt7pPxyeWEx0afCMo74FuYKrvA9Is4AgapcXEReFc9CRXADAqWOrF2qqxsXb/IkdLOrDGCMaopnyOiLioWQ== +podverse-shared@^4.13.14: + version "4.13.14" + resolved "https://registry.yarnpkg.com/podverse-shared/-/podverse-shared-4.13.14.tgz#7ef2ce8597d4e42374f10d3432ba2aa1ef7e8688" + integrity sha512-1ZW0ooZIz98tMSwtavgU7Kig5z9soaMkDbni/nPilfgJ5rcqIkyi1O9OIwcb1xRExBMO2U2vrQqANpAjbt0l0Q== dependencies: he "^1.2.0" html-entities "^2.3.2"