From 761739d2f8c8bb6becf0f0644fb89e302ea4de28 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 13 Mar 2024 11:15:12 +0100 Subject: [PATCH 1/4] feat(algolia): index explorer views to Algolia --- Makefile | 1 + baker/algolia/configureAlgolia.ts | 16 ++ baker/algolia/indexExplorerViewsToAlgolia.ts | 199 +++++++++++++++++++ baker/algolia/indexExplorersToAlgolia.ts | 2 +- explorer/ExplorerDecisionMatrix.ts | 12 +- site/search/searchTypes.ts | 2 + 6 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 baker/algolia/indexExplorerViewsToAlgolia.ts diff --git a/Makefile b/Makefile index aa073fab758..b42a0822298 100644 --- a/Makefile +++ b/Makefile @@ -358,6 +358,7 @@ reindex: itsJustJavascript node --enable-source-maps itsJustJavascript/baker/algolia/indexToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js + node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsToAlgolia.js clean: rm -rf node_modules itsJustJavascript diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index baebc8a8803..e6b6a02c409 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -131,6 +131,22 @@ export const configureAlgolia = async () => { disableTypoToleranceOnAttributes: ["text"], }) + const explorerViewsIndex = client.initIndex( + getIndexName(SearchIndexName.ExplorerViews) + ) + + await explorerViewsIndex.setSettings({ + ...baseSettings, + searchableAttributes: [ + "unordered(viewTitle)", + "unordered(viewSettings)", + ], + customRanking: ["desc(score)", "asc(viewIndexWithinExplorer)"], + attributeForDistinct: "viewTitleAndExplorerSlug", + distinct: true, + minWordSizefor1Typo: 6, + }) + const synonyms = [ ["kids", "children"], ["pork", "pigmeat"], diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts new file mode 100644 index 00000000000..630283c7c41 --- /dev/null +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -0,0 +1,199 @@ +import * as db from "../../db/db.js" +import { ExplorerBlockGraphers } from "./indexExplorersToAlgolia.js" +import { DecisionMatrix } from "../../explorer/ExplorerDecisionMatrix.js" +import { tsvFormat } from "d3-dsv" +import { + ExplorerChoiceParams, + ExplorerControlType, +} from "../../explorer/ExplorerConstants.js" +import { GridBoolean } from "../../gridLang/GridLangConstants.js" +import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" +import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" +import { getAlgoliaClient } from "./configureAlgolia.js" +import { getIndexName } from "../../site/search/searchClient.js" +import { SearchIndexName } from "../../site/search/searchTypes.js" + +interface ExplorerViewEntry { + viewTitle: string + viewSubtitle: string + viewSettings: string[] + viewQueryParams: string + + viewGrapherId?: number + + // Potential ranking criteria + viewIndexWithinExplorer: number + titleLength: number + numNonDefaultSettings: number + // viewViews_7d: number +} + +interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { + explorerSlug: string + explorerTitle: string + explorerViews_7d: number + viewTitleAndExplorerSlug: string // used for deduplication: `viewTitle | explorerSlug` + + score: number + + objectID?: string +} + +const explorerChoiceToViewSettings = ( + choices: ExplorerChoiceParams, + decisionMatrix: DecisionMatrix +): string[] => { + return Object.entries(choices).map(([choiceName, choiceValue]) => { + const choiceControlType = + decisionMatrix.choiceNameToControlTypeMap.get(choiceName) + if (choiceControlType === ExplorerControlType.Checkbox) + return choiceValue === GridBoolean.true ? choiceName : "" + else return choiceValue + }) +} + +const getExplorerViewRecordsForExplorerSlug = async ( + trx: db.KnexReadonlyTransaction, + slug: string +): Promise => { + const explorerConfig = await trx + .table("explorers") + .select("config") + .where({ slug }) + .first() + .then((row) => JSON.parse(row.config) as any) + + const explorerGrapherBlock: ExplorerBlockGraphers = + explorerConfig.blocks.filter( + (block: any) => block.type === "graphers" + )[0] as ExplorerBlockGraphers + + if (explorerGrapherBlock === undefined) + throw new Error(`Explorer ${slug} has no grapher block`) + + // TODO: Maybe make DecisionMatrix accept JSON directly + const tsv = tsvFormat(explorerGrapherBlock.block) + const explorerDecisionMatrix = new DecisionMatrix(tsv) + + console.log( + `Processing explorer ${slug} (${explorerDecisionMatrix.numRows} rows)` + ) + + const defaultSettings = explorerDecisionMatrix.defaultSettings + + const records = explorerDecisionMatrix + .allDecisionsAsQueryParams() + .map((choice, i) => { + explorerDecisionMatrix.setValuesFromChoiceParams(choice) + + // Check which choices are non-default, i.e. are not the first available option in a dropdown/radio + const nonDefaultSettings = Object.entries( + explorerDecisionMatrix.availableChoiceOptions + ).filter(([choiceName, choiceOptions]) => { + // Keep only choices which are not the default, which is: + // - either the options marked as `default` in the decision matrix + // - or the first available option in the decision matrix + return ( + choiceOptions.length > 1 && + !(defaultSettings[choiceName] !== undefined + ? defaultSettings[choiceName] === choice[choiceName] + : choice[choiceName] === choiceOptions[0]) + ) + }) + + // TODO: Handle grapherId and fetch title/subtitle + // TODO: Handle indicator-based explorers + + const record: ExplorerViewEntry = { + viewTitle: explorerDecisionMatrix.selectedRow.title, + viewSubtitle: explorerDecisionMatrix.selectedRow.subtitle, + viewSettings: explorerChoiceToViewSettings( + choice, + explorerDecisionMatrix + ), + viewGrapherId: explorerDecisionMatrix.selectedRow.grapherId, + viewQueryParams: explorerDecisionMatrix.toString(), + + viewIndexWithinExplorer: i, + titleLength: explorerDecisionMatrix.selectedRow.title?.length, + numNonDefaultSettings: nonDefaultSettings.length, + } + return record + }) + + return records +} + +const getExplorerViewRecords = async ( + trx: db.KnexReadonlyTransaction +): Promise => { + const publishedExplorers = Object.values( + await db.getPublishedExplorersBySlug(trx) + ) + + const pageviews = await getAnalyticsPageviewsByUrlObj(trx) + + let records = [] as ExplorerViewEntryWithExplorerInfo[] + for (const explorerInfo of publishedExplorers) { + const explorerViewRecords = await getExplorerViewRecordsForExplorerSlug( + trx, + explorerInfo.slug + ) + + const explorerPageviews = + pageviews[`/explorers/${explorerInfo.slug}`]?.views_7d ?? 0 + records = records.concat( + explorerViewRecords.map( + (record, i): ExplorerViewEntryWithExplorerInfo => ({ + ...record, + explorerSlug: explorerInfo.slug, + explorerTitle: explorerInfo.title, + explorerViews_7d: explorerPageviews, + viewTitleAndExplorerSlug: `${record.viewTitle} | ${explorerInfo.slug}`, + // Scoring function + score: + explorerPageviews * 10 - + record.numNonDefaultSettings * 50 - + record.titleLength, + + objectID: `${explorerInfo.slug}-${i}`, + }) + ) + ) + } + + return records +} + +const indexExplorerViewsToAlgolia = async () => { + if (!ALGOLIA_INDEXING) return + + const client = getAlgoliaClient() + if (!client) { + console.error( + `Failed indexing explorer views (Algolia client not initialized)` + ) + return + } + + try { + const index = client.initIndex( + getIndexName(SearchIndexName.ExplorerViews) + ) + + const records = await db.knexReadonlyTransaction( + getExplorerViewRecords, + db.TransactionCloseMode.Close + ) + await index.replaceAllObjects(records) + } catch (e) { + console.log("Error indexing explorer views to Algolia:", e) + } +} + +process.on("unhandledRejection", (e) => { + console.error(e) + process.exit(1) +}) + +void indexExplorerViewsToAlgolia() diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts index 2ac7580a1fd..9085cf1dab9 100644 --- a/baker/algolia/indexExplorersToAlgolia.ts +++ b/baker/algolia/indexExplorersToAlgolia.ts @@ -22,7 +22,7 @@ type ExplorerBlockColumns = { block: { name: string; additionalInfo?: string }[] } -type ExplorerBlockGraphers = { +export type ExplorerBlockGraphers = { type: "graphers" block: { title?: string diff --git a/explorer/ExplorerDecisionMatrix.ts b/explorer/ExplorerDecisionMatrix.ts index 0b0ff8b802e..2f2f454df5b 100644 --- a/explorer/ExplorerDecisionMatrix.ts +++ b/explorer/ExplorerDecisionMatrix.ts @@ -86,7 +86,7 @@ export class DecisionMatrix { table: CoreTable @observable currentParams: ExplorerChoiceParams = {} constructor(delimited: string, hash = "") { - this.choices = makeChoicesMap(delimited) + this.choiceNameToControlTypeMap = makeChoicesMap(delimited) this.table = new CoreTable(parseDelimited(dropColumnTypes(delimited)), [ // todo: remove col def? { @@ -141,7 +141,7 @@ export class DecisionMatrix { ) } - private choices: Map + choiceNameToControlTypeMap: Map hash: string toConstrainedOptions(): ExplorerChoiceParams { @@ -243,7 +243,7 @@ export class DecisionMatrix { } @computed private get choiceNames(): ChoiceName[] { - return Array.from(this.choices.keys()) + return Array.from(this.choiceNameToControlTypeMap.keys()) } @computed private get allChoiceOptions(): ChoiceMap { @@ -256,7 +256,7 @@ export class DecisionMatrix { return choiceMap } - @computed private get availableChoiceOptions(): ChoiceMap { + @computed get availableChoiceOptions(): ChoiceMap { const result: ChoiceMap = {} this.choiceNames.forEach((choiceName) => { result[choiceName] = this.allChoiceOptions[choiceName].filter( @@ -317,7 +317,7 @@ export class DecisionMatrix { } // The first row with defaultView column value of "true" determines the default view to use - private get defaultSettings() { + get defaultSettings() { const hits = this.rowsWith({ [GrapherGrammar.defaultView.keyword]: "true", }) @@ -373,7 +373,7 @@ export class DecisionMatrix { constrainedOptions ) ) - const type = this.choices.get(title)! + const type = this.choiceNameToControlTypeMap.get(title)! return { title, diff --git a/site/search/searchTypes.ts b/site/search/searchTypes.ts index bb23b325138..1491bb6c190 100644 --- a/site/search/searchTypes.ts +++ b/site/search/searchTypes.ts @@ -68,6 +68,7 @@ export type IChartHit = Hit & ChartRecord export enum SearchIndexName { Explorers = "explorers", + ExplorerViews = "explorer-views", Charts = "charts", Pages = "pages", } @@ -85,4 +86,5 @@ export const indexNameToSubdirectoryMap: Record = { [SearchIndexName.Pages]: "", [SearchIndexName.Charts]: "/grapher", [SearchIndexName.Explorers]: "/explorers", + [SearchIndexName.ExplorerViews]: "/explorers", } From 7ad2723671da55ce8c8736864390eff4de59054d Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 14 Mar 2024 18:44:37 +0100 Subject: [PATCH 2/4] enhance(algolia): explorer views based on grapherIds are handled --- baker/algolia/indexExplorerViewsToAlgolia.ts | 45 ++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index 630283c7c41..e9c9f89b3d5 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -12,6 +12,7 @@ import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" import { getAlgoliaClient } from "./configureAlgolia.js" import { getIndexName } from "../../site/search/searchClient.js" import { SearchIndexName } from "../../site/search/searchTypes.js" +import { keyBy } from "lodash" interface ExplorerViewEntry { viewTitle: string @@ -39,6 +40,9 @@ interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { objectID?: string } +// Creates a search-ready string from a choice. +// Special handling is pretty much only necessary for checkboxes: If they are not ticked, then their name is not included. +// Imagine a "Per capita" checkbox, for example. If it's not ticked, then we don't want searches for "per capita" to wrongfully match it. const explorerChoiceToViewSettings = ( choices: ExplorerChoiceParams, decisionMatrix: DecisionMatrix @@ -101,9 +105,6 @@ const getExplorerViewRecordsForExplorerSlug = async ( ) }) - // TODO: Handle grapherId and fetch title/subtitle - // TODO: Handle indicator-based explorers - const record: ExplorerViewEntry = { viewTitle: explorerDecisionMatrix.selectedRow.title, viewSubtitle: explorerDecisionMatrix.selectedRow.subtitle, @@ -121,6 +122,44 @@ const getExplorerViewRecordsForExplorerSlug = async ( return record }) + // Enrich `grapherId`-powered views with title/subtitle + const grapherIds = records + .filter((record) => record.viewGrapherId !== undefined) + .map((record) => record.viewGrapherId as number) + + if (grapherIds.length) { + console.log( + `Fetching grapher info from ${grapherIds.length} graphers for explorer ${slug}` + ) + const grapherIdToTitle = await trx + .table("charts") + .select( + "id", + trx.raw("config->>'$.title' as title"), + trx.raw("config->>'$.subtitle' as subtitle") + ) + .whereIn("id", grapherIds) + .andWhereRaw("config->>'$.isPublished' = 'true'") + .then((rows) => keyBy(rows, "id")) + + for (const record of records) { + if (record.viewGrapherId !== undefined) { + const grapherInfo = grapherIdToTitle[record.viewGrapherId] + if (grapherInfo === undefined) { + console.warn( + `Grapher id ${record.viewGrapherId} not found for explorer ${slug}` + ) + continue + } + record.viewTitle = grapherInfo.title + record.viewSubtitle = grapherInfo.subtitle + record.titleLength = grapherInfo.title?.length + } + } + } + + // TODO: Handle indicator-based explorers + return records } From e62157f13be877f5fe391c79b3daf7e8157049c7 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Mon, 18 Mar 2024 16:42:49 +0100 Subject: [PATCH 3/4] feat(search): surface explorer views in autocomplete --- site/search/Autocomplete.tsx | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/site/search/Autocomplete.tsx b/site/search/Autocomplete.tsx index 6a5ae89cd91..8185f1a26b4 100644 --- a/site/search/Autocomplete.tsx +++ b/site/search/Autocomplete.tsx @@ -73,7 +73,12 @@ const getItemUrl: AutocompleteSource["getItemUrl"] = ({ item }) => const prependSubdirectoryToAlgoliaItemUrl = (item: BaseItem): string => { const indexName = parseIndexName(item.__autocomplete_indexName as string) const subdirectory = indexNameToSubdirectoryMap[indexName] - return `${subdirectory}/${item.slug}` + switch (indexName) { + case SearchIndexName.ExplorerViews: + return `${subdirectory}/${item.explorerSlug}${item.viewQueryParams}` + default: + return `${subdirectory}/${item.slug}` + } } const FeaturedSearchesSource: AutocompleteSource = { @@ -133,6 +138,14 @@ const AlgoliaSource: AutocompleteSource = { distinct: true, }, }, + { + indexName: getIndexName(SearchIndexName.ExplorerViews), + query, + params: { + hitsPerPage: 1, + distinct: true, + }, + }, { indexName: getIndexName(SearchIndexName.Explorers), query, @@ -152,11 +165,20 @@ const AlgoliaSource: AutocompleteSource = { item.__autocomplete_indexName as string ) const indexLabel = - index === SearchIndexName.Charts - ? "Chart" - : index === SearchIndexName.Explorers - ? "Explorer" - : pageTypeDisplayNames[item.type as PageType] + index === SearchIndexName.Charts ? ( + "Chart" + ) : index === SearchIndexName.Explorers ? ( + "Explorer" + ) : index === SearchIndexName.ExplorerViews ? ( + <> + in {item.explorerTitle} Data Explorer + + ) : ( + pageTypeDisplayNames[item.type as PageType] + ) + + const mainAttribute = + index === SearchIndexName.ExplorerViews ? "viewTitle" : "title" return (
= { From 3c15ba9440dd7f9f21b5e97ab2f2f9d13617dbbe Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Mon, 18 Mar 2024 19:16:12 +0100 Subject: [PATCH 4/4] feat(search): surface explorer views in search results --- site/search/Search.scss | 21 ++++++++++++-- site/search/SearchPanel.tsx | 55 +++++++++++++++++++++++++++++++++++++ site/search/searchTypes.ts | 8 ++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/site/search/Search.scss b/site/search/Search.scss index 1c2f1454fb9..9751d6c9f5e 100644 --- a/site/search/Search.scss +++ b/site/search/Search.scss @@ -193,6 +193,7 @@ .search-results__pages-list, .search-results__explorers-list, +.search-results__explorer-views-list, .search-results__charts-list { gap: var(--grid-gap); @include sm-only { @@ -227,7 +228,8 @@ display: block; } -.search-results__explorer-hit a { +.search-results__explorer-hit a, +.search-results__explorer-view-hit a { background-color: $blue-10; height: 100%; padding: 24px; @@ -326,6 +328,7 @@ .search-results__pages, .search-results__explorers, +.search-results__explorer-views, .search-results__charts { display: none; } @@ -333,6 +336,7 @@ .search-results[data-active-filter="all"] { .search-results__pages, .search-results__explorers, + .search-results__explorer-views, .search-results__charts { // both needed for .search-results__show-more-container absolute-positioning display: inline-block; @@ -377,10 +381,21 @@ } } -.search-results[data-active-filter="explorers"] .search-results__explorers { +.search-results__explorer-view-hit { + display: none; + + &:nth-child(-n + 4) { + display: inline; + } +} + +.search-results[data-active-filter="explorers"] .search-results__explorers, +.search-results[data-active-filter="explorers"] + .search-results__explorer-views { display: inline; - .search-results__explorer-hit { + .search-results__explorer-hit, + .search-results__explorer-view-hit { display: inline; } } diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index 1482fe17ec5..a03b0640404 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -35,6 +35,7 @@ import { searchCategoryFilters, IPageHit, pageTypeDisplayNames, + IExplorerViewHit, } from "./searchTypes.js" import { EXPLORERS_ROUTE_FOLDER } from "../../explorer/ExplorerConstants.js" import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" @@ -121,6 +122,23 @@ function ChartHit({ hit }: { hit: IChartHit }) { ) } +function ExplorerViewHit({ hit }: { hit: IExplorerViewHit }) { + return ( + +

{hit.viewTitle}

+ + in {hit.explorerTitle} Data Explorer + + {/* Explorer subtitles are mostly useless at the moment, so we're only showing titles */} + + ) +} + function ExplorerHit({ hit }: { hit: IExplorerHit }) { return ( get(results, ["results", "hits", "length"], 0) ) + hitsLengthByIndexName[getIndexName("all")] = Object.values( hitsLengthByIndexName ).reduce((a: number, b: number) => a + b, 0) + hitsLengthByIndexName[getIndexName(SearchIndexName.Explorers)] = + hitsLengthByIndexName[getIndexName(SearchIndexName.Explorers)] + + hitsLengthByIndexName[getIndexName(SearchIndexName.ExplorerViews)] + return (
    { + + + +
    +
    +

    + Data Explorer views +

    + +
    + +
    +     +     +export type IExplorerViewHit = Hit & { + objectID: string + explorerSlug: string + viewTitle: string + explorerTitle: string + viewQueryParams: string +} + export type IExplorerHit = Hit & { objectID: string slug: string