From 6be2870974c5ddf94c2bd4b0ceb2179e1898abb2 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 16 Apr 2024 08:56:04 +0200 Subject: [PATCH 1/5] feat(search): surface explorer views in search results (#3429) * feat(search): surface explorer views in autocomplete * feat(search): surface explorer views in search results * refactor(search): don't display results from `explorers` index * feat(search): card styles for explorer views * refactor(search): clean up explorer card code * enhance(search): explorer card styles * enhance(search): compute number of distinct explorers * feat(search): basic explorer cards * enhance(search): mobile styles for explorer cards * enhance(search): "Explore all indicators" on mobile * enhance(search): margin below subtitles * enhance(search): show charts above explorers * enhance(search): show 2 explorer results * enhance(search): add analytics tracking * enhance(search): stop querying the `Explorers` index * enhance(search): track clicks on mobile, too * enhance(search): remove detailed explorer info from Autocomplete * fix(algolia): send analytics events with index-specific `queryID` --- site/SiteAnalytics.ts | 6 + site/search/Autocomplete.scss | 9 +- site/search/Autocomplete.tsx | 16 +- site/search/Search.scss | 152 ++++++++++++------- site/search/SearchPanel.tsx | 265 ++++++++++++++++++++++++++++------ site/search/searchClient.ts | 7 +- site/search/searchTypes.ts | 18 ++- 7 files changed, 370 insertions(+), 103 deletions(-) diff --git a/site/SiteAnalytics.ts b/site/SiteAnalytics.ts index 9af87e4e533..80d0ce31c07 100644 --- a/site/SiteAnalytics.ts +++ b/site/SiteAnalytics.ts @@ -30,11 +30,15 @@ export class SiteAnalytics extends GrapherAnalytics { position, url, positionInSection, + cardPosition, + positionWithinCard, filter, }: { query: string position: string positionInSection: string + cardPosition?: string + positionWithinCard?: string url: string filter: SearchCategoryFilter }) { @@ -45,6 +49,8 @@ export class SiteAnalytics extends GrapherAnalytics { query, position, positionInSection, + cardPosition, + positionWithinCard, filter, }), eventTarget: url, diff --git a/site/search/Autocomplete.scss b/site/search/Autocomplete.scss index f8cff8723c0..42827f718c1 100644 --- a/site/search/Autocomplete.scss +++ b/site/search/Autocomplete.scss @@ -215,9 +215,14 @@ section[data-autocomplete-source-id="recentSearchesPlugin"] } } -section[data-autocomplete-source-id="autocomplete"] +section[data-autocomplete-source-id="autocomplete"] { + .aa-ItemWrapper { + text-wrap: pretty; + } + .aa-ItemWrapper__contentType { - color: $blue-50; + color: $blue-50; + } } section[data-autocomplete-source-id="runSearch"] { diff --git a/site/search/Autocomplete.tsx b/site/search/Autocomplete.tsx index 9fe5271441d..a8ba7f01711 100644 --- a/site/search/Autocomplete.tsx +++ b/site/search/Autocomplete.tsx @@ -76,7 +76,12 @@ const getItemUrl: AutocompleteSource["getItemUrl"] = ({ item }) => const prependSubdirectoryToAlgoliaItemUrl = (item: BaseItem): string => { const indexName = parseIndexName(item.__autocomplete_indexName as string) const subdirectory = indexNameToSubdirectoryMap[indexName] - return `${subdirectory}/${item.slug}` + switch (indexName) { + case SearchIndexName.ExplorerViews: + return `${subdirectory}/${item.explorerSlug}${item.viewQueryParams}` + default: + return `${subdirectory}/${item.slug}` + } } const FeaturedSearchesSource: AutocompleteSource = { @@ -142,7 +147,7 @@ const AlgoliaSource: AutocompleteSource = { }, }, { - indexName: getIndexName(SearchIndexName.Explorers), + indexName: getIndexName(SearchIndexName.ExplorerViews), query, params: { hitsPerPage: 1, @@ -162,10 +167,13 @@ const AlgoliaSource: AutocompleteSource = { const indexLabel = index === SearchIndexName.Charts ? "Chart" - : index === SearchIndexName.Explorers + : index === SearchIndexName.ExplorerViews ? "Explorer" : pageTypeDisplayNames[item.type as PageType] + const mainAttribute = + index === SearchIndexName.ExplorerViews ? "viewTitle" : "title" + return (
= { diff --git a/site/search/Search.scss b/site/search/Search.scss index 1c2f1454fb9..6d68718a7ed 100644 --- a/site/search/Search.scss +++ b/site/search/Search.scss @@ -192,7 +192,7 @@ } .search-results__pages-list, -.search-results__explorers-list, +.search-results__explorer-list, .search-results__charts-list { gap: var(--grid-gap); @include sm-only { @@ -227,16 +227,11 @@ display: block; } -.search-results__explorer-hit a { +.search-results__explorer-hit { background-color: $blue-10; height: 100%; padding: 24px; display: block; - transition: background-color 0.1s; - - &:hover { - background-color: $blue-20; - } h4 { margin: 0; @@ -244,7 +239,86 @@ } p { - color: $blue-60; + color: $blue-50; + font-weight: 400; + } + + .search-results__explorer-hit-link-mobile { + @include owid-link-60; + margin-top: 16px; + display: block; + } + + .search-results__explorer-hit-header { + display: flex; + justify-content: space-between; + + .search-results__explorer-hit-title-container { + min-width: 0; + } + + .search-results__explorer-hit-title { + display: inline; + color: $blue-90; + margin: 0 8px 0 0; + } + + .search-results__explorer-hit-subtitle { + margin: 0 8px 0 0; + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; + } + + .search-results__explorer-hit-link { + @include owid-link-60; + margin-top: 0; + flex: 1 0 auto; + text-align: right; + } + } + + .search-results__explorer-views-list { + margin-left: 20px; + margin-top: 5px; + list-style: none; + gap: 16px; + + .search-results__explorer-view { + overflow: hidden; + + .search-results__explorer-view-title-container { + color: $blue-90; + text-decoration: underline; + @include owid-link-90; + + svg { + font-size: 0.75em; + margin-left: 6px; + } + } + + .search-results__explorer-view-subtitle { + margin-top: 0; + margin-bottom: 0; + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; + } + + @include sm-only { + // *If on "All" tab on small screens*, only show the first two views on mobile, and hide view subtitles + @at-root .search-results[data-active-filter="all"] & { + &:nth-child(n + 3) { + display: none; + } + + .search-results__explorer-view-subtitle { + display: none; + } + } + } + } } } @@ -330,57 +404,35 @@ display: none; } -.search-results[data-active-filter="all"] { - .search-results__pages, - .search-results__explorers, - .search-results__charts { - // both needed for .search-results__show-more-container absolute-positioning - display: inline-block; - position: relative; - width: 100%; - } -} - -.search-results__page-hit { - display: none; -} - -.search-results__page-hit { - &:nth-child(-n + 4) { - display: inline; - } -} - -.search-results[data-active-filter="pages"] .search-results__pages { - display: inline; +// Show results depending on active tab +.search-results { + &[data-active-filter="all"] { + .search-results__pages, + .search-results__explorers, + .search-results__charts { + // both needed for .search-results__show-more-container absolute-positioning + display: inline-block; + position: relative; + width: 100%; + } - .search-results__page-hit { - display: inline; + // On the "All" tab, limit to the first 4 pages, 4 charts, and 2 explorer views + .search-results__page-hit:nth-child(n + 5), + .search-results__chart-hit:nth-child(n + 5), + .search-results__explorer-hit:nth-child(n + 3) { + display: none; + } } -} -.search-results[data-active-filter="charts"] .search-results__charts { - display: inline; - - .search-results__chart-hit { + &[data-active-filter="pages"] .search-results__pages { display: inline; } -} -.search-results__explorer-hit { - display: none; -} - -.search-results__explorer-hit { - &:nth-child(-n + 2) { + &[data-active-filter="charts"] .search-results__charts { display: inline; } -} - -.search-results[data-active-filter="explorers"] .search-results__explorers { - display: inline; - .search-results__explorer-hit { + &[data-active-filter="explorer-views"] .search-results__explorers { display: inline; } } diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index fc39b2495f2..4d9f85c866b 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -8,6 +8,8 @@ import { mapValues, isElementHidden, sortBy, + groupBy, + uniqBy, } from "@ourworldindata/utils" import { InstantSearch, @@ -19,6 +21,7 @@ import { Snippet, useInstantSearch, PoweredBy, + useHits, } from "react-instantsearch" import algoliasearch, { SearchClient } from "algoliasearch" import { @@ -30,18 +33,22 @@ import { import { action, observable } from "mobx" import { observer } from "mobx-react" import { - IExplorerHit, IChartHit, SearchCategoryFilter, SearchIndexName, searchCategoryFilters, IPageHit, pageTypeDisplayNames, + IExplorerViewHit, PageRecord, } from "./searchTypes.js" import { EXPLORERS_ROUTE_FOLDER } from "../../explorer/ExplorerConstants.js" import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" -import { faHeartBroken, faSearch } from "@fortawesome/free-solid-svg-icons" +import { + faArrowRight, + faHeartBroken, + faSearch, +} from "@fortawesome/free-solid-svg-icons" import { DEFAULT_SEARCH_PLACEHOLDER, getIndexName, @@ -55,6 +62,7 @@ import { DEFAULT_GRAPHER_HEIGHT, DEFAULT_GRAPHER_WIDTH, } from "@ourworldindata/grapher" +import type { SearchResults as AlgoliaSearchResultsType } from "algoliasearch-helper" import { SiteAnalytics } from "../SiteAnalytics.js" import { extractRegionNamesFromSearchQuery } from "./SearchUtils.js" @@ -128,17 +136,152 @@ function ChartHit({ hit }: { hit: IChartHit }) { ) } -function ExplorerHit({ hit }: { hit: IExplorerHit }) { +interface ExplorerViewHitWithPosition extends IExplorerViewHit { + // Analytics data + // Position of this hit in the search results: For example, if there is one card with 3 views, and a second card with 2 views, the first card will have hitPosition 0, 1, and 2, and the second card will have hitPosition 3 and 4. + hitPositionOverall: number + // Position of this hit within the card: For example, if there are 3 views in a card, they will have positions 0, 1, and 2. + hitPositionWithinCard: number +} + +interface GroupedExplorerViews { + explorerSlug: string + explorerTitle: string + explorerSubtitle: string + numViewsWithinExplorer: number + views: ExplorerViewHitWithPosition[] +} + +const getNumberOfExplorerHits = (rawHits: IExplorerViewHit[]) => + uniqBy(rawHits, "explorerSlug").length + +function ExplorerViewHits() { + const { hits } = useHits() + + const groupedHits = useMemo(() => { + const groupedBySlug = groupBy(hits, "explorerSlug") + const arr = Object.values(groupedBySlug).map((explorerViews) => { + const firstView = explorerViews[0] + return { + explorerSlug: firstView.explorerSlug, + explorerTitle: firstView.explorerTitle, + explorerSubtitle: firstView.explorerSubtitle, + numViewsWithinExplorer: firstView.numViewsWithinExplorer, + + // Run uniq, so if we end up in a situation where multiple views with the same title + // are returned, we only show the first of them + views: uniqBy(explorerViews, "viewTitle"), + } + }) + let totalHits = 0 + arr.forEach((group) => { + group.views = group.views.map((view, index) => ({ + ...view, + hitPositionWithinCard: index, + hitPositionOverall: totalHits + index, + })) as ExplorerViewHitWithPosition[] + totalHits += group.views.length + }) + return arr as GroupedExplorerViews[] + }, [hits]) + return ( - +
+ {groupedHits.map((group, i) => ( + + ))} +
+
+ ) +} + +function ExplorerHit({ + groupedHit, + cardPosition, +}: { + groupedHit: GroupedExplorerViews + cardPosition: number +}) { + const firstHit = groupedHit.views[0] + + const exploreAllProps = { + href: `${BAKED_BASE_URL}/${EXPLORERS_ROUTE_FOLDER}/${groupedHit.explorerSlug}`, + "data-algolia-index": getIndexName(SearchIndexName.ExplorerViews), + "data-algolia-object-id": firstHit.objectID, + "data-algolia-position": firstHit.hitPositionOverall, + "data-algolia-card-position": cardPosition, + "data-algolia-position-within-card": 0, + "data-algolia-event-name": "click_explorer", + } + + return ( + ) } @@ -147,11 +290,13 @@ function ShowMore({ cutoffNumber, activeCategoryFilter, handleCategoryFilterClick, + getTotalNumberOfHits, }: { category: SearchIndexName cutoffNumber: number activeCategoryFilter: SearchCategoryFilter handleCategoryFilterClick: (x: SearchIndexName) => void + getTotalNumberOfHits?: (results: AlgoliaSearchResultsType) => number }) { const { results } = useInstantSearch() // Hide if we're on the same tab as the category this button is for @@ -163,13 +308,16 @@ function ShowMore({ handleCategoryFilterClick(category) } - const numberShowing = Math.min(cutoffNumber, results.hits.length) - const isShowingAllResults = numberShowing === results.hits.length + const totalNumberOfHits = + getTotalNumberOfHits?.(results) ?? results.hits.length + + const numberShowing = Math.min(cutoffNumber, totalNumberOfHits) + const isShowingAllResults = numberShowing === totalNumberOfHits const message = isShowingAllResults ? numberShowing <= 2 ? "Showing all results" : `Showing all ${numberShowing} results` - : `Showing ${numberShowing} of the top ${results.hits.length} results` + : `Showing ${numberShowing} of the top ${totalNumberOfHits} results` return (
@@ -201,6 +349,13 @@ function Filters({ const hitsLengthByIndexName = mapValues(resultsByIndexName, (results) => get(results, ["results", "hits", "length"], 0) ) + + hitsLengthByIndexName[getIndexName(SearchIndexName.ExplorerViews)] = + getNumberOfExplorerHits( + resultsByIndexName[getIndexName(SearchIndexName.ExplorerViews)] + ?.results?.hits ?? [] + ) + hitsLengthByIndexName[getIndexName("all")] = Object.values( hitsLengthByIndexName ).reduce((a: number, b: number) => a + b, 0) @@ -272,16 +427,19 @@ const PAGES_ATTRIBUTES_TO_SEARCH_NO_FULLTEXT: (keyof PageRecord)[] = [ ] // Should be a subset of the `searchableAttributes` set up in `configureAlgolia` for the `pages` index; minus the "content" attribute const SearchResults = (props: SearchResultsProps) => { - const { - results: { queryID }, - } = useInstantSearch() + const { scopedResults } = useInstantSearch() const { activeCategoryFilter, isHidden, handleCategoryFilterClick } = props + const queryIdByIndexName = useMemo( + () => + new Map(scopedResults.map((r) => [r.indexId, r.results?.queryID])), + [scopedResults] + ) + // Listen to all clicks, if user clicks on a hit (and has consented to analytics - grep "hasClickAnalyticsConsent"), // Extract the pertinent hit data from the HTML and log the click to Algolia const handleHitClick = useCallback( (event: MouseEvent) => { - if (!queryID) return let target = event.target as HTMLElement | null if (target) { let isHit = false @@ -296,6 +454,9 @@ const SearchResults = (props: SearchResultsProps) => { const objectId = target.getAttribute( "data-algolia-object-id" ) + const eventName = + target.getAttribute("data-algolia-event-name") ?? + undefined const allVisibleHits = Array.from( document.querySelectorAll( @@ -309,18 +470,35 @@ const SearchResults = (props: SearchResultsProps) => { const positionInSection = target.getAttribute( "data-algolia-position" ) + + // Optional (only for explorers); Starts from 1 + const cardPosition = + target.getAttribute("data-algolia-card-position") ?? + undefined + + // Optional (only for explorers); Starts from 1 in each card; or 0 for the full explorer link + const positionWithinCard = + target.getAttribute( + "data-algolia-position-within-card" + ) ?? undefined + const index = target.getAttribute("data-algolia-index") const href = target.getAttribute("href") const query = props.query + const queryID = index + ? queryIdByIndexName.get(index) + : undefined if ( objectId && + queryID && positionInSection && index && href && query ) { logSiteSearchClickToAlgoliaInsights({ + eventName, index, queryID, objectIDs: [objectId], @@ -330,6 +508,8 @@ const SearchResults = (props: SearchResultsProps) => { query, position: String(globalPosition), positionInSection, + cardPosition, + positionWithinCard, url: href, filter: activeCategoryFilter, }) @@ -337,12 +517,12 @@ const SearchResults = (props: SearchResultsProps) => { } } }, - [queryID, activeCategoryFilter, props.query] + [activeCategoryFilter, props.query, queryIdByIndexName] ) useEffect(() => { document.addEventListener("click", handleHitClick) return () => document.removeEventListener("click", handleHitClick) - }, [queryID, handleHitClick]) + }, [handleHitClick]) const searchQueryRegionsMatches = useMemo(() => { const extractedRegions = extractRegionNamesFromSearchQuery(props.query) @@ -354,6 +534,7 @@ const SearchResults = (props: SearchResultsProps) => { const hasClickAnalyticsConsent = getPreferenceValue( PreferenceType.Analytics ) + return (
{ /> - + { } /* Hack: This is the only way to _not_ send `restrictSearchableAttributes` along for this index */ /> -
+

- Data Explorers + Charts

{
- + -
+

- Charts + Data Explorers

+ ) => getNumberOfExplorerHits(results.hits)} />
- +
diff --git a/site/search/searchClient.ts b/site/search/searchClient.ts index 0098ce8c16b..6859b2e7109 100644 --- a/site/search/searchClient.ts +++ b/site/search/searchClient.ts @@ -42,10 +42,13 @@ export const parseIndexName = (index: string): SearchIndexName => { } export const logSiteSearchClickToAlgoliaInsights = ( - event: Omit + event: Omit & { eventName?: string } ) => { const client = getInsightsClient() - client("clickedObjectIDsAfterSearch", { ...event, eventName: "click" }) + client("clickedObjectIDsAfterSearch", { + ...event, + eventName: event.eventName ?? "click", + }) } export const DEFAULT_SEARCH_PLACEHOLDER = diff --git a/site/search/searchTypes.ts b/site/search/searchTypes.ts index 1491bb6c190..a7dec624360 100644 --- a/site/search/searchTypes.ts +++ b/site/search/searchTypes.ts @@ -36,6 +36,22 @@ export interface PageRecord { export type IPageHit = PageRecord & Hit +export type IExplorerViewHit = Hit & { + objectID: string + + // Explorer-wide fields + explorerSlug: string + explorerTitle: string + explorerSubtitle: string + numViewsWithinExplorer: number + + // View-specific fields + viewTitle: string + viewSubtitle: string + viewQueryParams: string + viewTitleIndexWithinExplorer: number +} + export type IExplorerHit = Hit & { objectID: string slug: string @@ -78,8 +94,8 @@ export type SearchCategoryFilter = SearchIndexName | "all" export const searchCategoryFilters: [string, SearchCategoryFilter][] = [ ["All", "all"], ["Research & Writing", SearchIndexName.Pages], - ["Data Explorers", SearchIndexName.Explorers], ["Charts", SearchIndexName.Charts], + ["Data Explorers", SearchIndexName.ExplorerViews], ] export const indexNameToSubdirectoryMap: Record = { From c5f3491e4bc082277d327d9cdd1984533cced827 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 16 Apr 2024 09:00:32 +0200 Subject: [PATCH 2/5] feat(search): pre-select explorer entities if they appear in search query (#3483) --- baker/algolia/configureAlgolia.ts | 8 +- site/search/SearchPanel.tsx | 120 +++++++++++++++++++++--------- 2 files changed, 93 insertions(+), 35 deletions(-) diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 25e9abca2b1..c83ae4d15b0 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -9,7 +9,7 @@ import { ALGOLIA_INDEXING, ALGOLIA_SECRET_KEY, } from "../../settings/serverSettings.js" -import { countries } from "@ourworldindata/utils" +import { countries, regions } from "@ourworldindata/utils" import { SearchIndexName } from "../../site/search/searchTypes.js" import { getIndexName } from "../../site/search/searchClient.js" @@ -25,6 +25,11 @@ export const getAlgoliaClient = (): SearchClient | undefined => { return client } +const allCountryNamesAndVariants = regions.flatMap((region) => [ + region.name, + ...(("variantNames" in region && region.variantNames) || []), +]) + // This function initializes and applies settings to the Algolia search indices // Algolia settings should be configured here rather than in the Algolia dashboard UI, as then // they are recorded and transferrable across dev/prod instances @@ -164,6 +169,7 @@ export const configureAlgolia = async () => { attributeForDistinct: "explorerSlug", distinct: 4, minWordSizefor1Typo: 6, + optionalWords: allCountryNamesAndVariants, }) const synonyms = [ diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index 4d9f85c866b..6ac82a8c3bc 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -10,6 +10,9 @@ import { sortBy, groupBy, uniqBy, + EntityName, + Url, + Region, } from "@ourworldindata/utils" import { InstantSearch, @@ -61,6 +64,7 @@ import { import { DEFAULT_GRAPHER_HEIGHT, DEFAULT_GRAPHER_WIDTH, + setSelectedEntityNamesParam, } from "@ourworldindata/grapher" import type { SearchResults as AlgoliaSearchResultsType } from "algoliasearch-helper" import { SiteAnalytics } from "../SiteAnalytics.js" @@ -95,6 +99,22 @@ function PagesHit({ hit }: { hit: IPageHit }) { ) } +const getEntityQueryStr = ( + entities: EntityName[] | null | undefined, + existingQueryStr: string = "" +) => { + if (!entities?.length) return existingQueryStr + else { + return setSelectedEntityNamesParam( + // If we have any entities pre-selected, we want to show the chart tab + Url.fromQueryStr(existingQueryStr).updateQueryParams({ + tab: "chart", + }), + entities + ).queryStr + } +} + function ChartHit({ hit }: { hit: IChartHit }) { const [imgLoaded, setImgLoaded] = useState(false) const [imgError, setImgError] = useState(false) @@ -155,7 +175,11 @@ interface GroupedExplorerViews { const getNumberOfExplorerHits = (rawHits: IExplorerViewHit[]) => uniqBy(rawHits, "explorerSlug").length -function ExplorerViewHits() { +function ExplorerViewHits({ + countriesRegionsToSelect, +}: { + countriesRegionsToSelect?: Region[] +}) { const { hits } = useHits() const groupedHits = useMemo(() => { @@ -193,6 +217,7 @@ function ExplorerViewHits() { groupedHit={group} key={group.explorerSlug} cardPosition={i} + countriesRegionsToSelect={countriesRegionsToSelect} /> ))}
@@ -203,14 +228,25 @@ function ExplorerViewHits() { function ExplorerHit({ groupedHit, cardPosition, + countriesRegionsToSelect, }: { groupedHit: GroupedExplorerViews cardPosition: number + countriesRegionsToSelect?: Region[] }) { const firstHit = groupedHit.views[0] + // If the explorer title contains something like "Ukraine" already, don't bother selecting Ukraine in it + const entitiesToSelectExcludingExplorerTitle = + countriesRegionsToSelect?.filter( + (e) => !groupedHit.explorerTitle.includes(e.name) + ) + const queryStr = getEntityQueryStr( + entitiesToSelectExcludingExplorerTitle?.map((e) => e.name) + ) + const exploreAllProps = { - href: `${BAKED_BASE_URL}/${EXPLORERS_ROUTE_FOLDER}/${groupedHit.explorerSlug}`, + href: `${BAKED_BASE_URL}/${EXPLORERS_ROUTE_FOLDER}/${groupedHit.explorerSlug}${queryStr}`, "data-algolia-index": getIndexName(SearchIndexName.ExplorerViews), "data-algolia-object-id": firstHit.objectID, "data-algolia-position": firstHit.hitPositionOverall, @@ -242,38 +278,52 @@ function ExplorerHit({
{ ) => getNumberOfExplorerHits(results.hits)} /> - + From 1215d1e6aa84755e22d79ee2109bfccf90c0e664 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 16 Apr 2024 09:02:54 +0200 Subject: [PATCH 3/5] refactor(algolia): remove old explorer indexing code (#3472) --- Makefile | 1 - baker/algolia/configureAlgolia.ts | 21 -- baker/algolia/indexExplorerViewsToAlgolia.ts | 10 +- baker/algolia/indexExplorersToAlgolia.ts | 209 ------------------- site/search/searchTypes.ts | 11 - 5 files changed, 9 insertions(+), 243 deletions(-) delete mode 100644 baker/algolia/indexExplorersToAlgolia.ts diff --git a/Makefile b/Makefile index fc3bc5644e0..3eac17b6a23 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,6 @@ reindex: itsJustJavascript node --enable-source-maps itsJustJavascript/baker/algolia/configureAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js - node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsToAlgolia.js clean: diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index c83ae4d15b0..9fa27d01f94 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -130,24 +130,6 @@ export const configureAlgolia = async () => { disablePrefixOnAttributes: ["content"], }) - const explorersIndex = client.initIndex( - getIndexName(SearchIndexName.Explorers) - ) - - await explorersIndex.setSettings({ - ...baseSettings, - searchableAttributes: [ - "unordered(slug)", - "unordered(title)", - "unordered(subtitle)", - "unordered(text)", - ], - customRanking: ["desc(views_7d)"], - attributeForDistinct: "slug", - attributesForFaceting: [], - disableTypoToleranceOnAttributes: ["text"], - }) - const explorerViewsIndex = client.initIndex( getIndexName(SearchIndexName.ExplorerViews) ) @@ -334,9 +316,6 @@ export const configureAlgolia = async () => { await chartsIndex.saveSynonyms(algoliaSynonyms, { replaceExistingSynonyms: true, }) - await explorersIndex.saveSynonyms(algoliaSynonyms, { - replaceExistingSynonyms: true, - }) await explorerViewsIndex.saveSynonyms(algoliaSynonyms, { replaceExistingSynonyms: true, }) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index b5288e3d27d..6039a6e1b21 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -1,5 +1,4 @@ import * as db from "../../db/db.js" -import { ExplorerBlockGraphers } from "./indexExplorersToAlgolia.js" import { DecisionMatrix } from "../../explorer/ExplorerDecisionMatrix.js" import { tsvFormat } from "d3-dsv" import { @@ -15,6 +14,15 @@ import { SearchIndexName } from "../../site/search/searchTypes.js" import { groupBy, keyBy, orderBy } from "lodash" import { MarkdownTextWrap } from "@ourworldindata/components" +export type ExplorerBlockGraphers = { + type: "graphers" + block: { + title?: string + subtitle?: string + grapherId?: number + }[] +} + interface ExplorerViewEntry { viewTitle: string viewSubtitle: string diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts deleted file mode 100644 index 9085cf1dab9..00000000000 --- a/baker/algolia/indexExplorersToAlgolia.ts +++ /dev/null @@ -1,209 +0,0 @@ -import cheerio from "cheerio" -import { isArray } from "lodash" -import { match } from "ts-pattern" -import { - GrapherInterface, - DbRawChart, - checkIsPlainObjectWithGuard, - identity, - keyBy, - parseChartConfig, -} from "@ourworldindata/utils" -import { getAlgoliaClient } from "./configureAlgolia.js" -import * as db from "../../db/db.js" -import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" -import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" -import { chunkParagraphs } from "../chunk.js" -import { SearchIndexName } from "../../site/search/searchTypes.js" -import { getIndexName } from "../../site/search/searchClient.js" - -type ExplorerBlockColumns = { - type: "columns" - block: { name: string; additionalInfo?: string }[] -} - -export type ExplorerBlockGraphers = { - type: "graphers" - block: { - title?: string - subtitle?: string - grapherId?: number - }[] -} - -type ExplorerEntry = { - slug: string - title: string - subtitle: string - views_7d: number - blocks: string // (ExplorerBlockLineChart | ExplorerBlockColumns | ExplorerBlockGraphers)[] -} - -type ExplorerRecord = { - slug: string - title: string - subtitle: string - views_7d: number - text: string -} - -function extractTextFromExplorer( - blocksString: string, - graphersUsedInExplorers: Record -): string { - const blockText = new Set() - const blocks = JSON.parse(blocksString) - - if (isArray(blocks)) { - for (const block of blocks) { - if (checkIsPlainObjectWithGuard(block) && "type" in block) { - match(block) - .with( - { type: "columns" }, - (columns: ExplorerBlockColumns) => { - columns.block.forEach( - ({ name = "", additionalInfo = "" }) => { - blockText.add(name) - blockText.add(additionalInfo) - } - ) - } - ) - .with( - { type: "graphers" }, - (graphers: ExplorerBlockGraphers) => { - graphers.block.forEach( - ({ - title = "", - subtitle = "", - grapherId = undefined, - }) => { - blockText.add(title) - blockText.add(subtitle) - - if (grapherId !== undefined) { - const chartConfig = - graphersUsedInExplorers[grapherId] - - if (chartConfig) { - blockText.add( - chartConfig.title ?? "" - ) - blockText.add( - chartConfig.subtitle ?? "" - ) - } - } - } - ) - } - ) - .otherwise(() => { - // type: "tables" - // do nothing - }) - } - } - } - - return [...blockText].filter(identity).join(" ") -} - -function getNullishJSONValueAsPlaintext(value: string): string { - return value !== "null" ? cheerio.load(value)("body").text() : "" -} - -const getExplorerRecords = async ( - knex: db.KnexReadonlyTransaction -): Promise => { - const pageviews = await getAnalyticsPageviewsByUrlObj(knex) - - // Fetch info about all charts used in explorers, as linked by the explorer_charts table - const graphersUsedInExplorers = await db - .knexRaw>( - knex, - `-- sql - SELECT config FROM charts - INNER JOIN ( - SELECT DISTINCT chartId AS id FROM explorer_charts - ) AS ec - USING (id) - ` - ) - .then((charts) => charts.map((c) => parseChartConfig(c.config))) - .then((charts) => keyBy(charts, "id")) - - const explorerRecords = await db - .knexRaw>( - knex, - `-- sql - SELECT slug, - COALESCE(config->>"$.explorerSubtitle", "null") AS subtitle, - COALESCE(config->>"$.explorerTitle", "null") AS title, - COALESCE(config->>"$.blocks", "null") AS blocks - FROM explorers - WHERE isPublished = true - ` - ) - .then((results) => - results.flatMap(({ slug, title, subtitle, blocks }) => { - const textFromExplorer = extractTextFromExplorer( - blocks, - graphersUsedInExplorers - ) - const uniqueTextTokens = new Set([ - ...textFromExplorer.split(" "), - ]) - const textChunks = chunkParagraphs( - [...uniqueTextTokens].join(" "), - 1000 - ) - - // In case we don't have any text for this explorer, we still want to index it - const textChunksForIteration = textChunks.length - ? textChunks - : [""] - - const formattedTitle = `${getNullishJSONValueAsPlaintext( - title - )} Data Explorer` - - return textChunksForIteration.map((chunk, i) => ({ - slug, - title: formattedTitle, - subtitle: getNullishJSONValueAsPlaintext(subtitle), - views_7d: pageviews[`/explorers/${slug}`]?.views_7d ?? 0, - text: chunk, - objectID: `${slug}-${i}`, - })) - }) - ) - - return explorerRecords -} - -const indexExplorersToAlgolia = async () => { - if (!ALGOLIA_INDEXING) return - - const client = getAlgoliaClient() - if (!client) { - console.error( - `Failed indexing explorers (Algolia client not initialized)` - ) - return - } - - try { - const index = client.initIndex(getIndexName(SearchIndexName.Explorers)) - - const records = await db.knexReadonlyTransaction( - getExplorerRecords, - db.TransactionCloseMode.Close - ) - await index.replaceAllObjects(records) - } catch (e) { - console.log("Error indexing explorers to Algolia: ", e) - } -} - -void indexExplorersToAlgolia() diff --git a/site/search/searchTypes.ts b/site/search/searchTypes.ts index a7dec624360..0d4a71ad430 100644 --- a/site/search/searchTypes.ts +++ b/site/search/searchTypes.ts @@ -52,15 +52,6 @@ export type IExplorerViewHit = Hit & { viewTitleIndexWithinExplorer: number } -export type IExplorerHit = Hit & { - objectID: string - slug: string - subtitle: string - text: string - title: string - views_7d: number -} - export interface ChartRecord { objectID: string chartId: number @@ -83,7 +74,6 @@ export interface ChartRecord { export type IChartHit = Hit & ChartRecord export enum SearchIndexName { - Explorers = "explorers", ExplorerViews = "explorer-views", Charts = "charts", Pages = "pages", @@ -101,6 +91,5 @@ export const searchCategoryFilters: [string, SearchCategoryFilter][] = [ export const indexNameToSubdirectoryMap: Record = { [SearchIndexName.Pages]: "", [SearchIndexName.Charts]: "/grapher", - [SearchIndexName.Explorers]: "/explorers", [SearchIndexName.ExplorerViews]: "/explorers", } From f7e23da44693bcb41d30950a31a729772c266884 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 16 Apr 2024 09:04:30 +0200 Subject: [PATCH 4/5] Explorer search: Incorporate design feedback (#3493) * enhance(search): disallow line break between explorer view title and icon * enhance(search): subtitle for explorers section * fix(search): fix show more button styles on mobile * enhance(search): smaller "Explore all indicators" link * enhance(search): set line-height for chart title * enhance(search): "Data Explorer" suffix for all data explorers * enhance(search): search bar padding & placeholder styles * enhance(search): explicitly set white background color iOS Safari was using some grey as its default * enhance(search): bigger row-gap for charts * enhance(search): feature country-searching in search placeholder * enhance(search): update search placeholder --- site/gdocs/components/centered-article.scss | 4 + site/search/Search.scss | 71 +++++++--- site/search/SearchPanel.tsx | 147 +++++++++++--------- site/search/searchClient.ts | 2 +- 4 files changed, 142 insertions(+), 82 deletions(-) diff --git a/site/gdocs/components/centered-article.scss b/site/gdocs/components/centered-article.scss index e2e4daf8ab3..eeeb97b8223 100644 --- a/site/gdocs/components/centered-article.scss +++ b/site/gdocs/components/centered-article.scss @@ -1124,6 +1124,10 @@ div.raw-html-table__container { &::placeholder { color: $blue-40; } + + &:placeholder-shown { + text-overflow: ellipsis; + } } .aa-InputWrapperPrefix { diff --git a/site/search/Search.scss b/site/search/Search.scss index 6d68718a7ed..54dcbd53bf3 100644 --- a/site/search/Search.scss +++ b/site/search/Search.scss @@ -1,3 +1,6 @@ +$reset-button-size: 24px; +$reset-button-margin: 16px; + .search-page-container { min-height: calc(100vh - $header-height-sm); @include sm-up { @@ -29,11 +32,19 @@ } .ais-SearchBox-input { + background-color: white; width: 100%; height: 56px; padding-left: 16px; - // To conceal the placeholder text underneath the svg buttons on mobile - padding-right: 100px; + // To give room to the "Clear" button to the right + padding-right: $reset-button-size + $reset-button-margin + 4px; + + // If the placeholder is shown, then the Clear button is not shown + &:placeholder-shown { + padding-right: 16px; + text-overflow: ellipsis; + } + border: none; outline: $blue-20 1px solid; outline-offset: 0px; @@ -46,7 +57,7 @@ .ais-SearchBox-reset { display: inline-block; position: absolute; - right: 16px; + right: $reset-button-margin; top: 30%; border: none; border-radius: 16px; @@ -122,14 +133,19 @@ } } +.search-results__header-container { + margin-top: 24px; + margin-bottom: 24px; +} + .search-results__header { display: flex; align-items: baseline; } .search-results__section-title { - margin-top: 24px; - margin-bottom: 24px; + margin-top: 0; + margin-bottom: 0; margin-right: 16px; color: $blue-50; @@ -138,6 +154,11 @@ } } +.search-results__section-subtitle { + margin-top: 4px; + color: $blue-90; +} + .search-results__show-more-container { @include body-3-medium; em { @@ -151,6 +172,10 @@ cursor: pointer; } + &.show-more-btn-hidden button { + visibility: hidden; // Don't show the button, but keep the margin/padding for mobile layout + } + @include sm-only { position: absolute; display: flex; @@ -200,6 +225,11 @@ } } +// For charts, make it clearer that the title etc. belong to the chart above +.search-results__charts-list { + row-gap: calc(2 * var(--grid-gap)); +} + .search-results__page-hit { list-style: none; @@ -244,6 +274,7 @@ } .search-results__explorer-hit-link-mobile { + @include body-3-medium; @include owid-link-60; margin-top: 16px; display: block; @@ -271,25 +302,28 @@ } .search-results__explorer-hit-link { - @include owid-link-60; margin-top: 0; - flex: 1 0 auto; + flex: 0 0 auto; text-align: right; + + a { + @include body-3-medium; + @include owid-link-60; + } } } .search-results__explorer-views-list { - margin-left: 20px; - margin-top: 5px; + margin-top: 16px; list-style: none; gap: 16px; .search-results__explorer-view { + margin-left: 24px; overflow: hidden; .search-results__explorer-view-title-container { color: $blue-90; - text-decoration: underline; @include owid-link-90; svg { @@ -334,14 +368,17 @@ } } -.search-results__chart-hit-highlight { - line-height: 21px; - color: $blue-90; -} +.search-results__chart-hit-title-container { + line-height: 20px; -.search-results__chart-hit-variant { - color: $blue-60; - font-size: 0.9em; + .search-results__chart-hit-highlight { + color: $blue-90; + } + + .search-results__chart-hit-variant { + color: $blue-60; + font-size: 0.9em; + } } @keyframes chartErrorFadeIn { diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index 6ac82a8c3bc..6abaa0ba910 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -143,15 +143,17 @@ function ChartHit({ hit }: { hit: IChartHit }) { onError={() => setImgError(true)} /> - {" "} - - {hit.variantName} - +
+ {" "} + + {hit.variantName} + +
) } @@ -249,8 +251,8 @@ function ExplorerHit({ href: `${BAKED_BASE_URL}/${EXPLORERS_ROUTE_FOLDER}/${groupedHit.explorerSlug}${queryStr}`, "data-algolia-index": getIndexName(SearchIndexName.ExplorerViews), "data-algolia-object-id": firstHit.objectID, - "data-algolia-position": firstHit.hitPositionOverall, - "data-algolia-card-position": cardPosition, + "data-algolia-position": firstHit.hitPositionOverall + 1, + "data-algolia-card-position": cardPosition + 1, "data-algolia-position-within-card": 0, "data-algolia-event-name": "click_explorer", } @@ -263,19 +265,19 @@ function ExplorerHit({

- {groupedHit.explorerTitle} + {groupedHit.explorerTitle} Data Explorer

{groupedHit.explorerSubtitle}

- - Explore all {groupedHit.numViewsWithinExplorer} indicators - +
    {groupedHit.views.map((view) => { @@ -316,7 +318,11 @@ function ExplorerHit({ highlightedTagName="strong" className="search-results__explorer-view-title" /> - + + ‍ + {/* Zero-width joiner to prevent line break between title and icon */} + +

    {view.viewSubtitle} @@ -370,13 +376,15 @@ function ShowMore({ : `Showing ${numberShowing} of the top ${totalNumberOfHits} results` return ( -

    +
    {message} - {!isShowingAllResults && ( - - )} +
    ) } @@ -604,18 +612,20 @@ const SearchResults = (props: SearchResultsProps) => { />
    -
    -

    - Research & Writing -

    - +
    +
    +

    + Research & Writing +

    + +
    { />
    -
    -

    - Charts -

    - +
    +
    +

    + Charts +

    + +
    { />
    -
    -

    - Data Explorers -

    - - ) => getNumberOfExplorerHits(results.hits)} - /> +
    +
    +

    + Data Explorers +

    + + ) => getNumberOfExplorerHits(results.hits)} + /> +
    +

    + Interactive visualization tools to explore a + wide range of related indicators. +

    + diff --git a/site/search/searchClient.ts b/site/search/searchClient.ts index 6859b2e7109..f8a7fc5302f 100644 --- a/site/search/searchClient.ts +++ b/site/search/searchClient.ts @@ -52,4 +52,4 @@ export const logSiteSearchClickToAlgoliaInsights = ( } export const DEFAULT_SEARCH_PLACEHOLDER = - "Try “Life expectancy”, “Economic Growth”, “Homicide rate”, “Biodiversity”…" + "Try “Life expectancy”, “Poverty Nigeria Vietnam”, “CO2 France”…" From 7f88da5a849c81d62369c5a6c8d1960eaea20fa5 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 16 Apr 2024 09:10:50 +0200 Subject: [PATCH 5/5] feat(search): match geographic entities in search (#3388) * chore(search): make country name variants one-way synonyms * feat(search): match geographic entities within search * fix(algolia): sort entity names with variant names first, so Algolia synonyms can work * enhance(search): refine entity-picking logic * enhance(search): sort entity names * perf(algolia): optimize chart indexing code a bit * enhance(search): show entities as comma-separated list * enhance(search): use settings for URLs * fix(search): recognize strings like "high-income countries" --- .env.example-full | 2 + baker/algolia/configureAlgolia.ts | 23 ++++++--- baker/algolia/indexChartsToAlgolia.ts | 35 ++++++++++++- packages/@ourworldindata/utils/src/Util.ts | 6 +++ packages/@ourworldindata/utils/src/index.ts | 1 + settings/clientSettings.ts | 4 ++ site/search/Search.scss | 18 +++++++ site/search/SearchPanel.tsx | 34 +++++++++++-- site/search/SearchUtils.tsx | 55 +++++++++++++++++++++ 9 files changed, 167 insertions(+), 11 deletions(-) diff --git a/.env.example-full b/.env.example-full index 25cf0e85ca8..6ab3e0a6aef 100644 --- a/.env.example-full +++ b/.env.example-full @@ -42,6 +42,8 @@ IMAGE_HOSTING_R2_SECRET_ACCESS_KEY='' # optional OPENAI_API_KEY='' +GRAPHER_DYNAMIC_THUMBNAIL_URL='' # optional; can set this to https://ourworldindata.org/grapher/thumbnail to use the live thumbnail worker + # enable search (readonly) ALGOLIA_ID='' # optional ALGOLIA_SEARCH_KEY='' # optional diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 9fa27d01f94..3c87cfc70a9 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -9,7 +9,7 @@ import { ALGOLIA_INDEXING, ALGOLIA_SECRET_KEY, } from "../../settings/serverSettings.js" -import { countries, regions } from "@ourworldindata/utils" +import { countries, regions, excludeUndefined } from "@ourworldindata/utils" import { SearchIndexName } from "../../site/search/searchTypes.js" import { getIndexName } from "../../site/search/searchClient.js" @@ -296,12 +296,6 @@ export const configureAlgolia = async () => { ["solar", "photovoltaic", "photovoltaics", "pv"], ] - // Send all our country variant names to algolia as synonyms - for (const country of countries) { - if (country.variantNames) - synonyms.push([country.name].concat(country.variantNames)) - } - const algoliaSynonyms = synonyms.map((s) => { return { objectID: s.join("-"), @@ -310,6 +304,21 @@ export const configureAlgolia = async () => { } as Synonym }) + // Send all our country variant names to algolia as one-way synonyms + for (const country of countries) { + const alternatives = excludeUndefined([ + country.shortName, + ...(country.variantNames ?? []), + ]) + for (const alternative of alternatives) + algoliaSynonyms.push({ + objectID: `${alternative}->${country.name}`, + type: "oneWaySynonym", + input: alternative, + synonyms: [country.name], + }) + } + await pagesIndex.saveSynonyms(algoliaSynonyms, { replaceExistingSynonyms: true, }) diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 1fa897c455c..8d2475ec7d7 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -8,6 +8,9 @@ import { OwidGdocLinkType, excludeNullish, isNil, + countries, + orderBy, + removeTrailingParenthetical, } from "@ourworldindata/utils" import { MarkdownTextWrap } from "@ourworldindata/components" import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" @@ -20,6 +23,35 @@ const computeScore = (record: Omit): number => { return numRelatedArticles * 500 + views_7d } +const countriesWithVariantNames = new Set( + countries + .filter((country) => country.variantNames?.length || country.shortName) + .map((country) => country.name) +) + +const processAvailableEntities = (availableEntities: string[] | null) => { + if (!availableEntities) return [] + + // Algolia is a bit weird with synonyms: + // If we have a synonym "USA" -> "United States", and we search for "USA", + // then it seems that Algolia can only find that within `availableEntities` + // if "USA" is within the first 100-or-so entries of the array. + // So, the easy solution is to sort the entities to ensure that countries + // with variant names are at the top. + // - @marcelgerber, 2024-03-25 + return orderBy( + availableEntities, + [ + (entityName) => + countriesWithVariantNames.has( + removeTrailingParenthetical(entityName) + ), + (entityName) => entityName, + ], + ["desc", "asc"] + ) +} + const getChartsRecords = async ( knex: db.KnexReadonlyTransaction ): Promise => { @@ -81,7 +113,7 @@ const getChartsRecords = async ( if (c.entityNames.length < 12000) c.entityNames = excludeNullish( JSON.parse(c.entityNames as string) as (string | null)[] - ) + ) as string[] else { console.info( `Chart ${c.id} has too many entities, skipping its entities` @@ -89,6 +121,7 @@ const getChartsRecords = async ( c.entityNames = [] } } + c.entityNames = processAvailableEntities(c.entityNames) c.tags = JSON.parse(c.tags) c.keyChartForTags = JSON.parse(c.keyChartForTags as string).filter( diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 1f892bafee0..794d202962e 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1856,6 +1856,12 @@ export function cartesian(matrix: T[][]): T[][] { ) } +// Remove any parenthetical content from _the end_ of a string +// E.g. "Africa (UN)" -> "Africa" +export function removeTrailingParenthetical(str: string): string { + return str.replace(/\s*\(.*\)$/, "") +} + export function isElementHidden(element: Element | null): boolean { if (!element) return false const computedStyle = window.getComputedStyle(element) diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index fccb5e5b961..9e585bbab97 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -120,6 +120,7 @@ export { checkIsDataInsight, checkIsAuthor, cartesian, + removeTrailingParenthetical, isElementHidden, } from "./Util.js" diff --git a/settings/clientSettings.ts b/settings/clientSettings.ts index 2be034f7284..88fb763f32a 100644 --- a/settings/clientSettings.ts +++ b/settings/clientSettings.ts @@ -34,6 +34,10 @@ export const BAKED_GRAPHER_EXPORTS_BASE_URL: string = export const BAKED_SITE_EXPORTS_BASE_URL: string = process.env.BAKED_SITE_EXPORTS_BASE_URL ?? `${BAKED_BASE_URL}/exports` +export const GRAPHER_DYNAMIC_THUMBNAIL_URL: string = + process.env.GRAPHER_DYNAMIC_THUMBNAIL_URL ?? + `${BAKED_GRAPHER_URL}/thumbnail` + export const ADMIN_BASE_URL: string = process.env.ADMIN_BASE_URL ?? `http://${ADMIN_SERVER_HOST}:${ADMIN_SERVER_PORT}` diff --git a/site/search/Search.scss b/site/search/Search.scss index 54dcbd53bf3..f4745131810 100644 --- a/site/search/Search.scss +++ b/site/search/Search.scss @@ -431,6 +431,24 @@ $reset-button-margin: 16px; } } +.search-results__chart-hit-entities { + list-style: none; + font-size: 0.8em; + + li { + display: inline; + color: $blue-50; + + &::after { + content: ", "; + } + + &:last-child::after { + content: ""; + } + } +} + /* * Tabs / Filtering **/ diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index 6abaa0ba910..b82696c1afe 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -31,7 +31,9 @@ import { ALGOLIA_ID, ALGOLIA_SEARCH_KEY, BAKED_BASE_URL, + BAKED_GRAPHER_EXPORTS_BASE_URL, BAKED_GRAPHER_URL, + GRAPHER_DYNAMIC_THUMBNAIL_URL, } from "../../settings/clientSettings.js" import { action, observable } from "mobx" import { observer } from "mobx-react" @@ -68,7 +70,10 @@ import { } from "@ourworldindata/grapher" import type { SearchResults as AlgoliaSearchResultsType } from "algoliasearch-helper" import { SiteAnalytics } from "../SiteAnalytics.js" -import { extractRegionNamesFromSearchQuery } from "./SearchUtils.js" +import { + extractRegionNamesFromSearchQuery, + pickEntitiesForChartHit, +} from "./SearchUtils.js" const siteAnalytics = new SiteAnalytics() @@ -119,9 +124,24 @@ function ChartHit({ hit }: { hit: IChartHit }) { const [imgLoaded, setImgLoaded] = useState(false) const [imgError, setImgError] = useState(false) + const entities = useMemo( + () => pickEntitiesForChartHit(hit), + // eslint-disable-next-line react-hooks/exhaustive-deps + [hit._highlightResult?.availableEntities] + ) + const queryStr = useMemo(() => getEntityQueryStr(entities), [entities]) + const previewUrl = queryStr + ? `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${hit.slug}${queryStr}` + : `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${hit.slug}.svg` + + useEffect(() => { + setImgLoaded(false) + setImgError(false) + }, [previewUrl]) + return ( )} setImgLoaded(true)} onError={() => setImgError(true)} /> @@ -154,6 +175,13 @@ function ChartHit({ hit }: { hit: IChartHit }) { {hit.variantName}
    + {entities.length > 0 && ( +
      + {entities.map((entity) => ( +
    • {entity}
      • + ))} +
    + )}     ) } diff --git a/site/search/SearchUtils.tsx b/site/search/SearchUtils.tsx index 2587d56e6f7..16700087f3c 100644 --- a/site/search/SearchUtils.tsx +++ b/site/search/SearchUtils.tsx @@ -1,8 +1,13 @@ +import { HitAttributeHighlightResult } from "instantsearch.js" +import { IChartHit } from "./searchTypes.js" +import { EntityName } from "@ourworldindata/types" import { Region, getRegionByNameOrVariantName, regions, + countries, escapeRegExp, + removeTrailingParenthetical, } from "@ourworldindata/utils" const allCountryNamesAndVariants = regions.flatMap((c) => [ @@ -22,3 +27,53 @@ export const extractRegionNamesFromSearchQuery = (query: string) => { if (regionNames.length === 0) return null return regionNames.map(getRegionByNameOrVariantName) as Region[] } + +const removeHighlightTags = (text: string) => + text.replace(/<\/?(mark|strong)>/g, "") + +export function pickEntitiesForChartHit(hit: IChartHit): EntityName[] { + const availableEntitiesHighlighted = hit._highlightResult + ?.availableEntities as HitAttributeHighlightResult[] | undefined + + const pickedEntities = availableEntitiesHighlighted + ?.filter((highlightEntry) => { + if (highlightEntry.matchLevel === "none") return false + + // Remove any trailing parentheses, e.g. "Africa (UN)" -> "Africa" + const entityNameWithoutTrailingParens = removeTrailingParenthetical( + removeHighlightTags(highlightEntry.value) + ) + + // The sequence of words that Algolia matched; could be something like ["arab", "united", "republic"] + // which we want to check against the entity name + const matchedSequenceLowerCase = highlightEntry.matchedWords + .join(" ") + .toLowerCase() + + // Pick entity if the matched sequence contains the full entity name + if ( + matchedSequenceLowerCase.startsWith( + entityNameWithoutTrailingParens + .replaceAll("-", " ") // makes "high-income countries" into "high income countries", enabling a match + .toLowerCase() + ) + ) + return true + + const country = countries.find( + (c) => c.name === entityNameWithoutTrailingParens + ) + if (country?.variantNames) { + // Pick entity if the matched sequence contains any of the variant names + return country.variantNames.some((variant) => + matchedSequenceLowerCase.includes(variant.toLowerCase()) + ) + } + + return false + }) + .map((highlightEntry) => removeHighlightTags(highlightEntry.value)) + .sort() + + return pickedEntities ?? [] +}