Merge pull request #120 from EdJoPaTo/language-code

feat: LanguageCode = string

Author: maxlath

docs/general_helpers.md

@@ -25,7 +25,7 @@
 - [Sitelink helpers](#sitelink-helpers)
   - [getSitelinkUrl](#getsitelinkurl)
   - [getSitelinkData](#getsitelinkdata)
-  - [isSitelinkKey](#issitelinkkey)
+  - [isSite](#issite)
 - [Wikibase Time converters](#wikibase-time-converters)
   - [wikibaseTimeToDateObject](#wikibasetimetodateobject)
   - [wikibaseTimeToEpochTime](#wikibasetimetoepochtime)
@@ -220,23 +220,23 @@ getSitelinkData('wikidatawiki')
 // => { lang: 'en', project: 'wikidata', key: 'wikidatawiki' }
 ```
 
-### isSitelinkKey
+### isSite
 ```js
-import { isSitelinkKey } from 'wikibase-sdk'
+import { isSite } from 'wikibase-sdk'
 
-isSitelinkKey('frwiki')
+isSite('frwiki')
 // => true
-isSitelinkKey('dewikiquote')
+isSite('dewikiquote')
 // => true
-isSitelinkKey('commons')
+isSite('commons')
 // => true
 // Accepting wikidata as a valid sitelink for convenience
-isSitelinkKey('wikidata')
+isSite('wikidata')
 // => true
-isSitelinkKey('frwikilinpinpin')
+isSite('frwikilinpinpin')
 // => false
 // /!\ langs are loosly validated
-isSitelinkKey('imaginarylangwiki')
+isSite('imaginarylangwiki')
 // => true
 ```
 

package.json

@@ -35,7 +35,7 @@
     "prepublishOnly": "git checkout main",
     "prepack": "npm run build && npm run lint && npm test",
     "postpublish": "./scripts/postpublish",
-    "update-sitelinks-languages": "./scripts/sitelinks_languages/update_sitelinks_languages",
+    "update-wikimedia-constants": "./scripts/update_wikimedia_constants.ts",
     "update-toc": "./scripts/update_toc",
     "watch": "tsc --watch"
   },

scripts/sitelinks_languages/generate_sitelinks_languages.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env ts-node
+import { writeFileSync } from 'fs'
+import { uniq } from '../src/utils/utils.js'
+
+interface Parameter {
+  name: string
+  type: string[]
+}
+
+function stringifyArray (input: string[]) {
+  return JSON.stringify(uniq(input), null, 2)
+    // Prevent linting errors
+    .replace(/"/g, '\'')
+    .replace(/'\n/, '\',\n')
+}
+
+function stringifySimpleRecord (input: Record<string, string>) {
+  let output = '{\n'
+  output += Object.entries(input).map(([ key, value ]) => '  ' + key + ': ' + '\'' + value + '\',\n').join('')
+  output += '}'
+  return output
+}
+
+doit()
+async function doit () {
+  const response = await fetch('https://www.wikidata.org/w/api.php?action=paraminfo&modules=wbgetentities&format=json')
+  const data = await response.json()
+
+  const parameters = data.paraminfo.modules[0].parameters as Parameter[]
+
+  const sites = parameters.find(o => o.name === 'sites')?.type
+  const languages = parameters.find(o => o.name === 'languages')?.type
+  if (!sites || !languages) throw new Error('paraminfo format changed')
+
+  const specialSites: Record<string, string> = {}
+  for (const site of sites) {
+    const project = site.match(/^(.+)wiki$/)?.[1]
+    if (!project) continue
+    if (!languages.includes(project.replace(/_/g, '-'))) {
+      specialSites[site] = project
+    }
+  }
+
+  const output = [
+    "// Generated by 'npm run update-wikimedia-constants'",
+    [
+      'export type Site = typeof sites[number]',
+      'export type WikimediaLanguageCode = typeof wikimediaLanguageCodes[number]',
+    ].join('\n'),
+    'export const specialSites = ' + stringifySimpleRecord(specialSites) + ' as const',
+    'export const sites = ' + stringifyArray(sites) + ' as const',
+    'export const wikimediaLanguageCodes = ' + stringifyArray(languages) + ' as const',
+  ].join('\n\n') + '\n'
+
+  writeFileSync('./src/helpers/wikimedia_constants.ts', output, 'utf-8')
+}

scripts/sitelinks_languages/get_sitelinks_sites

@@ -1,19 +1,18 @@
 import { typedEntries } from '../utils/utils.js'
-import type { WmLanguageCode } from '../types/options.js'
 import type { Aliases, Descriptions, Glosses, Labels, Lemmas, Representations, SimplifiedAliases, SimplifiedDescriptions, SimplifiedGlosses, SimplifiedLabels, SimplifiedLemmas, SimplifiedRepresentations } from '../types/terms.js'
 
 type InValue<T> = { readonly value: T }
 
-function singleValue<V> (data: Partial<Readonly<Record<WmLanguageCode, InValue<V>>>>) {
-  const simplified: Partial<Record<WmLanguageCode, V>> = {}
+function singleValue<K extends string, V> (data: Partial<Readonly<Record<K, InValue<V>>>>) {
+  const simplified: Partial<Record<K, V>> = {}
   for (const [ lang, obj ] of typedEntries(data)) {
     simplified[lang] = obj != null ? obj.value : null
   }
   return simplified
 }
 
-function multiValue<V> (data: Partial<Readonly<Record<WmLanguageCode, ReadonlyArray<InValue<V>>>>>) {
-  const simplified: Partial<Record<WmLanguageCode, readonly V[]>> = {}
+function multiValue<K extends string, V> (data: Partial<Readonly<Record<K, ReadonlyArray<InValue<V>>>>>) {
+  const simplified: Partial<Record<K, readonly V[]>> = {}
   for (const [ lang, obj ] of typedEntries(data)) {
     simplified[lang] = obj != null ? obj.map(o => o.value) : []
   }

scripts/sitelinks_languages/update_sitelinks_languages

@@ -1,14 +1,15 @@
 import { fixedEncodeURIComponent, isAKey, isOfType, rejectObsoleteInterface, replaceSpaceByUnderscores } from '../utils/utils.js'
-import { languages } from './sitelinks_languages.js'
-import { specialSites } from './special_sites.js'
-import type { WmLanguageCode } from '../types/options.js'
-import type { Site } from '../types/sitelinks.js'
+import { specialSites, type Site, sites } from './wikimedia_constants.js'
+import type { LanguageCode } from '../types/options.js'
 import type { Url } from '../utils/build_url.js'
 
 const wikidataBase = 'https://www.wikidata.org/wiki/'
 
+type ValueOf<T> = T[keyof T]
+type SpecialSiteProjectName = ValueOf<typeof specialSites>
+
 export interface GetSitelinkUrlOptions {
-  site: Site
+  site: Site | SpecialSiteProjectName
   title: string
 }
 
@@ -34,20 +35,23 @@ export function getSitelinkUrl ({ site, title }: GetSitelinkUrlOptions): Url {
 
 const wikimediaSite = (subdomain: string) => (title: string) => `https://${subdomain}.wikimedia.org/wiki/${title}`
 
-const siteUrlBuilders = {
+const siteUrlBuilders: Readonly<Record<SpecialSiteProjectName, (s: string) => string>> = {
   commons: wikimediaSite('commons'),
-  mediawiki: (title: string) => `https://www.mediawiki.org/wiki/${title}`,
+  foundation: wikimediaSite('foundation'),
+  mediawiki: title => `https://www.mediawiki.org/wiki/${title}`,
   meta: wikimediaSite('meta'),
+  outreach: wikimediaSite('outreach'),
+  sources: title => `https://wikisource.org/wiki/${title}`,
   species: wikimediaSite('species'),
-  wikidata: (entityId: string) => {
+  wikidata: entityId => {
     const prefix = prefixByEntityLetter[entityId[0]]
     let title = prefix ? `${prefix}:${entityId}` : entityId
     // Required for forms and senses
     title = title.replace('-', '#')
     return `${wikidataBase}${title}`
   },
   wikimania: wikimediaSite('wikimania'),
-} as const
+}
 
 const prefixByEntityLetter = {
   E: 'EntitySchema',
@@ -58,7 +62,7 @@ const prefixByEntityLetter = {
 const sitelinkUrlPattern = /^https?:\/\/([\w-]{2,10})\.(\w+)\.org\/\w+\/(.*)/
 
 export interface SitelinkData {
-  lang: WmLanguageCode
+  lang: LanguageCode
   project: Project
   key: string
   title?: string
@@ -72,59 +76,52 @@ export function getSitelinkData (site: Site | Url): SitelinkData {
     if (!matchData) throw new Error(`invalid sitelink url: ${url}`)
     let [ lang, project, title ] = matchData.slice(1)
     title = decodeURIComponent(title)
-    let key: string
+    if (lang === 'commons') {
+      return { lang: 'en', project: 'commons', key: 'commons', title, url }
+    }
+
+    if (!isOfType(projectNames, project)) {
+      throw new Error(`project is unknown: ${project}`)
+    }
+
     // Known case: wikidata, mediawiki
     if (lang === 'www') {
-      lang = 'en'
-      key = project
-    } else if (lang === 'commons') {
-      lang = 'en'
-      project = key = 'commons'
-    } else {
-      // Support multi-parts language codes, such as be_x_old
-      lang = lang.replace(/-/g, '_')
-      key = `${lang}${project}`.replace('wikipedia', 'wiki')
+      return { lang: 'en', project, key: project, title, url }
     }
-    // @ts-expect-error
+
+    // Support multi-parts language codes, such as be_x_old
+    const sitelang = lang.replace(/-/g, '_')
+    const key = `${sitelang}${project}`.replace('wikipedia', 'wiki')
     return { lang, project, key, title, url }
   } else {
-    const key = site
     if (isAKey(specialSites, site)) {
       const project = specialSites[site]
-      return { lang: 'en', project, key }
+      return { lang: 'en', project, key: site }
     }
 
-    let [ lang, projectSuffix, rest ] = key.split('wik')
+    if (!isOfType(sites, site)) {
+      throw new Error(`site not found: ${site}. Updating wikibase-sdk to a more recent version might fix the issue.`)
+    }
 
-    // Detecting cases like 'frwikiwiki' that would return [ 'fr', 'i', 'i' ]
-    if (rest != null) throw new Error(`invalid sitelink key: ${key}`)
+    let [ lang, projectSuffix, rest ] = site.split('wik')
 
-    if (!isOfType(languages, lang)) {
-      throw new Error(`sitelink lang not found: ${lang}. Updating wikibase-sdk to a more recent version might fix the issue.`)
-    }
+    // Detecting cases like 'frwikiwiki' that would return [ 'fr', 'i', 'i' ]
+    if (rest != null) throw new Error(`invalid sitelink key: ${site}`)
 
     // Support keys such as be_x_oldwiki, which refers to be-x-old.wikipedia.org
     lang = lang.replace(/_/g, '-')
 
     const project = projectsBySuffix[projectSuffix]
     if (!project) throw new Error(`sitelink project not found: ${project}`)
 
-    // @ts-expect-error
-    return { lang, project, key }
+    return { lang, project, key: site }
   }
 }
 
-export const isSitelinkKey = (site: string): boolean => {
-  try {
-    // relies on getSitelinkData validation
-    getSitelinkData(site)
-    return true
-  } catch (err) {
-    return false
-  }
-}
+export const isSite = (site: string): site is Site => isOfType(sites, site)
 
-export const wikimediaLanguageCodes = languages
+/** @deprecated use isSite */
+export const isSitelinkKey = isSite
 
 const projectsBySuffix = {
   i: 'wikipedia',