Adapt UnicodeSearch into EmojiSearch

Author: fileformat

.gitignore

@@ -4,7 +4,7 @@ dist
 *.env
 *.log
 node_modules
-public/ucd.json
+public/emoji.json
 tmp
 *.tmp
 

LICENSE.txt

@@ -629,7 +629,7 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    UnicodeSearch - search through the Unicode character database
+    EmojiSearch - search through Emoji
     Copyright (C) 2025 by Andrew Marcuse
 
     This program is free software: you can redistribute it and/or modify

README.md

@@ -1,15 +1,15 @@
-# UnicodeSearch <img alt="VectorLogoZone logo" src="public/favicon.svg" height="90" align="right" />
+# EmojiSearch <img alt="VectorLogoZone logo" src="public/favicon.svg" height="90" align="right" />
 
-[![deploy](https://github.com/FileFormatInfo/unicodesearch/actions/workflows/ghpages-deploy.yaml/badge.svg)](https://github.com/FileFormatInfo/unicodesearch/actions/workflows/ghpages-deploy.yaml)
+[![deploy](https://github.com/FileFormatInfo/emojisearch/actions/workflows/ghpages-deploy.yaml/badge.svg)](https://github.com/FileFormatInfo/emojisearch/actions/workflows/ghpages-deploy.yaml)
 
-A web app for searching and browsing the Unicode character database.
+A web app for searching and browsing the Unicode emoji database.
 
-See it in action: [unicodesearch.org](https://www.unicodesearch.org/)
+See it in action: [emojisearch.org](https://www.emojisearch.org/)
 
 ## Links
 
-- [Unicode Character Database](https://www.unicode.org/Public/17.0.0/ucd/) - XML data source for this app.
-- [Unicode Annex #42](https://www.unicode.org/reports/tr42/) - what the fields in the UCD mean.
+- [Emoji at the Unicode Character Database](https://www.unicode.org/Public/UCD/latest/ucd/emoji/) - data source for this app.
+- [Unicode Technical Standard #41: Emoji](https://unicode.org/reports/tr51/) - what the fields in the UCD mean.
 
 ## Credits
 
@@ -19,20 +19,14 @@ See it in action: [unicodesearch.org](https://www.unicodesearch.org/)
 [![Google Noto Emoji](https://www.vectorlogo.zone/logos/google/google-ar21.svg)](https://github.com/googlefonts/noto-emoji/blob/43f47be9404018cd9d8f73a227363a8f20acdab5/svg/emoji_u1f984.svg "Icon")
 [![Node.js](https://www.vectorlogo.zone/logos/nodejs/nodejs-ar21.svg)](https://nodejs.org/ "Application Server")
 [![npm](https://www.vectorlogo.zone/logos/npmjs/npmjs-ar21.svg)](https://www.npmjs.com/ "JS Package Management")
-[![Unicode](https://www.vectorlogo.zone/logos/unicode/unicode-ar21.svg)](https://www.unicode.org/Public/17.0.0/ucdxml/ "Unicode Character Database")
+[![Unicode](https://www.vectorlogo.zone/logos/unicode/unicode-ar21.svg)](https://www.unicode.org/Public/17.0.0/ "Unicode Character Database")
 [![Tabulator](https://www.vectorlogo.zone/logos/tabulatorinfo/tabulatorinfo-ar21.svg)](https://tabulator.info/ "Grid widget")
 [![TypeScript](https://www.vectorlogo.zone/logos/typescriptlang/typescriptlang-ar21.svg)](https://www.typescriptlang.org/ "Programming Language")
 [![VectorLogoZone](https://www.vectorlogo.zone/logos/vectorlogozone/vectorlogozone-ar21.svg)](https://www.vectorlogo.zone/ "Logos")
 [![Vite](https://www.vectorlogo.zone/logos/vitejsdev/vitejsdev-ar21.svg)](https://vitejs.dev/ "Bundler")
 
 ## To Do
 
-- [ ] handle `#` in name
-- [ ] accurate total count
-- [ ] preview column (SVGs)
 - [ ] bigger font for example column
-- [ ] status (for filter/sort) in footer
 - [ ] test dark mode
 - [ ] share links in footer
-- [ ] tags: bidi, letter, number
-- [ ] tag filters

bin/ucd_download.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Download and unzip the Unicode Character Database XML file
+# Download and unzip the emoji test file
 # from Unicode.org.
 #
 
@@ -23,12 +23,9 @@ fi
 
 curl \
 	--location \
-	--output "${TMP_DIR}/ucd.all.flat.zip" \
+	--output "${TMP_DIR}/emoji-test.txt" \
 	--show-error \
 	--silent \
-	https://www.unicode.org/Public/latest/ucdxml/ucd.all.flat.zip
-
-cd "${TMP_DIR}"
-unzip ucd.all.flat.zip
+	https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt
 
 echo "INFO: completed download at $(date -u +"%Y-%m-%dT%H:%M:%SZ")"

bin/ucd_to_json.ts

@@ -9,105 +9,83 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 type SearchEntry = {
-	code: string;
-	name: string;
-	age: string;
-	block: string;
-	category: string;
-	script: string;
-	tags?: string[];
+	codepoints: string;
+	qualification: string;
+	version: string;
+	emoji: string;
+	description: string;
+	group: string;
+	subgroup: string;
 }
 
 type SearchData = {
 	success: boolean;
+	lastmod: string;
 	data: SearchEntry[];
 }
 
 async function main() {
 	console.log(`INFO: starting at ${new Date().toISOString()}`);
 
-	const xmlPath = path.join( __dirname, '..', 'tmp', 'ucd.all.flat.xml' );
-	const jsonPath = path.join( __dirname, '..', 'public', 'ucd.json' );
+	const txtPath = path.join( __dirname, '..', 'tmp', 'emoji-test.txt' );
+	const jsonPath = path.join( __dirname, '..', 'public', 'emoji.json' );
 
 	try {
-		await fs.access(xmlPath);
+		await fs.access(txtPath);
 	} catch (err) {
-		console.log(`INFO: XML file does not exist in ${xmlPath}`);
+		console.log(`INFO: txt file does not exist in ${txtPath}`);
 		process.exit(1);
 	}
 
 	// Read and parse the XML file
-	console.log(`INFO: reading XML file from ${xmlPath}`);
-	const xmlData = await fs.readFile(xmlPath, 'utf-8');
-	console.log(`INFO: parsing XML data`);
-	const parser = new XMLParser({
-		ignoreAttributes: false,
-		attributeNamePrefix: '',
-	});
-	const jsonObj = parser.parse(xmlData);
-
-	console.log(`INFO: parsed ${jsonObj.ucd.repertoire.char.length} characters`);
-
-	if (true) {
-		fs.writeFile(
-			path.join(__dirname, "..", "tmp", "ucd.all.flat.json"),
-			JSON.stringify(jsonObj, null, 2),
-			"utf-8"
-		);
-	}
-
-	console.log(`INFO: generating JSON data`);
-	const entries: SearchEntry[] = [];
-
-	for (const charData of jsonObj.ucd.repertoire.char) {
-
-		if (!charData.cp || charData.cp.length === 0) {
-			if (!charData['first-cp']) {	// some private use area ranges mixed in
-				console.log(`WARN: skipping entry with no code point (${JSON.stringify(charData)})`);
-			}
+	console.log(`INFO: reading XML file from ${txtPath}`);
+	const txtData = await fs.readFile(txtPath, 'utf-8');
+	console.log(`INFO: parsing txt data`);
+
+	const data: SearchEntry[] = [];
+	let currentGroup = '';
+	let currentSubgroup = '';
+
+	const lines = txtData.split(/\r?\n/);
+	for (const line of lines) {
+		// console.log(`LINE: ${line}`);
+		if (line.length == 0) {
 			continue;
 		}
-
-		const tags: string[] = [];
-		if (charData.WSpace === 'Y') {
-			tags.push('Whitespace');
-		}
-		if (charData.Emoji === 'Y') {
-			tags.push('Emoji');
-		}
-		if (charData.Dep === 'Y') {
-			tags.push('Deprecated');
-		}
-		if (charData.QMark === 'Y') {
-			tags.push('Quote');
-		}
-		if (charData.Dash === 'Y') {
-			tags.push('Dash');
+		if (line.startsWith('#')) {
+			// comment line
+			if (line.startsWith('# group: ')) {
+				currentGroup = line.replace('# group: ', '').trim();
+			} else if (line.startsWith('# subgroup: ')) {
+				currentSubgroup = line.replace('# subgroup: ', '').trim();
+			}
+			continue;
 		}
 
-		var name = charData.na || charData.na1;
-		if (!name && charData['name-alias']) {
-			name = charData['name-alias'][0].alias;
+		const match = line.match(/^([0-9A-F ]+)\s*;\s*(fully-qualified|minimally-qualified|unqualified|component)\s*# (.+) E([0-9]+[.][0-9]+) (.+)$/);
+		if (match) {
+			data.push( {
+				codepoints: match[1].trim(),
+				qualification: match[2].trim(),
+				emoji: match[3].trim(),
+				description: match[5].trim(),
+				version: match[4].trim(),
+				group: currentGroup,
+				subgroup: currentSubgroup,
+				});
+		} else {
+			console.log(`DEBUG: no match for line: ${line}`);
 		}
-
-		entries.push({
-			code: charData.cp,
-			name: name || "(no name)",
-			age: charData.age,
-			block: charData.blk.replaceAll('_', ' '),
-			category: charData.gc,
-			script: charData.sc,
-			tags: tags.length ? tags : undefined,
-		});
 	}
 
 	const output: SearchData = {
 		success: true,
-		data: entries,
+		lastmod: new Date().toISOString(),
+		data,
 	};
 
 	// Write the JSON data to a file
-	console.log(`INFO: writing JSON data to ${jsonPath}`);
+	console.log(`INFO: writing ${data.length} emoji data to ${jsonPath}`);
 	await fs.writeFile(jsonPath, JSON.stringify(output, null, 2), 'utf-8');
 	console.log(`INFO: wrote JSON data to ${jsonPath}`);
 }

package.json

@@ -1,6 +1,6 @@
 {
   "author": "[email protected]",
-  "description": "UnicodeSearch",
+  "description": "EmojiSearch",
   "devDependencies": {
     "@types/node": "^24.3.1",
     "@types/tabulator-tables": "^6.2.11",