Files
memos/web/scripts/generate-tag-unicode-data.mjs
johnnyjoygh 5f3969a830 feat(reactions): add full emoji picker
Let people use any Unicode emoji through a searchable, virtualized
picker.
Keep reaction counts accurate and requests optimistic and idempotent.
Refresh demo content so signed-in visitors can participate.

BREAKING CHANGE: Remove the MEMO_RELATED instance setting and
memoRelatedSetting configuration value. Remove their declarations and
lifecycle references from configuration files; existing reactions are
preserved.
2026-11-14 20:46:46 +08:00

134 lines
5.4 KiB
JavaScript

import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
const UNICODE_RELEASE = "17.0.0";
const sources = {
derived: `https://www.unicode.org/Public/${UNICODE_RELEASE}/ucd/DerivedCoreProperties.txt`,
emoji: `https://www.unicode.org/Public/${UNICODE_RELEASE}/emoji/emoji-test.txt`,
unicodeData: `https://www.unicode.org/Public/${UNICODE_RELEASE}/ucd/UnicodeData.txt`,
};
const fetchText = async (url) => {
const response = await fetch(url);
if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status}`);
return response.text();
};
const parseRange = (value) => {
const [from, to = from] = value.split("..").map((part) => Number.parseInt(part, 16));
return [from, to];
};
const mergeRanges = (ranges) => {
const merged = [];
for (const [from, to] of ranges.sort((left, right) => left[0] - right[0])) {
const previous = merged.at(-1);
if (previous && from <= previous[1] + 1) previous[1] = Math.max(previous[1], to);
else merged.push([from, to]);
}
return merged;
};
const parseDerivedProperty = (source, property) =>
mergeRanges(
source
.split("\n")
.map((line) => line.match(/^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*([A-Za-z_]+)/))
.filter((match) => match?.[2] === property)
.map((match) => parseRange(match[1])),
);
const parseCombiningMarkRanges = (source) => {
const ranges = [];
let pending;
for (const line of source.split("\n")) {
if (!line) continue;
const fields = line.split(";");
const codePoint = Number.parseInt(fields[0], 16);
const name = fields[1];
const category = fields[2];
if (name.endsWith(", First>")) {
pending = { codePoint, category };
} else if (name.endsWith(", Last>") && pending) {
if (pending.category === "Mn" || pending.category === "Mc") ranges.push([pending.codePoint, codePoint]);
pending = undefined;
} else if (category === "Mn" || category === "Mc") {
ranges.push([codePoint, codePoint]);
}
}
return mergeRanges(ranges);
};
const parseEmoji = (source) =>
source
.split("\n")
.map((line) => line.match(/^([0-9A-F ]+)\s*;\s*fully-qualified\b/))
.filter(Boolean)
.map((match) =>
String.fromCodePoint(
...match[1]
.trim()
.split(/\s+/)
.map((part) => Number.parseInt(part, 16)),
),
);
const formatRanges = (name, ranges) => {
const values = ranges.flatMap(([from, to]) => [`0x${from.toString(16)}`, `0x${to.toString(16)}`]);
const lines = [];
for (let index = 0; index < values.length; index += 12) lines.push(` ${values.slice(index, index + 12).join(", ")},`);
return `export const ${name}: readonly number[] = [\n${lines.join("\n")}\n];`;
};
const formatEmoji = (emoji) => {
const lines = [];
for (let index = 0; index < emoji.length; index += 12) {
lines.push(
` ${emoji
.slice(index, index + 12)
.map((value) => JSON.stringify(value))
.join(", ")},`,
);
}
return `export const FULLY_QUALIFIED_EMOJI: readonly string[] = [\n${lines.join("\n")}\n];`;
};
const unicodeDataDigest = (xidContinue, defaultIgnorable, combiningMarks, emoji) => {
const canonical = [];
const addRanges = (name, ranges) => {
for (const [from, to] of ranges) canonical.push(`${name}:${from.toString(16)}-${to.toString(16)}\n`);
};
addRanges("xid", xidContinue);
addRanges("ignorable", defaultIgnorable);
addRanges("combining", combiningMarks);
canonical.push(
...emoji.map((value) => `emoji:${[...value].map((character) => character.codePointAt(0).toString(16)).join(",")}\n`).sort(),
);
return createHash("sha256").update(canonical.join("")).digest("hex");
};
const [derived, emojiTest, unicodeData] = await Promise.all(Object.values(sources).map(fetchText));
const xidContinue = parseDerivedProperty(derived, "XID_Continue");
const defaultIgnorable = parseDerivedProperty(derived, "Default_Ignorable_Code_Point");
const combiningMarks = parseCombiningMarkRanges(unicodeData);
const emoji = parseEmoji(emojiTest);
const output = `// Generated by scripts/generate-tag-unicode-data.mjs. Do not edit.\n// Unicode and Emoji ${UNICODE_RELEASE} data used by Memos tag recognition.\n// Unicode data SHA-256: ${unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks, emoji)}\n// biome-ignore-all format: generated data is kept in compact source order\n\n${formatRanges(
"XID_CONTINUE_RANGES",
xidContinue,
)}\n\n${formatRanges("DEFAULT_IGNORABLE_RANGES", defaultIgnorable)}\n\n${formatRanges("COMBINING_MARK_RANGES", combiningMarks)}\n\n${formatEmoji(emoji)}\n`;
const outputURL = new URL("../src/utils/tag-unicode-data.ts", import.meta.url);
await writeFile(fileURLToPath(outputURL), output);
const namedEmoji = emojiTest.split("\n").flatMap((line) => {
const match = line.match(/^([0-9A-F ]+)\s*;\s*fully-qualified\s*#\s*\S+\s+E[\d.]+\s+(.+)$/);
if (!match) return [];
const glyph = String.fromCodePoint(...match[1].trim().split(/\s+/).map((part) => Number.parseInt(part, 16)));
return [[glyph, match[2]]];
});
await writeFile(
new URL("../src/lib/emoji-data.ts", import.meta.url),
`// Generated by scripts/generate-tag-unicode-data.mjs. Do not edit.\n// Unicode Emoji ${UNICODE_RELEASE}: ${sources.emoji}\n// biome-ignore-all format: generated Unicode names\nexport const EMOJI: readonly (readonly [string, string])[] = [\n${namedEmoji.map((entry) => ` ${JSON.stringify(entry)},`).join("\n")}\n];\n`,
);