diff --git a/packages/producer/src/services/deterministicFonts-textSubset.test.ts b/packages/producer/src/services/deterministicFonts-textSubset.test.ts index abcc811e17..5023e69e93 100644 --- a/packages/producer/src/services/deterministicFonts-textSubset.test.ts +++ b/packages/producer/src/services/deterministicFonts-textSubset.test.ts @@ -15,38 +15,41 @@ async function requestedGoogleFontUrl(html: string): Promise { return new URL(requestedUrl); } +async function subsetTextFor(html: string): Promise { + const url = await requestedGoogleFontUrl(html); + return url.searchParams.get("text") ?? ""; +} + describe("Google Fonts text subsetting", () => { it("sends the composition character set to the CSS API", async () => { - const url = await requestedGoogleFontUrl( + const text = await subsetTextFor( `

旅行ランキング

`, ); - const text = url.searchParams.get("text") ?? ""; for (const character of new Set("旅行ランキング")) { expect(text).toContain(character); } }); it("includes decoded HTML entities from visible composition text", async () => { - const url = await requestedGoogleFontUrl( + const text = await subsetTextFor( `

旅行

`, ); - expect(url.searchParams.get("text")).toContain("旅行"); + expect(text).toContain("旅行"); }); it("includes case variants for transformed supplemental alias weights", async () => { - const url = await requestedGoogleFontUrl( + const text = await subsetTextFor( `

Your Kidney Transplant:
What Happens Next

`, ); - const text = url.searchParams.get("text") ?? ""; expect(encodeURIComponent(text).length).toBeLessThan(700); for (const character of new Set("YOUR KIDNEY TRANSPLANT:WHAT HAPPENS NEXT")) { expect(text).toContain(character); @@ -54,13 +57,12 @@ describe("Google Fonts text subsetting", () => { }); it("covers capitalized words through the same case closure", async () => { - const url = await requestedGoogleFontUrl( + const text = await subsetTextFor( `

hello world

`, ); - const text = url.searchParams.get("text") ?? ""; expect(text).toContain("H"); expect(text).toContain("W"); }); @@ -81,4 +83,151 @@ describe("Google Fonts text subsetting", () => { expect(url.searchParams.has("text")).toBe(false); }); + + it("includes Turkish İ and ı when lang=tr is present", async () => { + const text = await subsetTextFor( + `

istanbul

`, + ); + + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); + + it("does not include Turkish İ/ı without a Turkish lang attribute", async () => { + const text = await subsetTextFor( + `

istanbul

`, + ); + + expect(text).not.toContain("İ"); + expect(text).not.toContain("ı"); + }); + + it("includes Azeri locale variants when lang=az is present", async () => { + const text = await subsetTextFor( + `

iyi

`, + ); + + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); + + it("maps ASCII to fullwidth equivalents when full-width appears in the source", async () => { + const text = await subsetTextFor( + `

ABC

`, + ); + + expect(text).toContain("Ａ"); + expect(text).toContain("Ｂ"); + expect(text).toContain("Ｃ"); + }); + + it("does not add fullwidth variants without full-width in the source", async () => { + const text = await subsetTextFor( + `

ABC

`, + ); + + expect(text).not.toContain("Ａ"); + }); + + it("maps small kana to full-size equivalents when full-size-kana appears in the source", async () => { + const text = await subsetTextFor( + `

ぁっょ

`, + ); + + expect(text).toContain("あ"); + expect(text).toContain("つ"); + expect(text).toContain("よ"); + }); + + it("maps small katakana to full-size when full-size-kana appears in the source", async () => { + const text = await subsetTextFor( + `

ァヵ

`, + ); + + expect(text).toContain("ア"); + expect(text).toContain("カ"); + }); + + it("stays within the URL budget for a realistic mixed-script composition with all transforms", async () => { + const latin = + "The Quick Brown Fox Jumps Over The Lazy Dog — Your Kidney Transplant: What Happens Next"; + const cjk = "旅行ランキング東京大阪京都名古屋福岡"; + const kana = "ぁぃぅぇぉっゃゅょゎァィゥェォッャュョヮヵヶ"; + + const withTransforms = await subsetTextFor( + `

${latin}

${cjk}${kana}

`, + ); + + const withoutTransforms = await subsetTextFor( + `

${latin}

${cjk}${kana}

`, + ); + + const transformCost = + encodeURIComponent(withTransforms).length - encodeURIComponent(withoutTransforms).length; + expect(transformCost).toBeLessThan(900); + expect(encodeURIComponent(withTransforms).length).toBeLessThanOrEqual(1700); + }); + + it("collects lang from nested elements, not just the root", async () => { + const text = await subsetTextFor( + `

hello

istanbul

`, + ); + + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); + + it("skips invalid lang attributes without crashing", async () => { + const text = await subsetTextFor( + `

hello

world

`, + ); + + expect(text.length).toBeGreaterThan(0); + expect(text).toContain("h"); + }); + + it("does not trigger fullwidth from an unrelated text-transform plus a full-width class", async () => { + const text = await subsetTextFor( + `

ABC

`, + ); + + expect(text).not.toContain("Ａ"); + }); + + it("triggers fullwidth for case-insensitive text-transform: FULL-WIDTH", async () => { + const text = await subsetTextFor( + `

ABC

`, + ); + + expect(text).toContain("Ａ"); + }); }); diff --git a/packages/producer/src/services/deterministicFonts.ts b/packages/producer/src/services/deterministicFonts.ts index 7b99aad6e1..ec5dab9f0e 100644 --- a/packages/producer/src/services/deterministicFonts.ts +++ b/packages/producer/src/services/deterministicFonts.ts @@ -1198,22 +1198,127 @@ export interface InjectDeterministicFontFacesOptions { // collapsing repeated prose and base64 assets to a tiny set. const GOOGLE_FONTS_TEXT_MAX_ENCODED_LENGTH = 1_700; +const SMALL_TO_FULL_KANA: ReadonlyMap = new Map([ + ["ぁ", "あ"], + ["ぃ", "い"], + ["ぅ", "う"], + ["ぇ", "え"], + ["ぉ", "お"], + ["っ", "つ"], + ["ゃ", "や"], + ["ゅ", "ゆ"], + ["ょ", "よ"], + ["ゎ", "わ"], + ["ァ", "ア"], + ["ィ", "イ"], + ["ゥ", "ウ"], + ["ェ", "エ"], + ["ォ", "オ"], + ["ッ", "ツ"], + ["ャ", "ヤ"], + ["ュ", "ユ"], + ["ョ", "ヨ"], + ["ヮ", "ワ"], + ["ヵ", "カ"], + ["ヶ", "ケ"], +]); + +function collectLangAttributes(document: { + querySelectorAll(selector: string): Iterable<{ getAttribute(name: string): string | null }>; +}): Set { + const locales = new Set(); + for (const element of document.querySelectorAll("[lang]")) { + const lang = element.getAttribute("lang"); + if (!lang) continue; + const primary = lang.split("-")[0]!.toLowerCase(); + try { + Intl.getCanonicalLocales(primary); + locales.add(primary); + } catch { + // Invalid BCP-47 tag (e.g. lang="en_US", lang="x") — skip silently. + } + } + return locales; +} + +function addCaseClosure(out: Set, character: string, locales: ReadonlySet): void { + out.add(character); + for (const variant of `${character.toUpperCase()}${character.toLowerCase()}`) { + out.add(variant); + } + for (const locale of locales) { + for (const variant of `${character.toLocaleUpperCase(locale)}${character.toLocaleLowerCase(locale)}`) { + out.add(variant); + } + } +} + +function addFullwidthVariants(chars: Set): void { + for (const character of [...chars]) { + const code = character.codePointAt(0) ?? 0; + if (code >= 0x0021 && code <= 0x007e) { + chars.add(String.fromCodePoint(code + 0xfee0)); + } + } +} + +function addFullSizeKanaVariants(chars: Set): void { + for (const character of [...chars]) { + const full = SMALL_TO_FULL_KANA.get(character); + if (full) chars.add(full); + } +} + +const FULL_WIDTH_KEYWORD_RE = /\bfull-width\b/; +const FULL_SIZE_KANA_KEYWORD_RE = /\bfull-size-kana\b/; +const DECLARATION_BOUNDARY_RE = /[;{}]/; + +function skipWhitespace(s: string, pos: number): number { + while (pos < s.length && " \t\n\r\f\v".includes(s[pos]!)) pos += 1; + return pos; +} + +function findDeclarationEnd(s: string, pos: number): number { + const match = DECLARATION_BOUNDARY_RE.exec(s.slice(pos)); + return match ? pos + match.index : s.length; +} + +// Linear indexOf/slice scan: a `text-transform\s*:[^;{}]*\bkw\b` regex backtracks +// O(n²) on input with many `text-transform:` runs (js/polynomial-redos). +function hasTextTransformKeyword(html: string, keyword: RegExp): boolean { + const haystack = html.toLowerCase(); + const property = "text-transform"; + let from = 0; + for (;;) { + const at = haystack.indexOf(property, from); + if (at === -1) return false; + const afterProp = skipWhitespace(haystack, at + property.length); + if (haystack[afterProp] !== ":") { + from = at + property.length; + continue; + } + const end = findDeclarationEnd(haystack, afterProp + 1); + if (keyword.test(haystack.slice(afterProp + 1, end))) return true; + from = end; + } +} + function extractGoogleFontsText(html: string): string | undefined { const { document } = parseHTML(html); const decodedBodyText = document.body?.textContent ?? ""; - // Source + decoded text is an intentional over-approximation: base64, scripts, and class names - // collapse in the Set, while decoded entities contribute the glyphs the browser actually paints. - const characters = [...Array.from(html), ...Array.from(decodedBodyText)]; + const locales = collectLangAttributes(document); + + // Intentional over-approximation: raw html includes base64, scripts, and + // class names, but they collapse in the Set and the budget gate catches bloat. const uniqueCharacters = new Set(); - for (const character of characters) { - uniqueCharacters.add(character); - // This closes locale-independent Unicode casing, including multi-code-point expansions such as - // ß -> SS. Locale/context transforms (for example Turkish İ) and CSS full-width/full-size-kana - // need a transform-aware follow-up rather than pretending this code-point closure is exhaustive. - for (const variant of `${character.toUpperCase()}${character.toLowerCase()}`) { - uniqueCharacters.add(variant); - } + for (const character of new Set([...Array.from(html), ...Array.from(decodedBodyText)])) { + addCaseClosure(uniqueCharacters, character, locales); } + + if (hasTextTransformKeyword(html, FULL_WIDTH_KEYWORD_RE)) addFullwidthVariants(uniqueCharacters); + if (hasTextTransformKeyword(html, FULL_SIZE_KANA_KEYWORD_RE)) + addFullSizeKanaVariants(uniqueCharacters); + const fontText = [...uniqueCharacters].join(""); return encodeURIComponent(fontText).length <= GOOGLE_FONTS_TEXT_MAX_ENCODED_LENGTH ? fontText