From b99185b4bf3e75685ed6c1f2b17b37adbdd781e9 Mon Sep 17 00:00:00 2001 From: ste__cal Date: Mon, 25 May 2026 17:22:47 +0200 Subject: [PATCH 1/2] fix: prevent unknown character corruption in translations (#68) * fix: prevent unknown character corruption in translations ## Changed - Set contentType explicitly per parser so Lara no longer auto-detects TextBlock[] as HTML and replaces non-ASCII chars with literal `?` - Engine splits each batch by detected content type so values containing inline HTML are sent as text/html and plain values as text/plain - Retry U+FFFD-corrupted translations up to 3 times as solo calls before failing with a neutral "please retry" message - Bump version to 1.3.4 ## New - contentType utility (hasHtmlMarkup / resolveContentType) and per-parser getContentType() exposed through ParserFactory - Parameterized integration tests covering content-type routing for all 10 supported file formats - Deterministic test reproducing the upstream UTF-8 streaming bug in @translated/lara, plus integration coverage for the retry guard * chore: use #utils path alias in contentType test imports ## Changed - Replace relative `../../utils/contentType.js` import with `#utils/contentType.js` to match the convention used by sibling utility tests (e.g. entities.test.ts). --- README.md | 2 +- package.json | 2 +- .../android-xml.integration.test.ts | 34 +- .../batch-translation.integration.test.ts | 58 +++ .../content-type-routing.integration.test.ts | 378 ++++++++++++++++++ .../utf8-retry.integration.test.ts | 170 ++++++++ .../parsers/android-xml.parser.test.ts | 6 + src/__tests__/parsers/json.parser.test.ts | 6 + src/__tests__/parsers/markdown.parser.test.ts | 6 + src/__tests__/parsers/parser.factory.test.ts | 19 + src/__tests__/parsers/po.parser.test.ts | 6 + src/__tests__/parsers/ts.parser.test.ts | 6 + src/__tests__/parsers/txt.parser.test.ts | 6 + src/__tests__/parsers/vue.parser.test.ts | 6 + .../parsers/xcode-strings.parser.test.ts | 6 + .../parsers/xcode-stringsdict.parser.test.ts | 6 + .../parsers/xcode-xcstrings.parser.test.ts | 6 + .../sdk-utf8-streaming.repro.test.ts | 133 ++++++ src/__tests__/utils/contentType.test.ts | 45 +++ src/interface/parser.ts | 12 + src/messages/messages.ts | 1 + src/modules/translation/translation.engine.ts | 159 ++++++-- src/parsers/android-xml.parser.ts | 4 + src/parsers/json.parser.ts | 4 + src/parsers/markdown.parser.ts | 4 + src/parsers/parser.factory.ts | 8 + src/parsers/po.parser.ts | 4 + src/parsers/ts.parser.ts | 4 + src/parsers/txt.parser.ts | 4 + src/parsers/vue.parser.ts | 4 + src/parsers/xcode-strings.parser.ts | 4 + src/parsers/xcode-stringsdict.parser.ts | 4 + src/parsers/xcode-xcstrings.parser.ts | 4 + src/utils/contentType.ts | 24 ++ 34 files changed, 1113 insertions(+), 32 deletions(-) create mode 100644 src/__tests__/integration/content-type-routing.integration.test.ts create mode 100644 src/__tests__/integration/utf8-retry.integration.test.ts create mode 100644 src/__tests__/sdk-utf8-streaming.repro.test.ts create mode 100644 src/__tests__/utils/contentType.test.ts create mode 100644 src/utils/contentType.ts diff --git a/README.md b/README.md index 828be18..da040ca 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Lara Cli automates translation of your i18n files with a single command, preserv Supports multiple file formats including JSON, PO (gettext), TypeScript, Vue I18n single-file components, Markdown and MDX files, Android XML string resource files, Xcode localization files (.strings, .stringsdict, .xcstrings), and plain text (.txt) files. See [Supported Formats](docs/config/formats.md) for details. -[![Version](https://img.shields.io/badge/version-1.3.3-blue.svg)](https://github.com/translated/lara-cli) +[![Version](https://img.shields.io/badge/version-1.3.4-blue.svg)](https://github.com/translated/lara-cli) diff --git a/package.json b/package.json index 5936f8a..8db04b4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@translated/lara-cli", "type": "module", - "version": "1.3.3", + "version": "1.3.4", "description": "CLI tool for automated i18n file translation using Lara Translate", "repository": { "type": "git", diff --git a/src/__tests__/integration/android-xml.integration.test.ts b/src/__tests__/integration/android-xml.integration.test.ts index 5784369..7b37844 100644 --- a/src/__tests__/integration/android-xml.integration.test.ts +++ b/src/__tests__/integration/android-xml.integration.test.ts @@ -6,7 +6,7 @@ import { fileURLToPath } from 'url'; import yaml from 'yaml'; -import { executeCommand } from './test-helpers.js'; +import { executeCommand, mockTranslateBatchWithFallback } from './test-helpers.js'; import initCommand from '../../cli/cmd/init/init.js'; import translateCommand from '../../cli/cmd/translate/translate.js'; import { ConfigProvider } from '#modules/config/config.provider.js'; @@ -744,4 +744,36 @@ describe('Android XML Repository Integration Tests', () => { expect(contentAfter).toContain('[it] Hello World'); expect(contentAfter).toContain('[it] Welcome to the app'); }); + + // Android string resources support inline HTML markup (, , ,
), + // so the engine should declare contentType=text/html — that tells Lara to + // preserve the markup instead of stripping or escaping it. + it('passes contentType=text/html for Android XML sources', async () => { + mockTranslateBatchWithFallback.mockClear(); + await mkdir(path.join(testDir, 'res', 'en'), { recursive: true }); + await writeFile( + path.join(testDir, 'res', 'en', 'strings.xml'), + ` + + Hello +` + ); + + await executeCommand(initCommand, [ + '--non-interactive', + '--source', + 'en', + '--target', + 'it', + '--paths', + 'res/[locale]/strings.xml', + ]); + (ConfigProvider as any).instance = null; + + await executeCommand(translateCommand, []); + + expect(mockTranslateBatchWithFallback).toHaveBeenCalled(); + const [, , , batchOptions] = mockTranslateBatchWithFallback.mock.calls[0]!; + expect((batchOptions as any).contentType).toBe('text/html'); + }); }); diff --git a/src/__tests__/integration/batch-translation.integration.test.ts b/src/__tests__/integration/batch-translation.integration.test.ts index c8e17ba..ea20589 100644 --- a/src/__tests__/integration/batch-translation.integration.test.ts +++ b/src/__tests__/integration/batch-translation.integration.test.ts @@ -254,4 +254,62 @@ describe('Batch translation', () => { a: '[it] third', }); }); + + // Regression test: Lara auto-detects TextBlock[] input as HTML-flavored content + // unless contentType is set explicitly. For JSON sources the values are plain + // text, so the engine must pass contentType: 'text/plain' to avoid the API + // mangling characters into '?' placeholders. + it('passes contentType=text/plain for JSON sources', async () => { + await writeJsonSource({ a: 'one', b: 'two' }); + await initJson(); + + await executeCommand(translateCommand, []); + + expect(mockTranslateBatchWithFallback).toHaveBeenCalledTimes(1); + const [, , , batchOptions] = mockTranslateBatchWithFallback.mock.calls[0]!; + expect((batchOptions as any).contentType).toBe('text/plain'); + }); + + // When a JSON file mixes plain values with values that contain inline HTML + // markup, the engine must split them into two API calls — one with + // contentType=text/plain (so non-HTML text isn't mangled) and one with + // contentType=text/html (so the tags survive the round trip). + it('splits mixed plain + HTML values into two batch calls', async () => { + await writeJsonSource({ + plain1: 'Hello world', + htmlA: 'Click here', + plain2: 'Goodbye', + htmlB: 'Be bold', + }); + await initJson(); + + await executeCommand(translateCommand, []); + + expect(mockTranslateBatchWithFallback).toHaveBeenCalledTimes(2); + + const callsByType = new Map(); + for (const [textBlocks, , , options] of mockTranslateBatchWithFallback.mock.calls) { + const contentType = (options as any).contentType as string; + callsByType.set( + contentType, + textBlocks.map((b) => b.text) + ); + } + + expect(callsByType.get('text/plain')?.sort()).toEqual(['Goodbye', 'Hello world']); + expect(callsByType.get('text/html')?.sort()).toEqual([ + 'Be bold', + 'Click here', + ]); + + const itContent = JSON.parse( + await readFile(path.join(testDir, 'i18n', 'locales', 'it.json'), 'utf-8') + ); + expect(itContent).toEqual({ + plain1: '[it] Hello world', + htmlA: '[it] Click here', + plain2: '[it] Goodbye', + htmlB: '[it] Be bold', + }); + }); }); diff --git a/src/__tests__/integration/content-type-routing.integration.test.ts b/src/__tests__/integration/content-type-routing.integration.test.ts new file mode 100644 index 0000000..179f833 --- /dev/null +++ b/src/__tests__/integration/content-type-routing.integration.test.ts @@ -0,0 +1,378 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { mkdir, writeFile, rm, unlink } from 'fs/promises'; +import { existsSync } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +import { executeCommand, mockTranslate, mockTranslateBatchWithFallback } from './test-helpers.js'; +import initCommand from '../../cli/cmd/init/init.js'; +import translateCommand from '../../cli/cmd/translate/translate.js'; +import { ConfigProvider } from '#modules/config/config.provider.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// Plain source values (no HTML markup). Includes a curly apostrophe to mirror +// the real bug report — these characters are exactly what triggered the `??` +// corruption when Lara auto-detected the batch as HTML-flavored content. +const PLAIN_A = 'Hello world'; +const PLAIN_B = 'A Customer’s continued use'; + +// HTML-bearing source values. Tags are chosen without inner double quotes so +// they can be embedded in any of the format-specific source files without +// escaping headaches. +const HTML_A = 'Click here'; +const HTML_B = 'Be bold'; + +type FormatCase = { + name: string; + paths: string; + setup: (dir: string) => Promise; + // Plain values are expected to be sent with the parser's default contentType. + plainValues: string[]; + // HTML values are always expected to be sent with contentType=text/html + // (the per-value detection overrides the parser default). + htmlValues: string[]; + defaultContentType: 'text/plain' | 'text/html'; +}; + +const FORMATS: FormatCase[] = [ + { + name: 'JSON', + paths: 'i18n/[locale].json', + setup: async (dir) => { + await mkdir(path.join(dir, 'i18n'), { recursive: true }); + await writeFile( + path.join(dir, 'i18n', 'en.json'), + JSON.stringify({ p1: PLAIN_A, p2: PLAIN_B, h1: HTML_A, h2: HTML_B }, null, 2) + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'PO', + paths: 'locales/[locale]/messages.po', + setup: async (dir) => { + await mkdir(path.join(dir, 'locales', 'en'), { recursive: true }); + await writeFile( + path.join(dir, 'locales', 'en', 'messages.po'), + `msgid "" +msgstr "" +"Content-Type: text/plain; charset=UTF-8\\n" + +msgid "p1" +msgstr "${PLAIN_A}" + +msgid "p2" +msgstr "${PLAIN_B}" + +msgid "h1" +msgstr "${HTML_A}" + +msgid "h2" +msgstr "${HTML_B}" +` + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'TypeScript', + paths: 'src/i18n.ts', + setup: async (dir) => { + await mkdir(path.join(dir, 'src'), { recursive: true }); + await writeFile( + path.join(dir, 'src', 'i18n.ts'), + `const messages = { + en: { + p1: '${PLAIN_A}', + p2: '${PLAIN_B}', + h1: '${HTML_A}', + h2: '${HTML_B}', + }, +}; + +export default messages; +` + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'Vue', + paths: 'src/components/*.vue', + setup: async (dir) => { + await mkdir(path.join(dir, 'src', 'components'), { recursive: true }); + await writeFile( + path.join(dir, 'src', 'components', 'HelloWorld.vue'), + ` + +${JSON.stringify({ en: { p1: PLAIN_A, p2: PLAIN_B, h1: HTML_A, h2: HTML_B } }, null, 2)} + +` + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'Markdown', + paths: 'docs/[locale]/guide.md', + setup: async (dir) => { + await mkdir(path.join(dir, 'docs', 'en'), { recursive: true }); + // Note: inline HTML in markdown becomes an `html` AST node and is + // intentionally skipped by the parser. So we only assert on plain text + // segments here — there is no path that would route a markdown value + // through `text/html`. + await writeFile( + path.join(dir, 'docs', 'en', 'guide.md'), + `# ${PLAIN_A} + +${PLAIN_B}. +` + ); + }, + plainValues: [PLAIN_A, `${PLAIN_B}.`], + htmlValues: [], + defaultContentType: 'text/plain', + }, + { + name: 'Android XML', + paths: 'res/[locale]/strings.xml', + setup: async (dir) => { + await mkdir(path.join(dir, 'res', 'en'), { recursive: true }); + // Inline `` / `` tags must be entity-escaped in the raw XML; + // fast-xml-parser decodes them back to literal `` / `` strings + // when the engine reads the values. + const escape = (s: string) => s.replace(//g, '>'); + await writeFile( + path.join(dir, 'res', 'en', 'strings.xml'), + ` + + ${PLAIN_A} + ${PLAIN_B} + ${escape(HTML_A)} + ${escape(HTML_B)} + +` + ); + }, + // Android string resources legitimately allow inline HTML, so the parser's + // default is text/html. All four values — including the plain ones — + // should be sent with contentType=text/html. + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/html', + }, + { + name: 'Xcode .strings', + paths: '[locale].lproj/Localizable.strings', + setup: async (dir) => { + await mkdir(path.join(dir, 'en.lproj'), { recursive: true }); + await writeFile( + path.join(dir, 'en.lproj', 'Localizable.strings'), + `"p1" = "${PLAIN_A}"; +"p2" = "${PLAIN_B}"; +"h1" = "${HTML_A}"; +"h2" = "${HTML_B}"; +` + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'Xcode .stringsdict', + paths: '[locale].lproj/Localizable.stringsdict', + setup: async (dir) => { + await mkdir(path.join(dir, 'en.lproj'), { recursive: true }); + const escape = (s: string) => s.replace(//g, '>'); + // stringsdict translatable values are the plural-form strings. We put + // a plain value in `one` and an HTML-bearing value in `other`. + await writeFile( + path.join(dir, 'en.lproj', 'Localizable.stringsdict'), + ` + + + + greeting + + NSStringLocalizedFormatKey + %#@variant@ + variant + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + ${PLAIN_A} + other + ${escape(HTML_A)} + + + + +` + ); + }, + plainValues: [PLAIN_A], + htmlValues: [HTML_A], + defaultContentType: 'text/plain', + }, + { + name: 'Xcode .xcstrings', + paths: 'Localizable.xcstrings', + setup: async (dir) => { + await writeFile( + path.join(dir, 'Localizable.xcstrings'), + JSON.stringify( + { + sourceLanguage: 'en', + version: '1.0', + strings: { + p1: { + localizations: { en: { stringUnit: { state: 'translated', value: PLAIN_A } } }, + }, + p2: { + localizations: { en: { stringUnit: { state: 'translated', value: PLAIN_B } } }, + }, + h1: { localizations: { en: { stringUnit: { state: 'translated', value: HTML_A } } } }, + h2: { localizations: { en: { stringUnit: { state: 'translated', value: HTML_B } } } }, + }, + }, + null, + 2 + ) + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, + { + name: 'TXT', + paths: 'texts/[locale]/messages.txt', + setup: async (dir) => { + await mkdir(path.join(dir, 'texts', 'en'), { recursive: true }); + await writeFile( + path.join(dir, 'texts', 'en', 'messages.txt'), + `${PLAIN_A}\n${PLAIN_B}\n${HTML_A}\n${HTML_B}\n` + ); + }, + plainValues: [PLAIN_A, PLAIN_B], + htmlValues: [HTML_A, HTML_B], + defaultContentType: 'text/plain', + }, +]; + +describe('Content-type routing per file format', () => { + let testDir: string; + let originalCwd: string; + let originalEnv: NodeJS.ProcessEnv; + let originalExit: typeof process.exit; + + beforeEach(async () => { + originalCwd = process.cwd(); + originalEnv = { ...process.env }; + originalExit = process.exit; + + process.exit = vi.fn() as any; + + testDir = path.join( + __dirname, + '..', + '..', + '..', + 'tmp', + `test-ctype-${Date.now()}-${Math.random().toString(36).substring(7)}` + ); + await mkdir(testDir, { recursive: true }); + process.chdir(testDir); + + process.env.LARA_ACCESS_KEY_ID = 'test-key-id'; + process.env.LARA_ACCESS_KEY_SECRET = 'test-key-secret'; + + (ConfigProvider as any).instance = null; + mockTranslate.mockClear(); + mockTranslateBatchWithFallback.mockClear(); + }); + + afterEach(async () => { + process.chdir(originalCwd); + process.env = originalEnv; + process.exit = originalExit; + + if (existsSync(testDir)) { + await rm(testDir, { recursive: true, force: true }); + } + const lockFilePath = path.join(originalCwd, 'lara.lock'); + if (existsSync(lockFilePath)) { + await unlink(lockFilePath).catch(() => {}); + } + + (ConfigProvider as any).instance = null; + }); + + // Collects every text -> contentType pair seen by the mocked translator + // (across solo translate() and batched translateBatchWithFallback()). + function collectCalls(): Array<{ text: string; contentType: string }> { + const out: Array<{ text: string; contentType: string }> = []; + const sources: Array<{ mock: { calls: any[] } }> = [ + mockTranslate as unknown as { mock: { calls: any[] } }, + mockTranslateBatchWithFallback as unknown as { mock: { calls: any[] } }, + ]; + for (const m of sources) { + for (const [textBlocks, , , options] of m.mock.calls) { + const contentType = (options as any).contentType as string; + for (const block of textBlocks as Array<{ text: string }>) { + out.push({ text: block.text, contentType }); + } + } + } + return out; + } + + for (const fmt of FORMATS) { + it(`${fmt.name}: plain values go to ${fmt.defaultContentType}, HTML values go to text/html`, async () => { + await fmt.setup(testDir); + + await executeCommand(initCommand, [ + '--non-interactive', + '--source', + 'en', + '--target', + 'it', + '--paths', + fmt.paths, + ]); + (ConfigProvider as any).instance = null; + + await executeCommand(translateCommand, []); + + const pairs = collectCalls(); + const textToType = new Map>(); + for (const { text, contentType } of pairs) { + if (!textToType.has(text)) textToType.set(text, new Set()); + textToType.get(text)!.add(contentType); + } + + for (const text of fmt.plainValues) { + expect(textToType.get(text), `plain value missing from API calls: ${text}`).toBeDefined(); + expect([...(textToType.get(text) ?? [])]).toEqual([fmt.defaultContentType]); + } + + for (const text of fmt.htmlValues) { + expect(textToType.get(text), `HTML value missing from API calls: ${text}`).toBeDefined(); + expect([...(textToType.get(text) ?? [])]).toEqual(['text/html']); + } + }); + } +}); diff --git a/src/__tests__/integration/utf8-retry.integration.test.ts b/src/__tests__/integration/utf8-retry.integration.test.ts new file mode 100644 index 0000000..d5e4c2f --- /dev/null +++ b/src/__tests__/integration/utf8-retry.integration.test.ts @@ -0,0 +1,170 @@ +/** + * U+FFFD retry guard — integration tests. + * + * The upstream Lara SDK has a UTF-8 streaming bug that corrupts multi-byte + * characters split across TCP chunks (see `sdk-utf8-streaming.repro.test.ts`). + * `TranslationEngine.executeTasks` mitigates it by detecting `�` in any + * translation result and re-issuing the affected texts as solo calls — whose + * tiny responses almost never span a chunk boundary. + * + * These tests drive the mitigation through the mocked TranslationService: + * 1. Batch comes back with U+FFFD in one entry → solo retry is issued and + * the corrupted value is replaced with a clean one in the output file. + * 2. Solo retry STILL returns U+FFFD → translate throws a clear error. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { mkdir, writeFile, readFile, rm, unlink } from 'fs/promises'; +import { existsSync } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +import { executeCommand, mockTranslate, mockTranslateBatchWithFallback } from './test-helpers.js'; +import initCommand from '../../cli/cmd/init/init.js'; +import translateCommand from '../../cli/cmd/translate/translate.js'; +import { ConfigProvider } from '#modules/config/config.provider.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +describe('U+FFFD retry guard', () => { + let testDir: string; + let originalCwd: string; + let originalEnv: NodeJS.ProcessEnv; + let originalExit: typeof process.exit; + + beforeEach(async () => { + originalCwd = process.cwd(); + originalEnv = { ...process.env }; + originalExit = process.exit; + process.exit = vi.fn() as any; + + testDir = path.join( + __dirname, + '..', + '..', + '..', + 'tmp', + `test-utf8-retry-${Date.now()}-${Math.random().toString(36).substring(7)}` + ); + await mkdir(testDir, { recursive: true }); + process.chdir(testDir); + + process.env.LARA_ACCESS_KEY_ID = 'test-key-id'; + process.env.LARA_ACCESS_KEY_SECRET = 'test-key-secret'; + + (ConfigProvider as any).instance = null; + mockTranslate.mockReset(); + mockTranslateBatchWithFallback.mockReset(); + }); + + afterEach(async () => { + process.chdir(originalCwd); + process.env = originalEnv; + process.exit = originalExit; + + if (existsSync(testDir)) { + await rm(testDir, { recursive: true, force: true }); + } + const lockFilePath = path.join(originalCwd, 'lara.lock'); + if (existsSync(lockFilePath)) { + await unlink(lockFilePath).catch(() => {}); + } + (ConfigProvider as any).instance = null; + }); + + async function writeSource(keys: Record): Promise { + await mkdir(path.join(testDir, 'i18n', 'locales'), { recursive: true }); + await writeFile( + path.join(testDir, 'i18n', 'locales', 'en.json'), + JSON.stringify(keys, null, 2) + ); + } + + async function initJson(): Promise { + await executeCommand(initCommand, [ + '--non-interactive', + '--source', + 'en', + '--target', + 'bg', + '--paths', + 'i18n/locales/[locale].json', + ]); + (ConfigProvider as any).instance = null; + } + + it('retries U+FFFD-corrupted batch entries as solo calls and recovers', async () => { + // Batch returns a result where one entry is corrupted with U+FFFD — + // the same shape we observed in the wild (`тез��` in Bulgarian). + mockTranslateBatchWithFallback.mockImplementationOnce( + async (textBlocks: { text: string; translatable: boolean }[]) => + textBlocks.map((block) => + block.text === 'these changes' + ? { text: 'тез�� изменения', translatable: true } + : { text: `[bg] ${block.text}`, translatable: true } + ) + ); + + // Solo retry returns the clean translation — what a small response gives. + mockTranslate.mockImplementationOnce( + async (textBlocks: { text: string; translatable: boolean }[]) => [ + { text: 'тези изменения', translatable: textBlocks[0]!.translatable }, + ] + ); + + await writeSource({ clean: 'hello world', dirty: 'these changes' }); + await initJson(); + + await executeCommand(translateCommand, []); + + // One batch call, one solo retry — exactly one of each. + expect(mockTranslateBatchWithFallback).toHaveBeenCalledTimes(1); + expect(mockTranslate).toHaveBeenCalledTimes(1); + + // The retry was issued for the corrupted text only. + const [retryBlocks] = mockTranslate.mock.calls[0]!; + expect(retryBlocks).toEqual([{ text: 'these changes', translatable: true }]); + + // Output file has the clean translation in place of the corrupted one. + const bgContent = JSON.parse( + await readFile(path.join(testDir, 'i18n', 'locales', 'bg.json'), 'utf-8') + ); + expect(bgContent.clean).toBe('[bg] hello world'); + expect(bgContent.dirty).toBe('тези изменения'); + expect(JSON.stringify(bgContent)).not.toContain('�'); + }); + + it('falls through to a neutral retry-please message after several silent retries', async () => { + mockTranslateBatchWithFallback.mockImplementationOnce( + async (textBlocks: { text: string; translatable: boolean }[]) => + textBlocks.map(() => ({ text: 'тез�� изменения', translatable: true })) + ); + // Every solo retry also comes back corrupt — extremely unlikely in real + // life, but the engine must give up after a few attempts and surface a + // short message instead of writing a broken file. + mockTranslate.mockImplementation(async () => [{ text: 'тез�� изменения', translatable: true }]); + + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await writeSource({ dirty: 'these changes' }); + await initJson(); + + // The translate command catches engine errors and logs them via + // console.error, then sets hasErrors and exits with code 1. + await expect(executeCommand(translateCommand, [])).rejects.toThrow( + /Process exited with code 1/ + ); + + // Three silent solo retries before giving up. + expect(mockTranslate).toHaveBeenCalledTimes(3); + + const loggedMessages = consoleErrorSpy.mock.calls.map((args) => args.join(' ')).join('\n'); + expect(loggedMessages).toMatch(/Translation failed for some keys\. Please retry\./); + // No SDK internals leak into user-facing output. + expect(loggedMessages).not.toMatch(/U\+FFFD/); + expect(loggedMessages).not.toMatch(/@translated\/lara/); + expect(loggedMessages).not.toMatch(/chunk\.toString/); + + consoleErrorSpy.mockRestore(); + }); +}); diff --git a/src/__tests__/parsers/android-xml.parser.test.ts b/src/__tests__/parsers/android-xml.parser.test.ts index fa0c95f..f87289d 100644 --- a/src/__tests__/parsers/android-xml.parser.test.ts +++ b/src/__tests__/parsers/android-xml.parser.test.ts @@ -1077,4 +1077,10 @@ describe('AndroidXmlParser', () => { expect(result).toBe('\n\n'); }); }); + + describe('getContentType', () => { + it('should return text/html (Android string resources allow inline HTML markup)', () => { + expect(parser.getContentType()).toBe('text/html'); + }); + }); }); diff --git a/src/__tests__/parsers/json.parser.test.ts b/src/__tests__/parsers/json.parser.test.ts index c017000..1be7c8f 100644 --- a/src/__tests__/parsers/json.parser.test.ts +++ b/src/__tests__/parsers/json.parser.test.ts @@ -398,4 +398,10 @@ describe('JsonParser', () => { expect(result).toBe('{}'); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/parsers/markdown.parser.test.ts b/src/__tests__/parsers/markdown.parser.test.ts index 2c4cb21..bca15e0 100644 --- a/src/__tests__/parsers/markdown.parser.test.ts +++ b/src/__tests__/parsers/markdown.parser.test.ts @@ -540,6 +540,12 @@ For help, contact [support](mailto:support@example.com).`; }); }); + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); + describe('parse and serialize roundtrip', () => { it('should maintain structure through parse and serialize', () => { const originalContent = `# Document Title diff --git a/src/__tests__/parsers/parser.factory.test.ts b/src/__tests__/parsers/parser.factory.test.ts index 657569d..2984e63 100644 --- a/src/__tests__/parsers/parser.factory.test.ts +++ b/src/__tests__/parsers/parser.factory.test.ts @@ -69,4 +69,23 @@ describe('ParserFactory', () => { expect(() => new ParserFactory('../file.po')).not.toThrow(); }); }); + + describe('getContentType', () => { + it('should return text/plain for plain-text formats', () => { + expect(new ParserFactory('/p/file.json').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.po').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.ts').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.vue').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.md').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.mdx').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.strings').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.stringsdict').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.xcstrings').getContentType()).toBe('text/plain'); + expect(new ParserFactory('/p/file.txt').getContentType()).toBe('text/plain'); + }); + + it('should return text/html for Android XML (inline HTML markup permitted)', () => { + expect(new ParserFactory('/p/file.xml').getContentType()).toBe('text/html'); + }); + }); }); diff --git a/src/__tests__/parsers/po.parser.test.ts b/src/__tests__/parsers/po.parser.test.ts index 389115d..0a2df29 100644 --- a/src/__tests__/parsers/po.parser.test.ts +++ b/src/__tests__/parsers/po.parser.test.ts @@ -832,4 +832,10 @@ describe('PoParser', () => { expect(result).toBe('msgid ""\nmsgstr ""\n'); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/parsers/ts.parser.test.ts b/src/__tests__/parsers/ts.parser.test.ts index 9f00629..40b1fe7 100644 --- a/src/__tests__/parsers/ts.parser.test.ts +++ b/src/__tests__/parsers/ts.parser.test.ts @@ -475,4 +475,10 @@ describe('TsParser', () => { expect(result).toBe('const messages = {};\n\nexport default messages;'); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/parsers/txt.parser.test.ts b/src/__tests__/parsers/txt.parser.test.ts index 8e45cc1..539a42f 100644 --- a/src/__tests__/parsers/txt.parser.test.ts +++ b/src/__tests__/parsers/txt.parser.test.ts @@ -250,6 +250,12 @@ describe('TxtParser', () => { }); }); + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); + describe('parse and serialize roundtrip', () => { it('should maintain structure through parse and serialize', () => { const originalContent = 'Hello World\n\nWelcome to our app.\n\nGoodbye!'; diff --git a/src/__tests__/parsers/vue.parser.test.ts b/src/__tests__/parsers/vue.parser.test.ts index 5dd9552..d61cfe7 100644 --- a/src/__tests__/parsers/vue.parser.test.ts +++ b/src/__tests__/parsers/vue.parser.test.ts @@ -535,6 +535,12 @@ describe('VueParser', () => { }); }); + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); + describe('hasI18nTag', () => { it('should return true when i18n tag exists', () => { const content = '\n\n{"key": "value"}\n'; diff --git a/src/__tests__/parsers/xcode-strings.parser.test.ts b/src/__tests__/parsers/xcode-strings.parser.test.ts index 0fdce75..ba15d78 100644 --- a/src/__tests__/parsers/xcode-strings.parser.test.ts +++ b/src/__tests__/parsers/xcode-strings.parser.test.ts @@ -422,4 +422,10 @@ describe('XcodeStringsParser', () => { expect(result).toBe(''); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/parsers/xcode-stringsdict.parser.test.ts b/src/__tests__/parsers/xcode-stringsdict.parser.test.ts index f5be445..d30146d 100644 --- a/src/__tests__/parsers/xcode-stringsdict.parser.test.ts +++ b/src/__tests__/parsers/xcode-stringsdict.parser.test.ts @@ -768,4 +768,10 @@ describe('XcodeStringsdictParser', () => { expect(result).toEqual({}); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/parsers/xcode-xcstrings.parser.test.ts b/src/__tests__/parsers/xcode-xcstrings.parser.test.ts index 2f2a560..94e6542 100644 --- a/src/__tests__/parsers/xcode-xcstrings.parser.test.ts +++ b/src/__tests__/parsers/xcode-xcstrings.parser.test.ts @@ -702,4 +702,10 @@ describe('XcodeXcstringsParser', () => { expect(parsed.version).toBe('1.0'); }); }); + + describe('getContentType', () => { + it('should return text/plain', () => { + expect(parser.getContentType()).toBe('text/plain'); + }); + }); }); diff --git a/src/__tests__/sdk-utf8-streaming.repro.test.ts b/src/__tests__/sdk-utf8-streaming.repro.test.ts new file mode 100644 index 0000000..e2430fe --- /dev/null +++ b/src/__tests__/sdk-utf8-streaming.repro.test.ts @@ -0,0 +1,133 @@ +/** + * Deterministic reproduction of the UTF-8 streaming bug in `@translated/lara`. + * + * The SDK reads the `/translate` streaming response with: + * + * res.on('data', chunk => { buffer += chunk.toString(); ... }) + * + * (`node_modules/@translated/lara/lib/net/lara/node-client.js:172`) + * + * `chunk.toString()` decodes each TCP chunk as UTF-8 independently. When a + * multi-byte character is split across two chunks, each orphaned byte becomes + * a U+FFFD replacement character. This is observable in the wild as `тез��` + * (Bulgarian "these" — the final `и` truncated) or similar in Cyrillic / + * Greek / CJK / Arabic translations. + * + * This test drives the SDK against a local HTTP server that writes the + * response in two flushes, with the byte boundary deliberately placed + * between the two UTF-8 bytes of `и` (`0xD0 0xB8`). The first assertion + * proves the bug exists; the second shows that `node:string_decoder` + * decodes the same wire bytes cleanly — i.e. the SDK's one-line fix. + * + * NOTE: this test depends on observable behaviour of an upstream package. + * If `@translated/lara` is patched to use `StringDecoder` (or `TextDecoder` + * with `stream: true`), the first assertion will fail — which is the + * desired outcome; delete this file at that point. + */ + +import { describe, it, expect } from 'vitest'; +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { StringDecoder } from 'node:string_decoder'; +import { Translator, AuthToken } from '@translated/lara'; + +// Build a structurally-valid, non-expired JWT so the SDK's auth bypass kicks +// in (`client.js` short-circuits when an AuthToken is provided). Only the +// `exp` claim in the payload is inspected. +function makeFakeJwt(): string { + const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from( + JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 3600 }) + ).toString('base64url'); + return `${header}.${payload}.signature-not-checked`; +} + +describe('@translated/lara: UTF-8 streaming response bug', () => { + it('produces U+FFFD when a multi-byte char straddles a TCP chunk boundary', async () => { + // The wire payload Lara would emit for a Bulgarian translation. The + // streaming endpoint sends NDJSON; each line is a JSON object whose `data` + // becomes the partial result. + const responseText = + JSON.stringify({ + status: 200, + data: { + content_type: 'text/plain', + source_language: 'en', + translation: [{ text: 'тези изменения', translatable: true }], + }, + }) + '\n'; + + const responseBytes = Buffer.from(responseText, 'utf8'); + + // Locate the `и` inside `изменения` and find the byte index that sits + // between its two UTF-8 bytes (0xD0 0xB8). Splitting the stream here is + // what triggers the bug. + const izmIndex = responseText.indexOf('изменения'); + const iCharIndex = izmIndex; // first char of 'изменения' is 'и' + const bytesUpToI = Buffer.from(responseText.slice(0, iCharIndex + 1), 'utf8'); + const splitAt = bytesUpToI.length - 1; // between the two bytes of 'и' + + const chunkA = responseBytes.subarray(0, splitAt); + const chunkB = responseBytes.subarray(splitAt); + + // Tiny HTTP server that flushes the response in two writes around the + // mid-character byte. `setNoDelay` + a setTimeout between writes ensures + // they arrive as separate `data` events on the client side. + const server = createServer((_req, res) => { + res.socket?.setNoDelay(true); + res.writeHead(200, { 'content-type': 'application/x-ndjson' }); + res.write(chunkA); + setTimeout(() => { + res.write(chunkB); + res.end(); + }, 20); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + + try { + const translator = new Translator(new AuthToken(makeFakeJwt(), ''), { + serverUrl: `http://127.0.0.1:${port}`, + }); + + const result = await translator.translate( + [{ text: 'these changes', translatable: true }], + 'en', + 'bg' + ); + + const translatedText = (result.translation as Array<{ text: string }>)[0]!.text; + + // ─── Bug demonstration ───────────────────────────────────────────── + // What we shipped on the wire: 'тези изменения' + // What the SDK reports back: 'тези зменения' + expect(translatedText).toContain('�'); + expect(translatedText).not.toContain('изменения'); + // Prefix and suffix are intact — only the chars at the boundary are lost. + expect(translatedText).toContain('тези'); + expect(translatedText).toContain('зменения'); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it('StringDecoder decodes the same chunk-split bytes correctly (the fix)', () => { + // Same wire payload, same chunk boundary — but decoded with + // `node:string_decoder`, which buffers incomplete UTF-8 sequences + // across chunk boundaries. This is the one-line fix the SDK needs. + const payload = JSON.stringify({ data: { translation: [{ text: 'тези изменения' }] } }) + '\n'; + const bytes = Buffer.from(payload, 'utf8'); + const izmIndex = payload.indexOf('изменения'); + const splitAt = Buffer.from(payload.slice(0, izmIndex + 1), 'utf8').length - 1; + + const decoder = new StringDecoder('utf8'); + const safe = + decoder.write(bytes.subarray(0, splitAt)) + + decoder.write(bytes.subarray(splitAt)) + + decoder.end(); + + expect(safe).not.toContain('�'); + expect(safe).toContain('тези изменения'); + }); +}); diff --git a/src/__tests__/utils/contentType.test.ts b/src/__tests__/utils/contentType.test.ts new file mode 100644 index 0000000..0fa936a --- /dev/null +++ b/src/__tests__/utils/contentType.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { hasHtmlMarkup, resolveContentType } from '#utils/contentType.js'; + +describe('hasHtmlMarkup', () => { + it('detects simple opening tags', () => { + expect(hasHtmlMarkup('Hello world')).toBe(true); + expect(hasHtmlMarkup('Click here')).toBe(true); + expect(hasHtmlMarkup('
')).toBe(true); + expect(hasHtmlMarkup('text')).toBe(true); + }); + + it('returns false for plain text', () => { + expect(hasHtmlMarkup('Hello world')).toBe(false); + expect(hasHtmlMarkup('')).toBe(false); + expect(hasHtmlMarkup('A Customer’s continued use')).toBe(false); + expect(hasHtmlMarkup('Multi-line\ntext')).toBe(false); + }); + + it('does not match bare angle brackets without a tag name', () => { + expect(hasHtmlMarkup('5 < 10')).toBe(false); + expect(hasHtmlMarkup('a < b > c')).toBe(false); + expect(hasHtmlMarkup('<>')).toBe(false); + }); + + it('does not match ICU-style placeholders', () => { + expect(hasHtmlMarkup('Hello {name}')).toBe(false); + expect(hasHtmlMarkup('{count, plural, one {# item} other {# items}}')).toBe(false); + }); +}); + +describe('resolveContentType', () => { + it('upgrades plain default to text/html when markup is present', () => { + expect(resolveContentType('Click here', 'text/plain')).toBe('text/html'); + }); + + it('keeps the default when no markup is present', () => { + expect(resolveContentType('Hello', 'text/plain')).toBe('text/plain'); + expect(resolveContentType('Hello', 'text/html')).toBe('text/html'); + }); + + it('returns text/html for markup regardless of the default', () => { + expect(resolveContentType('bold', 'text/html')).toBe('text/html'); + expect(resolveContentType('bold', 'text/plain')).toBe('text/html'); + }); +}); diff --git a/src/interface/parser.ts b/src/interface/parser.ts index 5ccd9d7..4a930b1 100644 --- a/src/interface/parser.ts +++ b/src/interface/parser.ts @@ -45,4 +45,16 @@ export interface Parser, TOptions = void> { * @returns The default content string */ getFallback(): string; + + /** + * Returns the Lara API content type to use when translating values + * parsed from this format. Use `"text/plain"` for formats whose values + * are plain strings, or `"text/html"` for formats whose values may + * contain inline markup that the API should preserve as HTML. + * + * Setting this explicitly prevents Lara from auto-detecting structured + * input as HTML-flavored content, which can otherwise corrupt plain + * text (replacing characters with `?` placeholders). + */ + getContentType(): string; } diff --git a/src/messages/messages.ts b/src/messages/messages.ts index 59ca708..6e4af71 100644 --- a/src/messages/messages.ts +++ b/src/messages/messages.ts @@ -28,6 +28,7 @@ export const Messages = { noSupportedFileTypes: 'No supported file types configured', emptyTranslationResult: (value: string) => `Translation service returned empty result for: ${value}`, + translationRetryFailed: 'Translation failed for some keys. Please retry.', maxRetriesExceeded: 'Maximum retry attempts exceeded. Please try again later.', envVarsNotSet: 'LARA_ACCESS_KEY_ID and LARA_ACCESS_KEY_SECRET must be set', fileAndTextMutuallyExclusive: '--file and --text cannot be used together', diff --git a/src/modules/translation/translation.engine.ts b/src/modules/translation/translation.engine.ts index 9090185..f9b22b9 100644 --- a/src/modules/translation/translation.engine.ts +++ b/src/modules/translation/translation.engine.ts @@ -5,6 +5,7 @@ import { calculateChecksum, commitChecksum, ChecksumState } from '#utils/checksu import { buildLocalePath, ensureDirectoryExists, readSafe } from '#utils/path.js'; import { detectFormatting } from '#utils/formatting.js'; import { normalizeEntities } from '#utils/entities.js'; +import { resolveContentType } from '#utils/contentType.js'; import { writeFile } from 'fs/promises'; import { progressWithOra } from '#utils/progressWithOra.js'; import { TextBlock } from './translation.service.js'; @@ -85,6 +86,10 @@ export class TranslationEngine { // Automatically detects the file format based on the input path extension. private readonly parser: ParserFactory; + // Default Lara content type for this format. Overridden per value when + // the source string contains inline HTML markup. + private readonly defaultContentType: string; + constructor(options: TranslationEngineOptions) { this.sourceLocale = options.sourceLocale; this.targetLocales = options.targetLocales; @@ -120,6 +125,7 @@ export class TranslationEngine { this.translatorService = TranslationService.getInstance(); this.parser = new ParserFactory(this.inputPath); + this.defaultContentType = this.parser.getContentType(); } public async translate() { @@ -257,19 +263,35 @@ export class TranslationEngine { targetLocale: string ): Promise> { const translations = new Map(); + // Tasks whose first translation came back containing the U+FFFD + // replacement character. Symptomatic of a UTF-8 streaming bug in + // @translated/lara (chunk.toString() per chunk loses bytes when a + // multi-byte character straddles a TCP chunk boundary). Retried below + // as solo calls, which produce tiny responses unlikely to span chunks. + const corruptedTasks: TranslateTask[] = []; + + const recordResult = (task: TranslateTask, translatedText: string): void => { + const normalized = normalizeEntities(task.text, translatedText); + if (normalized.includes('�')) { + corruptedTasks.push(task); + } else { + translations.set(task.key, normalized); + } + }; const soloPromises = classified.solo.map(async (task) => { + const contentType = resolveContentType(task.text, this.defaultContentType); const result = await this.translatorService.translate( [{ text: task.text, translatable: true }], this.sourceLocale, targetLocale, - this.buildTranslateOptions(task.instruction) + this.buildTranslateOptions(task.instruction, contentType) ); const translated = result[0]; if (!translated) { throw new Error(Messages.errors.emptyTranslationResult(task.text)); } - translations.set(task.key, normalizeEntities(task.text, translated.text)); + recordResult(task, translated.text); }); const batchPromises: Promise[] = []; @@ -277,43 +299,122 @@ export class TranslationEngine { // projectInstruction / none) — isKeySpecific=false implies this. Read it // off the first task instead of re-resolving. const batchInstruction = classified.batch[0]?.instruction; - const batchOptions = this.buildTranslateOptions(batchInstruction); - - for (let i = 0; i < classified.batch.length; i += this.batchSize) { - const chunk = classified.batch.slice(i, i + this.batchSize); - const textBlocks: TextBlock[] = chunk.map((task) => ({ - text: task.text, - translatable: true, - })); - - batchPromises.push( - (async () => { - const result = await this.translatorService.translateBatchWithFallback( - textBlocks, - this.sourceLocale, - targetLocale, - batchOptions - ); - chunk.forEach((task, idx) => { - const translated = result[idx]; - if (!translated) { - throw new Error(Messages.errors.emptyTranslationResult(task.text)); - } - translations.set(task.key, normalizeEntities(task.text, translated.text)); - }); - })() - ); + + // Group all batch tasks by detected content type before chunking. A JSON + // file can mix plain values with values that contain inline HTML (e.g. + // `"Click here"`); these two must be sent in separate + // API calls with the matching `contentType`, otherwise the plain pipeline + // strips tags or the HTML pipeline corrupts plain Cyrillic/Greek text. + // Grouping across all chunks (instead of within each chunk) minimises + // the number of API calls when one content type is sparse. + const groups = new Map(); + for (const task of classified.batch) { + const contentType = resolveContentType(task.text, this.defaultContentType); + const bucket = groups.get(contentType); + if (bucket) { + bucket.push(task); + } else { + groups.set(contentType, [task]); + } + } + + for (const [contentType, tasks] of groups) { + const groupOptions = this.buildTranslateOptions(batchInstruction, contentType); + for (let i = 0; i < tasks.length; i += this.batchSize) { + const chunk = tasks.slice(i, i + this.batchSize); + const textBlocks: TextBlock[] = chunk.map((task) => ({ + text: task.text, + translatable: true, + })); + + batchPromises.push( + (async () => { + const result = await this.translatorService.translateBatchWithFallback( + textBlocks, + this.sourceLocale, + targetLocale, + groupOptions + ); + chunk.forEach((task, idx) => { + const translated = result[idx]; + if (!translated) { + throw new Error(Messages.errors.emptyTranslationResult(task.text)); + } + recordResult(task, translated.text); + }); + })() + ); + } } await Promise.all([...soloPromises, ...batchPromises]); + + if (corruptedTasks.length > 0) { + await this.retryCorruptedTasks(corruptedTasks, targetLocale, translations); + } + return translations; } - private buildTranslateOptions(instruction: string | undefined): TranslateOptions { + /** + * Re-translate each task as a solo call. Solo responses are small enough + * that they almost never span a TCP chunk boundary, so the U+FFFD bug + * does not fire on them. We retry up to MAX_RETRIES times silently; if + * every attempt still comes back with U+FFFD we surface a short, neutral + * error so the user knows to re-run without exposing SDK internals. + */ + private async retryCorruptedTasks( + tasks: TranslateTask[], + targetLocale: string, + translations: Map + ): Promise { + const MAX_RETRIES = 3; + let anyStillCorrupted = false; + + await Promise.all( + tasks.map(async (task) => { + const contentType = resolveContentType(task.text, this.defaultContentType); + const options = this.buildTranslateOptions(task.instruction, contentType); + + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + const result = await this.translatorService.translate( + [{ text: task.text, translatable: true }], + this.sourceLocale, + targetLocale, + options + ); + const translated = result[0]; + if (!translated) { + throw new Error(Messages.errors.emptyTranslationResult(task.text)); + } + const normalized = normalizeEntities(task.text, translated.text); + if (!normalized.includes('�')) { + translations.set(task.key, normalized); + return; + } + } + + anyStillCorrupted = true; + }) + ); + + if (anyStillCorrupted) { + throw new Error(Messages.errors.translationRetryFailed); + } + } + + private buildTranslateOptions( + instruction: string | undefined, + contentType: string + ): TranslateOptions { return { instructions: instruction ? [instruction] : undefined, adaptTo: this.translationMemoryIds.length > 0 ? this.translationMemoryIds : [], // Always pass an array for adaptTo; an empty array prevents Lara from using translation memories when none are explicitly selected glossaries: this.glossaryIds.length > 0 ? this.glossaryIds : undefined, + // Setting contentType explicitly prevents Lara from auto-detecting + // TextBlock[] input as HTML-flavored content, which can otherwise + // corrupt plain text (e.g., replacing characters with `?`). + contentType, noTrace: this.noTrace || undefined, }; } diff --git a/src/parsers/android-xml.parser.ts b/src/parsers/android-xml.parser.ts index 28173ff..f7314c9 100644 --- a/src/parsers/android-xml.parser.ts +++ b/src/parsers/android-xml.parser.ts @@ -546,4 +546,8 @@ export class AndroidXmlParser implements Parser< getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/html'; + } } diff --git a/src/parsers/json.parser.ts b/src/parsers/json.parser.ts index ddfa231..a73ec2a 100644 --- a/src/parsers/json.parser.ts +++ b/src/parsers/json.parser.ts @@ -89,4 +89,8 @@ export class JsonParser implements Parser, JsonParserOpt getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/parsers/markdown.parser.ts b/src/parsers/markdown.parser.ts index d778583..dc21ca0 100644 --- a/src/parsers/markdown.parser.ts +++ b/src/parsers/markdown.parser.ts @@ -67,6 +67,10 @@ export class MarkdownParser implements Parser, MarkdownP return this.fallbackContent; } + getContentType(): string { + return 'text/plain'; + } + /** * Extracts all translatable text segments from the markdown AST. * diff --git a/src/parsers/parser.factory.ts b/src/parsers/parser.factory.ts index 2792d95..0ad69b2 100644 --- a/src/parsers/parser.factory.ts +++ b/src/parsers/parser.factory.ts @@ -130,4 +130,12 @@ export class ParserFactory { getFallback(): string { return this.parser.getFallback(); } + + /** + * Returns the Lara API content type to use when translating values + * parsed from this format. + */ + getContentType(): string { + return this.parser.getContentType(); + } } diff --git a/src/parsers/po.parser.ts b/src/parsers/po.parser.ts index 9b61713..f14bbd7 100644 --- a/src/parsers/po.parser.ts +++ b/src/parsers/po.parser.ts @@ -271,4 +271,8 @@ export class PoParser implements Parser, PoParserOptions getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/parsers/ts.parser.ts b/src/parsers/ts.parser.ts index 6915e1f..3a50e74 100644 --- a/src/parsers/ts.parser.ts +++ b/src/parsers/ts.parser.ts @@ -89,6 +89,10 @@ export class TsParser implements Parser, TsParserOptions return this.fallbackContent; } + getContentType(): string { + return 'text/plain'; + } + private extractMessagesObject(content: string): Record | null { try { // Parse the TypeScript file into an AST diff --git a/src/parsers/txt.parser.ts b/src/parsers/txt.parser.ts index ae06591..744ec23 100644 --- a/src/parsers/txt.parser.ts +++ b/src/parsers/txt.parser.ts @@ -67,4 +67,8 @@ export class TxtParser implements Parser, TxtParserOptio getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/parsers/vue.parser.ts b/src/parsers/vue.parser.ts index 0cece05..082a7ee 100644 --- a/src/parsers/vue.parser.ts +++ b/src/parsers/vue.parser.ts @@ -116,6 +116,10 @@ export class VueParser implements Parser, VueParserOptio return this.fallbackContent; } + getContentType(): string { + return 'text/plain'; + } + /** * Checks if a Vue file contains an i18n tag. * diff --git a/src/parsers/xcode-strings.parser.ts b/src/parsers/xcode-strings.parser.ts index 857c0a5..66d1dfe 100644 --- a/src/parsers/xcode-strings.parser.ts +++ b/src/parsers/xcode-strings.parser.ts @@ -247,4 +247,8 @@ export class XcodeStringsParser implements Parser< getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/parsers/xcode-stringsdict.parser.ts b/src/parsers/xcode-stringsdict.parser.ts index 1f9ce19..feca479 100644 --- a/src/parsers/xcode-stringsdict.parser.ts +++ b/src/parsers/xcode-stringsdict.parser.ts @@ -338,4 +338,8 @@ export class XcodeStringsdictParser implements Parser< getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/parsers/xcode-xcstrings.parser.ts b/src/parsers/xcode-xcstrings.parser.ts index c1386ce..479cb6a 100644 --- a/src/parsers/xcode-xcstrings.parser.ts +++ b/src/parsers/xcode-xcstrings.parser.ts @@ -283,4 +283,8 @@ export class XcodeXcstringsParser implements Parser< getFallback(): string { return this.fallbackContent; } + + getContentType(): string { + return 'text/plain'; + } } diff --git a/src/utils/contentType.ts b/src/utils/contentType.ts new file mode 100644 index 0000000..a6d9fd1 --- /dev/null +++ b/src/utils/contentType.ts @@ -0,0 +1,24 @@ +/** + * Heuristic detection of HTML-like markup in a string. Matches an opening tag + * that starts with an ASCII letter — e.g. ``, ``, `
`. + * Deliberately conservative: bare `<` followed by whitespace or a digit + * (e.g. `5 < 10`) does not match. + */ +const HTML_TAG_RE = /<[a-zA-Z][^>]*>/; + +export function hasHtmlMarkup(text: string): boolean { + return HTML_TAG_RE.test(text); +} + +/** + * Picks the Lara `contentType` for a single translatable value. + * + * If the value contains inline HTML markup, it must be sent as `text/html` + * regardless of the parser's default — otherwise Lara's plain-text path + * may strip or escape the tags. If the value has no markup, fall back to + * the parser's declared default (typically `text/plain` for JSON/PO/etc., + * or `text/html` for Android XML where the whole format permits markup). + */ +export function resolveContentType(text: string, defaultType: string): string { + return hasHtmlMarkup(text) ? 'text/html' : defaultType; +} From 5a94667ced82cad57806ef73ad79c5cef76f5396 Mon Sep 17 00:00:00 2001 From: ste__cal Date: Mon, 25 May 2026 17:53:24 +0200 Subject: [PATCH 2/2] chore: upgrade glob and patch brace-expansion CVE (#69) ## Changed - Bump `glob` from `^11.1.0` to `^13.0.6` to remove deprecation warning at install - Override transitive `brace-expansion` in vulnerable range `>=5.0.0 <5.0.6` to `^5.0.6` (CVE-2026-45149, GHSA-jxxr-4gwj-5jf2) --- package.json | 7 ++++++- pnpm-lock.yaml | 56 ++++++++++++-------------------------------------- 2 files changed, 19 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index 8db04b4..d5dbf8d 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "fast-xml-parser": "^5.5.9", "flat": "^6.0.1", "gettext-parser": "^8.0.0", - "glob": "^11.1.0", + "glob": "^13.0.6", "ora": "^8.2.0", "picomatch": "^4.0.4", "remark": "^15.0.1", @@ -87,5 +87,10 @@ "typescript-eslint": "^8.57.2", "vite": "^7.3.3", "vitest": "^4.1.2" + }, + "pnpm": { + "overrides": { + "brace-expansion@>=5.0.0 <5.0.6": "^5.0.6" + } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2298a3..8ef2ce5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + brace-expansion@>=5.0.0 <5.0.6: ^5.0.6 + importers: .: @@ -42,8 +45,8 @@ importers: specifier: ^8.0.0 version: 8.0.0 glob: - specifier: ^11.1.0 - version: 11.1.0 + specifier: ^13.0.6 + version: 13.0.6 ora: specifier: ^8.2.0 version: 8.2.0 @@ -499,10 +502,6 @@ packages: '@types/node': optional: true - '@isaacs/cliui@9.0.0': - resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==} - engines: {node: '>=18'} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -842,8 +841,8 @@ packages: brace-expansion@1.1.14: resolution: {integrity: sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==} - brace-expansion@5.0.5: - resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} + brace-expansion@5.0.6: + resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==} engines: {node: 18 || 20 || >=22} buffer@6.0.3: @@ -1109,10 +1108,6 @@ packages: flatted@3.4.2: resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} - foreground-child@3.3.1: - resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} - engines: {node: '>=14'} - form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -1145,11 +1140,9 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} - glob@11.1.0: - resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} - engines: {node: 20 || >=22} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me - hasBin: true + glob@13.0.6: + resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} + engines: {node: 18 || 20 || >=22} globals@14.0.0: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} @@ -1252,10 +1245,6 @@ packages: resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} engines: {node: '>=8'} - jackspeak@4.2.3: - resolution: {integrity: sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==} - engines: {node: 20 || >=22} - js-tokens@10.0.0: resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} @@ -1500,9 +1489,6 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} - package-json-from-dist@1.0.1: - resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} - parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -2155,8 +2141,6 @@ snapshots: optionalDependencies: '@types/node': 24.12.0 - '@isaacs/cliui@9.0.0': {} - '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -2488,7 +2472,7 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 - brace-expansion@5.0.5: + brace-expansion@5.0.6: dependencies: balanced-match: 4.0.4 @@ -2759,11 +2743,6 @@ snapshots: flatted@3.4.2: {} - foreground-child@3.3.1: - dependencies: - cross-spawn: 7.0.6 - signal-exit: 4.1.0 - form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -2808,13 +2787,10 @@ snapshots: dependencies: is-glob: 4.0.3 - glob@11.1.0: + glob@13.0.6: dependencies: - foreground-child: 3.3.1 - jackspeak: 4.2.3 minimatch: 10.2.4 minipass: 7.1.3 - package-json-from-dist: 1.0.1 path-scurry: 2.0.2 globals@14.0.0: {} @@ -2889,10 +2865,6 @@ snapshots: html-escaper: 2.0.2 istanbul-lib-report: 3.0.1 - jackspeak@4.2.3: - dependencies: - '@isaacs/cliui': 9.0.0 - js-tokens@10.0.0: {} js-tokens@4.0.0: {} @@ -3254,7 +3226,7 @@ snapshots: minimatch@10.2.4: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 5.0.6 minimatch@3.1.5: dependencies: @@ -3305,8 +3277,6 @@ snapshots: dependencies: p-limit: 3.1.0 - package-json-from-dist@1.0.1: {} - parent-module@1.0.1: dependencies: callsites: 3.1.0