fix(docs-i18n): escape model-added list markers

2026-08-02 15:11:32 +00:00 · 2026-07-23 06:54:27 -04:00
parent 9ae9fcb1cd
commit 7f550eaf50
3 changed files with 127 additions and 10 deletions
--- a/scripts/docs-i18n/doc_chunked_raw.go
+++ b/scripts/docs-i18n/doc_chunked_raw.go
@@ -60,13 +60,14 @@ func translateDocBodyChunked(ctx context.Context, translator docsTranslator, rel
 	mapping := map[string]string{}
 	maskedBody := maskMarkdownFencedLiterals(body, placeholderState.Next, &placeholders, mapping)
 	maskedBody = maskMarkdownDocSyntax(maskedBody, placeholderState.Next, &placeholders, mapping)
+	listPlaceholders := maskedListMarkerPlaceholders(mapping)
 	blocks := splitDocBodyIntoBlocks(maskedBody)
 	groups := groupDocBlocks(blocks, docsI18nDocChunkMaxBytes())
 	logDocChunkPlan(relPath, blocks, groups)
 	out := strings.Builder{}
 	for index, group := range groups {
 		chunkID := fmt.Sprintf("%s.chunk-%03d", relPath, index+1)
-		translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, placeholders, srcLang, tgtLang)
+		translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, placeholders, listPlaceholders, srcLang, tgtLang)
 		if err != nil {
 			return "", err
 		}
@@ -74,6 +75,7 @@ func translateDocBodyChunked(ctx context.Context, translator docsTranslator, rel
 	}
 	translatedBody := out.String()
 	translatedBody = normalizeMaskedListMarkerPlaceholders(translatedBody, mapping)
+	translatedBody = escapeUnexpectedMarkdownListMarkers(translatedBody, listPlaceholders)
 	if err := validatePlaceholders(translatedBody, placeholders); err != nil {
 		return "", fmt.Errorf("%s: restore fenced literals: %w", relPath, err)
 	}
@@ -126,14 +128,14 @@ func validateDocBodyFencedLiterals(source, translated string) error {
 	return nil
 }

-func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) {
+func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
 	source := strings.Join(blocks, "")
 	if strings.TrimSpace(source) == "" {
 		return source, nil
 	}
 	if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
 		logDocChunkPlanSplit(chunkID, plan, source)
-		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang)
+		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
 	}
 	normalizedSource, commonIndent := stripCommonIndent(source)
 	log.Printf("docs-i18n: chunk start %s blocks=%d bytes=%d", chunkID, len(blocks), len(source))
@@ -145,6 +147,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
 		translated = sanitizeDocChunkProtocolWrappers(source, translated)
 		translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
 		translated = reapplyCommonIndent(translated, commonIndent)
+		translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
 		if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil {
 			log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated))
 			return translated, nil
@@ -153,27 +156,27 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
 		}
 	}
 	if len(blocks) <= 1 {
-		if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, protectedPlaceholders, srcLang, tgtLang); fallbackErr == nil {
+		if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, protectedPlaceholders, listPlaceholders, srcLang, tgtLang); fallbackErr == nil {
 			return fallback, nil
 		}
 		if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
 			logDocChunkPlanSplit(chunkID, plan, source)
-			return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang)
+			return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
 		}
 		return "", fmt.Errorf("%s: %w", chunkID, err)
 	}
 	if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
 		logDocChunkSplit(chunkID, len(blocks), err)
-		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang)
+		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
 	}
 	if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok {
 		logDocChunkSplit(chunkID, len(blocks), err)
-		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang)
+		return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
 	}
 	return "", fmt.Errorf("%s: %w", chunkID, err)
 }

-func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) {
+func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
 	sourceStructure := summarizeDocChunkStructure(source)
 	if sourceStructure.fenceCount != 0 {
 		return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID)
@@ -194,6 +197,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk
 	translated = sanitizeDocChunkProtocolWrappers(source, translated)
 	translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
 	translated = reapplyCommonIndent(translated, commonIndent)
+	translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
 	if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
 		return "", validationErr
 	}
@@ -733,11 +737,11 @@ func containsProtocolWrapperToken(text string) bool {
 	return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart))
 }

-func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) {
+func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
 	var out strings.Builder
 	translatedGroups := make([]string, 0, len(groups))
 	for index, group := range groups {
-		translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, protectedPlaceholders, srcLang, tgtLang)
+		translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
 		if err != nil {
 			return "", err
 		}
--- a/scripts/docs-i18n/doc_mode_test.go
+++ b/scripts/docs-i18n/doc_mode_test.go
@@ -300,6 +300,18 @@ func (t *docSyntaxMaskingTranslator) TranslateRaw(_ context.Context, text, _, _

 func (t *docSyntaxMaskingTranslator) Close() {}

+type accidentalListMarkerTranslator struct{}
+
+func (accidentalListMarkerTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
+	return text, nil
+}
+
+func (accidentalListMarkerTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
+	return strings.ReplaceAll(text, "September begins the standard rate.", "1. September beginnt der Standardtarif."), nil
+}
+
+func (accidentalListMarkerTranslator) Close() {}
+
 type duplicateFirstFencedPlaceholderTranslator struct {
 	rawCalls int
 }
@@ -762,6 +774,37 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing.
 	}
 }

+func TestEscapeUnexpectedMarkdownListMarkersPreservesFencedExamples(t *testing.T) {
+	t.Parallel()
+
+	translated := strings.Join([]string{
+		"1. September beginnt der Standardtarif.",
+		"- Unbeabsichtigter Aufzählungspunkt.",
+		"> 2) Verschachtelte Nummerierung.",
+		"- __OC_I18N_000001__Maskierter Listeneintrag.",
+		"3. September mit __OC_I18N_000002__Inlinecode.",
+		"```md",
+		"1. Beispiel bleibt unverändert.",
+		"```",
+		"",
+	}, "\n")
+	want := strings.Join([]string{
+		`1\. September beginnt der Standardtarif.`,
+		`\- Unbeabsichtigter Aufzählungspunkt.`,
+		`> 2\) Verschachtelte Nummerierung.`,
+		"- __OC_I18N_000001__Maskierter Listeneintrag.",
+		`3\. September mit __OC_I18N_000002__Inlinecode.`,
+		"```md",
+		"1. Beispiel bleibt unverändert.",
+		"```",
+		"",
+	}, "\n")
+
+	if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]struct{}{"__OC_I18N_000001__": {}}); got != want {
+		t.Fatalf("unexpected escaped list markers:\n%s\nwant:\n%s", got, want)
+	}
+}
+
 func TestValidateDocChunkTranslationRejectsTranslatedInlineCode(t *testing.T) {
 	t.Parallel()

@@ -2363,6 +2406,27 @@ func TestTranslateDocBodyChunkedMasksInlineCodeAndListMarkers(t *testing.T) {
 	}
 }

+func TestTranslateDocBodyChunkedEscapesModelInventedListMarker(t *testing.T) {
+	t.Parallel()
+
+	body := "1. First step.\n2. Second step.\n\nSeptember begins the standard rate.\n"
+	translated, err := translateDocBodyChunked(
+		context.Background(), accidentalListMarkerTranslator{}, "concepts/model-failover.md", body, "en", "de",
+	)
+	if err != nil {
+		t.Fatalf("translateDocBodyChunked returned error: %v", err)
+	}
+	if !strings.Contains(translated, "1. First step.\n2. Second step.") {
+		t.Fatalf("expected source list markers to be restored:\n%s", translated)
+	}
+	if !strings.Contains(translated, `1\. September beginnt der Standardtarif.`) {
+		t.Fatalf("expected model-invented list marker to be escaped:\n%s", translated)
+	}
+	if err := validateDocBodyFencedLiterals(body, translated); err != nil {
+		t.Fatalf("expected repaired final structure to validate: %v", err)
+	}
+}
+
 func TestTranslateDocBodyChunkedRetriesSingletonFenceAfterValidationFailure(t *testing.T) {
 	body := strings.Join([]string{
 		"```md",
--- a/scripts/docs-i18n/masking.go
+++ b/scripts/docs-i18n/masking.go
@@ -150,6 +150,55 @@ func normalizeMaskedListMarkerPlaceholders(text string, mapping map[string]strin
 	return strings.Join(lines, "")
 }

+func maskedListMarkerPlaceholders(mapping map[string]string) map[string]struct{} {
+	placeholders := make(map[string]struct{})
+	for placeholder, original := range mapping {
+		markerSpan := listMarkerRe.FindStringIndex(original)
+		if markerSpan != nil && markerSpan[0] == 0 && markerSpan[1] == len(original) {
+			placeholders[placeholder] = struct{}{}
+		}
+	}
+	return placeholders
+}
+
+func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]struct{}) string {
+	ranges := markdownListMarkerRanges(text)
+	if len(ranges) == 0 {
+		return text
+	}
+	var out strings.Builder
+	position := 0
+	for _, span := range ranges {
+		lineEnd := strings.IndexByte(text[span[1]:], '\n')
+		if lineEnd < 0 {
+			lineEnd = len(text)
+		} else {
+			lineEnd += span[1]
+		}
+		if placeholder := placeholderRe.FindString(text[span[1]:lineEnd]); placeholder != "" {
+			if _, ok := listPlaceholders[placeholder]; ok && strings.HasPrefix(text[span[1]:lineEnd], placeholder) {
+				continue
+			}
+		}
+		value := text[span[0]:span[1]]
+		match := listMarkerRe.FindStringSubmatchIndex(value)
+		if len(match) < 6 {
+			continue
+		}
+		markerStart, markerEnd := match[4], match[5]
+		insertAt := markerStart
+		if markerEnd-markerStart > 1 {
+			insertAt = markerEnd - 1
+		}
+		absolute := span[0] + insertAt
+		out.WriteString(text[position:absolute])
+		out.WriteByte('\\')
+		position = absolute
+	}
+	out.WriteString(text[position:])
+	return out.String()
+}
+
 func protectedMarkdownLinkRanges(text string) [][2]int {
 	ranges := make([][2]int, 0)
 	for _, match := range linkLabelRe.FindAllStringSubmatchIndex(text, -1) {