diff --git a/scripts/docs-i18n/doc_chunked_raw.go b/scripts/docs-i18n/doc_chunked_raw.go index 96b7c2495fb7..5d415d731a39 100644 --- a/scripts/docs-i18n/doc_chunked_raw.go +++ b/scripts/docs-i18n/doc_chunked_raw.go @@ -60,13 +60,14 @@ func translateDocBodyChunked(ctx context.Context, translator docsTranslator, rel mapping := map[string]string{} maskedBody := maskMarkdownFencedLiterals(body, placeholderState.Next, &placeholders, mapping) maskedBody = maskMarkdownDocSyntax(maskedBody, placeholderState.Next, &placeholders, mapping) + listPlaceholders := maskedListMarkerPlaceholders(mapping) blocks := splitDocBodyIntoBlocks(maskedBody) groups := groupDocBlocks(blocks, docsI18nDocChunkMaxBytes()) logDocChunkPlan(relPath, blocks, groups) out := strings.Builder{} for index, group := range groups { chunkID := fmt.Sprintf("%s.chunk-%03d", relPath, index+1) - translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, placeholders, srcLang, tgtLang) + translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, placeholders, listPlaceholders, srcLang, tgtLang) if err != nil { return "", err } @@ -74,6 +75,7 @@ func translateDocBodyChunked(ctx context.Context, translator docsTranslator, rel } translatedBody := out.String() translatedBody = normalizeMaskedListMarkerPlaceholders(translatedBody, mapping) + translatedBody = escapeUnexpectedMarkdownListMarkers(translatedBody, listPlaceholders) if err := validatePlaceholders(translatedBody, placeholders); err != nil { return "", fmt.Errorf("%s: restore fenced literals: %w", relPath, err) } @@ -126,14 +128,14 @@ func validateDocBodyFencedLiterals(source, translated string) error { return nil } -func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) { +func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { source := strings.Join(blocks, "") if strings.TrimSpace(source) == "" { return source, nil } if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok { logDocChunkPlanSplit(chunkID, plan, source) - return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang) + return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } normalizedSource, commonIndent := stripCommonIndent(source) log.Printf("docs-i18n: chunk start %s blocks=%d bytes=%d", chunkID, len(blocks), len(source)) @@ -145,6 +147,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun translated = sanitizeDocChunkProtocolWrappers(source, translated) translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) translated = reapplyCommonIndent(translated, commonIndent) + translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil { log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated)) return translated, nil @@ -153,27 +156,27 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun } } if len(blocks) <= 1 { - if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, protectedPlaceholders, srcLang, tgtLang); fallbackErr == nil { + if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, protectedPlaceholders, listPlaceholders, srcLang, tgtLang); fallbackErr == nil { return fallback, nil } if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok { logDocChunkPlanSplit(chunkID, plan, source) - return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang) + return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } return "", fmt.Errorf("%s: %w", chunkID, err) } if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok { logDocChunkSplit(chunkID, len(blocks), err) - return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang) + return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok { logDocChunkSplit(chunkID, len(blocks), err) - return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, srcLang, tgtLang) + return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } return "", fmt.Errorf("%s: %w", chunkID, err) } -func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) { +func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { sourceStructure := summarizeDocChunkStructure(source) if sourceStructure.fenceCount != 0 { return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID) @@ -194,6 +197,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk translated = sanitizeDocChunkProtocolWrappers(source, translated) translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) translated = reapplyCommonIndent(translated, commonIndent) + translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil { return "", validationErr } @@ -733,11 +737,11 @@ func containsProtocolWrapperToken(text string) bool { return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart)) } -func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, srcLang, tgtLang string) (string, error) { +func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { var out strings.Builder translatedGroups := make([]string, 0, len(groups)) for index, group := range groups { - translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, protectedPlaceholders, srcLang, tgtLang) + translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) if err != nil { return "", err } diff --git a/scripts/docs-i18n/doc_mode_test.go b/scripts/docs-i18n/doc_mode_test.go index 1e4e8feed6f0..077825e10daf 100644 --- a/scripts/docs-i18n/doc_mode_test.go +++ b/scripts/docs-i18n/doc_mode_test.go @@ -300,6 +300,18 @@ func (t *docSyntaxMaskingTranslator) TranslateRaw(_ context.Context, text, _, _ func (t *docSyntaxMaskingTranslator) Close() {} +type accidentalListMarkerTranslator struct{} + +func (accidentalListMarkerTranslator) Translate(_ context.Context, text, _, _ string) (string, error) { + return text, nil +} + +func (accidentalListMarkerTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) { + return strings.ReplaceAll(text, "September begins the standard rate.", "1. September beginnt der Standardtarif."), nil +} + +func (accidentalListMarkerTranslator) Close() {} + type duplicateFirstFencedPlaceholderTranslator struct { rawCalls int } @@ -762,6 +774,37 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing. } } +func TestEscapeUnexpectedMarkdownListMarkersPreservesFencedExamples(t *testing.T) { + t.Parallel() + + translated := strings.Join([]string{ + "1. September beginnt der Standardtarif.", + "- Unbeabsichtigter Aufzählungspunkt.", + "> 2) Verschachtelte Nummerierung.", + "- __OC_I18N_000001__Maskierter Listeneintrag.", + "3. September mit __OC_I18N_000002__Inlinecode.", + "```md", + "1. Beispiel bleibt unverändert.", + "```", + "", + }, "\n") + want := strings.Join([]string{ + `1\. September beginnt der Standardtarif.`, + `\- Unbeabsichtigter Aufzählungspunkt.`, + `> 2\) Verschachtelte Nummerierung.`, + "- __OC_I18N_000001__Maskierter Listeneintrag.", + `3\. September mit __OC_I18N_000002__Inlinecode.`, + "```md", + "1. Beispiel bleibt unverändert.", + "```", + "", + }, "\n") + + if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]struct{}{"__OC_I18N_000001__": {}}); got != want { + t.Fatalf("unexpected escaped list markers:\n%s\nwant:\n%s", got, want) + } +} + func TestValidateDocChunkTranslationRejectsTranslatedInlineCode(t *testing.T) { t.Parallel() @@ -2363,6 +2406,27 @@ func TestTranslateDocBodyChunkedMasksInlineCodeAndListMarkers(t *testing.T) { } } +func TestTranslateDocBodyChunkedEscapesModelInventedListMarker(t *testing.T) { + t.Parallel() + + body := "1. First step.\n2. Second step.\n\nSeptember begins the standard rate.\n" + translated, err := translateDocBodyChunked( + context.Background(), accidentalListMarkerTranslator{}, "concepts/model-failover.md", body, "en", "de", + ) + if err != nil { + t.Fatalf("translateDocBodyChunked returned error: %v", err) + } + if !strings.Contains(translated, "1. First step.\n2. Second step.") { + t.Fatalf("expected source list markers to be restored:\n%s", translated) + } + if !strings.Contains(translated, `1\. September beginnt der Standardtarif.`) { + t.Fatalf("expected model-invented list marker to be escaped:\n%s", translated) + } + if err := validateDocBodyFencedLiterals(body, translated); err != nil { + t.Fatalf("expected repaired final structure to validate: %v", err) + } +} + func TestTranslateDocBodyChunkedRetriesSingletonFenceAfterValidationFailure(t *testing.T) { body := strings.Join([]string{ "```md", diff --git a/scripts/docs-i18n/masking.go b/scripts/docs-i18n/masking.go index e7ca457cc7c9..05d42f03f608 100644 --- a/scripts/docs-i18n/masking.go +++ b/scripts/docs-i18n/masking.go @@ -150,6 +150,55 @@ func normalizeMaskedListMarkerPlaceholders(text string, mapping map[string]strin return strings.Join(lines, "") } +func maskedListMarkerPlaceholders(mapping map[string]string) map[string]struct{} { + placeholders := make(map[string]struct{}) + for placeholder, original := range mapping { + markerSpan := listMarkerRe.FindStringIndex(original) + if markerSpan != nil && markerSpan[0] == 0 && markerSpan[1] == len(original) { + placeholders[placeholder] = struct{}{} + } + } + return placeholders +} + +func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]struct{}) string { + ranges := markdownListMarkerRanges(text) + if len(ranges) == 0 { + return text + } + var out strings.Builder + position := 0 + for _, span := range ranges { + lineEnd := strings.IndexByte(text[span[1]:], '\n') + if lineEnd < 0 { + lineEnd = len(text) + } else { + lineEnd += span[1] + } + if placeholder := placeholderRe.FindString(text[span[1]:lineEnd]); placeholder != "" { + if _, ok := listPlaceholders[placeholder]; ok && strings.HasPrefix(text[span[1]:lineEnd], placeholder) { + continue + } + } + value := text[span[0]:span[1]] + match := listMarkerRe.FindStringSubmatchIndex(value) + if len(match) < 6 { + continue + } + markerStart, markerEnd := match[4], match[5] + insertAt := markerStart + if markerEnd-markerStart > 1 { + insertAt = markerEnd - 1 + } + absolute := span[0] + insertAt + out.WriteString(text[position:absolute]) + out.WriteByte('\\') + position = absolute + } + out.WriteString(text[position:]) + return out.String() +} + func protectedMarkdownLinkRanges(text string) [][2]int { ranges := make([][2]int, 0) for _, match := range linkLabelRe.FindAllStringSubmatchIndex(text, -1) {