fix(i18n): restore masked list marker structure

This commit is contained in:
Peter Steinberger
2026-07-17 11:11:42 -04:00
parent 75697ad3bf
commit edef9822ee
3 changed files with 56 additions and 7 deletions

View File

@@ -73,6 +73,7 @@ func translateDocBodyChunked(ctx context.Context, translator docsTranslator, rel
out.WriteString(translated)
}
translatedBody := out.String()
translatedBody = normalizeMaskedListMarkerPlaceholders(translatedBody, mapping)
if err := validatePlaceholders(translatedBody, placeholders); err != nil {
return "", fmt.Errorf("%s: restore fenced literals: %w", relPath, err)
}

View File

@@ -294,7 +294,7 @@ func (t *docSyntaxMaskingTranslator) Translate(_ context.Context, text, _, _ str
func (t *docSyntaxMaskingTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
t.rawInputs = append(t.rawInputs, text)
translated := strings.ReplaceAll(text, "Visible prose", "Видимый текст")
translated = regexp.MustCompile(`(?m)^\s+(__OC_I18N_\d+__)`).ReplaceAllString(translated, "$1")
translated = regexp.MustCompile(`(?m)^(__OC_I18N_\d+__)`).ReplaceAllString(translated, " $1")
return translated, nil
}
@@ -733,6 +733,35 @@ func TestExtractMarkdownListMarkerPrefixesIgnoresFencedExamples(t *testing.T) {
}
}
func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing.T) {
t.Parallel()
mapping := map[string]string{
"__OC_I18N_000001__": "- ",
"__OC_I18N_000002__": " - ",
"__OC_I18N_000003__": "> 1. ",
"__OC_I18N_000004__": "`inline`",
}
translated := strings.Join([]string{
" __OC_I18N_000001__Top level",
"> __OC_I18N_000002__Nested",
" > __OC_I18N_000003__Quoted",
" __OC_I18N_000004__ prose",
"",
}, "\n")
want := strings.Join([]string{
"__OC_I18N_000001__Top level",
"__OC_I18N_000002__Nested",
"__OC_I18N_000003__Quoted",
" __OC_I18N_000004__ prose",
"",
}, "\n")
if got := normalizeMaskedListMarkerPlaceholders(translated, mapping); got != want {
t.Fatalf("normalized placeholders changed unexpectedly\nwant:\n%s\ngot:\n%s", want, got)
}
}
func TestValidateDocChunkTranslationRejectsTranslatedInlineCode(t *testing.T) {
t.Parallel()

View File

@@ -8,12 +8,13 @@ import (
)
var (
inlineCodeRe = regexp.MustCompile("`[^`]+`")
angleLinkRe = regexp.MustCompile(`<https?://[^>]+>`)
linkURLRe = regexp.MustCompile(`\[[^\]]*\]\(([^)]+)\)`)
linkLabelRe = regexp.MustCompile(`!?\[([^\]\r\n]+)\]\(([^)\r\n]+)\)`)
placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
listMarkerRe = regexp.MustCompile(`^([ \t]*(?:>[ \t]*)*)([-+*]|[0-9]+[.)])([ \t]+)`)
inlineCodeRe = regexp.MustCompile("`[^`]+`")
angleLinkRe = regexp.MustCompile(`<https?://[^>]+>`)
linkURLRe = regexp.MustCompile(`\[[^\]]*\]\(([^)]+)\)`)
linkLabelRe = regexp.MustCompile(`!?\[([^\]\r\n]+)\]\(([^)\r\n]+)\)`)
placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
listMarkerRe = regexp.MustCompile(`^([ \t]*(?:>[ \t]*)*)([-+*]|[0-9]+[.)])([ \t]+)`)
listContainerPrefixRe = regexp.MustCompile(`^[ \t]*(?:>[ \t]*)*$`)
// Hard validation stays limited to low-ambiguity composite literals. Plain numbers remain
// model-visible so target-language plurals and ordinals can change grammar without false failures.
numericValueRe = regexp.MustCompile(`(?:0[xX][0-9A-Za-z_]+|0[bB][0-9A-Za-z_]+|0[oO][0-9A-Za-z_]+|[0-9]+(?:\.[0-9]+)?(?::[0-9]+(?:\.[0-9]+)?)+|[0-9]+(?:\.[0-9]+)?(?:/[0-9]+(?:\.[0-9]+)?)+|(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)[eE][+-]?[0-9]+)`)
@@ -131,6 +132,24 @@ func extractMarkdownListMarkerPrefixes(text string) []string {
return prefixes
}
func normalizeMaskedListMarkerPlaceholders(text string, mapping map[string]string) string {
lines := strings.SplitAfter(text, "\n")
for index, line := range lines {
span := placeholderRe.FindStringIndex(line)
if span == nil || !listContainerPrefixRe.MatchString(line[:span[0]]) {
continue
}
placeholder := line[span[0]:span[1]]
original := mapping[placeholder]
markerSpan := listMarkerRe.FindStringIndex(original)
if markerSpan == nil || markerSpan[0] != 0 || markerSpan[1] != len(original) {
continue
}
lines[index] = line[span[0]:]
}
return strings.Join(lines, "")
}
func protectedMarkdownLinkRanges(text string) [][2]int {
ranges := make([][2]int, 0)
for _, match := range linkLabelRe.FindAllStringSubmatchIndex(text, -1) {