docs: clean up clawtributors generator

2026-05-06 07:20:43 +00:00 · 2026-04-15 14:18:47 -04:00
parent be7f4a2342
commit 84185cb3eb
4 changed files with 579 additions and 433 deletions
--- a/scripts/clawtributors-map.json
+++ b/scripts/clawtributors-map.json
@@ -19,7 +19,6 @@
    "jiulingyun"
  ],
  "seedCommit": "d6863f87",
-  "placeholderAvatar": "assets/avatar-placeholder.svg",
  "displayName": {
    "jdrhyne": "Jonathan D. Rhyne (DJ-D)"
  },
--- a/scripts/update-clawtributors.ts
+++ b/scripts/update-clawtributors.ts
@@ -5,6 +5,12 @@ import type { ApiContributor, Entry, MapConfig, User } from "./update-clawtribut

 const REPO = "openclaw/openclaw";
 const PER_LINE = 10;
+const AVATAR_PROBE_SIZE = 40;
+const AVATAR_SIZE = 48;
+const CLAWTRIBUTORS_START = "<!-- clawtributors:start -->";
+const CLAWTRIBUTORS_END = "<!-- clawtributors:end -->";
+const CLAWTRIBUTORS_HIDDEN_START = "<!-- clawtributors:hidden:start";
+const CLAWTRIBUTORS_HIDDEN_END = "clawtributors:hidden:end -->";

 const mapPath = resolve("scripts/clawtributors-map.json");
 const mapConfig = JSON.parse(readFileSync(mapPath, "utf8")) as MapConfig;
@@ -17,10 +23,13 @@ const ensureLogins = (mapConfig.ensureLogins ?? []).map((login) => login.toLower
 const readmePath = resolve("README.md");
 const seedCommit = mapConfig.seedCommit ?? null;
 const seedEntries = seedCommit ? parseReadmeEntries(run(`git show ${seedCommit}:README.md`)) : [];
+const currentReadme = readFileSync(readmePath, "utf8");
+const hiddenReadmeLogins = new Set(parseHiddenReadmeLogins(currentReadme));
 const raw = run(`gh api "repos/${REPO}/contributors?per_page=100&anon=1" --paginate`);
 const contributors = parsePaginatedJson(raw) as ApiContributor[];
 const apiByLogin = new Map<string, User>();
 const contributionsByLogin = new Map<string, number>();
+const defaultAvatarByLogin = new Map<string, Promise<boolean>>();

 for (const item of contributors) {
  if (!item?.login || !item?.html_url || !item?.avatar_url) {
@@ -45,11 +54,12 @@ for (const login of ensureLogins) {
  }
 }

-// %x1f = unit separator to avoid collisions with author names containing "|"
-const log = run("git log --reverse --format=%aN%x1f%aE%x1f%aI --numstat");
 const linesByLogin = new Map<string, number>();
 const firstCommitByLogin = new Map<string, string>();

+// %x1f = unit separator to avoid collisions with author names containing "|"
+const log = run("git log --reverse --format=%aN%x1f%aE%x1f%aI --numstat");
+
 let currentName: string | null = null;
 let currentEmail: string | null = null;

@@ -113,7 +123,6 @@ for (const login of ensureLogins) {
  }
 }

-// Fetch merged PRs and count per author
 const prsByLogin = new Map<string, number>();
 const prRaw = run(
  `gh pr list -R ${REPO} --state merged --limit 5000 --json author --jq '.[].author.login'`,
@@ -272,43 +281,49 @@ for (const [login, loc] of linesByLogin.entries()) {
 }

 const entries = Array.from(entriesByKey.values());
+const visibleEntries = await filterVisibleEntries(entries, hiddenReadmeLogins);

-entries.sort((a, b) => {
+visibleEntries.sort((a, b) => {
  if (b.score !== a.score) {
    return b.score - a.score;
  }
  return a.display.localeCompare(b.display);
 });

-const htmlLines: string[] = [];
-for (let i = 0; i < entries.length; i += PER_LINE) {
-  const chunk = entries.slice(i, i + PER_LINE);
+const markdownLines: string[] = [];
+for (let i = 0; i < visibleEntries.length; i += PER_LINE) {
+  const chunk = visibleEntries.slice(i, i + PER_LINE);
  const parts = chunk.map((entry) => {
-    return `<a href="${entry.html_url}"><img src="${entry.avatar_url}" width="48" height="48" alt="${entry.display}" title="${entry.display}"/></a>`;
+    return `[![${escapeMarkdownLabel(entry.display)}](${entry.avatar_url})](${entry.html_url})`;
  });
-  htmlLines.push(`  ${parts.join(" ")}`);
+  markdownLines.push(parts.join(" "));
 }

-const block = `${htmlLines.join("\n")}\n`;
-const readme = readFileSync(readmePath, "utf8");
-const start = readme.indexOf('<p align="left">');
-const end = readme.indexOf("</p>", start);
+const block = `${CLAWTRIBUTORS_START}\n${markdownLines.join("\n\n")}\n${CLAWTRIBUTORS_END}`;
+const hiddenBlock = buildHiddenReadmeBlock(entries, visibleEntries);
+const hiddenRange = findHiddenReadmeRange(currentReadme);
+const readmeWithoutMeta = hiddenRange
+  ? `${currentReadme.slice(0, hiddenRange.start)}${currentReadme.slice(hiddenRange.end)}`
+  : currentReadme;
+const range = findClawtributorsRange(readmeWithoutMeta);

-if (start === -1 || end === -1) {
+if (!range) {
  throw new Error("README.md missing clawtributors block");
 }

-const next = `${readme.slice(0, start)}<p align="left">\n${block}${readme.slice(end)}`;
+const next = `${readmeWithoutMeta.slice(0, range.start)}${block}\n${hiddenBlock}${readmeWithoutMeta.slice(range.end)}`;
 writeFileSync(readmePath, next);

-console.log(`Updated README clawtributors: ${entries.length} entries`);
+console.log(
+  `Updated README clawtributors: ${visibleEntries.length} visible (${entries.length - visibleEntries.length} default-avatar entries hidden)`,
+);
 console.log(`\nTop 25 by composite score: (commits*2 + PRs*10 + sqrt(LOC)) * tenure`);
 console.log(`  tenure = 1.0 + (days_since_first_commit / repo_age)^2 * 0.5`);
 console.log(
  `${"#".padStart(3)}  ${"login".padEnd(24)} ${"score".padStart(8)} ${"tenure".padStart(7)} ${"commits".padStart(8)} ${"PRs".padStart(6)} ${"LOC".padStart(10)}  first commit`,
 );
 console.log("-".repeat(85));
-for (const entry of entries.slice(0, 25)) {
+for (const [index, entry] of visibleEntries.slice(0, 25).entries()) {
  const login = (entry.login ?? entry.key).slice(0, 24);
  const fd = entry.firstCommitDate || "?";
  const daysIn =
@@ -316,7 +331,7 @@ for (const entry of entries.slice(0, 25)) {
  const tr = Math.min(1, daysIn / repoAgeDays);
  const tenure = 1.0 + tr * tr * 0.5;
  console.log(
-    `${entries.indexOf(entry) + 1}`.padStart(3) +
+    `${index + 1}`.padStart(3) +
      `  ${login.padEnd(24)} ${entry.score.toFixed(0).padStart(8)} ${tenure.toFixed(2).padStart(6)}x ${String(entry.commits).padStart(8)} ${String(entry.prs).padStart(6)} ${String(entry.lines).padStart(10)}  ${fd}`,
  );
 }
@@ -386,12 +401,15 @@ function normalizeAvatar(url: string): string {
  if (!/^https?:/i.test(url)) {
    return url;
  }
-  const lower = url.toLowerCase();
-  if (lower.includes("s=") || lower.includes("size=")) {
+  try {
+    const parsed = new URL(url);
+    parsed.searchParams.delete("s");
+    parsed.searchParams.delete("size");
+    parsed.searchParams.set("s", String(AVATAR_SIZE));
+    return parsed.toString();
+  } catch {
    return url;
  }
-  const sep = url.includes("?") ? "&" : "?";
-  return `${url}${sep}s=48`;
 }

 function fetchUser(login: string): User | null {
@@ -418,6 +436,159 @@ function fetchUser(login: string): User | null {
  }
 }

+function isDefaultGitHubAvatar(login: string): Promise<boolean> {
+  const normalized = normalizeLogin(login)?.toLowerCase();
+  if (!normalized) {
+    return Promise.resolve(false);
+  }
+  const cached = defaultAvatarByLogin.get(normalized);
+  if (cached) {
+    return cached;
+  }
+  const pending = probeDefaultGitHubAvatar(normalized);
+  defaultAvatarByLogin.set(normalized, pending);
+  return pending;
+}
+
+async function probeDefaultGitHubAvatar(login: string): Promise<boolean> {
+  try {
+    const response = await fetch(`https://github.com/${login}.png?size=${AVATAR_PROBE_SIZE}`, {
+      headers: { "user-agent": "openclaw-clawtributors" },
+      signal: AbortSignal.timeout(8000),
+    });
+    if (!response.ok) {
+      return false;
+    }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const dimensions = readImageDimensions(buffer);
+    return Boolean(
+      dimensions && (dimensions.width > AVATAR_PROBE_SIZE || dimensions.height > AVATAR_PROBE_SIZE),
+    );
+  } catch {
+    return false;
+  }
+}
+
+async function filterVisibleEntries(
+  entries: Entry[],
+  hiddenLogins: ReadonlySet<string>,
+): Promise<Entry[]> {
+  const results = await mapConcurrent(entries, 8, async (entry) => {
+    const login = entry.login ?? entry.key;
+    if (!login) {
+      return entry;
+    }
+    const normalized = normalizeLogin(login)?.toLowerCase();
+    if (normalized && hiddenLogins.has(normalized)) {
+      return null;
+    }
+    return (await isDefaultGitHubAvatar(login)) ? null : entry;
+  });
+  return results.filter((entry): entry is Entry => entry !== null);
+}
+
+async function mapConcurrent<T, R>(
+  items: T[],
+  limit: number,
+  mapper: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = [];
+  results.length = items.length;
+  let nextIndex = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex++;
+      results[index] = await mapper(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
+function readImageDimensions(buffer: Buffer): { width: number; height: number } | null {
+  if (isPng(buffer)) {
+    return readPngDimensions(buffer);
+  }
+  if (isJpeg(buffer)) {
+    return readJpegDimensions(buffer);
+  }
+  return null;
+}
+
+function isPng(buffer: Buffer): boolean {
+  return (
+    buffer.length >= 24 &&
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a
+  );
+}
+
+function readPngDimensions(buffer: Buffer): { width: number; height: number } | null {
+  if (buffer.length < 24) {
+    return null;
+  }
+  return {
+    width: buffer.readUInt32BE(16),
+    height: buffer.readUInt32BE(20),
+  };
+}
+
+function isJpeg(buffer: Buffer): boolean {
+  return buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8;
+}
+
+function readJpegDimensions(buffer: Buffer): { width: number; height: number } | null {
+  let offset = 2;
+  while (offset + 9 < buffer.length) {
+    if (buffer[offset] !== 0xff) {
+      offset += 1;
+      continue;
+    }
+
+    const marker = buffer[offset + 1];
+    offset += 2;
+
+    if (marker === 0xd8 || marker === 0xd9) {
+      continue;
+    }
+    if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
+      continue;
+    }
+    if (offset + 2 > buffer.length) {
+      return null;
+    }
+
+    const length = buffer.readUInt16BE(offset);
+    if (length < 2 || offset + length > buffer.length) {
+      return null;
+    }
+
+    if (
+      (marker >= 0xc0 && marker <= 0xc3) ||
+      (marker >= 0xc5 && marker <= 0xc7) ||
+      (marker >= 0xc9 && marker <= 0xcb) ||
+      (marker >= 0xcd && marker <= 0xcf)
+    ) {
+      if (length < 7) {
+        return null;
+      }
+      return {
+        height: buffer.readUInt16BE(offset + 3),
+        width: buffer.readUInt16BE(offset + 5),
+      };
+    }
+
+    offset += length;
+  }
+  return null;
+}
+
 function resolveLogin(
  name: string,
  email: string | null,
@@ -503,16 +674,27 @@ function normalizeIdentifier(value: string): string {
  return value.toLowerCase().replace(/[^a-z0-9]/g, "");
 }

+function escapeMarkdownLabel(value: string): string {
+  return value.replace(/([\\[\]])/g, "\\$1");
+}
+
 function parseReadmeEntries(
  content: string,
 ): Array<{ display: string; html_url: string; avatar_url: string }> {
-  const start = content.indexOf('<p align="left">');
-  const end = content.indexOf("</p>", start);
-  if (start === -1 || end === -1) {
+  const range = findClawtributorsRange(content);
+  if (!range) {
    return [];
  }
-  const block = content.slice(start, end);
+  const block = content.slice(range.start, range.end);
  const entries: Array<{ display: string; html_url: string; avatar_url: string }> = [];
+  const markdown = /\[!\[([^\]]+)\]\(([^)]+)\)\]\(([^)]+)\)/g;
+  for (const match of block.matchAll(markdown)) {
+    const [, alt, src, href] = match;
+    if (!href || !src || !alt) {
+      continue;
+    }
+    entries.push({ html_url: href, avatar_url: src, display: alt.replace(/\\([\\[\]])/g, "$1") });
+  }
  const linked = /<a href="([^"]+)"><img src="([^"]+)"[^>]*alt="([^"]+)"[^>]*>/g;
  for (const match of block.matchAll(linked)) {
    const [, href, src, alt] = match;
@@ -535,6 +717,70 @@ function parseReadmeEntries(
  return entries;
 }

+function parseHiddenReadmeLogins(content: string): string[] {
+  const range = findHiddenReadmeRange(content);
+  if (!range) {
+    return [];
+  }
+  const block = content.slice(range.start, range.end);
+  return block
+    .split("\n")
+    .map((line) => normalizeLogin(line.trim())?.toLowerCase() ?? null)
+    .filter((login): login is string => Boolean(login));
+}
+
+function buildHiddenReadmeBlock(entries: Entry[], visibleEntries: Entry[]): string {
+  const visibleLogins = new Set(
+    visibleEntries
+      .map((entry) => normalizeLogin(entry.login ?? entry.key)?.toLowerCase() ?? null)
+      .filter((login): login is string => Boolean(login)),
+  );
+  const hiddenLogins = entries
+    .map((entry) => normalizeLogin(entry.login ?? entry.key)?.toLowerCase() ?? null)
+    .filter((login): login is string => Boolean(login))
+    .filter((login) => !visibleLogins.has(login))
+    .toSorted((a, b) => a.localeCompare(b));
+  const notice =
+    "default-avatar-cache: hidden from the rendered wall because these users still use GitHub's default avatar";
+  if (hiddenLogins.length === 0) {
+    return `${CLAWTRIBUTORS_HIDDEN_START}\n${notice}\n${CLAWTRIBUTORS_HIDDEN_END}\n`;
+  }
+  return `${CLAWTRIBUTORS_HIDDEN_START}\n${notice}\n${hiddenLogins.join("\n")}\n${CLAWTRIBUTORS_HIDDEN_END}\n`;
+}
+
+function findClawtributorsRange(content: string): { start: number; end: number } | null {
+  const markerStart = content.indexOf(CLAWTRIBUTORS_START);
+  const markerEnd = content.indexOf(CLAWTRIBUTORS_END, markerStart);
+  if (markerStart !== -1 && markerEnd !== -1) {
+    return {
+      start: markerStart,
+      end: markerEnd + CLAWTRIBUTORS_END.length,
+    };
+  }
+
+  const legacyStart = content.indexOf('<p align="left">');
+  const legacyEnd = content.indexOf("</p>", legacyStart);
+  if (legacyStart === -1 || legacyEnd === -1) {
+    return null;
+  }
+  return {
+    start: legacyStart,
+    end: legacyEnd + "</p>".length,
+  };
+}
+
+function findHiddenReadmeRange(content: string): { start: number; end: number } | null {
+  const markerStart = content.indexOf(CLAWTRIBUTORS_HIDDEN_START);
+  const markerEnd = content.indexOf(CLAWTRIBUTORS_HIDDEN_END, markerStart);
+  if (markerStart === -1 || markerEnd === -1) {
+    return null;
+  }
+  return {
+    start: markerStart,
+    end: markerEnd + CLAWTRIBUTORS_HIDDEN_END.length,
+  };
+}
+
 function loginFromUrl(url: string): string | null {
  const match = /^https?:\/\/github\.com\/([^/?#]+)/i.exec(url);
  if (!match) {
--- a/scripts/update-clawtributors.types.ts
+++ b/scripts/update-clawtributors.types.ts
@@ -3,7 +3,6 @@ export type MapConfig = {
  displayName?: Record<string, string>;
  nameToLogin?: Record<string, string>;
  emailToLogin?: Record<string, string>;
-  placeholderAvatar?: string;
  seedCommit?: string;
 };