From 8775745d9d59efb67a24223c2f205a4478418810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Sun, 4 Oct 2026 21:00:55 -0400 Subject: [PATCH] Catalog: twitter-text counting, zoned times, whole PNGs, plain hosts, no links or code in READMEs Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 4 ++- README.md | 7 ++-- package.json | 3 +- pnpm-lock.yaml | 35 ++++++++++++++++++++ pnpm-workspace.yaml | 2 ++ scripts/catalog.mjs | 72 ++++++++++++++++++++++++++++++------------ tests/catalog.test.mjs | 41 +++++++++++++++++++++--- 7 files changed, 133 insertions(+), 31 deletions(-) create mode 100644 pnpm-workspace.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44dfdfc..c7b32c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,9 @@ Use your desktop's window capture, such as GNOME's Print Screen and its Window m - Describe the product: what it does, how it works, who it is for. - Links go only to the manifest's `website` or `source`, exactly. - Images are PNG files in the app folder, by name. -- No raw HTML, no code blocks, no install steps. +- No raw HTML, no code, inline or in blocks, and no install steps. +- Write characters as they are, not as `&...;` references. +- Any address in the text is the `website` or the `source`. `pnpm validate` checks the size, the links, the images and the markup. diff --git a/README.md b/README.md index 0183d34..3884c9c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ apps/ "platforms": ["macOS", "Linux"], "website": null, "source": "https://github.com/ada/tally", + "publishedAt": "2026-10-04T12:00:00Z", "previews": ["preview0.png", "preview1.png"] } ``` @@ -41,13 +42,13 @@ apps/ | `id` | Lowercase letters, digits, single hyphens. Matches the folder. | | `name` | The app's name, in its own capitals. | | `author` | A person or an organization. | -| `description` | At most 280 weighted characters, counted as Twitter counts them. | +| `description` | At most 280 weighted characters, counted by [twitter-text](https://github.com/twitter/twitter-text): an address counts 23, CJK and emoji 2. | | `category` | `dev` (Developer Tools), `terminal` (Terminal & Network), `system` (System & Desktop) or `work` (Productivity & Media). | | `platforms` | Some of `macOS`, `Windows`, `Linux`, `Web`. | -| `website`, `source` | Canonical `https` addresses, or `null`. One at least. | +| `website`, `source` | Canonical `https` addresses with a plain host, or `null`. One at least. | | `previews` | PNG file names in the folder. The first is the cover. | | `building` | Optional. `true` while the app is not out yet. | -| `publishedAt` | Optional. ISO 8601. Maintainers set it. | +| `publishedAt` | ISO 8601 with a zone, such as `2026-10-04T12:00:00Z`: when the app joins the catalog. Maintainers confirm it; later updates keep it. | | `stars`, `starsUpdatedAt` | CI writes them for GitHub sources. Leave them out. | Unknown fields fail validation. diff --git a/package.json b/package.json index 666e304..b3672eb 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "validate": "node scripts/validate.mjs" }, "dependencies": { - "marked": "18.0.14" + "marked": "18.0.14", + "twitter-text": "3.1.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd01493..677c3b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,14 +11,49 @@ importers: marked: specifier: 18.0.14 version: 18.0.14 + twitter-text: + specifier: 3.1.0 + version: 3.1.0 packages: + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + core-js@2.6.12: + resolution: {integrity: sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==} + deprecated: core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js. + marked@18.0.14: resolution: {integrity: sha512-mBHK6FBHuBAlhgRe88w9F0O1AbwwXJUcQibUbC/QcdTbVGAD7aWza+xt3N6oT/jCZx3/OMeS+8rnuiHZcQ9s7A==} engines: {node: '>= 20'} hasBin: true + punycode@1.4.1: + resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} + + twemoji-parser@11.0.2: + resolution: {integrity: sha512-5kO2XCcpAql6zjdLwRwJjYvAZyDy3+Uj7v1ipBzLthQmDL7Ce19bEqHr3ImSNeoSW2OA8u02XmARbXHaNO8GhA==} + + twitter-text@3.1.0: + resolution: {integrity: sha512-nulfUi3FN6z0LUjYipJid+eiwXvOLb8Ass7Jy/6zsXmZK3URte043m8fL3FyDzrK+WLpyqhHuR/TcARTN/iuGQ==} + snapshots: + '@babel/runtime@7.29.7': {} + + core-js@2.6.12: {} + marked@18.0.14: {} + + punycode@1.4.1: {} + + twemoji-parser@11.0.2: {} + + twitter-text@3.1.0: + dependencies: + '@babel/runtime': 7.29.7 + core-js: 2.6.12 + punycode: 1.4.1 + twemoji-parser: 11.0.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..10ed64a --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + core-js: false diff --git a/scripts/catalog.mjs b/scripts/catalog.mjs index e27f214..0987a3d 100644 --- a/scripts/catalog.mjs +++ b/scripts/catalog.mjs @@ -1,6 +1,8 @@ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { existsSync, lstatSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; +import { crc32, inflateSync } from "node:zlib"; import { Lexer } from "marked"; +import twitter from "twitter-text"; export const CATEGORIES = { dev: "Developer Tools", @@ -14,19 +16,23 @@ export const README_BYTES = 10240; const ID = /^[a-z0-9]+(-[a-z0-9]+)*$/; const FILE = /^[a-z0-9][a-z0-9._-]*$/; const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); -const REQUIRED = ["id", "name", "author", "description", "category", "platforms", "website", "source", "previews"]; -const OPTIONAL = ["building", "publishedAt", "stars", "starsUpdatedAt"]; -const TOKENS = new Set(["space", "heading", "paragraph", "text", "strong", "em", "del", "codespan", "br", "hr", "blockquote", "list", "list_item", "table", "escape", "checkbox", "def", "link", "image"]); +const REQUIRED = ["id", "name", "author", "description", "category", "platforms", "website", "source", "publishedAt", "previews"]; +const OPTIONAL = ["building", "stars", "starsUpdatedAt"]; +const TIME = /^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(\.\d+)?(Z|[+-]\d\d:\d\d)$/; +const ENTITY = /&(#\d+|#x[0-9a-f]+|[a-z][a-z0-9]*);/i; +const URLISH = /[a-z][a-z0-9+.-]*:\/\/[^\s<>()\[\]"'`]*/gi; +const TOKENS = new Set(["space", "heading", "paragraph", "text", "strong", "em", "del", "br", "hr", "blockquote", "list", "list_item", "table", "escape", "checkbox", "def", "link", "image"]); -/** Twitter's weighted length: these ranges count 1, all else 2. */ -export function weighted(text) { - const light = [[0, 4351], [8192, 8205], [8208, 8223], [8242, 8247]]; - let n = 0; - for (const ch of text.normalize("NFC")) { - const c = ch.codePointAt(0); - n += light.some(([a, b]) => c >= a && c <= b) ? 1 : 2; - } - return n; +/** Twitter's weighted length. */ +export const weighted = (text) => twitter.parseTweet(text).weightedLength; + +/** An ISO 8601 time with its zone, on a real date. */ +function time(value) { + const m = typeof value === "string" && value.match(TIME); + if (!m) return false; + const [y, mo, d, h, mi, sec] = m.slice(1, 7).map(Number); + const at = new Date(Date.UTC(y, mo - 1, d, h, mi, sec)); + return at.getUTCMonth() === mo - 1 && at.getUTCDate() === d && h < 24 && mi < 60 && sec < 60; } function fail(where, what) { @@ -46,7 +52,9 @@ function address(where, value, field) { } catch { fail(where, `${field} is not a URL: ${value}`); } - if (url.protocol !== "https:" || url.href !== value) fail(where, `${field} must be a canonical https URL: ${value}`); + if (url.protocol !== "https:" || url.href !== value || url.username || url.password || ENTITY.test(value) || !/^[a-z0-9.-]+$/.test(url.hostname)) { + fail(where, `${field} must be a canonical https URL with a plain host: ${value}`); + } } /** The owner and repo of a GitHub source, or null. */ @@ -58,8 +66,29 @@ export function github(source) { function png(where, dir, name) { if (!FILE.test(name) || !name.endsWith(".png")) fail(where, `${name} must be a lowercase .png file name in the app folder`); const path = join(dir, name); - if (!existsSync(path) || !statSync(path).isFile()) fail(where, `${name} is missing`); - if (!readFileSync(path).subarray(0, 8).equals(PNG)) fail(where, `${name} is not a PNG`); + if (!existsSync(path)) fail(where, `${name} is missing`); + const bytes = readFileSync(path); + if (!bytes.subarray(0, 8).equals(PNG)) fail(where, `${name} is not a PNG`); + const types = []; + const data = []; + for (let at = 8; at < bytes.length; ) { + if (at + 12 > bytes.length) fail(where, `${name} is cut short`); + const length = bytes.readUInt32BE(at); + const end = at + 8 + length; + if (end + 4 > bytes.length) fail(where, `${name} is cut short`); + const type = bytes.toString("latin1", at + 4, at + 8); + if (crc32(bytes.subarray(at + 4, end)) !== bytes.readUInt32BE(end)) fail(where, `${name} has a bad ${type} checksum`); + if (type === "IHDR" && (length !== 13 || !bytes.readUInt32BE(at + 8) || !bytes.readUInt32BE(at + 12))) fail(where, `${name} has a bad header`); + if (type === "IDAT") data.push(bytes.subarray(at + 8, end)); + types.push(type); + at = end + 4; + } + if (types[0] !== "IHDR" || types.at(-1) !== "IEND" || !data.length) fail(where, `${name} is not a whole PNG`); + try { + inflateSync(Buffer.concat(data)); + } catch (error) { + fail(where, `${name} has broken image data: ${error.message}`); + } } function manifest(dir, id) { @@ -78,7 +107,7 @@ function manifest(dir, id) { if (app.id !== id) fail(where, `id ${JSON.stringify(app.id)} must match the folder ${id}`); for (const key of ["name", "author", "description"]) text(where, app[key], key); if (weighted(app.description) > 280) fail(where, `description is ${weighted(app.description)} weighted characters, over 280`); - if (!Object.hasOwn(CATEGORIES, app.category)) fail(where, `category must be one of ${Object.keys(CATEGORIES).join(", ")}`); + if (typeof app.category !== "string" || !Object.hasOwn(CATEGORIES, app.category)) fail(where, `category must be one of ${Object.keys(CATEGORIES).join(", ")}`); if (!Array.isArray(app.platforms) || !app.platforms.length) fail(where, "platforms must be a non-empty list"); for (const p of app.platforms) if (!PLATFORMS.includes(p)) fail(where, `unknown platform ${p}; use ${PLATFORMS.join(", ")}`); if (new Set(app.platforms).size !== app.platforms.length) fail(where, "platforms repeat"); @@ -89,9 +118,7 @@ function manifest(dir, id) { if (new Set(app.previews).size !== app.previews.length) fail(where, "previews repeat"); for (const name of app.previews) png(where, dir, name); if ("building" in app && typeof app.building !== "boolean") fail(where, "building must be true or false"); - for (const key of ["publishedAt", "starsUpdatedAt"]) { - if (key in app && (typeof app[key] !== "string" || Number.isNaN(Date.parse(app[key])) || !/^\d{4}-\d\d-\d\dT/.test(app[key]))) fail(where, `${key} must be an ISO 8601 time`); - } + for (const key of ["publishedAt", "starsUpdatedAt"]) if (key in app && !time(app[key])) fail(where, `${key} must be an ISO 8601 time with its zone`); if ("stars" in app !== "starsUpdatedAt" in app) fail(where, "stars and starsUpdatedAt go together"); if ("stars" in app && (!Number.isInteger(app.stars) || app.stars < 0)) fail(where, "stars must be a whole number"); if ("stars" in app && !github(app.source)) fail(where, "stars need a GitHub source"); @@ -107,6 +134,8 @@ function readme(dir, app) { if (bytes.length > README_BYTES) fail(where, `${bytes.length} bytes, over ${README_BYTES}`); const markdown = bytes.toString("utf8"); const links = [app.website, app.source].filter(Boolean); + if (ENTITY.test(markdown)) fail(where, `write characters as they are, not as references: ${markdown.match(ENTITY)[0]}`); + for (const url of markdown.replaceAll("\\", "").match(URLISH) ?? []) if (!links.includes(url)) fail(where, `addresses go only to the website or source: ${url}`); const walk = (tokens) => { for (const t of tokens) { if (!TOKENS.has(t.type)) fail(where, `${t.type === "html" ? "raw HTML" : t.type === "code" ? "a code block" : t.type} is not allowed: ${t.raw.trim().slice(0, 60)}`); @@ -128,10 +157,11 @@ export function read(root) { for (const entry of readdirSync(appsDir).sort()) { if (entry === ".gitkeep") continue; const dir = join(appsDir, entry); - if (!statSync(dir).isDirectory()) fail(`apps/${entry}`, "only app folders belong in apps/"); + if (!lstatSync(dir).isDirectory()) fail(`apps/${entry}`, "only app folders belong in apps/, and no links"); if (!ID.test(entry)) fail(`apps/${entry}`, "folder names are lowercase kebab-case"); for (const file of readdirSync(dir)) { if (!["manifest.json", "README.md"].includes(file) && !FILE.test(file)) fail(`apps/${entry}/${file}`, "file names are lowercase"); + if (!lstatSync(join(dir, file)).isFile()) fail(`apps/${entry}/${file}`, "only plain files belong in an app folder"); } const app = manifest(dir, entry); apps.push({ ...app, readme: readme(dir, app) }); diff --git a/tests/catalog.test.mjs b/tests/catalog.test.mjs index 0d7617f..c8d111b 100644 --- a/tests/catalog.test.mjs +++ b/tests/catalog.test.mjs @@ -3,9 +3,23 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; +import { crc32, deflateSync } from "node:zlib"; import { read, weighted } from "../scripts/catalog.mjs"; -const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13]); +function chunk(type, data) { + const head = Buffer.alloc(8); + head.writeUInt32BE(data.length); + head.write(type, 4, "latin1"); + const sum = Buffer.alloc(4); + sum.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), data]))); + return Buffer.concat([head, data, sum]); +} + +/** A one-pixel gray PNG. */ +const header = Buffer.from([0, 0, 0, 1, 0, 0, 0, 1, 8, 0, 0, 0, 0]); +const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", header), chunk("IDAT", deflateSync(Buffer.from([0, 128]))), chunk("IEND", Buffer.alloc(0))]); +const broken = Buffer.from(PNG); +broken[broken.length - 20] ^= 0xff; const good = { id: "tally", @@ -16,6 +30,7 @@ const good = { platforms: ["macOS", "Linux"], website: null, source: "https://github.com/ada/tally", + publishedAt: "2026-10-04T00:00:00Z", previews: ["preview0.png"], }; @@ -61,6 +76,14 @@ const bad = [ ["a remote preview", { app: { ...good, previews: ["https://x.io/a.png"] } }, /lowercase .png file name/], ["a missing preview", { app: { ...good, previews: ["preview1.png"] } }, /preview1.png is missing/], ["a preview that is no PNG", { app: { ...good, previews: ["fake.png"] }, files: { "fake.png": "GIF89a" } }, /not a PNG/], + ["a PNG cut short", { app: { ...good, previews: ["cut.png"] }, files: { "cut.png": PNG.subarray(0, 40) } }, /cut short/], + ["a PNG with a bad checksum", { app: { ...good, previews: ["bad.png"] }, files: { "bad.png": broken } }, /checksum/], + ["a category list", { app: { ...good, category: ["work"] } }, /category must be one of/], + ["no publishedAt", { app: { ...good, publishedAt: undefined } }, /publishedAt is required/], + ["a time without its zone", { app: { ...good, publishedAt: "2026-10-04T00:00:00" } }, /with its zone/], + ["a day that does not exist", { app: { ...good, publishedAt: "2026-02-30T00:00:00Z" } }, /with its zone/], + ["an encoded host", { app: { ...good, website: "https://good.example@evil.com/" } }, /plain host/], + ["a host with credentials", { app: { ...good, website: "https://good.example@evil.com/" } }, /plain host/], ["a long description", { app: { ...good, description: "a".repeat(281) } }, /281 weighted characters/], ["stars without their time", { app: { ...good, stars: 3 } }, /go together/], ["stars on a source off GitHub", { app: { ...good, source: "https://gitlab.com/ada/tally", stars: 3, starsUpdatedAt: "2026-10-04T00:00:00Z" } }, /need a GitHub source/], @@ -69,10 +92,16 @@ const bad = [ ["a repeated featured id", { featured: ["tally", "tally"] }, /ids repeat/], ["raw HTML in a README", { files: { "README.md": "Hi \n" } }, /raw HTML is not allowed/], ["a code block in a README", { files: { "README.md": "```\ncurl x | sh\n```\n" } }, /a code block is not allowed/], - ["a foreign link in a README", { files: { "README.md": "[get it](https://evil.io)\n" } }, /links go only/], - ["a foreign autolink in a README", { files: { "README.md": "Visit https://evil.io now\n" } }, /links go only/], - ["a foreign reference in a README", { files: { "README.md": "[a][r]\n\n[r]: https://evil.io\n" } }, /links go only/], - ["a remote image in a README", { files: { "README.md": "![x](https://evil.io/a.png)\n" } }, /lowercase .png file name/], + ["a foreign link in a README", { files: { "README.md": "[get it](https://evil.io)\n" } }, /addresses go only/], + ["a foreign autolink in a README", { files: { "README.md": "Visit https://evil.io now\n" } }, /addresses go only/], + ["a foreign reference in a README", { files: { "README.md": "[a][r]\n\n[r]: https://evil.io\n" } }, /addresses go only/], + ["a remote image in a README", { files: { "README.md": "![x](https://evil.io/a.png)\n" } }, /addresses go only/], + ["a script link in a README", { files: { "README.md": "[x](javascript:alert(1))\n" } }, /links go only/], + ["a site-relative link in a README", { files: { "README.md": "[x](/docs)\n" } }, /links go only/], + ["a picture outside the folder in a README", { files: { "README.md": "![x](../a.png)\n" } }, /lowercase .png file name/], + ["inline code in a README", { files: { "README.md": "Run `cargo install tally`.\n" } }, /codespan is not allowed/], + ["a character reference in a README", { files: { "README.md": "Go https://evil.io/ now\n" } }, /not as references/], + ["an escaped address in a README", { files: { "README.md": "Go https\://evil.io/ now\n" } }, /addresses go only/], ["a large README", { files: { "README.md": "a".repeat(10241) } }, /over 10240/], ]; @@ -83,4 +112,6 @@ for (const [name, setup, error] of bad) { test("weighted length counts CJK twice", () => { assert.equal(weighted("abc"), 3); assert.equal(weighted("漢字"), 4); + assert.equal(weighted("a https://a.co"), 25); + assert.equal(weighted("👨‍👩‍👧"), 2); });