From 5a3abf6c89bee357dd0e00eddf490e99db7ff43a Mon Sep 17 00:00:00 2001 From: paul Date: Tue, 23 Jun 2026 12:28:08 +0000 Subject: [PATCH] fix: batch large-thread candidates + stop negative-caching uncapped names (closes task/chunk-large-thread-candidates) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source: task/chunk-large-thread-candidates (plan/steward-linker-quality) — content.js sent all candidates in one call, the server capped at 100, and names past the cap were permanently null-cached and never retried — silently breaking large threads. - content.js: resolve the per-scan ask set in batches of BATCH (100, <= server MAX_CANDIDATES) via a new pure Extract.chunk helper, so every candidate is actually answered and none is dropped by the resolver's slice. - content.js: cache only keys actually present in a response (hasOwnProperty); cap-overflow / partial-error / resolver-down names are left out of `resolved` and retried next scan instead of stuck at null. - extract.js: add pure chunk(arr, size) on the RSLExtract global; extract.test.cjs covers exact-multiple, remainder, under-size, empty, and lossless in-order concatenation. Co-Authored-By: Claude Opus 4.8 --- extension/content.js | 19 ++++++++++++++++--- extension/extract.js | 11 ++++++++++- extension/extract.test.cjs | 25 ++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/extension/content.js b/extension/content.js index c7f055e..b4e47cc 100644 --- a/extension/content.js +++ b/extension/content.js @@ -97,6 +97,12 @@ // ---- resolver client ------------------------------------------------------ const resolved = new Map(); // candidate string -> { name, spotify?, google?, primary } | null + // The resolver caps each /resolve call at MAX_CANDIDATES (service/resolve-core.js); names past the + // cap are silently dropped from the response. So split a large ask into batches of at most that cap + // and resolve each — every candidate is actually answered, none lost to the server-side slice. + // MUST stay <= the server's MAX_CANDIDATES. + const BATCH = 100; + // The network call goes through the background script — a page-context fetch to // http://localhost is blocked by reddit's CSP / mixed-content rules. async function resolve(names) { @@ -133,9 +139,16 @@ log("candidates found:", seen.size); const ask = [...seen].filter((c) => !resolved.has(c)); - if (ask.length) { - const res = await resolve(ask); - if (res) for (const c of ask) resolved.set(c, res[c] || null); // negative-cache misses + // Resolve in server-cap-sized batches so no candidate is dropped by the resolver's slice. + // Only cache keys the response actually carries: a name the server didn't answer (cap overflow, + // partial error, or a resolver-down null response) is left OUT of `resolved` so it is retried on + // the next scan, instead of being stuck at null and never retried again. + for (const batch of Extract.chunk(ask, BATCH)) { + const res = await resolve(batch); + if (!res) continue; // whole-batch failure (resolver down): leave every name un-cached for retry + for (const c of batch) { + if (Object.prototype.hasOwnProperty.call(res, c)) resolved.set(c, res[c] || null); + } } for (const node of nodes) if (node.isConnected) wrapNode(node); diff --git a/extension/extract.js b/extension/extract.js index 509c11b..8518732 100644 --- a/extension/extract.js +++ b/extension/extract.js @@ -221,6 +221,15 @@ return all; } + // Split an array into consecutive sub-arrays of at most `size` elements (the last is the + // remainder). Pure + position-preserving: concatenating the chunks reproduces the input in order. + // The resolver client uses it to keep each /resolve request at or under the server's per-call cap. + function chunk(arr, size) { + const out = []; + for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size)); + return out; + } + // True only for http/https URLs — reject javascript:, data:, garbage, empty. // The resolver's url flows into an injected link's href, so the content script // re-validates the scheme client-side before assigning it (defense in depth; the @@ -235,5 +244,5 @@ } } - return { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl }; + return { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl, chunk }; }); diff --git a/extension/extract.test.cjs b/extension/extract.test.cjs index 89fb805..892e3d3 100644 --- a/extension/extract.test.cjs +++ b/extension/extract.test.cjs @@ -2,7 +2,7 @@ // CJS so it can require() extract.js, which is a UMD script (also a content script). const { test } = require("node:test"); const assert = require("node:assert/strict"); -const { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl } = require("./extract.js"); +const { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl, chunk } = require("./extract.js"); test("matchesIn finds Title-Case / ALL-CAPS runs, skips stopwords", () => { // Note: "and" is a connector, so "X and Y" greedily joins into one run (by design, @@ -277,6 +277,29 @@ test("isSafeHttpUrl accepts http/https and rejects javascript/data/garbage/empty assert.ok(!isSafeHttpUrl(undefined)); }); +test("chunk splits an array into <= size batches, in order, losslessly", () => { + // exact multiple: 200 -> [100, 100] + const a = Array.from({ length: 200 }, (_, i) => i); + const ca = chunk(a, 100); + assert.deepEqual(ca.map((b) => b.length), [100, 100]); + assert.deepEqual([].concat(...ca), a); // concatenation reproduces the input in order + + // remainder: 250 -> [100, 100, 50] + const b = Array.from({ length: 250 }, (_, i) => i); + const cb = chunk(b, 100); + assert.deepEqual(cb.map((x) => x.length), [100, 100, 50]); + assert.deepEqual([].concat(...cb), b); + + // under-size: 5 -> [5] + const c = [1, 2, 3, 4, 5]; + const cc = chunk(c, 100); + assert.deepEqual(cc.map((x) => x.length), [5]); + assert.deepEqual([].concat(...cc), c); + + // empty: [] -> [] + assert.deepEqual(chunk([], 100), []); +}); + test("allMatchesIn merges cue + Title-Case + album and sorts ascending by index", () => { const text = `Check out devourment and Cattle Decapitation; the album "Human Jerky" rules`; const ms = allMatchesIn(text);