fix: batch large-thread candidates + stop negative-caching uncapped names (closes task/chunk-large-thread-candidates)
Source: task/chunk-large-thread-candidates (plan/steward-linker-quality) — content.js sent all candidates in one call, the server capped at 100, and names past the cap were permanently null-cached and never retried — silently breaking large threads. - content.js: resolve the per-scan ask set in batches of BATCH (100, <= server MAX_CANDIDATES) via a new pure Extract.chunk helper, so every candidate is actually answered and none is dropped by the resolver's slice. - content.js: cache only keys actually present in a response (hasOwnProperty); cap-overflow / partial-error / resolver-down names are left out of `resolved` and retried next scan instead of stuck at null. - extract.js: add pure chunk(arr, size) on the RSLExtract global; extract.test.cjs covers exact-multiple, remainder, under-size, empty, and lossless in-order concatenation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+16
-3
@@ -97,6 +97,12 @@
|
|||||||
// ---- resolver client ------------------------------------------------------
|
// ---- resolver client ------------------------------------------------------
|
||||||
const resolved = new Map(); // candidate string -> { name, spotify?, google?, primary } | null
|
const resolved = new Map(); // candidate string -> { name, spotify?, google?, primary } | null
|
||||||
|
|
||||||
|
// The resolver caps each /resolve call at MAX_CANDIDATES (service/resolve-core.js); names past the
|
||||||
|
// cap are silently dropped from the response. So split a large ask into batches of at most that cap
|
||||||
|
// and resolve each — every candidate is actually answered, none lost to the server-side slice.
|
||||||
|
// MUST stay <= the server's MAX_CANDIDATES.
|
||||||
|
const BATCH = 100;
|
||||||
|
|
||||||
// The network call goes through the background script — a page-context fetch to
|
// The network call goes through the background script — a page-context fetch to
|
||||||
// http://localhost is blocked by reddit's CSP / mixed-content rules.
|
// http://localhost is blocked by reddit's CSP / mixed-content rules.
|
||||||
async function resolve(names) {
|
async function resolve(names) {
|
||||||
@@ -133,9 +139,16 @@
|
|||||||
log("candidates found:", seen.size);
|
log("candidates found:", seen.size);
|
||||||
|
|
||||||
const ask = [...seen].filter((c) => !resolved.has(c));
|
const ask = [...seen].filter((c) => !resolved.has(c));
|
||||||
if (ask.length) {
|
// Resolve in server-cap-sized batches so no candidate is dropped by the resolver's slice.
|
||||||
const res = await resolve(ask);
|
// Only cache keys the response actually carries: a name the server didn't answer (cap overflow,
|
||||||
if (res) for (const c of ask) resolved.set(c, res[c] || null); // negative-cache misses
|
// partial error, or a resolver-down null response) is left OUT of `resolved` so it is retried on
|
||||||
|
// the next scan, instead of being stuck at null and never retried again.
|
||||||
|
for (const batch of Extract.chunk(ask, BATCH)) {
|
||||||
|
const res = await resolve(batch);
|
||||||
|
if (!res) continue; // whole-batch failure (resolver down): leave every name un-cached for retry
|
||||||
|
for (const c of batch) {
|
||||||
|
if (Object.prototype.hasOwnProperty.call(res, c)) resolved.set(c, res[c] || null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for (const node of nodes) if (node.isConnected) wrapNode(node);
|
for (const node of nodes) if (node.isConnected) wrapNode(node);
|
||||||
|
|
||||||
|
|||||||
+10
-1
@@ -221,6 +221,15 @@
|
|||||||
return all;
|
return all;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Split an array into consecutive sub-arrays of at most `size` elements (the last is the
|
||||||
|
// remainder). Pure + position-preserving: concatenating the chunks reproduces the input in order.
|
||||||
|
// The resolver client uses it to keep each /resolve request at or under the server's per-call cap.
|
||||||
|
function chunk(arr, size) {
|
||||||
|
const out = [];
|
||||||
|
for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
// True only for http/https URLs — reject javascript:, data:, garbage, empty.
|
// True only for http/https URLs — reject javascript:, data:, garbage, empty.
|
||||||
// The resolver's url flows into an injected link's href, so the content script
|
// The resolver's url flows into an injected link's href, so the content script
|
||||||
// re-validates the scheme client-side before assigning it (defense in depth; the
|
// re-validates the scheme client-side before assigning it (defense in depth; the
|
||||||
@@ -235,5 +244,5 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl };
|
return { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl, chunk };
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// CJS so it can require() extract.js, which is a UMD script (also a content script).
|
// CJS so it can require() extract.js, which is a UMD script (also a content script).
|
||||||
const { test } = require("node:test");
|
const { test } = require("node:test");
|
||||||
const assert = require("node:assert/strict");
|
const assert = require("node:assert/strict");
|
||||||
const { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl } = require("./extract.js");
|
const { matchesIn, cueMatchesIn, albumMatchesIn, allMatchesIn, isSafeHttpUrl, chunk } = require("./extract.js");
|
||||||
|
|
||||||
test("matchesIn finds Title-Case / ALL-CAPS runs, skips stopwords", () => {
|
test("matchesIn finds Title-Case / ALL-CAPS runs, skips stopwords", () => {
|
||||||
// Note: "and" is a connector, so "X and Y" greedily joins into one run (by design,
|
// Note: "and" is a connector, so "X and Y" greedily joins into one run (by design,
|
||||||
@@ -277,6 +277,29 @@ test("isSafeHttpUrl accepts http/https and rejects javascript/data/garbage/empty
|
|||||||
assert.ok(!isSafeHttpUrl(undefined));
|
assert.ok(!isSafeHttpUrl(undefined));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("chunk splits an array into <= size batches, in order, losslessly", () => {
|
||||||
|
// exact multiple: 200 -> [100, 100]
|
||||||
|
const a = Array.from({ length: 200 }, (_, i) => i);
|
||||||
|
const ca = chunk(a, 100);
|
||||||
|
assert.deepEqual(ca.map((b) => b.length), [100, 100]);
|
||||||
|
assert.deepEqual([].concat(...ca), a); // concatenation reproduces the input in order
|
||||||
|
|
||||||
|
// remainder: 250 -> [100, 100, 50]
|
||||||
|
const b = Array.from({ length: 250 }, (_, i) => i);
|
||||||
|
const cb = chunk(b, 100);
|
||||||
|
assert.deepEqual(cb.map((x) => x.length), [100, 100, 50]);
|
||||||
|
assert.deepEqual([].concat(...cb), b);
|
||||||
|
|
||||||
|
// under-size: 5 -> [5]
|
||||||
|
const c = [1, 2, 3, 4, 5];
|
||||||
|
const cc = chunk(c, 100);
|
||||||
|
assert.deepEqual(cc.map((x) => x.length), [5]);
|
||||||
|
assert.deepEqual([].concat(...cc), c);
|
||||||
|
|
||||||
|
// empty: [] -> []
|
||||||
|
assert.deepEqual(chunk([], 100), []);
|
||||||
|
});
|
||||||
|
|
||||||
test("allMatchesIn merges cue + Title-Case + album and sorts ascending by index", () => {
|
test("allMatchesIn merges cue + Title-Case + album and sorts ascending by index", () => {
|
||||||
const text = `Check out devourment and Cattle Decapitation; the album "Human Jerky" rules`;
|
const text = `Check out devourment and Cattle Decapitation; the album "Human Jerky" rules`;
|
||||||
const ms = allMatchesIn(text);
|
const ms = allMatchesIn(text);
|
||||||
|
|||||||
Reference in New Issue
Block a user