Sitelet https://github.com/NodeOps-app/treg2/commit/2ec100658054b876ef0558ff805866241657dfb0
Skip to content

Commit 2ec1006

Browse files
stonexerstonexer
committed
refactor(archive): retire the legacy noise comparison
1 parent f356d26 commit 2ec1006

4 files changed

Lines changed: 18 additions & 87 deletions

File tree

‎docs/context/architecture/archive.md‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -122,12 +122,10 @@ by min(30 d, the judged `cache.max_age_s`); changed ⇒ ×0.5, floored at 60 s.
122122
until a stable refetch resets it. The lookup prefers the learned timer (`ttl_s > 0`) over the
123123
fixed phase-1 guesses.
124124

125-
**Noise vs change (opt-in `legacy_noise` only).** When a found-to-found refetch differs, the diff's leaf paths (lists collapse to `[]`,
126-
bounded depth 6 / 400 paths) are compared to the previous diff-set (`volatile_paths`, kept per
127-
key). The SAME set repeating counts as noise ⇒ stable, under two guards: it must be a minor
128-
share (< 40%) of a body with ≥ 5 leaves — a tiny body whose one value moves every fetch is a
129-
price and stays "changed". First occurrence always counts as changed. Stored bytes are never
130-
touched; stripping exists only in comparison.
125+
**Strict comparison.** Result admission still selects the decisive baseline and controls which
126+
transitions train TTL. Among found-to-found observations, identical raw hashes count stable and
127+
differing hashes count changed. The legacy field-noise heuristic is removed; hash comparison
128+
never fetches an old R2 body.
131129

132130
## The refresh worker (PR 5)
133131

@@ -235,12 +233,10 @@ enable. Rollback in production is a dashboard env edit, no deploy.
235233

236234
## Conservative comparison and controlled serving (2026-09-08)
237235

238-
`TREG_ARCHIVE_COMPARISON_MODE` defaults to `strict`: only identical raw-byte SHA-256 hashes
239-
count as stable among found-to-found observations. Every differing positive response counts as changed, including whitespace, JSON field
240-
order and timestamps. `legacy_noise` explicitly restores `_noise_only`; unknown values select
241-
strict. The legacy heuristic can misclassify recurring business-field changes as noise and is
242-
not recommended for serving. Strict mode does not load or decompress the previous carrier to
243-
compare differing hashes. Stored bodies and exact-byte dedup are unchanged.
236+
`TREG_ARCHIVE_COMPARISON_MODE` accepts only `strict` (the default). `comparison_mode()` always
237+
returns strict. The legacy noise implementation is removed, including its JSON path traversal.
238+
Every differing found-to-found raw-byte hash counts changed, including whitespace, field order
239+
and timestamps; hash comparison needs no previous-body GET. Stored bodies and exact-byte dedup are unchanged.
244240

245241
TTL learning and lookup retain the original behavior: stable observations grow the timer by
246242
1.5, changed observations halve it, and TTL_NEVER remains respected. The fixed capability timer

‎src/treg/archive.py‎

Lines changed: 5 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ def serving() -> bool:
4646

4747

4848
def comparison_mode() -> str:
49-
return ("legacy_noise" if get_settings().archive_comparison_mode.strip().lower()
50-
== "legacy_noise" else "strict")
49+
# Retain the settings name for deploy compatibility; the legacy heuristic is retired.
50+
return "strict"
5151

5252

5353
def serve_endpoints() -> set[str]:
@@ -623,9 +623,6 @@ async def _store_locked(
623623
decisive = next_state in ("found", "empty") and origin != "async_terminal"
624624
if decisive and baseline is not None and (previous_state, next_state) != ("empty", "empty"):
625625
stable = previous_state == next_state == "found" and baseline.content_hash == ch
626-
if (not stable and previous_state == next_state == "found"
627-
and comparison_mode() == "legacy_noise"):
628-
stable = _noise_only(await _snapshot_body(s, baseline), body, key)
629626
if stable:
630627
key.stable_seen += 1
631628
else:
@@ -1002,74 +999,13 @@ async def _touch_write(key_hash: str) -> None:
1002999
TTL_CEILING_S = 30 * 86400
10031000
TTL_NEVER = -1 # the key marked itself never-cache: changes on every fetch
10041001
_NEVER_AFTER = 4 # consecutive changed refetches (no stables) before self-marking
1005-
_NOISE_MAX_LEAF_SHARE = 0.4 # a repeated identical diff-set is noise only if it is a MINOR
1006-
_NOISE_MIN_LEAVES = 5 # corner of a body with at least this many leaves — a tiny body
1007-
# whose one value moves (a price) must stay "changed", never noise.
1008-
1009-
1010-
def _leaf_paths(node: Any, prefix: str = "$", *, depth: int = 0, out: list | None = None) -> list[str]:
1011-
"""Dotted leaf paths of a JSON tree; list items collapse to `[]` so per-row ids do not
1012-
explode one logical path into hundreds. Bounded by depth and count — comparison machinery
1013-
must never be the expensive part of a recording."""
1014-
if out is None:
1015-
out = []
1016-
if len(out) >= 400 or depth > 6:
1017-
return out
1018-
if isinstance(node, dict):
1019-
for k, v in node.items():
1020-
_leaf_paths(v, f"{prefix}.{k}", depth=depth + 1, out=out)
1021-
elif isinstance(node, list):
1022-
for v in node[:50]:
1023-
_leaf_paths(v, f"{prefix}[]", depth=depth + 1, out=out)
1024-
else:
1025-
out.append(prefix)
1026-
return out
10271002

10281003

1029-
def _changed_paths(old: Any, new: Any, prefix: str = "$", *, depth: int = 0,
1030-
out: set | None = None) -> set[str]:
1031-
"""Leaf paths whose values differ between two JSON trees (same collapse rules as above)."""
1032-
if out is None:
1033-
out = set()
1034-
if len(out) >= 400 or depth > 6:
1035-
return out
1036-
if isinstance(old, dict) and isinstance(new, dict):
1037-
for k in set(old) | set(new):
1038-
_changed_paths(old.get(k), new.get(k), f"{prefix}.{k}", depth=depth + 1, out=out)
1039-
elif isinstance(old, list) and isinstance(new, list):
1040-
for a, b in zip(old[:50], new[:50]):
1041-
_changed_paths(a, b, f"{prefix}[]", depth=depth + 1, out=out)
1042-
if len(old) != len(new):
1043-
out.add(f"{prefix}[]")
1044-
elif old != new:
1045-
out.add(prefix)
1046-
return out
10471004

10481005

1049-
def _noise_only(old_body: bytes | None, new_body: bytes, key) -> bool:
1050-
"""True when this refetch's difference is the SAME small diff-set as last time — learned
1051-
request ids and server timestamps, not data. Two guards keep a real signal out of the noise
1052-
bin: the identical set must repeat (first occurrence always counts as changed, and gets
1053-
remembered as the candidate), and it must be a minor share (< 40%) of a body with at least
1054-
5 leaves — a tiny body whose one value moves every fetch is a PRICE, not noise."""
1055-
if not old_body:
1056-
return False
1057-
try:
1058-
old_json, new_json = json.loads(old_body), json.loads(new_body)
1059-
except (ValueError, UnicodeDecodeError):
1060-
return False
1061-
changed = _changed_paths(old_json, new_json)
1062-
if not changed:
1063-
return False
1064-
known = set(key.volatile_paths or [])
1065-
leaves = len(_leaf_paths(new_json))
1066-
is_noise = (changed <= known
1067-
and leaves >= _NOISE_MIN_LEAVES
1068-
and len(changed) / leaves < _NOISE_MAX_LEAF_SHARE)
1069-
# Remember this diff-set as the next candidate either way (bounded), so the SAME noise
1070-
# repeating is recognized from its second occurrence on.
1071-
key.volatile_paths = sorted(changed)[:50]
1072-
return is_noise
1006+
1007+
1008+
10731009

10741010

10751011
def learn(key, *, stable: bool, entry: dict[str, Any] | None) -> None:

‎src/treg/config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ def _async_pg_driver(cls, v: str) -> str:
309309
archive_mode: str = "off"
310310
# Strict compares raw bytes. The old heuristic is an explicit diagnostic opt-in only;
311311
# unknown values also select strict. It never changes stored response bytes.
312-
archive_comparison_mode: str = "strict" # strict | legacy_noise
312+
archive_comparison_mode: Literal["strict"] = "strict" # compatibility setting; comparison is always strict
313313
# Exact endpoint IDs, comma-separated. Empty means no serving, even in serve mode.
314314
archive_serve_endpoints: str = ""
315315
# Stable team/endpoint cohorts; 0 disables serving, 100 includes every team.

‎tests/test_archive.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ async def test_repeated_business_change_is_strict_by_default(clients: AsyncClien
547547
assert len(snaps) == 3 and all(s.body is not None for s in snaps)
548548

549549

550-
async def test_repeated_noise_counts_as_stable(clients: AsyncClient, shadow, monkeypatch):
550+
async def test_removed_noise_mode_cannot_weaken_strict_comparison(clients: AsyncClient, shadow, monkeypatch):
551551
monkeypatch.setattr(get_settings(), "archive_comparison_mode", "legacy_noise")
552552
monkeypatch.setitem(catalog_store.load().by_id[EP], "cache", "transient")
553553
from tests.test_marketplace_call import _fake_relay
@@ -558,10 +558,9 @@ async def test_repeated_noise_counts_as_stable(clients: AsyncClient, shadow, mon
558558
await clients.get(f"/call/{EP}?aweme_id=7")
559559
await archive.drain() # recordings must land in call order
560560
keys, _ = await _rows()
561-
# fetch 2 differs (first diff: counts changed, remembers the set); fetch 3 repeats the SAME
562-
# small diff-set ⇒ noise ⇒ stable.
563-
assert keys[0].change_seen == 1 and keys[0].stable_seen == 1
564-
assert keys[0].volatile_paths == ["$.req_id", "$.ts"]
561+
# A legacy configuration value cannot restore heuristic comparisons. Both changes count.
562+
assert keys[0].change_seen == 2 and keys[0].stable_seen == 0
563+
assert keys[0].volatile_paths == []
565564

566565

567566
async def test_always_changing_key_marks_itself_never_cache(clients: AsyncClient, serve, monkeypatch):

0 commit comments

Comments
 (0)