diff --git a/tools/histcapture.py b/tools/histcapture.py index 72b8f6c..0235677 100644 --- a/tools/histcapture.py +++ b/tools/histcapture.py @@ -14,26 +14,23 @@ PURPOSE DESTINATION: CUBE namespace 'hermes', one entry per message, named "HIST::" with a "TYPE: HIST-QNA" header + JSON body. -DEFAULT BEHAVIOR — FORWARD ONLY (no backfill) +DEFAULT BEHAVIOR — FORWARD ONLY (no backfill, no deletion) On first run with no state file, we DO NOT start from message id 0. We start from the CURRENT maximum message id, so only future exchanges are captured. This honors the standing "no backfill" rule. -BOUNDED SEED (explicit, user-approved) + Saved history is NEVER deleted by this script. The one-time cleanup of an + early over-backfill bug was a manual, one-off action and is NOT a recurring + retention limit — historical entries that land in the cube stay. + +BOUNDED SEED (explicit, user-approved, one-time) `--seed-hours N` populates the last N hours ONCE. This is the only - backfill path and is opt-in. The user approved a 24h seed. + backfill path and is opt-in (used to bootstrap the initial 24h window). IDEMPOTENCY - A state file records the highest message id pushed; reruns only push newer messages (safe under a timer). - - Writes check existence first and skip duplicates (so re-running a seed - or a crash mid-run does not create dupes). - -PRUNE - `--prune-older-than-hours N` removes entries this script would have written - that are older than N hours (used to undo an over-eager backfill). Names - are reconstructed deterministically from the DB, so we can target exactly - what this script wrote. + - Writes check existence first and skip duplicates. FAILURE If the cubed socket is down, cube_write returns False; we do not advance @@ -54,7 +51,6 @@ CUBED_SOCKET = os.environ.get("CUBED_SOCKET") or "/run/user/1000/cubelinux/cubed NAMESPACE = "hermes" STATE_FILE = "/home/CUBELinux/CUBELinux-2/tools/histcapture.state" WINDOW_SECONDS = 3600 -WALL1_GROUP = 1002 # cubelinux gid; we run as root, group gives socket access # --- cube RPC -------------------------------------------------------------- @@ -93,12 +89,6 @@ def cube_read(name: str): return None -def cube_delete(name: str) -> bool: - return bool(_rpc({ - "cmd": "delete", "namespace": NAMESPACE, "name": name, - }).get("ok")) - - # --- state ----------------------------------------------------------------- def read_last_id() -> int: try: @@ -138,12 +128,15 @@ def fetch_messages(since_id: int, before_ts: Optional[float] = None): def max_message_id() -> int: - con = sqlite3.connect(STATE_DB) - cur = con.cursor() - cur.execute("SELECT MAX(id) FROM messages") - m = cur.fetchone()[0] or 0 - con.close() - return m + con = None + try: + con = sqlite3.connect(STATE_DB) + cur = con.cursor() + cur.execute("SELECT MAX(id) FROM messages") + return cur.fetchone()[0] or 0 + finally: + if con: + con.close() # --- naming / entry -------------------------------------------------------- @@ -173,7 +166,6 @@ def capture(rows, dry_run: bool) -> int: seen: dict[str, int] = {} pushed_max = 0 written = 0 - skipped = 0 for r in rows: ts_iso = iso_from_ts(r["timestamp"]) seen[ts_iso] = seen.get(ts_iso, 0) + 1 @@ -182,7 +174,6 @@ def capture(rows, dry_run: bool) -> int: print(f"[dry-run] {name} ({r['role']})") continue if cube_read(name) is not None: - skipped += 1 pushed_max = max(pushed_max, r["id"]) continue if cube_write(name, entry): @@ -191,7 +182,7 @@ def capture(rows, dry_run: bool) -> int: else: print(f"WRITE FAILED id {r['id']} ({name})", file=sys.stderr) return pushed_max # stop; caller records progress - return pushed_max if (written or skipped) else 0 + return pushed_max if (written) else 0 # --- commands -------------------------------------------------------------- @@ -238,42 +229,11 @@ def cmd_seed(hours: int, dry_run: bool) -> int: return 0 -def cmd_prune(hours: int, dry_run: bool) -> int: - now = datetime.now(timezone.utc).timestamp() - cutoff = now - hours * WINDOW_SECONDS - # messages OLDER than the cutoff that this script would have written - con = sqlite3.connect(STATE_DB) - con.row_factory = sqlite3.Row - cur = con.cursor() - cur.execute( - "SELECT id, session_id, role, content, timestamp FROM messages " - "WHERE timestamp < ? AND role IN ('user','assistant') ORDER BY id ASC", - (cutoff,), - ) - rows = cur.fetchall() - con.close() - seen: dict[str, int] = {} - removed = 0 - for r in rows: - ts_iso = iso_from_ts(r["timestamp"]) - seen[ts_iso] = seen.get(ts_iso, 0) + 1 - name, _ = build_entry(r, seen[ts_iso]) - if dry_run: - print(f"[dry-run prune] would delete {name}") - continue - if cube_delete(name): - removed += 1 - print(f"pruned {removed} entries older than {hours}h") - return 0 - - def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dry-run", action="store_true") ap.add_argument("--seed-hours", type=int, metavar="N", help="one-time bounded population of last N hours") - ap.add_argument("--prune-older-than-hours", type=int, metavar="N", - help="remove entries this script wrote older than N hours") ap.add_argument("--reset", action="store_true", help="clear last_id state") args = ap.parse_args() @@ -281,11 +241,9 @@ def main() -> int: if args.reset: write_last_id(0) print("reset last_id -> 0") - if not (args.seed_hours or args.prune_older_than_hours): + if not args.seed_hours: return 0 - if args.prune_older_than_hours: - return cmd_prune(args.prune_older_than_hours, args.dry_run) if args.seed_hours: return cmd_seed(args.seed_hours, args.dry_run) return cmd_forward(args.dry_run) diff --git a/tools/histcapture.state b/tools/histcapture.state index 54224df..579c923 100644 --- a/tools/histcapture.state +++ b/tools/histcapture.state @@ -1 +1 @@ -12606 \ No newline at end of file +12631 \ No newline at end of file