tooling: remove prune path — history is never deleted
User clarified: the earlier prune was a ONE-TIME cleanup of an over-backfill bug, NOT a recurring retention cap. Saved history must persist. Removed cmd_prune / --prune-older-than-hours entirely; the script now only captures forward (default) or seeds a bounded window once. No code path deletes cube entries. Verified: help has no prune arg, forward/seed dry-runs change no state, live timer runs forward-only.
This commit is contained in:
CUBELinux-2
+19
-61
@@ -14,26 +14,23 @@ PURPOSE
|
||||
DESTINATION: CUBE namespace 'hermes', one entry per message, named
|
||||
"HIST:<ISO8601>:<seq>" with a "TYPE: HIST-QNA" header + JSON body.
|
||||
|
||||
DEFAULT BEHAVIOR — FORWARD ONLY (no backfill)
|
||||
DEFAULT BEHAVIOR — FORWARD ONLY (no backfill, no deletion)
|
||||
On first run with no state file, we DO NOT start from message id 0. We
|
||||
start from the CURRENT maximum message id, so only future exchanges are
|
||||
captured. This honors the standing "no backfill" rule.
|
||||
|
||||
BOUNDED SEED (explicit, user-approved)
|
||||
Saved history is NEVER deleted by this script. The one-time cleanup of an
|
||||
early over-backfill bug was a manual, one-off action and is NOT a recurring
|
||||
retention limit — historical entries that land in the cube stay.
|
||||
|
||||
BOUNDED SEED (explicit, user-approved, one-time)
|
||||
`--seed-hours N` populates the last N hours ONCE. This is the only
|
||||
backfill path and is opt-in. The user approved a 24h seed.
|
||||
backfill path and is opt-in (used to bootstrap the initial 24h window).
|
||||
|
||||
IDEMPOTENCY
|
||||
- A state file records the highest message id pushed; reruns only push
|
||||
newer messages (safe under a timer).
|
||||
- Writes check existence first and skip duplicates (so re-running a seed
|
||||
or a crash mid-run does not create dupes).
|
||||
|
||||
PRUNE
|
||||
`--prune-older-than-hours N` removes entries this script would have written
|
||||
that are older than N hours (used to undo an over-eager backfill). Names
|
||||
are reconstructed deterministically from the DB, so we can target exactly
|
||||
what this script wrote.
|
||||
- Writes check existence first and skip duplicates.
|
||||
|
||||
FAILURE
|
||||
If the cubed socket is down, cube_write returns False; we do not advance
|
||||
@@ -54,7 +51,6 @@ CUBED_SOCKET = os.environ.get("CUBED_SOCKET") or "/run/user/1000/cubelinux/cubed
|
||||
NAMESPACE = "hermes"
|
||||
STATE_FILE = "/home/CUBELinux/CUBELinux-2/tools/histcapture.state"
|
||||
WINDOW_SECONDS = 3600
|
||||
WALL1_GROUP = 1002 # cubelinux gid; we run as root, group gives socket access
|
||||
|
||||
|
||||
# --- cube RPC --------------------------------------------------------------
|
||||
@@ -93,12 +89,6 @@ def cube_read(name: str):
|
||||
return None
|
||||
|
||||
|
||||
def cube_delete(name: str) -> bool:
|
||||
return bool(_rpc({
|
||||
"cmd": "delete", "namespace": NAMESPACE, "name": name,
|
||||
}).get("ok"))
|
||||
|
||||
|
||||
# --- state -----------------------------------------------------------------
|
||||
def read_last_id() -> int:
|
||||
try:
|
||||
@@ -138,12 +128,15 @@ def fetch_messages(since_id: int, before_ts: Optional[float] = None):
|
||||
|
||||
|
||||
def max_message_id() -> int:
|
||||
con = sqlite3.connect(STATE_DB)
|
||||
cur = con.cursor()
|
||||
cur.execute("SELECT MAX(id) FROM messages")
|
||||
m = cur.fetchone()[0] or 0
|
||||
con.close()
|
||||
return m
|
||||
con = None
|
||||
try:
|
||||
con = sqlite3.connect(STATE_DB)
|
||||
cur = con.cursor()
|
||||
cur.execute("SELECT MAX(id) FROM messages")
|
||||
return cur.fetchone()[0] or 0
|
||||
finally:
|
||||
if con:
|
||||
con.close()
|
||||
|
||||
|
||||
# --- naming / entry --------------------------------------------------------
|
||||
@@ -173,7 +166,6 @@ def capture(rows, dry_run: bool) -> int:
|
||||
seen: dict[str, int] = {}
|
||||
pushed_max = 0
|
||||
written = 0
|
||||
skipped = 0
|
||||
for r in rows:
|
||||
ts_iso = iso_from_ts(r["timestamp"])
|
||||
seen[ts_iso] = seen.get(ts_iso, 0) + 1
|
||||
@@ -182,7 +174,6 @@ def capture(rows, dry_run: bool) -> int:
|
||||
print(f"[dry-run] {name} ({r['role']})")
|
||||
continue
|
||||
if cube_read(name) is not None:
|
||||
skipped += 1
|
||||
pushed_max = max(pushed_max, r["id"])
|
||||
continue
|
||||
if cube_write(name, entry):
|
||||
@@ -191,7 +182,7 @@ def capture(rows, dry_run: bool) -> int:
|
||||
else:
|
||||
print(f"WRITE FAILED id {r['id']} ({name})", file=sys.stderr)
|
||||
return pushed_max # stop; caller records progress
|
||||
return pushed_max if (written or skipped) else 0
|
||||
return pushed_max if (written) else 0
|
||||
|
||||
|
||||
# --- commands --------------------------------------------------------------
|
||||
@@ -238,42 +229,11 @@ def cmd_seed(hours: int, dry_run: bool) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_prune(hours: int, dry_run: bool) -> int:
|
||||
now = datetime.now(timezone.utc).timestamp()
|
||||
cutoff = now - hours * WINDOW_SECONDS
|
||||
# messages OLDER than the cutoff that this script would have written
|
||||
con = sqlite3.connect(STATE_DB)
|
||||
con.row_factory = sqlite3.Row
|
||||
cur = con.cursor()
|
||||
cur.execute(
|
||||
"SELECT id, session_id, role, content, timestamp FROM messages "
|
||||
"WHERE timestamp < ? AND role IN ('user','assistant') ORDER BY id ASC",
|
||||
(cutoff,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
con.close()
|
||||
seen: dict[str, int] = {}
|
||||
removed = 0
|
||||
for r in rows:
|
||||
ts_iso = iso_from_ts(r["timestamp"])
|
||||
seen[ts_iso] = seen.get(ts_iso, 0) + 1
|
||||
name, _ = build_entry(r, seen[ts_iso])
|
||||
if dry_run:
|
||||
print(f"[dry-run prune] would delete {name}")
|
||||
continue
|
||||
if cube_delete(name):
|
||||
removed += 1
|
||||
print(f"pruned {removed} entries older than {hours}h")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--seed-hours", type=int, metavar="N",
|
||||
help="one-time bounded population of last N hours")
|
||||
ap.add_argument("--prune-older-than-hours", type=int, metavar="N",
|
||||
help="remove entries this script wrote older than N hours")
|
||||
ap.add_argument("--reset", action="store_true",
|
||||
help="clear last_id state")
|
||||
args = ap.parse_args()
|
||||
@@ -281,11 +241,9 @@ def main() -> int:
|
||||
if args.reset:
|
||||
write_last_id(0)
|
||||
print("reset last_id -> 0")
|
||||
if not (args.seed_hours or args.prune_older_than_hours):
|
||||
if not args.seed_hours:
|
||||
return 0
|
||||
|
||||
if args.prune_older_than_hours:
|
||||
return cmd_prune(args.prune_older_than_hours, args.dry_run)
|
||||
if args.seed_hours:
|
||||
return cmd_seed(args.seed_hours, args.dry_run)
|
||||
return cmd_forward(args.dry_run)
|
||||
|
||||
@@ -1 +1 @@
|
||||
12606
|
||||
12631
|
||||
Reference in New Issue
Block a user