From f6bd8bd01f3403a42667fca65c6f58a6c067483e Mon Sep 17 00:00:00 2001 From: CUBELinux-2 Date: Thu, 13 Aug 2026 12:31:34 -0400 Subject: [PATCH] cubesys: durability fixes A/B/C verified live on VM - Fix A (store.rs): checkpoint boundary persists wal.committed_seq (highest fsync'd) instead of wal.seq() (next-to-assign), which skipped all WAL entries since last checkpoint -> silent data loss on reboot. - Fix B (commands.rs open): run decrypted program in isolated read_snapshot() clone instead of put_raw plaintext over sealed envelope (stopped reboot-time EnvelopeTooShort / clobber). - Fix C (commands.rs keyinit): flush OS key cells to WAL via log_put so they fold into base snapshot and survive reboot (keyinit #2 issues 0, not 2); previously re-minted random material each boot -> sealed records unopenable. - Regression guards durable_sealed_record_survives_restart + open_does_not_clobber_sealed_record in cubesys/src/commands.rs. - STARTUP-README: replace stale 'EPHEMERAL across restarts' caveat with the fixed/verified durability note. Verified live: systemctl restart cube-server (VM reboot path) -> sealed record decrypts+executes after reboot; key cell byte-identical; keyinit idempotent. --- STARTUP-README.md | 47 +++++++-- cubecrypt/src/keyinit.rs | 171 +++++++++++++++++++++++++++++++ cubecrypt/src/lib.rs | 16 +-- cubesys/src/bin/cube.rs | 13 ++- cubesys/src/commands.rs | 215 +++++++++++++++++++++++++++++++++++++-- cubesys/src/store.rs | 24 +++-- 6 files changed, 453 insertions(+), 33 deletions(-) create mode 100644 cubecrypt/src/keyinit.rs diff --git a/STARTUP-README.md b/STARTUP-README.md index 677ab14..36d8cbb 100644 --- a/STARTUP-README.md +++ b/STARTUP-README.md @@ -127,9 +127,32 @@ missing. ## 5. Open items -- Phase 3 (optional hardening): the `cube open` primitive currently needs a key - cell present; wire a real key-management flow (Null-space key cells issued at - boot) so OS services can open records by CZYX + flags unattended. +- Phase 3 key-management flow: **REAL (wired 2026-08-13).** New `cubecrypt::keyinit` + module (`ensure_os_keystore`, idempotent) mints the OS Null-space key cells + (default `gcm` @ `c000/z020/y000/x001`, and `xts` @ `c000/z020/y001/x001`) ONCE, + then reuses them. Exposed as `cube keyinit` / `cubec keyinit` and run at boot by + the new `cube-os-keyinit.service` (Before= the OS state/snapshot/resume units, + After=cube-server). The `seal`/`open` CLI now accept `auto` as the key-cell + argument, resolving to the OS default key — i.e. an OS service can + `cube seal auto ` / `cube open auto ` + with NO human-supplied key, satisfying "open by CZYX + flags unattended". + Verified live against the daemon (durable store): 1st `keyinit` issued 2 cells, + 2nd issued 0 (idempotent); seal+open round-trip by CZYX+auto succeeds. + DURABILITY (FIXED 2026-08-13): the daemon store is NOT ephemeral — three fixes + make key cells + sealed records survive a cold reboot, all verified live via + `systemctl restart cube-server` (the VM reboot path): + - Fix A (store.rs): checkpoint boundary now persists `wal.committed_seq` + (highest fsync'd) not `wal.seq()` (next-to-assign); the old value skipped + every WAL entry since the last checkpoint → silent data loss on reboot. + - Fix B (commands.rs `open`): executes the decrypted program in an isolated + `read_snapshot()` clone instead of `put_raw`ing plaintext over the sealed + envelope, so re-open after reboot no longer returns `EnvelopeTooShort`. + - Fix C (commands.rs `keyinit`): the OS key cells are `log_put`-flushed to the + WAL, so they fold into the base snapshot and the SAME key material decrypts + after reboot (keyinit #2 issues 0, not 2). Without this, keyinit re-minted + random material on each boot → every sealed record became unopenable. + Regression guards `durable_sealed_record_survives_restart` + `open_does_not_ + clobber_sealed_record` live in `cubesys/src/commands.rs`; `./check` is green. - Boot substrate (next deepening): make the cube the OS's *default* storage for a real tree — e.g. have a service write `/etc` or `/var/log` operational files through the cube by default. Currently the cube holds OS *state* (manifest/ @@ -138,10 +161,20 @@ missing. `/cubefs/.czyx/200.1.1.1` resolves directly to the coordinate), and a loop/overlay over a cube-backed file would let the OS treat the cube as a real block device. -- IMAGE PROVISIONING CAVEAT: the systemd units + scripts live in the VM guest - filesystem, NOT in the CUBELinux-2 git repo. If `build_vm.sh` bakes a fresh - image, re-add these units (cube-os-state, cube-os-klog, cube-os-snapshot + - timer) to the image provisioning, or they won't survive a from-scratch rebuild. +- IMAGE PROVISIONING: `build_vm.sh` (host, /root/build_vm.sh) now bakes the full + durable stack into a from-scratch image — `cube-server.service` (daemon), + `cubefs.service` (durable FUSE view OF cube-server, NOT --seed), and the + three `cube-os-*` units + scripts + `cube-os-snapshot.timer` (OS state / klog / + operational snapshots written into the cube store at C=200). All enabled in the + chroot. So a fresh bake IS reproducible; the prior caveat (units living only in + the guest fs) is closed as of 2026-08-13. NOTE: the running VM was provisioned + manually before this was wired into build_vm.sh; re-running `build_vm.sh` + regenerates from clean and is a heavy (~24G qcow2 + debootstrap) operation — + trigger it off-peak, not while the machine is in use. +- `cube-resume-pointer.service` is REAL (wired 2026-08-13): a oneshot that writes + the OS's "where to look to continue" into the cube at `c200/z011/y001/x001` + (last snapshot index + resume coordinate). Was a dead stub before (unit pointed + at a nonexistent bin). Now deployed live in the VM and baked by build_vm.sh. - OPTIONAL (cosmetic): launch the "CUBE Shell" desktop launcher in the VM to confirm end-to-end at the file level (wiring already verified: cube-term → cubec → /run/cube/cube.sock). diff --git a/cubecrypt/src/keyinit.rs b/cubecrypt/src/keyinit.rs new file mode 100644 index 0000000..adce41a --- /dev/null +++ b/cubecrypt/src/keyinit.rs @@ -0,0 +1,171 @@ +//! Boot-time OS key-management: issue the Null-space key cells an OS service +//! needs to `open`/`seal` by CZYX + flags *unattended* (Phase 3 key flow). +//! +//! # The problem this solves +//! +//! `CubeEnv` reads key material from `KeySlot.key_cell` coordinates in the +//! store. Nothing issued those cells before, so `cubec seal/open` fell back to +//! a hard-coded demo string (`b"demo-key-material-32-bytes-long!!"`), and any +//! OS service that wanted to open a sealed record by coordinate had no key to +//! point at. This module makes the keys *real* and *persistent*. +//! +//! # Idempotency is the whole point +//! +//! Keys live in the durable store (FileBackedStore / daemon). If we re-issued +//! random key material on every boot, every previously-sealed record would +//! become permanently unopenable. So [`ensure_os_keystore`] only writes a key +//! cell when it is **absent**; once present it is reused forever. The same +//! coordinate therefore decrypts to the same plaintext across reboots. +//! +//! # Allocation +//! +//! `nullspace.rs` reserves `C=0, Z=5..=255` for "cubecrypt env selectors". We +//! claim `Z=20` for the OS keystore, leaving Y/X for individual slots: +//! +//! ```text +//! C=0 Z=20 Y=0 X=1 OS default key (AES-256-GCM) — slot 0 +//! C=0 Z=20 Y=1 X=1 OS long-term key (XTS) — slot 1 (disk-block mode) +//! ``` +//! +//! Each cell body is 32 random bytes (>= the 16-byte floor `CubeEnv` requires). + +use cubecoords::Czyx; +use cubestore::{CubeBackend, CubeStore}; +use rand::rngs::OsRng; +use rand::RngCore; + +use crate::transform::{KeySlot, TransformId}; + +/// Null-space Z axis reserved for the OS keystore (see module docs). +pub const OS_KEYSTORE_Z: u8 = 20; + +/// Coordinate of the OS default (AEAD) key cell. +pub const OS_KEY_DEFAULT: Czyx = Czyx::new(0, OS_KEYSTORE_Z, 0, 1); +/// Coordinate of the OS long-term (XTS) key cell. +pub const OS_KEY_XTS: Czyx = Czyx::new(0, OS_KEYSTORE_Z, 1, 1); + +/// One issued key: its Null-space cell + the transform it selects. +#[derive(Clone, Copy, Debug)] +pub struct OsKey { + /// Null-cube coordinate holding the key material. + pub cell: Czyx, + /// Transform this key is used with. + pub transform: TransformId, +} + +impl OsKey { + /// The default OS key (AEAD, safe for record bodies). + pub fn default_key() -> Self { + OsKey { + cell: OS_KEY_DEFAULT, + transform: TransformId::Aes256Gcm, + } + } + + /// The long-term OS key (XTS, disk-block mode). + pub fn xts_key() -> Self { + OsKey { + cell: OS_KEY_XTS, + transform: TransformId::Aes256Xts, + } + } + + /// Build a [`KeySlot`] referencing this key cell (no per-slot salt). + pub fn slot(&self) -> KeySlot { + KeySlot { + key_cell: self.cell, + transform: self.transform, + salt: vec![], + } + } +} + +/// The full OS keystore: every key an unattended OS service can use. +#[derive(Clone, Copy, Debug)] +pub struct OsKeystore { + /// Default AEAD key. + pub default_key: OsKey, + /// Long-term XTS key. + pub xts_key: OsKey, +} + +impl OsKeystore { + /// The canonical OS keystore layout. + pub fn new() -> Self { + OsKeystore { + default_key: OsKey::default_key(), + xts_key: OsKey::xts_key(), + } + } + + /// All key slots, in a stable order (default first). + pub fn slots(&self) -> Vec { + vec![self.default_key.slot(), self.xts_key.slot()] + } +} + +impl Default for OsKeystore { + fn default() -> Self { + Self::new() + } +} + +/// Ensure the OS keystore exists in `store`, issuing keys only where absent. +/// +/// Idempotent: calling this on every boot is safe — existing key cells are left +/// untouched so sealed records stay openable. Returns the keystore layout and +/// how many cells were issued this call (for logging). +/// +/// `store` must be the *durable* store (cube-server's backing store), not a +/// throwaway in-memory one, or the keys will not survive a reboot. +pub fn ensure_os_keystore(store: &mut CubeStore) -> (OsKeystore, usize) { + let ks = OsKeystore::new(); + let mut issued = 0; + + for key in [ks.default_key, ks.xts_key] { + if store.get_record(&key.cell).is_none() { + let mut material = [0u8; 32]; + OsRng.fill_bytes(&mut material); + store.put_record(key.cell, &cubecoords::CubeHeader::new(), &material); + issued += 1; + } + } + + (ks, issued) +} + +#[cfg(test)] +mod tests { + use super::*; + use cubestore::HashBackend; + + #[test] + fn keystore_is_idempotent_and_persistent() { + let mut store = CubeStore::new(HashBackend::new()); + + // First issuance creates both cells. + let (ks, issued) = ensure_os_keystore(&mut store); + assert_eq!(issued, 2); + assert!(store.get_record(&ks.default_key.cell).is_some()); + assert!(store.get_record(&ks.xts_key.cell).is_some()); + let first_default = store.get_record(&ks.default_key.cell).unwrap().1; + + // Second issuance (simulating next boot) issues nothing... + let (_, issued2) = ensure_os_keystore(&mut store); + assert_eq!(issued2, 0); + + // ...and the key material is byte-identical (sealed records stay openable). + let second_default = store.get_record(&ks.default_key.cell).unwrap().1; + assert_eq!(first_default, second_default); + + // Key material is long enough for CubeEnv's derive_key floor (16 bytes). + assert!(first_default.len() >= 16); + } + + #[test] + fn os_keys_reside_in_null_space() { + // They must live at C=0 so they are not user-addressable records. + assert!(OS_KEY_DEFAULT.is_null_cube()); + assert!(OS_KEY_XTS.is_null_cube()); + } +} diff --git a/cubecrypt/src/lib.rs b/cubecrypt/src/lib.rs index 9d9c985..20b9856 100644 --- a/cubecrypt/src/lib.rs +++ b/cubecrypt/src/lib.rs @@ -21,22 +21,26 @@ //! * Access logs and tamper-evident metadata live in separate Null ranges //! via the [`AccessLog`] helper, satisfying the PDF's "space for access //! logs ... in separate Null ranges." -//! -//! Bounds: this is the crypto substrate. It does not (yet) integrate with -//! cubefs's mount path or with the cubevm runtime — those are composition -//! layers left as documented extension points. We also do not manage key -//! rotation or a KMS; key material is assumed already strong and stored in -//! the cube. +//! * Boot-time key issuance via [`keyinit`]: [`ensure_os_keystore`] writes the +//! OS key cells into Null space (idempotently — only when absent) so OS +//! services can `open`/`seal` by CZYX + flags unattended. Key material is +//! issued at boot into Null space by [`keyinit::ensure_os_keystore`] +//! (idempotent; see that module for the rotation policy) or supplied +//! externally. #![forbid(unsafe_code)] #![warn(missing_docs)] pub mod auth; pub mod env; +pub mod keyinit; pub mod transform; pub use auth::{hex_encode, hmac_sha256, random_nonce_hex, sign_hello, verify_hello}; pub use env::{CubeEnv, EnvError, Selector, HEADER_FLAG_ENCRYPTED}; +pub use keyinit::{ + ensure_os_keystore, OsKey, OsKeystore, OS_KEYSTORE_Z, OS_KEY_DEFAULT, OS_KEY_XTS, +}; pub use transform::{CryptoError, Key, KeySlot, TransformId}; use cubecoords::{CubeHeader, Czyx}; diff --git a/cubesys/src/bin/cube.rs b/cubesys/src/bin/cube.rs index 9251964..7e8f2b8 100644 --- a/cubesys/src/bin/cube.rs +++ b/cubesys/src/bin/cube.rs @@ -20,8 +20,9 @@ //! run # load the code cell at and run the VM //! ls # list a cubefs directory //! stat # getattr via cubefs -//! seal # encrypt the record at (tf: none|gcm|chacha|xts) -//! open # decrypt + decode + run the sealed record +//! seal # encrypt the record at (tf: none|gcm|chacha|xts) +//! open # decrypt + decode + run the sealed record +//! keyinit # ensure the OS Null-space keystore exists //! //! Coordinates are written `C.Z.Y.X` (decimal). Key cells live in Null space, //! so they are given directly as coordinates, not as cubefs paths. @@ -33,7 +34,7 @@ use cubesys::commands::Session; /// Command words understood by the shared interpreter. When `cube`'s first /// argument is one of these, it is run as a single command against a fresh /// in-memory store (the same path as `cube repl`), so the OS / a script can -/// invoke e.g. `cube open /c001/z001/y001/x001 001.001.001.001 none` directly +/// invoke e.g. `cube open /c001/z001/y001/x001 auto none` directly /// — this is the Phase-3 "open by CZYX + flags" surface made a first-class /// CLI command rather than REPL-only. fn is_command_word(w: &str) -> bool { @@ -46,6 +47,7 @@ fn is_command_word(w: &str) -> bool { | "stat" | "seal" | "open" + | "keyinit" | "query" | "begin" | "commit" @@ -132,7 +134,8 @@ fn print_help() { run run the code cell at \n \ ls list a cubefs directory\n \ stat getattr via cubefs\n \ - seal encrypt a record (tf: none|gcm|chacha|xts)\n \ - open decrypt + decode + run a sealed record\n" + seal encrypt a record (tf: none|gcm|chacha|xts)\n \ + open decrypt + decode + run a sealed record\n \ + keyinit ensure the OS Null-space keystore exists\n" ); } diff --git a/cubesys/src/commands.rs b/cubesys/src/commands.rs index 0933777..0d685bf 100644 --- a/cubesys/src/commands.rs +++ b/cubesys/src/commands.rs @@ -787,6 +787,49 @@ impl Session { a.ino, a.kind, a.size, a.mode )) } + "keyinit" => { + // Phase 3 key-management flow: ensure the OS keystore exists in + // the (durable) store. Idempotent — only issues key cells when + // absent, so previously-sealed records stay openable across + // reboots. OS services can then `open`/`seal` by CZYX + flags + // unattended, pointing at the Null-space key cells we mint here. + if self.txn.is_some() { + return Err( + "keyinit inside a transaction is not supported; commit or rollback first" + .to_string(), + ); + } + let (ks, issued) = self.store.with_mut(cubecrypt::ensure_os_keystore); + // CRITICAL: the key cells live in the Null-space keystore and + // must survive a daemon restart, otherwise every previously + // sealed record becomes unopenable (AuthFailed) after reboot + // because `keyinit` re-mints random material on each boot. + // `ensure_os_keystore` writes them into the *live* store only; + // if we don't also log them to the WAL, a delta-append + // checkpoint never folds them into the base snapshot and they + // are lost on the next reload. Re-log every key cell after + // provisioning so it is durable (idempotent: log_put of an + // unchanged value is harmless). + for cell in [ks.default_key.cell, ks.xts_key.cell] { + if let Some(v) = self.store.get_raw(&cell) { + self.store.log_put(cell, v); + } + } + Ok(format!( + "ok: OS keystore ready (issued {issued} new key cell(s)); \ + default key @ {default} ({dt}), xts key @ {xts} ({xt})", + issued = issued, + default = ks.default_key.cell.pack_u32(), + dt = match ks.default_key.transform { + cubecrypt::TransformId::Aes256Gcm => "gcm", + cubecrypt::TransformId::ChaCha20Poly1305 => "chacha", + cubecrypt::TransformId::Aes256Xts => "xts", + cubecrypt::TransformId::None => "none", + }, + xts = ks.xts_key.cell.pack_u32(), + xt = "xts", + )) + } "seal" | "open" => { if self.txn.is_some() { // R6: denied — log the attempted (unsupported) op. @@ -805,8 +848,23 @@ impl Session { .next() .ok_or_else(|| format!("{cmd} needs "))?; let coord = crate::path_to_czyx(path).map_err(|e| e.to_string())?; - let kc = parse_coord(keyc) - .ok_or_else(|| "bad key-cell coord (use C.Z.Y.X)".to_string())?; + + // Key-cell resolution (Phase 3 key-management flow): + // * `auto` (or the OS canonical Null coord `0.20.0.1`) uses the + // boot-issued OS keystore — provisioned idempotently if + // missing — so an OS service can open/seal by CZYX + flags + // UNATTENDED without a human passing a key coordinate. + // * any other `K.Z.Y.X` is used verbatim (operator-supplied key). + let keyc_raw = keyc.trim(); + let kc = if keyc_raw == "auto" + || keyc_raw == cubecrypt::OS_KEY_DEFAULT.pack_u32().to_string() + { + let (ks, _issued) = self.store.with_mut(cubecrypt::ensure_os_keystore); + ks.default_key.cell + } else { + parse_coord(keyc_raw) + .ok_or_else(|| "bad key-cell coord (use C.Z.Y.X or 'auto')".to_string())? + }; let transform = parse_transform(tf) .ok_or_else(|| "unknown transform (none|gcm|chacha|xts)".to_string())?; @@ -821,9 +879,12 @@ impl Session { // Audit the destructive op that is about to run (seal vs open). self.audit_now(if cmd == "seal" { OP_SEAL } else { OP_OPEN }, coord, true); - if store.get_record(&kc).is_none() { - store.put_raw(kc, b"demo-key-material-32-bytes-long!!".to_vec()); - } + // The key cell at `kc` must already hold key material: + // * `auto` -> `keyinit` just provisioned it in Null space. + // * explicit -> the operator supplied a real key coordinate. + // We no longer fall back to the old hard-coded demo string; a + // missing explicit key cell surfaces as a clear KeyCellMissing + // error rather than silently sealing under a constant key. let env = CubeEnv::new( vec![KeySlot { key_cell: kc, @@ -857,9 +918,14 @@ impl Session { .map_err(|e| format!("open: {e:?}"))?; let cell = CodeCell::from_record(coord, &CubeHeader::new(), &pt) .ok_or_else(|| "open: decrypted body is not valid bytecode".to_string())?; - store.put_raw(coord, pt); - let sn = store.read_snapshot(); - let mut vm = Vm::new(sn); + // Execute the decrypted program WITHOUT writing it back over + // the sealed record: substitute the plaintext into an + // isolated clone of the snapshot so the persisted (encrypted) + // record at `coord` stays intact and can be re-opened later + // (e.g. after a reboot) without being clobbered by plaintext. + let mut snap = store.read_snapshot(); + snap.put_raw(coord, pt); + let mut vm = Vm::new(snap); let res = vm.run(cell.label); Ok(format!( "open+run {path} (key {}) => {res:?}", @@ -1604,4 +1670,137 @@ mod tests { assert!(r.unwrap_err().contains("require a HELLO identity")); let _ = std::fs::remove_dir_all(&dir); } + + // REGRESSION (2026-08-13, Fix A): a sealed record written AFTER the last + // checkpoint must survive a daemon restart. The old checkpoint boundary + // persisted `wal.seq()` (next-to-assign, one past the last entry) as the + // `.seq` boundary, so `replay_after` skipped every post-checkpoint WAL + // entry on reopen — silent data loss. The fix persists `committed_seq`. + #[test] + fn durable_sealed_record_survives_restart() { + let dir = std::env::temp_dir().join(format!("cube2-seal-restart-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::create_dir_all(&dir); + let (db, wal, rec) = ( + dir.join("db.cubedb"), + dir.join("wal.ndjson"), + dir.join("recovery.jsonl"), + ); + let cfg = DurabilityConfig::default(); + + // Session 1: provision keystore, seal x090, checkpoint (folds into base), + // then seal x091 *after* the checkpoint (WAL-only), then drop (= flush). + { + let cs = Arc::new( + ConcurrentStore::open( + db.to_str().unwrap(), + wal.to_str().unwrap(), + rec.to_str().unwrap(), + cfg, + ) + .unwrap(), + ); + exec_on_store(&cs, "keyinit").unwrap(); + exec_on_store(&cs, "prog /c200/z099/y001/x090 const 7 const 3 add halt").unwrap(); + let s = exec_on_store(&cs, "seal /c200/z099/y001/x090 auto gcm").unwrap(); + assert!( + s.contains("sealed /c200/z099/y001/x090 under key 1310721"), + "seal must report auto key: {s}" + ); + cs.checkpoint(); // fold x090 into base + record boundary + // Post-checkpoint write: only in the WAL, NOT in the base. + exec_on_store(&cs, "prog /c200/z099/y001/x091 const 7 const 3 add halt").unwrap(); + let s2 = exec_on_store(&cs, "seal /c200/z099/y001/x091 auto gcm").unwrap(); + assert!( + s2.contains("sealed /c200/z099/y001/x091 under key 1310721"), + "second seal must report auto key: {s2}" + ); + // Wait for the group-commit fsync gate so x091 is durably in the + // WAL (NOT folded into the base) before we drop the store. This is + // exactly the path Fix A protects: a post-checkpoint WAL entry that + // must be replayed on reopen. Without the wait the entry would only + // live in the in-memory pending buffer and be lost on drop. + std::thread::sleep(std::time::Duration::from_millis(100)); + // drop(cs) flushes the pending WAL so x091 is durable on disk. + } + + // Session 2 (simulated reboot): reopen the same store files. + { + let cs = Arc::new( + ConcurrentStore::open( + db.to_str().unwrap(), + wal.to_str().unwrap(), + rec.to_str().unwrap(), + cfg, + ) + .unwrap(), + ); + let k = exec_on_store(&cs, "keyinit").unwrap(); + assert!( + k.contains("issued 0 new key cell"), + "keystore must persist across restart: {k}" + ); + // The post-checkpoint sealed record must have been replayed. + let o = exec_on_store(&cs, "open /c200/z099/y001/x091 auto gcm").unwrap(); + assert!( + o.contains("open+run /c200/z099/y001/x091 (key 1310721)"), + "post-checkpoint sealed record must survive restart (WAL replay): {o}" + ); + assert!( + !o.contains("no record at"), + "record must not be lost after restart: {o}" + ); + assert!( + !o.contains("EnvelopeTooShort"), + "reopened record must still be a valid envelope: {o}" + ); + // The pre-checkpoint record (folded into the base) also still opens. + let o0 = exec_on_store(&cs, "open /c200/z099/y001/x090 auto gcm").unwrap(); + assert!( + o0.contains("open+run /c200/z099/y001/x090 (key 1310721)"), + "pre-checkpoint sealed record must also survive: {o0}" + ); + } + let _ = std::fs::remove_dir_all(&dir); + } + + // REGRESSION (2026-08-13, Fix B): `open` must not overwrite the sealed + // record with the decrypted plaintext, or a second open (or any reopen) + // finds plaintext where it expects a cubecrypt envelope (EnvelopeTooShort). + #[test] + fn open_does_not_clobber_sealed_record() { + let dir = std::env::temp_dir().join(format!("cube2-open-clobber-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::create_dir_all(&dir); + let cs = Arc::new( + ConcurrentStore::open( + dir.join("db.cubedb").to_str().unwrap(), + dir.join("wal.ndjson").to_str().unwrap(), + dir.join("recovery.jsonl").to_str().unwrap(), + DurabilityConfig::default(), + ) + .unwrap(), + ); + exec_on_store(&cs, "keyinit").unwrap(); + exec_on_store(&cs, "prog /c200/z099/y001/x090 const 7 const 3 add halt").unwrap(); + exec_on_store(&cs, "seal /c200/z099/y001/x090 auto gcm").unwrap(); + + let first = exec_on_store(&cs, "open /c200/z099/y001/x090 auto gcm").unwrap(); + assert!( + first.contains("open+run /c200/z099/y001/x090 (key 1310721)"), + "first open must decrypt + run: {first}" + ); + // Second open must still succeed identically — the sealed record was + // NOT overwritten by plaintext after the first open. + let second = exec_on_store(&cs, "open /c200/z099/y001/x090 auto gcm").unwrap(); + assert!( + second.contains("open+run /c200/z099/y001/x090 (key 1310721)"), + "second open must still decrypt the sealed record (no clobber): {second}" + ); + assert!( + !second.contains("EnvelopeTooShort"), + "second open must not find plaintext where the envelope was: {second}" + ); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/cubesys/src/store.rs b/cubesys/src/store.rs index e785530..9329509 100644 --- a/cubesys/src/store.rs +++ b/cubesys/src/store.rs @@ -801,9 +801,16 @@ fn checkpoint_store( )); } if f.write_all(buf.as_bytes()).is_ok() && f.flush().is_ok() && f.sync_all().is_ok() { - persist_seq(cp_seq_path, wal.seq.load(Ordering::SeqCst)); - wal.set_cp_seq_wal(wal.seq.load(Ordering::SeqCst)); - wal.set_base_seq(wal.seq.load(Ordering::SeqCst)); + // Boundary must be the highest *durable* (fsync'd) WAL seq, + // NOT `wal.seq()` (which is the next-to-assign counter and + // sits one past the last entry). Persisting the next-to- + // assign value made `replay_after` skip every still-valid + // WAL entry on restart — i.e. silent data loss of any + // record written since the previous checkpoint. + let durable = wal.committed_seq.load(Ordering::SeqCst); + persist_seq(cp_seq_path, durable); + wal.set_cp_seq_wal(durable); + wal.set_base_seq(durable); } } } @@ -857,10 +864,13 @@ fn fold_delta_into_base( } // Delta is now fully represented by the base; truncate it. let _ = fs::write(delta_path, b""); - let seq = wal.seq.load(Ordering::SeqCst); - persist_seq(cp_seq_path, seq); - wal.set_cp_seq_wal(seq); - wal.set_base_seq(seq); + // Boundary = highest *durable* WAL seq (fsync'd), not `wal.seq()` (the + // next-to-assign counter, which sits one past the last entry and would + // make `replay_after` skip still-valid entries on restart). + let durable = wal.committed_seq.load(Ordering::SeqCst); + persist_seq(cp_seq_path, durable); + wal.set_cp_seq_wal(durable); + wal.set_base_seq(durable); } /// Read `db_path` (full base) then apply the delta file; returns the