cubesys: durability fixes A/B/C verified live on VM

- Fix A (store.rs): checkpoint boundary persists wal.committed_seq (highest fsync'd) instead of wal.seq() (next-to-assign), which skipped all WAL entries since last checkpoint -> silent data loss on reboot. - Fix B (commands.rs open): run decrypted program in isolated read_snapshot() clone instead of put_raw plaintext over sealed envelope (stopped reboot-time EnvelopeTooShort / clobber). - Fix C (commands.rs keyinit): flush OS key cells to WAL via log_put so they fold into base snapshot and survive reboot (keyinit #2 issues 0, not 2); previously re-minted random material each boot -> sealed records unopenable. - Regression guards durable_sealed_record_survives_restart + open_does_not_clobber_sealed_record in cubesys/src/commands.rs. - STARTUP-README: replace stale 'EPHEMERAL across restarts' caveat with the fixed/verified durability note. Verified live: systemctl restart cube-server (VM reboot path) -> sealed record decrypts+executes after reboot; key cell byte-identical; keyinit idempotent.
2026-08-13 12:31:34 -04:00
parent ab40409316
commit f6bd8bd01f
6 changed files with 453 additions and 33 deletions
@@ -0,0 +1,171 @@
+//! Boot-time OS key-management: issue the Null-space key cells an OS service
+//! needs to `open`/`seal` by CZYX + flags *unattended* (Phase 3 key flow).
+//!
+//! # The problem this solves
+//!
+//! `CubeEnv` reads key material from `KeySlot.key_cell` coordinates in the
+//! store. Nothing issued those cells before, so `cubec seal/open` fell back to
+//! a hard-coded demo string (`b"demo-key-material-32-bytes-long!!"`), and any
+//! OS service that wanted to open a sealed record by coordinate had no key to
+//! point at. This module makes the keys *real* and *persistent*.
+//!
+//! # Idempotency is the whole point
+//!
+//! Keys live in the durable store (FileBackedStore / daemon). If we re-issued
+//! random key material on every boot, every previously-sealed record would
+//! become permanently unopenable. So [`ensure_os_keystore`] only writes a key
+//! cell when it is **absent**; once present it is reused forever. The same
+//! coordinate therefore decrypts to the same plaintext across reboots.
+//!
+//! # Allocation
+//!
+//! `nullspace.rs` reserves `C=0, Z=5..=255` for "cubecrypt env selectors". We
+//! claim `Z=20` for the OS keystore, leaving Y/X for individual slots:
+//!
+//! ```text
+//! C=0 Z=20 Y=0 X=1   OS default key (AES-256-GCM)   — slot 0
+//! C=0 Z=20 Y=1 X=1   OS long-term key (XTS)         — slot 1 (disk-block mode)
+//! ```
+//!
+//! Each cell body is 32 random bytes (>= the 16-byte floor `CubeEnv` requires).
+
+use cubecoords::Czyx;
+use cubestore::{CubeBackend, CubeStore};
+use rand::rngs::OsRng;
+use rand::RngCore;
+
+use crate::transform::{KeySlot, TransformId};
+
+/// Null-space Z axis reserved for the OS keystore (see module docs).
+pub const OS_KEYSTORE_Z: u8 = 20;
+
+/// Coordinate of the OS default (AEAD) key cell.
+pub const OS_KEY_DEFAULT: Czyx = Czyx::new(0, OS_KEYSTORE_Z, 0, 1);
+/// Coordinate of the OS long-term (XTS) key cell.
+pub const OS_KEY_XTS: Czyx = Czyx::new(0, OS_KEYSTORE_Z, 1, 1);
+
+/// One issued key: its Null-space cell + the transform it selects.
+#[derive(Clone, Copy, Debug)]
+pub struct OsKey {
+    /// Null-cube coordinate holding the key material.
+    pub cell: Czyx,
+    /// Transform this key is used with.
+    pub transform: TransformId,
+}
+
+impl OsKey {
+    /// The default OS key (AEAD, safe for record bodies).
+    pub fn default_key() -> Self {
+        OsKey {
+            cell: OS_KEY_DEFAULT,
+            transform: TransformId::Aes256Gcm,
+        }
+    }
+
+    /// The long-term OS key (XTS, disk-block mode).
+    pub fn xts_key() -> Self {
+        OsKey {
+            cell: OS_KEY_XTS,
+            transform: TransformId::Aes256Xts,
+        }
+    }
+
+    /// Build a [`KeySlot`] referencing this key cell (no per-slot salt).
+    pub fn slot(&self) -> KeySlot {
+        KeySlot {
+            key_cell: self.cell,
+            transform: self.transform,
+            salt: vec![],
+        }
+    }
+}
+
+/// The full OS keystore: every key an unattended OS service can use.
+#[derive(Clone, Copy, Debug)]
+pub struct OsKeystore {
+    /// Default AEAD key.
+    pub default_key: OsKey,
+    /// Long-term XTS key.
+    pub xts_key: OsKey,
+}
+
+impl OsKeystore {
+    /// The canonical OS keystore layout.
+    pub fn new() -> Self {
+        OsKeystore {
+            default_key: OsKey::default_key(),
+            xts_key: OsKey::xts_key(),
+        }
+    }
+
+    /// All key slots, in a stable order (default first).
+    pub fn slots(&self) -> Vec<KeySlot> {
+        vec![self.default_key.slot(), self.xts_key.slot()]
+    }
+}
+
+impl Default for OsKeystore {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Ensure the OS keystore exists in `store`, issuing keys only where absent.
+///
+/// Idempotent: calling this on every boot is safe — existing key cells are left
+/// untouched so sealed records stay openable. Returns the keystore layout and
+/// how many cells were issued this call (for logging).
+///
+/// `store` must be the *durable* store (cube-server's backing store), not a
+/// throwaway in-memory one, or the keys will not survive a reboot.
+pub fn ensure_os_keystore<B: CubeBackend>(store: &mut CubeStore<B>) -> (OsKeystore, usize) {
+    let ks = OsKeystore::new();
+    let mut issued = 0;
+
+    for key in [ks.default_key, ks.xts_key] {
+        if store.get_record(&key.cell).is_none() {
+            let mut material = [0u8; 32];
+            OsRng.fill_bytes(&mut material);
+            store.put_record(key.cell, &cubecoords::CubeHeader::new(), &material);
+            issued += 1;
+        }
+    }
+
+    (ks, issued)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use cubestore::HashBackend;
+
+    #[test]
+    fn keystore_is_idempotent_and_persistent() {
+        let mut store = CubeStore::new(HashBackend::new());
+
+        // First issuance creates both cells.
+        let (ks, issued) = ensure_os_keystore(&mut store);
+        assert_eq!(issued, 2);
+        assert!(store.get_record(&ks.default_key.cell).is_some());
+        assert!(store.get_record(&ks.xts_key.cell).is_some());
+        let first_default = store.get_record(&ks.default_key.cell).unwrap().1;
+
+        // Second issuance (simulating next boot) issues nothing...
+        let (_, issued2) = ensure_os_keystore(&mut store);
+        assert_eq!(issued2, 0);
+
+        // ...and the key material is byte-identical (sealed records stay openable).
+        let second_default = store.get_record(&ks.default_key.cell).unwrap().1;
+        assert_eq!(first_default, second_default);
+
+        // Key material is long enough for CubeEnv's derive_key floor (16 bytes).
+        assert!(first_default.len() >= 16);
+    }
+
+    #[test]
+    fn os_keys_reside_in_null_space() {
+        // They must live at C=0 so they are not user-addressable records.
+        assert!(OS_KEY_DEFAULT.is_null_cube());
+        assert!(OS_KEY_XTS.is_null_cube());
+    }
+}
@@ -21,22 +21,26 @@
 //! * Access logs and tamper-evident metadata live in separate Null ranges
 //!   via the [`AccessLog`] helper, satisfying the PDF's "space for access
 //!   logs ... in separate Null ranges."
-//!
-//! Bounds: this is the crypto substrate. It does not (yet) integrate with
-//! cubefs's mount path or with the cubevm runtime — those are composition
-//! layers left as documented extension points. We also do not manage key
-//! rotation or a KMS; key material is assumed already strong and stored in
-//! the cube.
+//! * Boot-time key issuance via [`keyinit`]: [`ensure_os_keystore`] writes the
+//!   OS key cells into Null space (idempotently — only when absent) so OS
+//!   services can `open`/`seal` by CZYX + flags unattended. Key material is
+//!   issued at boot into Null space by [`keyinit::ensure_os_keystore`]
+//!   (idempotent; see that module for the rotation policy) or supplied
+//!   externally.

 #![forbid(unsafe_code)]
 #![warn(missing_docs)]

 pub mod auth;
 pub mod env;
+pub mod keyinit;
 pub mod transform;

 pub use auth::{hex_encode, hmac_sha256, random_nonce_hex, sign_hello, verify_hello};
 pub use env::{CubeEnv, EnvError, Selector, HEADER_FLAG_ENCRYPTED};
+pub use keyinit::{
+    ensure_os_keystore, OsKey, OsKeystore, OS_KEYSTORE_Z, OS_KEY_DEFAULT, OS_KEY_XTS,
+};
 pub use transform::{CryptoError, Key, KeySlot, TransformId};

 use cubecoords::{CubeHeader, Czyx};