store: a fold kept a deletion only when the record had no class

A fold turns the log's effects into the image's by keeping, per coordinate, the *last* entry in the
merged order — that is how "the later write wins" is implemented. The order came from a derive on
`Entry`, which compares fields in declaration order, and `flags` was declared before `seq`. So the
tie-break was not arrival at all: it was the class mask.

A `del` is written with no class (0). A **sealed** record is written with `0x0800`. For the same
coordinate that put the removal *before* the record it removed, so the record won and the fold wrote
it back — a deleted record returning from the fold, with its old value, on any store whose writes are
sealed. Found on the box on 2026-09-25: put → present, del → "(not found)", `cube-fold` → "folded the
log into the image" (generation 2647 → 2648, log emptied), get → the value, back. It reproduced
through a sealed front-end and an unsealed one, and the live store still holds the test records it
resurrected.

Why it hid: an unsealed store writes both entries with flags 0, the tie falls through to `seq`, and
the order is right — which is exactly what the guest gate did before this, so the gate agreed with a
bug it could not see. The shape that fails is the shape the machine uses.

`Entry` now orders by `(space, key, seq, …)` explicitly, with the reason written beside it, because
"the fields happen to be declared in this order" is what went wrong. The userspace store cannot have
this bug and does not: it applies the log into a `HashMap`, where a removal *is* the removal of the
key, so no ordering decides anything.

drivers/cube/cube_syscall.c

@@ -34,6 +34,13 @@ int cubelinux_kernel_put(const __u8 *space, __u64 x, __u64 y, __u64 z,
 ssize_t cubelinux_kernel_get(const __u8 *space, __u64 x, __u64 y, __u64 z,
 			     void *buf, size_t len, __u16 *out_flags);
 int cubelinux_kernel_del(const __u8 *space, __u64 x, __u64 y, __u64 z);
+/*
+ * Capture a request this layer refused, as a classed record in the events space. The refusals worth
+ * recording are the ones that leave the store usable — a caller asking for something the interface
+ * does not offer — because those are the moments the machine said no and carried on. It takes the
+ * driver's write lock itself, so it must be called from here and not from inside an op.
+ */
+int cubelinux_kernel_capture_refusal(__u64 op, int errno, const __u8 *kind, size_t kind_len);
 int cubelinux_kernel_sync(void);
 
 /*
@@ -156,8 +163,10 @@ static long cube_args_op(unsigned int op, void __user *uargs)
 	 * The size is the caller's, and it must be the one this kernel implements: a caller
 	 * built against a later block would otherwise have fields silently ignored.
 	 */
-	if (args.size != sizeof(struct cube_args))
+	if (args.size != sizeof(struct cube_args)) {
+		cubelinux_kernel_capture_refusal(op, EINVAL, "bad-size", 8);
 		return -EINVAL;
+	}
 
 	switch (op) {
 	case CUBE_OP_PUT:
@@ -167,12 +176,15 @@ static long cube_args_op(unsigned int op, void __user *uargs)
 	case CUBE_OP_SYNC:
 		break;
 	default:
+		cubelinux_kernel_capture_refusal(op, EINVAL, "bad-op", 6);
 		return -EINVAL;
 	}
 
 	if (op == CUBE_OP_PUT || op == CUBE_OP_GET) {
-		if (args.len > CUBE_MAX_VALUE)
+		if (args.len > CUBE_MAX_VALUE) {
+			cubelinux_kernel_capture_refusal(op, E2BIG, "too-big", 7);
 			return -E2BIG;
+		}
 		if (args.len > 0) {
 			buf = kvmalloc(args.len, GFP_KERNEL);
 			if (!buf)

drivers/cube/cubelinux_store.rs

@@ -41,7 +41,7 @@
 //! ```
 
 use core::fmt::{self, Write};
-use core::sync::atomic::{AtomicU32, Ordering};
+use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, Ordering};
 
 // The store's format, in one file, shared with userspace.
 //
@@ -302,8 +302,6 @@ const BOOT_FLAGS: u16 = 1 << 7;
 /// needs. History would be a second record per boot, and nobody has asked for one.
 const BOOT_POINT: (u64, u64, u64) = (0, 0, 0);
 
-/// Whether this boot has already been recorded. One attempt, claimed with a swap so two callers
-/// racing into their first write cannot write two records.
 /// How many attempts this boot has made at recording itself, and the ceiling on them.
 ///
 /// A count rather than a flag, because a flag that is claimed *before* the write cannot un-claim
@@ -316,6 +314,68 @@ const BOOT_POINT: (u64, u64, u64) = (0, 0, 0);
 static BOOT_RECORD_ATTEMPTS: AtomicU32 = AtomicU32::new(0);
 const BOOT_RECORD_MAX_ATTEMPTS: u32 = 4;
 
+/// The wall clock and the monotonic clock as they stood when this boot's record was written.
+///
+/// Kept so a later write can tell whether the *epoch* in that record is still the truth. The record
+/// is written at the first write of a boot, which is exactly when the wall clock is least likely to
+/// have been corrected: this box arrives from a night powered off with a clock tens of seconds out,
+/// so the first record carried a time it had no way to qualify. Two clocks are the whole mechanism,
+/// and neither needs a reference: the wall clock can be set from anywhere, the monotonic clock
+/// cannot be set at all, so a divergence between them over the same interval is a clock correction
+/// and nothing else. `uptime` in the record comes from the monotonic side, which means the instant
+/// the boot began (`boot - uptime`) stays recoverable however wrong the wall clock was.
+static BOOT_RECORD_WALL: AtomicI64 = AtomicI64::new(0);
+static BOOT_RECORD_MONO: AtomicI64 = AtomicI64::new(0);
+
+/// Whether the record has been rewritten after a correction, and how many tries that has taken.
+static BOOT_RECORD_CLOCK_SET: AtomicBool = AtomicBool::new(false);
+static BOOT_RECORD_REFRESHES: AtomicU32 = AtomicU32::new(0);
+const BOOT_RECORD_MAX_REFRESHES: u32 = 2;
+
+/// How far the wall clock must move relative to the monotonic clock before it counts as corrected.
+/// Both are whole seconds read one after the other, so a second of slack belongs here.
+const CLOCK_SET_TOLERANCE_SECS: i64 = 2;
+
+/// The space the **kernel captures its own events into**: `0xFB` repeated, reserved for events.
+///
+/// The same shape and the same reason as [`BOOT_SPACE`]: the driver cannot link `cube-core`, so the
+/// tag is written out on both sides and `verify-event-capture` is what holds them together — the
+/// guest scans this space at the raw `0xFB` and userspace scans it by the name `events`, and the
+/// bytes have to come back the same.
+const EVENT_SPACE: [u8; SPACE_ID_LEN] = [0xFB; SPACE_ID_LEN];
+
+/// The class bits the kernel stamps on the events it captures, spelled out from the events
+/// vocabulary in `cube-core::wordflags::EventFlags` and checked against it by the same gate.
+///
+/// `ERROR` (1<<5) is the vocabulary's mark that an event went wrong — it *combines*, which is why a
+/// refused request is `ERROR` and a request refused while writing the record about a boot is
+/// `ERROR | BOOT`.
+const EVENT_ERROR: u16 = 1 << 5;
+/// The class the kernel's own boot record already carries. The vocabulary's boot class.
+const EVENT_BOOT: u16 = 1 << 7;
+
+/// The coordinate the next captured event goes to: a sequence, so events accumulate.
+///
+/// This is the one place the kernel's own writing differs from its boot record, and deliberately:
+/// the boot record is *one* record at `(0,0,0)` saying "this boot", overwritten each boot, because
+/// that is what "which boot is this" needs. Events are a series, so they get distinct coordinates
+/// and are not overwritten — which is also why they need a ceiling (below).
+static EVENT_SEQ: AtomicU64 = AtomicU64::new(0);
+
+/// How many events this kernel may write by itself in one boot, and how many refusals among them.
+///
+/// The bound is not tidiness. A kernel that appends a record per event is a kernel that can turn a
+/// storm of refused requests into a storm of *writes*, and this store has already been through that
+/// from the other side: a walk with no store behind it served about two hundred thousand invented
+/// records a minute, which became two hundred thousand writes once the walk was made to answer. The
+/// ceiling means the kernel's own account of itself can never be the thing that fills the store —
+/// past it the kernel logs and stops, and says so in the log rather than quietly shaping the record.
+const EVENT_BUDGET: u32 = 32;
+static EVENTS_WRITTEN: AtomicU32 = AtomicU32::new(0);
+/// Refusals have their own, smaller ceiling: they are the class a caller or a bug can drive.
+const REFUSAL_BUDGET: u32 = 8;
+static REFUSALS_WRITTEN: AtomicU32 = AtomicU32::new(0);
+
 
 /// FNV-1a offset basis.
 const FNV_OFFSET: u64 = cube_format::FNV_OFFSET;
@@ -878,7 +938,7 @@ fn resolve_layout(b: &[u8]) -> Result<Layout, &'static str> {
 
 /// One record on its way into the merged store. Fixed size, so a `KVVec` of these sorts
 /// in place; values live in a pool beside them and are referenced by offset.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Clone, Copy, PartialEq, Eq)]
 struct Entry {
     space: [u8; 32],
     key: [u8; 24],
@@ -892,6 +952,38 @@ struct Entry {
     deleted: bool,
 }
 
+/// Order by `(space, key, seq, …)` — **`seq` before `flags`**, which is not the order the fields are
+/// declared in and cannot be left to a derive.
+///
+/// A fold keeps, for each coordinate, the *last* entry in this order: that is how "the later write
+/// wins" is implemented. With `flags` compared first, "later" is not what the order means: a `del`
+/// carries no class (0) and a **sealed** record carries `0x0800`, so the removal sorted *before* the
+/// record it removed and the record won — a deleted record came back from the fold, with its old
+/// value, on a store whose writes are sealed. Found on the box on 2026-09-25 by `put` → `del` →
+/// `cube-fold` → the record returning; the guest gate missed it because unsealed writes share one
+/// flag value, which is why the gate now deletes a *classed* record.
+///
+/// `flags` stays in the order, after `seq`: the sort also has to be total and deterministic, and two
+/// entries can share a coordinate and a sequence only if they came from the same source.
+impl Ord for Entry {
+    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
+        self.space
+            .cmp(&other.space)
+            .then_with(|| self.key.cmp(&other.key))
+            .then_with(|| self.seq.cmp(&other.seq))
+            .then_with(|| self.flags.cmp(&other.flags))
+            .then_with(|| self.value_off.cmp(&other.value_off))
+            .then_with(|| self.value_len.cmp(&other.value_len))
+            .then_with(|| self.deleted.cmp(&other.deleted))
+    }
+}
+
+impl PartialOrd for Entry {
+    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 /// The store as a sorted set of records, which is what a checkpoint writes and what the
 /// digest is taken over.
 struct Merged {
@@ -1972,24 +2064,223 @@ fn c_len(p: *const u8, max: usize) -> usize {
     n
 }
 
+/// Which clock the epoch in a boot record was read from.
+///
+/// `Raw` is the reading the first write of the boot took, before anything has had a chance to
+/// correct the clock. `Set` is a reading taken after a correction was *observed*, which is the only
+/// way this record can claim a time it knows was not provisional.
+#[derive(Clone, Copy)]
+enum ClockState {
+    Raw,
+    Set,
+}
+
+impl ClockState {
+    fn word(self) -> &'static [u8] {
+        match self {
+            ClockState::Raw => b"raw",
+            ClockState::Set => b"set",
+        }
+    }
+}
+
+/// Both clocks, read as close together as two calls can be.
+fn now_clocks() -> (i64, i64) {
+    let mut wall = bindings::timespec64 {
+        tv_sec: 0,
+        tv_nsec: 0,
+    };
+    let mut mono = bindings::timespec64 {
+        tv_sec: 0,
+        tv_nsec: 0,
+    };
+    // SAFETY: both are live, writable `timespec64`s, and neither call retains a pointer to one.
+    unsafe { bindings::ktime_get_real_ts64(&mut wall) };
+    // SAFETY: as above.
+    unsafe { bindings::ktime_get_ts64(&mut mono) };
+    (wall.tv_sec as i64, mono.tv_sec as i64)
+}
+
+/// Whether the wall clock has been set or adjusted since this boot's record was written.
+///
+/// The wall clock can be set from anywhere and the monotonic clock cannot be set at all, so if the
+/// wall clock has moved by a different amount than the monotonic clock over the same interval, then
+/// something set the wall clock — and the epoch in the record is a reading from before it.
+fn wall_clock_was_set() -> bool {
+    let wall_at = BOOT_RECORD_WALL.load(Ordering::Acquire);
+    if wall_at == 0 {
+        return false;
+    }
+    let mono_at = BOOT_RECORD_MONO.load(Ordering::Acquire);
+    let (wall, mono) = now_clocks();
+    ((wall - wall_at) - (mono - mono_at)).abs() > CLOCK_SET_TOLERANCE_SECS
+}
+
+/// Rewrite this boot's record when the clock it was read from has since been corrected.
+///
+/// The record lives at one coordinate and each boot overwrites the last, so this is the same write
+/// the first one was, with a clock reading that is no longer provisional. It is bounded like the
+/// first: a store that will not take the rewrite logs and stops asking.
+fn refresh_boot_record_if_the_clock_was_set() {
+    if BOOT_RECORD_CLOCK_SET.load(Ordering::Acquire) || !wall_clock_was_set() {
+        return;
+    }
+    if BOOT_RECORD_REFRESHES.fetch_add(1, Ordering::AcqRel) >= BOOT_RECORD_MAX_REFRESHES {
+        return;
+    }
+    let (wall, mono) = now_clocks();
+    let value = match boot_record_value(ClockState::Set, wall, mono) {
+        Some(v) => v,
+        None => return,
+    };
+    let mutation = Mutation {
+        space: BOOT_SPACE,
+        key: morton_encode(BOOT_POINT.0, BOOT_POINT.1, BOOT_POINT.2),
+        flags: BOOT_FLAGS,
+        value,
+    };
+    match append_mutation(&mutation, 1) {
+        Ok(_) => {
+            BOOT_RECORD_CLOCK_SET.store(true, Ordering::Release);
+            BOOT_RECORD_WALL.store(wall, Ordering::Release);
+            BOOT_RECORD_MONO.store(mono, Ordering::Release);
+            pr_info!("cubelinux: the clock was set, so this boot's record was rewritten with the corrected time\n");
+            // And the event goes in the store, not only in the log: the record about the boot now
+            // carries a time that was provisional for a while, and this is the record that says so.
+            if !capture_event(
+                EVENT_BOOT,
+                b"clock-set",
+                b"the boot record's epoch was rewritten after a correction",
+            ) {
+                pr_warn!("cubelinux: could not capture the clock-set event (budget or store)\n");
+            }
+        }
+        Err(_) => pr_warn!("cubelinux: could not rewrite this boot's record after a clock correction\n"),
+    }
+}
+
+/// Capture one event into the events space, as a record like any other.
+///
+/// The value is `event=<kind> uptime=<seconds since boot>[ <detail>]`, and it is classed by the
+/// caller's mask. Nothing about it is privileged: it is a record in a space, written through the
+/// same bounded append every other write takes, readable back through `cube(2)` and findable with a
+/// class scan — which is the whole point. The kernel's account of what happened to it stops being
+/// something you have to have been watching `dmesg` to know.
+///
+/// **Called with `STORE_OP` held** (the write path holds it, and so does the C-facing wrapper
+/// below), and bounded twice — see [`EVENT_BUDGET`] and [`REFUSAL_BUDGET`]. Returns whether the
+/// event was actually written, because a caller that logs "captured" when the budget was spent
+/// would be lying in the log about the store.
+fn capture_event(flags: u16, kind: &[u8], detail: &[u8]) -> bool {
+    if EVENTS_WRITTEN.load(Ordering::Acquire) >= EVENT_BUDGET {
+        return false;
+    }
+    if flags & EVENT_ERROR != 0 && REFUSALS_WRITTEN.load(Ordering::Acquire) >= REFUSAL_BUDGET {
+        return false;
+    }
+    let seq = EVENT_SEQ.fetch_add(1, Ordering::AcqRel);
+    let (_, mono) = now_clocks();
+
+    let mut value = KVVec::<u8>::new();
+    if push(&mut value, b"event=").is_none()
+        || push(&mut value, kind).is_none()
+        || push(&mut value, b" uptime=").is_none()
+        || push_dec(&mut value, mono.max(0) as u64).is_none()
+    {
+        return false;
+    }
+    if !detail.is_empty()
+        && (push(&mut value, b" ").is_none() || push(&mut value, detail).is_none())
+    {
+        return false;
+    }
+
+    let mutation = Mutation {
+        space: EVENT_SPACE,
+        key: morton_encode(seq, 0, 0),
+        flags,
+        value,
+    };
+    match append_mutation(&mutation, 1) {
+        Ok(_) => {
+            EVENTS_WRITTEN.fetch_add(1, Ordering::AcqRel);
+            if flags & EVENT_ERROR != 0 {
+                REFUSALS_WRITTEN.fetch_add(1, Ordering::AcqRel);
+            }
+            true
+        }
+        Err(_) => false,
+    }
+}
+
+/// Capture a request the kernel refused, called from the syscall layer.
+///
+/// The refusals worth recording are the ones that leave the store *usable*: a caller asking for
+/// something the interface does not offer (a bad size, a mode that is not a mode, a space that is
+/// not there). Those are the moments where the machine said no and carried on, which is exactly the
+/// behaviour this store is built around — and until now nothing recorded that it happened. A store
+/// that cannot be read at all is the one refusal that cannot be recorded *into the store*, and this
+/// does not pretend otherwise: it returns 0 and the log keeps that story.
+///
+/// Takes `STORE_OP` itself, so it must be called from outside the driver's write path — the C
+/// layer's argument checks are all before the driver is entered, which is the only place it is
+/// called from.
+///
+/// # Safety
+/// `kind` must point to `kind_len` readable bytes when `kind_len` is non-zero.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn cubelinux_kernel_capture_refusal(
+    op: u64,
+    errno: i32,
+    kind: *const u8,
+    kind_len: usize,
+) -> i32 {
+    let kind = if kind_len == 0 || kind.is_null() {
+        &b"refused"[..]
+    } else {
+        // SAFETY: the caller guarantees `kind_len` readable bytes at `kind`.
+        unsafe { core::slice::from_raw_parts(kind, kind_len) }
+    };
+    let mut detail = KVVec::<u8>::new();
+    if push(&mut detail, b"op=").is_none()
+        || push_dec(&mut detail, op).is_none()
+        || push(&mut detail, b" errno=").is_none()
+        || push_dec(&mut detail, errno.unsigned_abs() as u64).is_none()
+    {
+        return 0;
+    }
+    let _op = STORE_OP.lock();
+    if capture_event(EVENT_ERROR, kind, detail.as_slice()) {
+        1
+    } else {
+        0
+    }
+}
+
 /// The kernel's own account of the boot it is having, as one line:
-/// `boot=<seconds since the epoch> device=<the store it resolved> kernel=<its version banner>`.
+/// `boot=<seconds since the epoch> uptime=<seconds since boot> clock=<raw|set> device=<the store it
+/// resolved> kernel=<its version banner>`.
 ///
 /// The banner is last and unquoted because it contains spaces, so everything after the final `=`
 /// is the kernel's own words rather than a field this code parsed. The time is raw epoch seconds:
 /// rendering a calendar date in the kernel is date arithmetic, and a caller with a clock can do it
 /// without a kernel bug being the reason a timestamp is wrong.
-fn boot_record_value() -> Option<KVVec<u8>> {
+///
+/// `uptime` is carried beside it because the epoch alone does not say whether it was read at the
+/// start of the boot or an hour into it, and because `boot - uptime` is the instant the boot began
+/// irrespective of what the wall clock was doing. `clock` says which of the two readings this is —
+/// see [`ClockState`]. A reader that wants a time it can trust takes `clock=set`; a reader that
+/// wants the boot instant takes `boot - uptime`; a reader that wants both takes them from the same
+/// line and does not have to guess which one it got.
+fn boot_record_value(clock: ClockState, wall: i64, mono: i64) -> Option<KVVec<u8>> {
     let mut out = KVVec::<u8>::new();
-    let mut ts = bindings::timespec64 {
-        tv_sec: 0,
-        tv_nsec: 0,
-    };
-    // SAFETY: `ts` is a live, writable `timespec64`, and the call retains no pointer to it.
-    unsafe { bindings::ktime_get_real_ts64(&mut ts) };
 
     push(&mut out, b"boot=")?;
-    push_dec(&mut out, ts.tv_sec as u64)?;
+    push_dec(&mut out, wall.max(0) as u64)?;
+    push(&mut out, b" uptime=")?;
+    push_dec(&mut out, mono.max(0) as u64)?;
+    push(&mut out, b" clock=")?;
+    push(&mut out, clock.word())?;
     push(&mut out, b" device=")?;
 
     // SAFETY: `store_device` returns a static NUL-terminated buffer, valid for the life of the
@@ -2042,14 +2333,20 @@ fn ensure_boot_record() {
     // cannot both write a record: the loser returns, exactly as the swap did. The winner's attempt
     // is counted whether it succeeds or fails, which is what makes a failure retryable.
     let made = BOOT_RECORD_ATTEMPTS.load(Ordering::Acquire);
-    if made >= BOOT_RECORD_MAX_ATTEMPTS
-        || BOOT_RECORD_ATTEMPTS
-            .compare_exchange(made, made + 1, Ordering::AcqRel, Ordering::Acquire)
-            .is_err()
+    if made >= BOOT_RECORD_MAX_ATTEMPTS {
+        // The record is in the store. The last thing that can make it untrue is the clock it was
+        // read from, so that is the only thing still checked from here on.
+        refresh_boot_record_if_the_clock_was_set();
+        return;
+    }
+    if BOOT_RECORD_ATTEMPTS
+        .compare_exchange(made, made + 1, Ordering::AcqRel, Ordering::Acquire)
+        .is_err()
     {
         return;
     }
-    let value = match boot_record_value() {
+    let (wall, mono) = now_clocks();
+    let value = match boot_record_value(ClockState::Raw, wall, mono) {
         Some(v) => v,
         None => {
             pr_warn!("cubelinux: not enough memory to build this boot's record\n");
@@ -2066,9 +2363,26 @@ fn ensure_boot_record() {
     // log's head, not the image.
     match append_mutation(&mutation, 1) {
         Ok(_) => {
-            // Recorded. Park the count at the ceiling so no later write asks again.
+            // Recorded, with the two clock readings that go with it, so a later write can tell
+            // whether the epoch it just wrote is still the truth. Park the count at the ceiling so
+            // no later write asks again.
+            BOOT_RECORD_WALL.store(wall, Ordering::Release);
+            BOOT_RECORD_MONO.store(mono, Ordering::Release);
             BOOT_RECORD_ATTEMPTS.store(BOOT_RECORD_MAX_ATTEMPTS, Ordering::Release);
             pr_info!("cubelinux: recorded this boot in the store\n");
+            // A boot whose record took more than one attempt is a boot where the switch was on and
+            // the effect was off for a while — the defect this retry exists for. The record that
+            // lands is the same record either way, so without this event the difference between
+            // "recorded first time" and "recorded on the third try" would be invisible in the store.
+            if made > 0
+                && !capture_event(
+                    EVENT_ERROR | EVENT_BOOT,
+                    b"boot-record-late",
+                    b"the record took more than one attempt",
+                )
+            {
+                pr_warn!("cubelinux: could not capture the late-boot-record event (budget or store)\n");
+            }
         }
         Err(_) => pr_warn!("cubelinux: the store would not take this boot's record\n"),
     }