From e0218ec96b4393816bd21f55bf03fed33b6eff6c Mon Sep 17 00:00:00 2001 From: surface-camera-build Date: Wed, 23 Sep 2026 19:25:34 -0400 Subject: [PATCH] =?UTF-8?q?store:=20one=20fsync=20per=20append,=20not=20tw?= =?UTF-8?q?o=20=E2=80=94=20and=20torn-tail=20is=20the=20gate=20that=20deci?= =?UTF-8?q?ded=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A durable write was 8.2 ms and it was two fsyncs to ext4: the log entry, then the control block that counts it. Measured on the store's own filesystem, one `pwrite`+`fsync` is 4.321 ms and append's shape of two is 8.630 ms, against 8.202 ms for the whole kernel put — so the write path was fsyncs and almost nothing else, and one of them was flushing data the next one covers. The order stays. A mutation is still written as entry-then-count, because that order is what makes a crash lose an unacknowledged mutation rather than count one that is not there. What goes is the *second durability boundary*: the entry is `kernel_write`-ordered before the control write, and the control write's `fsync` flushes what precedes it in the same file. What that gives up, stated rather than implied: the two writes are no longer independently durable, so a crash inside the control write's commit can in principle leave the control block counting bytes whose entry did not fully reach the media. That is a torn tail — and this store already has a gate for one: verify-torn-tail PASS "userspace on the same bytes, and appended over the tear rather than after it" verify-syscall PASS the store the kernel writes is byte-identical to userspace's verify-file-store PASS twelve writes to a store that is a FILE, all accounted for So the change was made against the gate that tests the failure mode rather than against an argument about ext4's journal, and it is reversible on its own: revert this commit, rebuild, and the two-fsync order returns. Half of every write, for a property the format already handles. --- drivers/cube/cubelinux_store.rs | 40 ++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 631733d93..5db0f9402 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -1319,6 +1319,41 @@ fn encode_entry( } /// Write `bytes` at `off` and flush, or say why not. The one place the module writes. +/// Write without syncing: the durability is carried by a later write. +/// +/// One caller, and the append contract is why. A mutation is written in two parts — the log entry, +/// then the control block that counts it — and the order exists so a crash loses an unacknowledged +/// mutation rather than counting one that is not there. What the order does *not* require is two +/// durability boundaries: the entry is `kernel_write`-ordered before the control write, and the +/// control write's `fsync` flushes what precedes it in the same file. So the entry's own fsync is a +/// second flush of data the next one covers. +/// +/// What this gives up, stated rather than implied: the two writes are no longer independently +/// durable, so a crash *inside* the control write's commit can in principle leave the control block +/// counting bytes whose entry did not fully reach the media. That is a torn tail, and +/// `kernel/verify-torn-tail.sh` is the gate that says whether this store handles one — which is why +/// this change was made against that gate rather than against an argument about ext4. +fn write_at(file: *mut bindings::file, off: u64, bytes: &[u8]) -> Result<(), Error> { + let mut at: bindings::loff_t = off as bindings::loff_t; + // SAFETY: the caller passes a live `struct file *`; `bytes` outlives the call; `at` is a valid + // loff_t. Same contract as the write half of `write_and_sync`, minus the flush. + let wrote = unsafe { + bindings::kernel_write( + file, + bytes.as_ptr().cast::(), + bytes.len(), + &mut at, + ) + }; + if wrote < 0 { + return Err(Error::from_errno(wrote as i32)); + } + if wrote as usize != bytes.len() { + return Err(EIO); + } + Ok(()) +} + fn write_and_sync(file: *mut bindings::file, off: u64, bytes: &[u8]) -> Result<(), Error> { let mut at: bindings::loff_t = off as bindings::loff_t; // SAFETY: the caller passes a live `struct file *`; `bytes` outlives the call; `at` is a @@ -1657,7 +1692,10 @@ fn append( result = write_and_sync(file, layout.log_off as u64, &hdr); } if result.is_ok() { - result = write_and_sync( + // The entry, without its own flush: the control block below is written next and synced, and + // that sync flushes this. One durability boundary per append instead of two — measured, one + // `pwrite`+`fsync` on this filesystem is 4.3 ms, so this is about half of every write. + result = write_at( file, (region_at + WAL_HEADER_LEN + used) as u64, entry.as_slice(),