diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index d991ec734..b14a670b0 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -1341,6 +1341,13 @@ fn encode_entry( /// control write's `fsync` flushes what precedes it in the same file. So the entry's own fsync is a /// second flush of data the next one covers. /// +/// **Only where that next write exists.** A store with no control block has no count to write, so +/// nothing after the entry is flushed and this call would be the last word on a mutation that is +/// only in the page cache — an acknowledgement of a write the machine may never get back. `append` +/// therefore calls this only when `layout.control` is `Some`, and takes the entry's own `fsync` +/// otherwise; the two cases are one boolean apart and both are gated +/// (`kernel/verify-kernel-append.sh`, over an unformatted device and a formatted one). +/// /// What this gives up, stated rather than implied: the two writes are no longer independently /// durable, so a crash *inside* the control write's commit can in principle leave the control block /// counting bytes whose entry did not fully reach the media. That is a torn tail, and @@ -1691,6 +1698,18 @@ fn append( // valid file or an error pointer, which is checked. O_RDWR is 2. let file = store_file()?; + // Whether this append gets its durability from its own flush or from the control block's. + // + // The collapse to one `fsync` per append rests on a sentence: the entry goes down unsynced and + // the control write that counts it — written next, to the same file — carries the flush for + // both. That is true where there is a control block to write, and is measured to be true + // (`kernel/verify-kernel-append.sh` over a formatted device: the entry is on the device after a + // SIGKILL). It is *not* true of a store with no control block: the count write never happens, + // so nothing on this path is flushed at all and the caller is told a write it may never get + // back. That shape is not exotic — it is what a store the CLI builds looks like, and it is the + // shape that gate's own device has, where four acknowledged writes did not survive a kill. + // So the entry carries its own flush exactly when no later write in this operation will. + let synced_by_the_count = layout.control.is_some(); let mut result: Result<(), Error> = Ok(()); // A log region that has never been written is zeros, not a log: give it a header, the same // thing the userspace log does when its file does not exist. A header that does not already @@ -1705,14 +1724,16 @@ fn append( result = write_and_sync(file, layout.log_off as u64, &hdr); } if result.is_ok() { - // The entry, without its own flush: the control block below is written next and synced, and - // that sync flushes this. One durability boundary per append instead of two — measured, one - // `pwrite`+`fsync` on this filesystem is 4.3 ms, so this is about half of every write. - result = write_at( - file, - (region_at + WAL_HEADER_LEN + used) as u64, - entry.as_slice(), - ); + // One durability boundary per append where the control block can carry it — measured, one + // `pwrite`+`fsync` on this filesystem is 4.3 ms, so this is about half of every write. Where + // it cannot, this write is the boundary, and a write that is acknowledged without having + // been flushed is not a write. + let at = (region_at + WAL_HEADER_LEN + used) as u64; + result = if synced_by_the_count { + write_at(file, at, entry.as_slice()) + } else { + write_and_sync(file, at, entry.as_slice()) + }; } // The count second, so replay can never be told about an entry that is not there.