diff --git a/Makefile b/Makefile index b419d9892..e4def6095 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ NAME = CUBELinux # reject a version whose first character is not numeric, so a release spelled `CUBELinux.0.6` is # one the machine cannot load modules for. The box's own kernel works around this the same way # (`6.19.3-cube+`), so CUBELinux does too: the Linux base, then `-cubelinux`, then our version. -CUBELINUX_VERSION = 6.19.3-cubelinux0.6 +CUBELINUX_VERSION = 6.19.3-cubelinux0.7 # *DOCUMENTATION* # To see a list of typical targets execute "make help" diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 57e3ab7d5..8b18be6dd 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -41,7 +41,7 @@ //! ``` use core::fmt::{self, Write}; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicU32, Ordering}; // The store's format, in one file, shared with userspace. // @@ -166,6 +166,26 @@ fn store_device() -> *const u8 { /// still being a bound: pointing this at a whole 119 GiB disk is still refused. const MAX_BYTES: usize = 128 * 1024 * 1024; +/// The log size at which a write folds the log before appending to it. +/// +/// This is a ceiling on a READER's per-call cost, which is why it lives in the write path at all. +/// Every read reads the whole log window before it can answer anything — `Addressed::open` reads +/// `WAL_HEADER_LEN + log_used` bytes — so an unfolded log is not a bill paid once, at the fold. It is +/// a tax on every call, and it grows all day. +/// +/// The timer's policy said the opposite, and it was right for what it was written about: "the log +/// grows at roughly a megabyte a day against a 50 MB region, and a fold rewrites the whole image, so +/// folding more often would buy nothing and cost I/O". That is a data workload's arithmetic, where +/// the log is a recovery artefact and nobody reads it. Here the log is on the read path, and a 50 MB +/// log turns every coordinate read into a 50 MB read — a cliff that arrives at the end of the +/// region's life, not a slope. So the log is kept under this line instead. +/// +/// The trade, named rather than hidden: the WRITE that crosses the line pays a fold — once, bounded, +/// and rare — instead of every reader paying an ever-larger log. It is not free, and a fold is still +/// O(store) and still serialised against other operations by [`STORE_OP`]; making the fold itself +/// cheap is a separate piece of work. +const LOG_FOLD_BYTES: u64 = 256 * 1024; + /// The log's own magic, distinct from the image's so a reader that opens the wrong one /// cannot mistake it for the other. const WAL_MAGIC: [u8; 4] = *cube_format::WAL_MAGIC; @@ -267,7 +287,17 @@ const BOOT_POINT: (u64, u64, u64) = (0, 0, 0); /// Whether this boot has already been recorded. One attempt, claimed with a swap so two callers /// racing into their first write cannot write two records. -static BOOT_RECORDED: AtomicBool = AtomicBool::new(false); +/// How many attempts this boot has made at recording itself, and the ceiling on them. +/// +/// A count rather than a flag, because a flag that is claimed *before* the write cannot un-claim +/// itself, and the first attempt of a boot can fail for reasons that do not last. Measured on the +/// box: with `cube_boot_record=1` on the command line and a store image that the first client of the +/// boot could not open for writing, the kernel logged `could not read the store to record this boot` +/// and the store held no record at all — the switch was on and the effect was off, for that whole +/// boot, because the claim was made first and never retried. Bounded, so a store that is genuinely +/// broken logs a few warnings rather than one per write. +static BOOT_RECORD_ATTEMPTS: AtomicU32 = AtomicU32::new(0); +const BOOT_RECORD_MAX_ATTEMPTS: u32 = 4; /// FNV-1a offset basis. @@ -1291,16 +1321,164 @@ fn older_copy(generation: u64) -> usize { /// caller, *before* it reads the layout this writes against. Taking it here is too late: writers /// that each read `log_used = 0` before queueing on a lock inside this function still append at /// the same offset. Measured with the lock here: one 68-byte entry and twelve `ok`s. +/// The bytes an append reads against, and where they sit in the device. +/// +/// A store device's write path needs the control block and the log's head, and nothing else: the +/// image is tens of megabytes of records an append never looks at. It used to be handed the whole +/// device, so every put read all of it — 100 MB here — to append a few dozen bytes, and a write cost +/// **60–131 ms** on the box while a read cost 1–2 ms. Measured before this change, ten puts through +/// `cube(2)`: 71, 131, 66, 120, 60, 94, 117, 104, 116, 63 ms. +/// +/// The whole-image form stays, because two callers genuinely read it: a *bare* store, whose log runs +/// to the end of the file with nothing recording its length, and the v1→v4 migration, which folds +/// and therefore rewrites the image. +struct AppendSource<'a> { + /// The absolute offset `region` starts at — where the writes go. + at: usize, + /// The log region's bytes, from `at`. + region: &'a [u8], + /// The region's capacity. Comes from the control block when there is one; for a bare store the + /// region is everything to the end of the file, so its own length is the capacity. + capacity: usize, + /// The whole image, for the two callers that have it. + image: Option<&'a [u8]>, +} + +impl<'a> AppendSource<'a> { + /// The whole device or image, which is what a bare store's append reads and what the v1 + /// migration folds. + fn whole_image(image: &'a [u8], layout: &Layout) -> Self { + // The log starts after the bytes in use on a store device, after the image on a bare one. + let at = core::cmp::min(layout.log_off, image.len()); + let region = &image[at..]; + AppendSource { + at, + region, + capacity: region.len(), + image: Some(image), + } + } + + /// A store device's log head: the control block already gave the offsets and the capacity, so + /// these few bytes are all the append needs to see. + fn log_head(at: usize, region: &'a [u8], capacity: usize) -> Self { + AppendSource { + at, + region, + capacity, + image: None, + } + } +} + +/// Everything a device append consults, read without the image: the control block, and the log's +/// head. Both are small, and neither depends on how much the store holds. +struct WriteView { + layout: Layout, + /// The log region's first `WAL_HEADER_LEN` bytes — the header that decides how the entry is + /// framed. A region that has never been written reads as zeros, which is a log with no header + /// rather than a broken one. + head: KVVec, + capacity: usize, +} + +impl WriteView { + fn source(&self) -> AppendSource<'_> { + AppendSource::log_head(self.layout.log_off, self.head.as_slice(), self.capacity) + } +} + +/// Read what a write needs: the control block and the log's head, not the image. +/// +/// Returns `Ok(None)` for a store with no control block — a bare image — whose append must read the +/// whole thing because nothing there records where its log ends. +fn write_view() -> Result, Error> { + let file = store_file()?; + let mut scratch = KVVec::::with_capacity(4096, GFP_KERNEL)?; + scratch.resize(4096, 0, GFP_KERNEL)?; + + let mut head = KVVec::::new(); + read_exact_at(file, 0, CTL_COPY_B + CTL_COPY_LEN, &mut head, &mut scratch)?; + if head.as_slice().len() < 4 || &head.as_slice()[0..4] != CTL_MAGIC.as_slice() { + return Ok(None); + } + let control = read_control(head.as_slice()).ok_or(EINVAL)?; + let layout = Layout { + image_off: control.image_off() as usize, + log_off: control.log_off() as usize, + log_used: control.log_used as usize, + control: Some(control), + }; + + let mut log_head = KVVec::::new(); + read_exact_at(file, control.log_off(), WAL_HEADER_LEN, &mut log_head, &mut scratch)?; + Ok(Some(WriteView { + layout, + head: log_head, + capacity: control.log_capacity as usize, + })) +} + +/// Fold the log because it has grown past [`LOG_FOLD_BYTES`]. +/// +/// A fold rewrites the image, so unlike every other write this one reads the whole store. That is +/// the point of doing it here and rarely, rather than letting the cost sit on the read path. +fn fold_for_headroom(log_used: u64) -> Result<(), Error> { + let mut image = KVVec::::new(); + read_image(&mut image)?; + let layout = resolve_layout(image.as_slice()).map_err(|what| { + pr_err!("cubelinux: {}\n", what); + EINVAL + })?; + fold_now(image.as_slice(), &layout)?; + pr_info!( + "cubelinux: folded the log at {} bytes, because every read pays for it\n", + log_used + ); + Ok(()) +} + +/// Append one mutation, reading only what the store's shape requires. +/// +/// This is the entry point every write uses. It exists so that the bounded read and the whole-image +/// fallback are one decision in one place, rather than four callers each choosing. +fn append_mutation(m: &Mutation, op: u8) -> Result, Error> { + append_mutation_inner(m, op, true) +} + +/// `may_fold` is false on the way back from a fold, so a log that did not shrink cannot make this +/// recurse for ever: one fold per append, whether or not it helped. +fn append_mutation_inner(m: &Mutation, op: u8, may_fold: bool) -> Result, Error> { + match write_view()? { + Some(view) => { + if may_fold && view.layout.log_used as u64 > LOG_FOLD_BYTES { + let used = view.layout.log_used as u64; + fold_for_headroom(used)?; + // The fold emptied the log, so the write re-reads what it needs and appends to a + // fresh one — once only, so a fold that did not shrink anything cannot loop. + return append_mutation_inner(m, op, false); + } + let source = view.source(); + append(&source, &view.layout, m, op) + } + // A bare store: no control block, and a log whose end is the file's end. + None => { + let (image, layout) = device_and_layout()?; + append(&AppendSource::whole_image(image.as_slice(), &layout), &layout, m, op) + } + } +} + fn append( - image: &[u8], + source: &AppendSource<'_>, layout: &Layout, m: &Mutation, op: u8, ) -> Result, Error> { // Where it goes: after the bytes in use on a store device, after the valid prefix on a // bare image, where nothing records the length. - let region_at = core::cmp::min(layout.log_off, image.len()); - let region = &image[region_at..]; + let region_at = source.at; + let region = source.region; let (used, capacity) = match layout.control { // `log_used` counts entry bytes *after* the log's header, so the header is not // something an append has to account for in the count. @@ -1316,7 +1494,7 @@ fn append( return Err(EINVAL); } }; - (used, region.len()) + (used, source.capacity) } }; @@ -1344,9 +1522,29 @@ fn append( Some(WAL_VERSION) if used > 0 => { if layout.control.is_some() { // [`STORE_OP`] is held by the op that reached here, so these are plain calls. - fold_now(image, layout)?; + // + // The v1→v4 migration is the one device append that needs the image, because a fold + // rewrites it. Pay for the whole read here — once per store, at the migration — and + // keep it off the path every other write takes. + let mut read_here = KVVec::::new(); + match source.image { + Some(image) => fold_now(image, layout)?, + None => { + read_image(&mut read_here)?; + let layout = resolve_layout(read_here.as_slice()).map_err(|what| { + pr_err!("cubelinux: {}\n", what); + EINVAL + })?; + fold_now(read_here.as_slice(), &layout)?; + } + } let (device, layout) = device_and_layout()?; - return append(&device, &layout, m, op); + return append( + &AppendSource::whole_image(device.as_slice(), &layout), + &layout, + m, + op, + ); } WAL_VERSION } @@ -1692,25 +1890,36 @@ fn boot_record_value() -> Option> { Some(out) } -/// Record this boot, once, at the first write the kernel is asked to make. +/// Record this boot, at a write, and try again if the store would not take it. /// /// Called from every entry point that writes. It is not called from the read paths, so a kernel that /// only reads leaves no record — see the note above `BOOT_SPACE`. A failure here is logged and never /// propagated: a record of the boot is worth having, and is not a precondition for the caller's -/// write. It is attempted once per boot rather than retried on every write, because a store that -/// will not take it will not take it later either, and one warning is information where a stream of -/// them is noise. +/// write. /// -/// The first write pays one extra read of the store's image (this hook resolves the device and -/// layout for itself, and then the caller resolves it again for its own mutation). That is once per -/// boot, and it buys keeping the hook out of the callers' paths entirely. +/// It retries, up to [`BOOT_RECORD_MAX_ATTEMPTS`], because "a store that will not take it will not +/// take it later either" is not true of the failure it actually had: the first write of a boot can +/// arrive before the store is reachable by the credentials of the process that made it. That is not +/// a property of the store, and treating it as one cost a boot its record silently. +/// +/// What it costs: the control block and the log's head, through the same bounded read every other +/// write uses. It used to resolve the whole device — tens of megabytes — for the record and then +/// the caller resolved it again for its own mutation, which is why a retry was not affordable then +/// and is now. fn ensure_boot_record() { // SAFETY: a flag the C side set from the kernel command line during boot. if !unsafe { cubelinux_boot_record_enabled() } { return; } - // Claimed with a swap: two callers racing into their first write must not write two records. - if BOOT_RECORDED.swap(true, Ordering::AcqRel) { + // Claim one attempt with a compare-exchange, so two callers racing into their first write + // cannot both write a record: the loser returns, exactly as the swap did. The winner's attempt + // is counted whether it succeeds or fails, which is what makes a failure retryable. + let made = BOOT_RECORD_ATTEMPTS.load(Ordering::Acquire); + if made >= BOOT_RECORD_MAX_ATTEMPTS + || BOOT_RECORD_ATTEMPTS + .compare_exchange(made, made + 1, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { return; } let value = match boot_record_value() { @@ -1720,21 +1929,20 @@ fn ensure_boot_record() { return; } }; - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, - Err(_) => { - pr_warn!("cubelinux: could not read the store to record this boot\n"); - return; - } - }; let mutation = Mutation { space: BOOT_SPACE, key: morton_encode(BOOT_POINT.0, BOOT_POINT.1, BOOT_POINT.2), flags: BOOT_FLAGS, value, }; - match append(&device, &layout, &mutation, 1) { - Ok(_) => pr_info!("cubelinux: recorded this boot in the store\n"), + // The record goes down the same bounded path every other write takes: the control block and the + // log's head, not the image. + match append_mutation(&mutation, 1) { + Ok(_) => { + // Recorded. Park the count at the ceiling so no later write asks again. + BOOT_RECORD_ATTEMPTS.store(BOOT_RECORD_MAX_ATTEMPTS, Ordering::Release); + pr_info!("cubelinux: recorded this boot in the store\n"); + } Err(_) => pr_warn!("cubelinux: the store would not take this boot's record\n"), } } @@ -1780,11 +1988,7 @@ pub unsafe extern "C" fn cubelinux_kernel_put( flags, value: bytes, }; - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, - Err(e) => return -(e.to_errno() as i32), - }; - match append(&device, &layout, &mutation, 1) { + match append_mutation(&mutation, 1) { Ok(_) => 0, Err(e) => -(e.to_errno() as i32), } @@ -2722,6 +2926,9 @@ impl Addressed { /// The table is sorted by space and fixed-size, so this is a binary search over arithmetic /// addresses — the same reason the index works. fn space_entry(&mut self, space: &[u8; SPACE_ID_LEN]) -> Result> { + // One buffer for the whole search, not one per probe: a binary search over a thousand + // spaces allocates a thousand times otherwise, and an allocator stall is a latency spike + // with no bound on it — the opposite of what a control loop is allowed to see. let mut buf = KVVec::::new(); let mut lo = 0u64; let mut hi = self.header.space_count; @@ -2805,6 +3012,7 @@ impl Addressed { first: u64, records: u64, ) -> Result> { + // One buffer for the whole search — see the note in `space_entry`. let mut buf = KVVec::::new(); let mut lo = 0u64; let mut hi = records; @@ -2838,6 +3046,7 @@ impl Addressed { first: u64, records: u64, ) -> Result { + // One buffer for the whole search — see the note in `space_entry`. let mut buf = KVVec::::new(); let mut lo = 0u64; let mut hi = records; @@ -4201,13 +4410,10 @@ pub unsafe extern "C" fn cubelinux_kernel_del(space: *const u8, x: u64, y: u64, flags: 0, value: KVVec::new(), }; - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, - Err(e) => return -(e.to_errno() as i32), - }; // A delete is an entry with op=2; `append` writes op=1, so build it here from the same - // framing, and let the log's own reader be the judge of it. - match append(&device, &layout, &mutation, 2) { + // framing, and let the log's own reader be the judge of it. Like a put, it appends through the + // bounded read: a delete does not touch the records either. + match append_mutation(&mutation, 2) { Ok(_) => 0, Err(e) => -(e.to_errno() as i32), } @@ -4313,17 +4519,8 @@ impl MiscDevice for CubeStore { ensure_boot_record(); let me = kiocb.file(); - // Both operations need the device: its layout says where the image and log are. - let mut device = KVVec::::new(); - read_image(&mut device)?; - let layout = match resolve_layout(&device) { - Ok(l) => l, - Err(what) => { - dev_err!(me.dev, "cubelinux: cannot resolve the store layout: {}\n", what); - return Err(EINVAL); - } - }; - + // A put needs the control block and the log's head; a fold needs everything. So the whole + // device is read inside the branch that folds, and not before the branch that does not. match bytes[0] { OP_PUT => { if bytes.len() < 1 + 32 + 24 + 4 { @@ -4351,7 +4548,7 @@ impl MiscDevice for CubeStore { flags: 0, value, }; - match append(&device, &layout, &mutation, 1) { + match append_mutation(&mutation, 1) { Ok(_) => { dev_info!( me.dev, @@ -4368,6 +4565,18 @@ impl MiscDevice for CubeStore { } } OP_SYNC => { + // The one write that reads the whole store: a fold rewrites the image, so it has to + // hold it. This is the tail of the latency budget, and it is why a fold belongs on a + // timer rather than on the path a caller waits behind. + let mut device = KVVec::::new(); + read_image(&mut device)?; + let layout = match resolve_layout(&device) { + Ok(l) => l, + Err(what) => { + dev_err!(me.dev, "cubelinux: cannot resolve the store layout: {}\n", what); + return Err(EINVAL); + } + }; let ctl = match layout.control { Some(c) => c, None => {