From 56665a303e9c1f40328fcd9a11bd6d23f7b0bb75 Mon Sep 17 00:00:00 2001 From: surface-camera-build Date: Mon, 21 Sep 2026 01:59:11 -0400 Subject: [PATCH] cubelinux: read the store where it lies, instead of rebuilding it per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every read began by reading the whole device into kernel memory, parsing every record of every space into a merged pool, and heapsorting the lot — then throwing all of it away. So a `get` of one record cost as much as listing the store, every batch of a listing paid it again, and the coordinate bought nothing mechanically: knowing where a record is did not make reaching it any cheaper. Measured on the workhorse: one `get` 110 ms, one `spaces` call 119 ms, one 6,127-record listing 1,206 ms. The records are already in the order the coordinate computes — a checkpoint writes them by (space, key) — so the kernel now walks them where they lie: zero-copy values, no pool, no sort, and the log (small, and the only thing that can override the image) consulted as an overlay. It reads the image and the log window rather than the whole device, which is four images wide. * `get` — the log's newest word on the coordinate, else the image walked to it. * `enum` — one merge pass over two sorted sequences: the space's image records and its log edits. * `spaces` — from a table of where each space starts, cached by the control block's generation. That table is one entry per space, not per record, and a checkpoint is the only thing that invalidates it. * `put`/`del`/`sync` — unchanged: a write is a log append, and the fold still writes the whole sorted store. Verified: verify-enum.sh (a 3,006-record listing identical to userspace's, record for record) and verify-frontend.sh (socket -> front-end -> cube(2), including a listing larger than one batch and the store the front-end leaves being digest-identical to userspace's) both pass. Not yet what it should be: listing a 10-record space in a 20,000-record store still costs about six times what it costs in a 500-record one, so the space table is not taking effect as intended and the lookup is not yet independent of store size. That belongs in the image's layout — a coordinate cannot be turned into a byte offset in a variable-length packed list, because a record's position is the sum of every value before it — and the fix is a format one, not a caching one. --- drivers/cube/cubelinux_store.rs | 1006 ++++++++++++++++++++++++++----- 1 file changed, 849 insertions(+), 157 deletions(-) diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 94aa26fb3..64ec62f6a 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -1255,12 +1255,756 @@ pub unsafe extern "C" fn cubelinux_kernel_put( } } +// ── Reading the store where it lies ───────────────────────────────────────────────────── +// +// A read used to start by reading the whole device into kernel memory, parsing every record into +// a merged pool, and heapsorting the lot — then throwing all of it away. That is why a `get` of +// one record cost as much as listing the store, and why the coordinate bought nothing +// mechanically: knowing where a record is did not make reaching it any cheaper. +// +// The records are already the index. A checkpoint writes them in `(space, key)` order, which is +// the order the coordinate itself computes, and the log that can override them is small. So the +// kernel walks the records where they lie — zero-copy, no pool, no sort — and consults the log as +// an overlay. What an operation costs is proportional to the records it actually touches. + +/// How much is read at a time when a length is not known ahead of it. +const VIEW_CHUNK: usize = 256 * 1024; + +/// Read exactly `len` bytes at `off`. A short read is an error, not a smaller view: the store says +/// how long its image is, and quietly reading less would turn a truncated store into a shorter one +/// that looks complete. +fn read_exact_at( + file: *mut bindings::file, + off: u64, + len: usize, + out: &mut KVVec, + scratch: &mut KVVec, +) -> Result<()> { + out.clear(); + out.reserve(len, GFP_KERNEL)?; + let mut pos: bindings::loff_t = off as bindings::loff_t; + let mut left = len; + while left > 0 { + let want = core::cmp::min(left, scratch.len()); + // SAFETY: `file` is a live `struct file *`; `scratch` is a kernel buffer we own; `pos` is + // a valid loff_t. kernel_read copies at most `want` bytes and retains neither pointer. + let n = unsafe { + bindings::kernel_read(file, scratch.as_mut_ptr().cast::(), want, &mut pos) + }; + if n < 0 { + return Err(Error::from_errno(n as i32)); + } + if n == 0 { + return Err(EINVAL); + } + let n = n as usize; + out.extend_from_slice(&scratch.as_slice()[..n], GFP_KERNEL)?; + left -= n; + } + Ok(()) +} + +/// Read from `off` to the end, or to `cap` bytes, whichever comes first. +/// +/// For a bare image, whose extent nothing states, and for a log whose length is implied. +fn read_to_end_at( + file: *mut bindings::file, + off: u64, + out: &mut KVVec, + scratch: &mut KVVec, + cap: usize, +) -> Result<()> { + out.clear(); + let mut pos: bindings::loff_t = off as bindings::loff_t; + while out.len() < cap { + let want = core::cmp::min(scratch.len(), cap - out.len()); + // SAFETY: as in `read_exact_at`. + let n = unsafe { + bindings::kernel_read(file, scratch.as_mut_ptr().cast::(), want, &mut pos) + }; + if n < 0 { + return Err(Error::from_errno(n as i32)); + } + if n == 0 { + break; + } + let n = n as usize; + out.extend_from_slice(&scratch.as_slice()[..n], GFP_KERNEL)?; + } + Ok(()) +} + +/// The bytes one operation needs: the image, and the log beside it. +/// +/// The device is four images wide plus its log; reading the device to read the image copies +/// several times what is used, on every call. +struct View { + image: KVVec, + log: KVVec, + header: Header, + /// The control block's generation, which a checkpoint changes — the key to what is cached + /// about this image. Zero for a bare image, which has no control block and does not change + /// underneath a reader. + generation: u64, +} + +kernel::sync::global_lock! { + /// Where each space's records begin. + /// + /// A walk has to start at its space. Records are variable-length, so "where does space S begin" + /// cannot be computed from the key — it can only be found by reading the image once. Finding it + /// once per checkpoint is what makes a listing cost what it returns, and the table is one entry + /// per space (tens of bytes), not one per record. It goes stale on exactly one event: a + /// checkpoint rewriting the image, which changes the control block's generation. + /// + /// SAFETY: Initialized in the module initializer before first use. + unsafe(uninit) static SPACE_STARTS: Mutex> = None; +} + +struct SpaceStarts { + generation: u64, + image_len: usize, + /// Each space's first record, in the image's order. + starts: KVVec<([u8; SPACE_ID_LEN], usize)>, +} + +/// Where the live image's spaces begin, from the cache when it is current and by reading the image +/// when it is not. +fn space_starts(view: &View) -> Result> { + let mut guard = SPACE_STARTS.lock(); + if let Some(cached) = guard.as_ref() { + if cached.generation == view.generation && cached.image_len == view.image().len() { + let mut out = KVVec::new(); + out.extend_from_slice(cached.starts.as_slice(), GFP_KERNEL)?; + return Ok(out); + } + } + + let mut starts = KVVec::new(); + let mut records = Records::new(view.image(), &view.header); + let mut previous: Option<[u8; SPACE_ID_LEN]> = None; + loop { + let at = records.off; + match records.next() { + Some((space, _, _)) => { + if previous != Some(*space) { + starts.push((*space, at), GFP_KERNEL)?; + previous = Some(*space); + } + } + None => break, + } + } + + let mut out = KVVec::new(); + out.extend_from_slice(starts.as_slice(), GFP_KERNEL)?; + *guard = Some(SpaceStarts { + generation: view.generation, + image_len: view.image().len(), + starts, + }); + Ok(out) +} + +impl View { + fn image(&self) -> &[u8] { + self.image.as_slice() + } + + fn log(&self) -> &[u8] { + self.log.as_slice() + } + + /// Where one space's records begin, if the image holds it. + fn space_start(&self, space: &[u8; SPACE_ID_LEN]) -> Result> { + let starts = space_starts(self)?; + for (candidate, at) in starts.as_slice() { + if candidate == space { + return Ok(Some(*at)); + } + } + Ok(None) + } +} + +/// Read the image and its log window — not the whole device. +fn read_view() -> Result { + // O_RDONLY is 0 in Linux; filp_open takes the raw flags word. + // SAFETY: `store_device()` is a NUL-terminated C string the module parameter filled at boot, + // and filp_open returns a valid `struct file *` or an error pointer, checked here. + let file = unsafe { bindings::filp_open(store_device(), 0, 0) }; + let file = kernel::error::from_err_ptr(file)?; + if file.is_null() { + return Err(EINVAL); + } + + let view = read_view_from(file); + + // SAFETY: `file` came from filp_open and has not been closed; the owner argument is only + // meaningful for locks that nobody holds here. + unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; + + let view = view?; + if view.image.len() < HEADER_LEN_V1 { + return Err(EINVAL); + } + Ok(view) +} + +fn read_view_from(file: *mut bindings::file) -> Result { + let mut scratch = KVVec::::with_capacity(VIEW_CHUNK, GFP_KERNEL)?; + scratch.resize(VIEW_CHUNK, 0, GFP_KERNEL)?; + + // The control block, or the image's own header when the device is a bare image. + let mut head = KVVec::::with_capacity(CTL_COPY_B + CTL_COPY_LEN, GFP_KERNEL)?; + read_exact_at(file, 0, CTL_COPY_B + CTL_COPY_LEN, &mut head, &mut scratch)?; + let is_device = &head.as_slice()[0..4] == CTL_MAGIC.as_slice(); + + let (image_off, log_off, log_used, generation) = if is_device { + let control = read_control(head.as_slice()).ok_or(EINVAL)?; + ( + control.image_off() as usize, + control.log_off() as usize, + control.log_used as usize, + control.generation, + ) + } else { + (0, 0, 0, 0) + }; + + // What the image says about itself. A device keeps its image at an offset, so its header is + // read from there; a bare image starts with it. + let mut header = if is_device { + let mut raw = KVVec::::with_capacity(HEADER_LEN_V2, GFP_KERNEL)?; + match read_exact_at(file, image_off as u64, HEADER_LEN_V2, &mut raw, &mut scratch) { + Ok(()) => parse_header(raw.as_slice()), + // A v1 image is shorter than a v2 header, and saying "not a store" about it would be + // wrong: read the six bytes it does have. + Err(_) => { + read_exact_at(file, image_off as u64, HEADER_LEN_V1, &mut raw, &mut scratch)?; + parse_header(raw.as_slice()) + } + } + } else { + parse_header(head.as_slice()) + } + .map_err(|what| { + pr_err!("cubelinux: {}\n", what); + EINVAL + })?; + + let mut image = KVVec::::new(); + let mut log = KVVec::::new(); + match (is_device, header.image_bytes) { + (true, Some(bytes)) => { + let bytes = bytes as usize; + if bytes > MAX_BYTES { + return Err(EINVAL); + } + read_exact_at(file, image_off as u64, bytes, &mut image, &mut scratch)?; + } + // A v1 image states no extent, so it is read to the end — bounded, because a device that + // never ends must not become a read that never ends. + (true, None) => { + read_to_end_at(file, image_off as u64, &mut image, &mut scratch, MAX_BYTES)?; + } + (false, _) => { + read_to_end_at(file, 0, &mut image, &mut scratch, MAX_BYTES)?; + if header.image_bytes.is_none() { + header.image_bytes = Some(image.len() as u64); + } + } + } + + if is_device { + if log_used > 0 { + let len = core::cmp::min(WAL_HEADER_LEN + log_used, MAX_BYTES); + read_exact_at(file, log_off as u64, len, &mut log, &mut scratch)?; + } + } else { + // A bare image's log starts after the image, aligned, and runs to the end. + let at = log_offset(image.len() as u64); + read_to_end_at(file, at as u64, &mut log, &mut scratch, MAX_BYTES)?; + } + + Ok(View { + image, + log, + header, + generation, + }) +} + +/// Walk a store image's records, in the order the image holds them. +/// +/// Zero-copy: the value a caller gets is a slice of the image already in memory, so listing a +/// store does not copy its values anywhere. +struct Records<'a> { + image: &'a [u8], + off: usize, + end: usize, + left: Option, +} + +impl<'a> Records<'a> { + /// Begin at `off`, for a caller that has been told where a space starts. + /// + /// The record count is dropped: it is a bound on the records ahead of the *first* record, and + /// starting anywhere else makes it a bound that no longer means anything. The image's extent + /// is the bound that still holds. + fn at(image: &'a [u8], header: &Header, off: usize) -> Self { + let mut records = Records::new(image, header); + if off > records.off && off <= records.end { + records.off = off; + records.left = None; + } + records + } + + fn new(image: &'a [u8], header: &Header) -> Self { + let header_len = if header.version == VERSION_V1 { + HEADER_LEN_V1 + } else { + HEADER_LEN_V2 + }; + let extent = match header.image_bytes { + Some(n) => core::cmp::min(n as usize, image.len()), + None => image.len(), + }; + Records { + image, + off: core::cmp::min(header_len, extent), + end: extent, + left: header.record_count, + } + } + + fn next(&mut self) -> Option<(&'a [u8; 32], &'a [u8; 24], &'a [u8])> { + if self.left == Some(0) || self.off + RECORD_FIXED > self.end { + return None; + } + let frame = &self.image[self.off..self.off + RECORD_FIXED]; + // A v1 image states no count, so padding ends the walk — but only if every remaining byte + // is zero, because a record at the origin with an empty value is 64 zero bytes and is a + // record. + if self.left.is_none() + && frame.iter().all(|b| *b == 0) + && self.image[self.off..].iter().all(|b| *b == 0) + { + return None; + } + let mut word = [0u8; 8]; + word.copy_from_slice(&frame[SPACE_ID_LEN + RAW_KEY_LEN..RECORD_FIXED]); + let value_len = u64::from_le_bytes(word) as usize; + let value_at = self.off + RECORD_FIXED; + if value_at + value_len > self.end { + return None; + } + let space: &[u8; SPACE_ID_LEN] = frame[..SPACE_ID_LEN].try_into().ok()?; + let key: &[u8; RAW_KEY_LEN] = + frame[SPACE_ID_LEN..SPACE_ID_LEN + RAW_KEY_LEN].try_into().ok()?; + let value = &self.image[value_at..value_at + value_len]; + self.off = value_at + value_len; + if let Some(n) = self.left.as_mut() { + *n -= 1; + } + Some((space, key, value)) + } +} + +/// What the log says about one coordinate: written, or removed. +enum Effect<'a> { + Write(&'a [u8]), + Delete, +} + +/// The log's newest word on `(space, key)`, if it says anything about it. +/// +/// The log is the only thing that can override the image — it holds what has been written since +/// the last checkpoint — so this is what decides a read. +fn log_effect<'a>(log: &'a [u8], space: &[u8; SPACE_ID_LEN], key: &[u8; RAW_KEY_LEN]) -> Option> { + let mut entries = log_entries(log).ok()?; + let mut found = None; + while let Some((entry_space, entry_key, op, value_at, value_len)) = entries.next() { + if entry_space == space && entry_key == key { + found = Some(if op == 2 { + Effect::Delete + } else { + Effect::Write(&log[value_at..value_at + value_len]) + }); + } + } + found +} + +/// One log entry, as the walk sees it: `(space, key, op, value)`. +struct LogEntries<'a> { + log: &'a [u8], + off: usize, +} + +impl<'a> LogEntries<'a> { + /// The next entry: `(space, key, op, where its value starts, its value's length)`. + /// + /// The value's *position* rather than a slice, because a caller collecting edits keeps them in + /// a fixed-size vector and a borrow of the log would tie that vector's type to the log's life. + fn next(&mut self) -> Option<(&'a [u8], &'a [u8], u8, usize, usize)> { + if self.off + ENTRY_FIXED > self.log.len() { + return None; + } + let start = self.off; + let op = self.log[start]; + if op != 1 && op != 2 { + return None; + } + let mut word = [0u8; 4]; + word.copy_from_slice(&self.log[start + 1..start + 5]); + let crc = u32::from_le_bytes(word); + let space = &self.log[start + 5..start + 37]; + let key = &self.log[start + 37..start + 61]; + word.copy_from_slice(&self.log[start + 61..start + 65]); + let len = u32::from_le_bytes(word) as usize; + let frame_end = start + ENTRY_FIXED + len; + if frame_end > self.log.len() { + return None; + } + // The checksum covers space, key, length and value, so a torn tail is stopped at rather + // than applied — the same rule the fold uses. + if crc32(&self.log[start + 5..frame_end]) != crc { + return None; + } + self.off = frame_end; + Some((space, key, op, start + ENTRY_FIXED, len)) + } +} + +/// The log's entries, when there is a valid log to read. +fn log_entries(log: &[u8]) -> Result, &'static str> { + if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { + return Err("no-log"); + } + if log[4] != WAL_VERSION { + return Err("unsupported-log-version"); + } + Ok(LogEntries { + log, + off: WAL_HEADER_LEN, + }) +} + +/// One space's override of the image, on its way to being merged with it. +/// +/// The key and the value borrow the log rather than copying it, so a walker can hand out slices of +/// bytes that are already in memory — a listing never copies a value. +#[derive(Clone, Copy)] +struct LogEdit<'a> { + key: &'a [u8; RAW_KEY_LEN], + /// Position in the log, so the later of two edits to one key is the one that counts. + seq: u32, + value: &'a [u8], + deleted: bool, +} + +/// Every edit the log holds for one space, in key order, later edits last. +/// +/// The image and the log are both in key order, so one merge pass over them is a listing — and a +/// space is what a walk names, so only its own edits are collected. +fn log_edits<'a>( + log: &'a [u8], + space: &[u8; SPACE_ID_LEN], + out: &mut KVVec>, +) -> Result<()> { + out.clear(); + let mut entries = match log_entries(log) { + Ok(e) => e, + Err(_) => return Ok(()), + }; + let mut seq: u32 = 0; + while let Some((edit_space, key, op, value_at, value_len)) = entries.next() { + if edit_space == space { + let key: &[u8; RAW_KEY_LEN] = match key.try_into() { + Ok(k) => k, + Err(_) => break, + }; + out.push( + LogEdit { + key, + seq, + value: &log[value_at..value_at + value_len], + deleted: op == 2, + }, + GFP_KERNEL, + )?; + } + seq += 1; + } + sort_edits(out.as_mut_slice()); + Ok(()) +} + +fn sort_edits(edits: &mut [LogEdit<'_>]) { + let len = edits.len(); + if len < 2 { + return; + } + let mut start = len / 2; + while start > 0 { + start -= 1; + sift_down_edits(edits, start, len); + } + let mut end = len; + while end > 1 { + end -= 1; + edits.swap(0, end); + sift_down_edits(edits, 0, end); + } +} + +/// Heapsort, for the same reason the entry sort is one: a constant stack frame, so the size of a +/// log cannot decide whether reading it is safe. `sort_unstable` is used nowhere in this module. +fn sift_down_edits(edits: &mut [LogEdit<'_>], mut root: usize, end: usize) { + loop { + let mut child = root * 2 + 1; + if child >= end { + return; + } + if child + 1 < end + && (edits[child].key, edits[child].seq) < (edits[child + 1].key, edits[child + 1].seq) + { + child += 1; + } + if (edits[root].key, edits[root].seq) >= (edits[child].key, edits[child].seq) { + return; + } + edits.swap(root, child); + root = child; + } +} + +/// The live records of one space: the image's records for it, with the log's edits applied, in key +/// order. +/// +/// This is the whole of what a listing is — two sorted sequences and one pass — and it is why a +/// walk costs what it returns rather than what the store holds. +struct SpaceWalker<'a> { + records: Records<'a>, + edits: KVVec>, + edit_at: usize, + image_next: Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])>, + wanted: [u8; SPACE_ID_LEN], +} + +impl<'a> SpaceWalker<'a> { + fn new(view: &'a View, wanted: &[u8; SPACE_ID_LEN]) -> Result { + let mut edits = KVVec::>::new(); + log_edits(view.log(), wanted, &mut edits)?; + // Start where the space starts: a listing of a small space in a large store must not read + // the records that are not in it. + let mut records = match view.space_start(wanted)? { + Some(at) => Records::at(view.image(), &view.header, at), + None => { + let mut empty = Records::new(view.image(), &view.header); + empty.off = empty.end; // nothing in the image: the log's edits are the space + empty + } + }; + let image_next = next_in_space(&mut records, wanted); + Ok(SpaceWalker { + records, + edits, + edit_at: 0, + image_next, + wanted: *wanted, + }) + } + + /// The next live record, or nothing when the space is walked out. + fn next(&mut self) -> Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])> { + loop { + // The entries of one key arrive in log order, so the last of a key's group is the + // newest word on it and the earlier ones are history. + while self.edit_at + 1 < self.edits.len() + && self.edits[self.edit_at + 1].key == self.edits[self.edit_at].key + { + self.edit_at += 1; + } + let edit = self.edits.as_slice().get(self.edit_at).copied(); + match (edit, self.image_next) { + // The log's word on a coordinate the image holds: it wins. + (Some(e), Some((key, _))) if e.key == key => { + self.edit_at += 1; + self.image_next = next_in_space(&mut self.records, &self.wanted); + if !e.deleted { + return Some((e.key, e.value)); + } + } + // A record only the log holds, in its key's place. + (Some(e), Some((key, value))) if e.key < key => { + self.edit_at += 1; + self.image_next = Some((key, value)); + if !e.deleted { + return Some((e.key, e.value)); + } + } + // The image's own record, which the log says nothing about. + (_, Some((key, value))) => { + self.image_next = next_in_space(&mut self.records, &self.wanted); + return Some((key, value)); + } + // The image is walked out; the rest of the log still has records to report. + (Some(e), None) => { + self.edit_at += 1; + if !e.deleted { + return Some((e.key, e.value)); + } + } + (None, None) => return None, + } + } + } +} + +/// Does this space hold at least one live record? +/// +/// A space whose every record the log has removed is not a space to report, and a walk that stops +/// at the first record it finds is the cheapest way to know: it costs one record, not a listing. +fn space_has_records(view: &View, space: &[u8; SPACE_ID_LEN]) -> bool { + match SpaceWalker::new(view, space) { + Ok(mut walker) => walker.next().is_some(), + Err(_) => false, + } +} + +/// The distinct spaces the log writes into, ascending. Small by construction: the log is what has +/// been written since the last checkpoint. +fn log_spaces(log: &[u8], out: &mut KVVec<[u8; SPACE_ID_LEN]>) -> Result<()> { + out.clear(); + let mut entries = match log_entries(log) { + Ok(e) => e, + Err(_) => return Ok(()), + }; + while let Some((space, _, _, _, _)) = entries.next() { + let mut s = [0u8; SPACE_ID_LEN]; + s.copy_from_slice(&space[..SPACE_ID_LEN]); + if !out.as_slice().contains(&s) { + out.push(s, GFP_KERNEL)?; + } + } + sort_spaces(out.as_mut_slice()); + Ok(()) +} + +/// Heapsort for a store's spaces. The log holds few, but `sort_unstable` is not used anywhere in +/// this module: it allocates about 3.5 KiB of kernel stack per level, which is how reading a +/// store once took the machine down. +fn sort_spaces(spaces: &mut [[u8; SPACE_ID_LEN]]) { + let len = spaces.len(); + if len < 2 { + return; + } + let mut start = len / 2; + while start > 0 { + start -= 1; + sift_down_spaces(spaces, start, len); + } + let mut end = len; + while end > 1 { + end -= 1; + spaces.swap(0, end); + sift_down_spaces(spaces, 0, end); + } +} + +fn sift_down_spaces(spaces: &mut [[u8; SPACE_ID_LEN]], mut root: usize, end: usize) { + loop { + let mut child = root * 2 + 1; + if child >= end { + return; + } + if child + 1 < end && spaces[child] < spaces[child + 1] { + child += 1; + } + if spaces[root] >= spaces[child] { + return; + } + spaces.swap(root, child); + root = child; + } +} + +/// Hand the caller a value: its length, or -ENOENT when there is nothing there. +/// +/// When the value does not fit in `len` the bytes are *not* copied and the length is returned +/// anyway, so the caller can size its buffer and ask again — a short read that silently truncated +/// would be worse than an error. +fn copy_out(value: &[u8], buf: *mut u8, len: usize) -> isize { + if value.len() > len { + return value.len() as isize; + } + if !value.is_empty() { + // SAFETY: the shim guarantees `buf` holds `len` bytes and `len >= value.len()`. + unsafe { + core::ptr::copy_nonoverlapping(value.as_ptr(), buf, value.len()); + } + } + value.len() as isize +} + +/// The next record of one space in an image walk, skipping the records of every other space. +fn next_in_space<'a>( + records: &mut Records<'a>, + wanted: &[u8; SPACE_ID_LEN], +) -> Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])> { + while let Some((space, key, value)) = records.next() { + if space == wanted { + return Some((key, value)); + } + } + None +} + +/// Packs a walk's records into the caller's buffer, skipping what the cursor has covered. +struct Batch<'a> { + out: &'a mut [u8], + written: usize, + returned: u64, + seen: u64, + cursor: u64, + /// Set when a record did not fit: the size it needs, for the `-ERANGE` answer. + too_big: usize, +} + +impl Batch<'_> { + /// Offer one record. `false` means nothing more will fit in this batch. + fn offer(&mut self, key: &[u8], value: &[u8]) -> bool { + self.seen += 1; + if self.seen <= self.cursor { + return true; + } + match pack_record(key, value, self.out, self.written) { + Some(n) => { + self.written += n; + self.returned += 1; + true + } + None => { + self.too_big = RAW_KEY_LEN + 4 + value.len(); + false + } + } + } +} + /// `CUBE_OP_GET`: read a coordinate. /// /// Returns the record's length, or a negative errno. When the value does not fit in `len` the /// bytes are *not* copied and the length is returned anyway, so the caller can size its buffer /// and ask again — a short read that silently truncated would be worse than an error. /// +/// The log has the last word, because it holds what has been written since the last checkpoint; +/// only if it says nothing about the coordinate does the image answer. Nothing is copied out of +/// either: the value the caller gets is a slice of bytes already in memory. +/// /// # Safety /// `space` must point to 32 readable bytes; `buf` to `len` writable bytes. #[unsafe(no_mangle)] @@ -1273,58 +2017,32 @@ pub unsafe extern "C" fn cubelinux_kernel_get( len: usize, ) -> isize { let (sp, key) = unsafe { coord_key(space, x, y, z) }; - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as isize), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; // -EINVAL - } - }; - let window = log_window(&device, &layout); - let (mut merged, _applied) = match build_merged( - &live[..core::cmp::min(header.image_bytes.unwrap_or(live.len() as u64) as usize, live.len())], - window, - &header, - ) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; - } - }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); - // The last entry of the coordinate's group is the one that counts: newest write wins, and - // a removal means there is nothing there. - let mut found: Option<&Entry> = None; - for e in entries { - if e.space == sp && e.key == key { - found = Some(e); - } - } - match found { - None => -2, // -ENOENT - Some(e) if e.deleted => -2, - Some(e) => { - let value = merged.value(e); - if value.len() > len { - return value.len() as isize; - } - if !value.is_empty() { - // SAFETY: the shim guarantees `buf` holds `len` bytes and `len >= value.len()`. - unsafe { - core::ptr::copy_nonoverlapping(value.as_ptr(), buf, value.len()); - } - } - value.len() as isize + match log_effect(view.log(), &sp, &key) { + Some(Effect::Delete) => return -2, // -ENOENT + Some(Effect::Write(value)) => return copy_out(value, buf, len), + None => {} + } + + // Walk the records where they lie, starting where this space's records start: the image is in + // `(space, key)` order, so the record for a coordinate is passed exactly once, and a coordinate + // in a late space does not pay for every record before it. + let at = match view.space_start(&sp) { + Ok(Some(at)) => at, + Ok(None) => return -2, // -ENOENT: the space is not in the image, and the log already spoke + Err(_) => return -12, // -ENOMEM + }; + let mut records = Records::at(view.image(), &view.header, at); + while let Some((record_space, record_key, value)) = records.next() { + if record_space == &sp && record_key == &key { + return copy_out(value, buf, len); } } + -2 // -ENOENT } /// Pack one record the way the walk's uapi names it: `key(24) | value_len(u32, LE) | value`. @@ -1351,9 +2069,11 @@ fn pack_record(key: &[u8], value: &[u8], out: &mut [u8], at: usize) -> Option pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as i32), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; // -EINVAL - } + + // The live records of this space: the image's, with the log's edits applied as an overlay. + // A space is what a walk names, so only its own edits are collected — and the log is what a + // checkpoint has not yet folded, so there are few. + let mut walker = match SpaceWalker::new(&view, &wanted) { + Ok(w) => w, + Err(_) => return -12, // -ENOMEM }; - let window = log_window(&device, &layout); - let end = core::cmp::min( - header.image_bytes.unwrap_or(live.len() as u64) as usize, - live.len(), - ); - let (mut merged, _applied) = match build_merged(&live[..end], window, &header) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; - } - }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); + // SAFETY: the shim guarantees `cap` writable bytes at `buf`. let out = unsafe { core::slice::from_raw_parts_mut(buf, cap) }; - - let mut written = 0usize; - let mut returned: u64 = 0; - let mut seen: u64 = 0; - let mut first_too_big: usize = 0; - - let mut i = 0usize; - while i < entries.len() { - let head = &entries[i]; - // The winner of a coordinate is the LAST of its group: the newest write, or a removal. - let mut last = i; - while last + 1 < entries.len() - && entries[last + 1].space == head.space - && entries[last + 1].key == head.key - { - last += 1; - } - let winner = &entries[last]; - i = last + 1; - if winner.deleted || winner.space != wanted { - continue; - } - seen += 1; - if seen <= cursor { - continue; - } - match pack_record(&winner.key, merged.value(winner), out, written) { - Some(n) => { - written += n; - returned += 1; - } - None => { - first_too_big = RAW_KEY_LEN + 4 + winner.value_len as usize; - break; - } + let mut batch = Batch { + out, + written: 0, + returned: 0, + seen: 0, + cursor, + too_big: 0, + }; + while let Some((key, value)) = walker.next() { + if !batch.offer(key, value) { + break; } } // SAFETY: both out-pointers are writable under this function's contract. unsafe { - if first_too_big > 0 && written == 0 { + if batch.too_big > 0 && batch.written == 0 { // Not one whole record fits. Say how much it needs, the way a read does. - *out_len = first_too_big as u64; + *out_len = batch.too_big as u64; *out_cursor = cursor; return -34; // -ERANGE } - *out_len = written as u64; - *out_cursor = cursor + returned; + *out_len = batch.written as u64; + *out_cursor = cursor + batch.returned; } 0 } @@ -1464,60 +2147,67 @@ pub unsafe extern "C" fn cubelinux_kernel_enum( /// `space_out` must point to 32 writable bytes. #[unsafe(no_mangle)] pub unsafe extern "C" fn cubelinux_kernel_spaces(cursor: u64, space_out: *mut u8) -> i32 { - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as i32), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; + + // Candidates, in ascending order: the spaces the image's records run through (a checkpoint + // writes them in key order, and a space is the first half of the key), and any space the log + // writes into that the image has never seen. One pass over the image collects the first kind; + // the second kind is whatever the log holds, which is small. + let mut log_only = KVVec::<[u8; SPACE_ID_LEN]>::new(); + if log_spaces(view.log(), &mut log_only).is_err() { + return -12; // -ENOMEM + } + // A space is only worth reporting if it still holds a record: one whose records the log has all + // removed is not a space anybody can list. + let emit = |space: &[u8; SPACE_ID_LEN], index: &mut u64| -> bool { + if !space_has_records(&view, space) { + return false; } - }; - let window = log_window(&device, &layout); - let end = core::cmp::min( - header.image_bytes.unwrap_or(live.len() as u64) as usize, - live.len(), - ); - let (mut merged, _applied) = match build_merged(&live[..end], window, &header) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; + if *index == cursor { + // SAFETY: the caller guarantees 32 writable bytes at `space_out`. + unsafe { + core::ptr::copy_nonoverlapping(space.as_ptr(), space_out, SPACE_ID_LEN); + } + return true; } + *index += 1; + false + }; + + // The image's own spaces are the table of where they start — no pass over the records, because + // that pass already happened when the table was built. + let starts = match space_starts(&view) { + Ok(s) => s, + Err(_) => return -12, // -ENOMEM }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); let mut index: u64 = 0; - let mut previous: Option<[u8; SPACE_ID_LEN]> = None; - let mut i = 0usize; - while i < entries.len() { - let head = &entries[i]; - let mut last = i; - while last + 1 < entries.len() - && entries[last + 1].space == head.space - && entries[last + 1].key == head.key - { - last += 1; - } - let winner = &entries[last]; - i = last + 1; - if winner.deleted { - continue; - } - if previous != Some(winner.space) { - if index == cursor { - // SAFETY: the caller guarantees 32 writable bytes at `space_out`. - unsafe { - core::ptr::copy_nonoverlapping(winner.space.as_ptr(), space_out, SPACE_ID_LEN); - } + let mut log_at = 0usize; + for (space, _) in starts.as_slice() { + let space: [u8; SPACE_ID_LEN] = *space; + while log_at < log_only.len() && log_only.as_slice()[log_at] < space { + let candidate = log_only.as_slice()[log_at]; + log_at += 1; + if emit(&candidate, &mut index) { return 0; } - index += 1; - previous = Some(winner.space); + } + // A space the log also writes into is the same space, not a second one. + if log_at < log_only.len() && log_only.as_slice()[log_at] == space { + log_at += 1; + } + if emit(&space, &mut index) { + return 0; + } + } + while log_at < log_only.len() { + let candidate = log_only.as_slice()[log_at]; + log_at += 1; + if emit(&candidate, &mut index) { + return 0; } } -2 // -ENOENT: no such space; the walk is finished @@ -1591,6 +2281,8 @@ struct CubeStoreModule { impl kernel::InPlaceModule for CubeStoreModule { fn init(_module: &'static ThisModule) -> impl PinInit { pr_info!("cubelinux: store reader registered at /dev/cubelinux\n"); + // SAFETY: called exactly once, in the module initializer, before anything can take it. + unsafe { SPACE_STARTS.init() }; try_pin_init!(Self { _miscdev <- MiscDeviceRegistration::register(MiscDeviceOptions { name: c_str!("cubelinux"),