diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 94aa26fb3..64ec62f6a 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -1255,12 +1255,756 @@ pub unsafe extern "C" fn cubelinux_kernel_put( } } +// ── Reading the store where it lies ───────────────────────────────────────────────────── +// +// A read used to start by reading the whole device into kernel memory, parsing every record into +// a merged pool, and heapsorting the lot — then throwing all of it away. That is why a `get` of +// one record cost as much as listing the store, and why the coordinate bought nothing +// mechanically: knowing where a record is did not make reaching it any cheaper. +// +// The records are already the index. A checkpoint writes them in `(space, key)` order, which is +// the order the coordinate itself computes, and the log that can override them is small. So the +// kernel walks the records where they lie — zero-copy, no pool, no sort — and consults the log as +// an overlay. What an operation costs is proportional to the records it actually touches. + +/// How much is read at a time when a length is not known ahead of it. +const VIEW_CHUNK: usize = 256 * 1024; + +/// Read exactly `len` bytes at `off`. A short read is an error, not a smaller view: the store says +/// how long its image is, and quietly reading less would turn a truncated store into a shorter one +/// that looks complete. +fn read_exact_at( + file: *mut bindings::file, + off: u64, + len: usize, + out: &mut KVVec, + scratch: &mut KVVec, +) -> Result<()> { + out.clear(); + out.reserve(len, GFP_KERNEL)?; + let mut pos: bindings::loff_t = off as bindings::loff_t; + let mut left = len; + while left > 0 { + let want = core::cmp::min(left, scratch.len()); + // SAFETY: `file` is a live `struct file *`; `scratch` is a kernel buffer we own; `pos` is + // a valid loff_t. kernel_read copies at most `want` bytes and retains neither pointer. + let n = unsafe { + bindings::kernel_read(file, scratch.as_mut_ptr().cast::(), want, &mut pos) + }; + if n < 0 { + return Err(Error::from_errno(n as i32)); + } + if n == 0 { + return Err(EINVAL); + } + let n = n as usize; + out.extend_from_slice(&scratch.as_slice()[..n], GFP_KERNEL)?; + left -= n; + } + Ok(()) +} + +/// Read from `off` to the end, or to `cap` bytes, whichever comes first. +/// +/// For a bare image, whose extent nothing states, and for a log whose length is implied. +fn read_to_end_at( + file: *mut bindings::file, + off: u64, + out: &mut KVVec, + scratch: &mut KVVec, + cap: usize, +) -> Result<()> { + out.clear(); + let mut pos: bindings::loff_t = off as bindings::loff_t; + while out.len() < cap { + let want = core::cmp::min(scratch.len(), cap - out.len()); + // SAFETY: as in `read_exact_at`. + let n = unsafe { + bindings::kernel_read(file, scratch.as_mut_ptr().cast::(), want, &mut pos) + }; + if n < 0 { + return Err(Error::from_errno(n as i32)); + } + if n == 0 { + break; + } + let n = n as usize; + out.extend_from_slice(&scratch.as_slice()[..n], GFP_KERNEL)?; + } + Ok(()) +} + +/// The bytes one operation needs: the image, and the log beside it. +/// +/// The device is four images wide plus its log; reading the device to read the image copies +/// several times what is used, on every call. +struct View { + image: KVVec, + log: KVVec, + header: Header, + /// The control block's generation, which a checkpoint changes — the key to what is cached + /// about this image. Zero for a bare image, which has no control block and does not change + /// underneath a reader. + generation: u64, +} + +kernel::sync::global_lock! { + /// Where each space's records begin. + /// + /// A walk has to start at its space. Records are variable-length, so "where does space S begin" + /// cannot be computed from the key — it can only be found by reading the image once. Finding it + /// once per checkpoint is what makes a listing cost what it returns, and the table is one entry + /// per space (tens of bytes), not one per record. It goes stale on exactly one event: a + /// checkpoint rewriting the image, which changes the control block's generation. + /// + /// SAFETY: Initialized in the module initializer before first use. + unsafe(uninit) static SPACE_STARTS: Mutex> = None; +} + +struct SpaceStarts { + generation: u64, + image_len: usize, + /// Each space's first record, in the image's order. + starts: KVVec<([u8; SPACE_ID_LEN], usize)>, +} + +/// Where the live image's spaces begin, from the cache when it is current and by reading the image +/// when it is not. +fn space_starts(view: &View) -> Result> { + let mut guard = SPACE_STARTS.lock(); + if let Some(cached) = guard.as_ref() { + if cached.generation == view.generation && cached.image_len == view.image().len() { + let mut out = KVVec::new(); + out.extend_from_slice(cached.starts.as_slice(), GFP_KERNEL)?; + return Ok(out); + } + } + + let mut starts = KVVec::new(); + let mut records = Records::new(view.image(), &view.header); + let mut previous: Option<[u8; SPACE_ID_LEN]> = None; + loop { + let at = records.off; + match records.next() { + Some((space, _, _)) => { + if previous != Some(*space) { + starts.push((*space, at), GFP_KERNEL)?; + previous = Some(*space); + } + } + None => break, + } + } + + let mut out = KVVec::new(); + out.extend_from_slice(starts.as_slice(), GFP_KERNEL)?; + *guard = Some(SpaceStarts { + generation: view.generation, + image_len: view.image().len(), + starts, + }); + Ok(out) +} + +impl View { + fn image(&self) -> &[u8] { + self.image.as_slice() + } + + fn log(&self) -> &[u8] { + self.log.as_slice() + } + + /// Where one space's records begin, if the image holds it. + fn space_start(&self, space: &[u8; SPACE_ID_LEN]) -> Result> { + let starts = space_starts(self)?; + for (candidate, at) in starts.as_slice() { + if candidate == space { + return Ok(Some(*at)); + } + } + Ok(None) + } +} + +/// Read the image and its log window — not the whole device. +fn read_view() -> Result { + // O_RDONLY is 0 in Linux; filp_open takes the raw flags word. + // SAFETY: `store_device()` is a NUL-terminated C string the module parameter filled at boot, + // and filp_open returns a valid `struct file *` or an error pointer, checked here. + let file = unsafe { bindings::filp_open(store_device(), 0, 0) }; + let file = kernel::error::from_err_ptr(file)?; + if file.is_null() { + return Err(EINVAL); + } + + let view = read_view_from(file); + + // SAFETY: `file` came from filp_open and has not been closed; the owner argument is only + // meaningful for locks that nobody holds here. + unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; + + let view = view?; + if view.image.len() < HEADER_LEN_V1 { + return Err(EINVAL); + } + Ok(view) +} + +fn read_view_from(file: *mut bindings::file) -> Result { + let mut scratch = KVVec::::with_capacity(VIEW_CHUNK, GFP_KERNEL)?; + scratch.resize(VIEW_CHUNK, 0, GFP_KERNEL)?; + + // The control block, or the image's own header when the device is a bare image. + let mut head = KVVec::::with_capacity(CTL_COPY_B + CTL_COPY_LEN, GFP_KERNEL)?; + read_exact_at(file, 0, CTL_COPY_B + CTL_COPY_LEN, &mut head, &mut scratch)?; + let is_device = &head.as_slice()[0..4] == CTL_MAGIC.as_slice(); + + let (image_off, log_off, log_used, generation) = if is_device { + let control = read_control(head.as_slice()).ok_or(EINVAL)?; + ( + control.image_off() as usize, + control.log_off() as usize, + control.log_used as usize, + control.generation, + ) + } else { + (0, 0, 0, 0) + }; + + // What the image says about itself. A device keeps its image at an offset, so its header is + // read from there; a bare image starts with it. + let mut header = if is_device { + let mut raw = KVVec::::with_capacity(HEADER_LEN_V2, GFP_KERNEL)?; + match read_exact_at(file, image_off as u64, HEADER_LEN_V2, &mut raw, &mut scratch) { + Ok(()) => parse_header(raw.as_slice()), + // A v1 image is shorter than a v2 header, and saying "not a store" about it would be + // wrong: read the six bytes it does have. + Err(_) => { + read_exact_at(file, image_off as u64, HEADER_LEN_V1, &mut raw, &mut scratch)?; + parse_header(raw.as_slice()) + } + } + } else { + parse_header(head.as_slice()) + } + .map_err(|what| { + pr_err!("cubelinux: {}\n", what); + EINVAL + })?; + + let mut image = KVVec::::new(); + let mut log = KVVec::::new(); + match (is_device, header.image_bytes) { + (true, Some(bytes)) => { + let bytes = bytes as usize; + if bytes > MAX_BYTES { + return Err(EINVAL); + } + read_exact_at(file, image_off as u64, bytes, &mut image, &mut scratch)?; + } + // A v1 image states no extent, so it is read to the end — bounded, because a device that + // never ends must not become a read that never ends. + (true, None) => { + read_to_end_at(file, image_off as u64, &mut image, &mut scratch, MAX_BYTES)?; + } + (false, _) => { + read_to_end_at(file, 0, &mut image, &mut scratch, MAX_BYTES)?; + if header.image_bytes.is_none() { + header.image_bytes = Some(image.len() as u64); + } + } + } + + if is_device { + if log_used > 0 { + let len = core::cmp::min(WAL_HEADER_LEN + log_used, MAX_BYTES); + read_exact_at(file, log_off as u64, len, &mut log, &mut scratch)?; + } + } else { + // A bare image's log starts after the image, aligned, and runs to the end. + let at = log_offset(image.len() as u64); + read_to_end_at(file, at as u64, &mut log, &mut scratch, MAX_BYTES)?; + } + + Ok(View { + image, + log, + header, + generation, + }) +} + +/// Walk a store image's records, in the order the image holds them. +/// +/// Zero-copy: the value a caller gets is a slice of the image already in memory, so listing a +/// store does not copy its values anywhere. +struct Records<'a> { + image: &'a [u8], + off: usize, + end: usize, + left: Option, +} + +impl<'a> Records<'a> { + /// Begin at `off`, for a caller that has been told where a space starts. + /// + /// The record count is dropped: it is a bound on the records ahead of the *first* record, and + /// starting anywhere else makes it a bound that no longer means anything. The image's extent + /// is the bound that still holds. + fn at(image: &'a [u8], header: &Header, off: usize) -> Self { + let mut records = Records::new(image, header); + if off > records.off && off <= records.end { + records.off = off; + records.left = None; + } + records + } + + fn new(image: &'a [u8], header: &Header) -> Self { + let header_len = if header.version == VERSION_V1 { + HEADER_LEN_V1 + } else { + HEADER_LEN_V2 + }; + let extent = match header.image_bytes { + Some(n) => core::cmp::min(n as usize, image.len()), + None => image.len(), + }; + Records { + image, + off: core::cmp::min(header_len, extent), + end: extent, + left: header.record_count, + } + } + + fn next(&mut self) -> Option<(&'a [u8; 32], &'a [u8; 24], &'a [u8])> { + if self.left == Some(0) || self.off + RECORD_FIXED > self.end { + return None; + } + let frame = &self.image[self.off..self.off + RECORD_FIXED]; + // A v1 image states no count, so padding ends the walk — but only if every remaining byte + // is zero, because a record at the origin with an empty value is 64 zero bytes and is a + // record. + if self.left.is_none() + && frame.iter().all(|b| *b == 0) + && self.image[self.off..].iter().all(|b| *b == 0) + { + return None; + } + let mut word = [0u8; 8]; + word.copy_from_slice(&frame[SPACE_ID_LEN + RAW_KEY_LEN..RECORD_FIXED]); + let value_len = u64::from_le_bytes(word) as usize; + let value_at = self.off + RECORD_FIXED; + if value_at + value_len > self.end { + return None; + } + let space: &[u8; SPACE_ID_LEN] = frame[..SPACE_ID_LEN].try_into().ok()?; + let key: &[u8; RAW_KEY_LEN] = + frame[SPACE_ID_LEN..SPACE_ID_LEN + RAW_KEY_LEN].try_into().ok()?; + let value = &self.image[value_at..value_at + value_len]; + self.off = value_at + value_len; + if let Some(n) = self.left.as_mut() { + *n -= 1; + } + Some((space, key, value)) + } +} + +/// What the log says about one coordinate: written, or removed. +enum Effect<'a> { + Write(&'a [u8]), + Delete, +} + +/// The log's newest word on `(space, key)`, if it says anything about it. +/// +/// The log is the only thing that can override the image — it holds what has been written since +/// the last checkpoint — so this is what decides a read. +fn log_effect<'a>(log: &'a [u8], space: &[u8; SPACE_ID_LEN], key: &[u8; RAW_KEY_LEN]) -> Option> { + let mut entries = log_entries(log).ok()?; + let mut found = None; + while let Some((entry_space, entry_key, op, value_at, value_len)) = entries.next() { + if entry_space == space && entry_key == key { + found = Some(if op == 2 { + Effect::Delete + } else { + Effect::Write(&log[value_at..value_at + value_len]) + }); + } + } + found +} + +/// One log entry, as the walk sees it: `(space, key, op, value)`. +struct LogEntries<'a> { + log: &'a [u8], + off: usize, +} + +impl<'a> LogEntries<'a> { + /// The next entry: `(space, key, op, where its value starts, its value's length)`. + /// + /// The value's *position* rather than a slice, because a caller collecting edits keeps them in + /// a fixed-size vector and a borrow of the log would tie that vector's type to the log's life. + fn next(&mut self) -> Option<(&'a [u8], &'a [u8], u8, usize, usize)> { + if self.off + ENTRY_FIXED > self.log.len() { + return None; + } + let start = self.off; + let op = self.log[start]; + if op != 1 && op != 2 { + return None; + } + let mut word = [0u8; 4]; + word.copy_from_slice(&self.log[start + 1..start + 5]); + let crc = u32::from_le_bytes(word); + let space = &self.log[start + 5..start + 37]; + let key = &self.log[start + 37..start + 61]; + word.copy_from_slice(&self.log[start + 61..start + 65]); + let len = u32::from_le_bytes(word) as usize; + let frame_end = start + ENTRY_FIXED + len; + if frame_end > self.log.len() { + return None; + } + // The checksum covers space, key, length and value, so a torn tail is stopped at rather + // than applied — the same rule the fold uses. + if crc32(&self.log[start + 5..frame_end]) != crc { + return None; + } + self.off = frame_end; + Some((space, key, op, start + ENTRY_FIXED, len)) + } +} + +/// The log's entries, when there is a valid log to read. +fn log_entries(log: &[u8]) -> Result, &'static str> { + if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { + return Err("no-log"); + } + if log[4] != WAL_VERSION { + return Err("unsupported-log-version"); + } + Ok(LogEntries { + log, + off: WAL_HEADER_LEN, + }) +} + +/// One space's override of the image, on its way to being merged with it. +/// +/// The key and the value borrow the log rather than copying it, so a walker can hand out slices of +/// bytes that are already in memory — a listing never copies a value. +#[derive(Clone, Copy)] +struct LogEdit<'a> { + key: &'a [u8; RAW_KEY_LEN], + /// Position in the log, so the later of two edits to one key is the one that counts. + seq: u32, + value: &'a [u8], + deleted: bool, +} + +/// Every edit the log holds for one space, in key order, later edits last. +/// +/// The image and the log are both in key order, so one merge pass over them is a listing — and a +/// space is what a walk names, so only its own edits are collected. +fn log_edits<'a>( + log: &'a [u8], + space: &[u8; SPACE_ID_LEN], + out: &mut KVVec>, +) -> Result<()> { + out.clear(); + let mut entries = match log_entries(log) { + Ok(e) => e, + Err(_) => return Ok(()), + }; + let mut seq: u32 = 0; + while let Some((edit_space, key, op, value_at, value_len)) = entries.next() { + if edit_space == space { + let key: &[u8; RAW_KEY_LEN] = match key.try_into() { + Ok(k) => k, + Err(_) => break, + }; + out.push( + LogEdit { + key, + seq, + value: &log[value_at..value_at + value_len], + deleted: op == 2, + }, + GFP_KERNEL, + )?; + } + seq += 1; + } + sort_edits(out.as_mut_slice()); + Ok(()) +} + +fn sort_edits(edits: &mut [LogEdit<'_>]) { + let len = edits.len(); + if len < 2 { + return; + } + let mut start = len / 2; + while start > 0 { + start -= 1; + sift_down_edits(edits, start, len); + } + let mut end = len; + while end > 1 { + end -= 1; + edits.swap(0, end); + sift_down_edits(edits, 0, end); + } +} + +/// Heapsort, for the same reason the entry sort is one: a constant stack frame, so the size of a +/// log cannot decide whether reading it is safe. `sort_unstable` is used nowhere in this module. +fn sift_down_edits(edits: &mut [LogEdit<'_>], mut root: usize, end: usize) { + loop { + let mut child = root * 2 + 1; + if child >= end { + return; + } + if child + 1 < end + && (edits[child].key, edits[child].seq) < (edits[child + 1].key, edits[child + 1].seq) + { + child += 1; + } + if (edits[root].key, edits[root].seq) >= (edits[child].key, edits[child].seq) { + return; + } + edits.swap(root, child); + root = child; + } +} + +/// The live records of one space: the image's records for it, with the log's edits applied, in key +/// order. +/// +/// This is the whole of what a listing is — two sorted sequences and one pass — and it is why a +/// walk costs what it returns rather than what the store holds. +struct SpaceWalker<'a> { + records: Records<'a>, + edits: KVVec>, + edit_at: usize, + image_next: Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])>, + wanted: [u8; SPACE_ID_LEN], +} + +impl<'a> SpaceWalker<'a> { + fn new(view: &'a View, wanted: &[u8; SPACE_ID_LEN]) -> Result { + let mut edits = KVVec::>::new(); + log_edits(view.log(), wanted, &mut edits)?; + // Start where the space starts: a listing of a small space in a large store must not read + // the records that are not in it. + let mut records = match view.space_start(wanted)? { + Some(at) => Records::at(view.image(), &view.header, at), + None => { + let mut empty = Records::new(view.image(), &view.header); + empty.off = empty.end; // nothing in the image: the log's edits are the space + empty + } + }; + let image_next = next_in_space(&mut records, wanted); + Ok(SpaceWalker { + records, + edits, + edit_at: 0, + image_next, + wanted: *wanted, + }) + } + + /// The next live record, or nothing when the space is walked out. + fn next(&mut self) -> Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])> { + loop { + // The entries of one key arrive in log order, so the last of a key's group is the + // newest word on it and the earlier ones are history. + while self.edit_at + 1 < self.edits.len() + && self.edits[self.edit_at + 1].key == self.edits[self.edit_at].key + { + self.edit_at += 1; + } + let edit = self.edits.as_slice().get(self.edit_at).copied(); + match (edit, self.image_next) { + // The log's word on a coordinate the image holds: it wins. + (Some(e), Some((key, _))) if e.key == key => { + self.edit_at += 1; + self.image_next = next_in_space(&mut self.records, &self.wanted); + if !e.deleted { + return Some((e.key, e.value)); + } + } + // A record only the log holds, in its key's place. + (Some(e), Some((key, value))) if e.key < key => { + self.edit_at += 1; + self.image_next = Some((key, value)); + if !e.deleted { + return Some((e.key, e.value)); + } + } + // The image's own record, which the log says nothing about. + (_, Some((key, value))) => { + self.image_next = next_in_space(&mut self.records, &self.wanted); + return Some((key, value)); + } + // The image is walked out; the rest of the log still has records to report. + (Some(e), None) => { + self.edit_at += 1; + if !e.deleted { + return Some((e.key, e.value)); + } + } + (None, None) => return None, + } + } + } +} + +/// Does this space hold at least one live record? +/// +/// A space whose every record the log has removed is not a space to report, and a walk that stops +/// at the first record it finds is the cheapest way to know: it costs one record, not a listing. +fn space_has_records(view: &View, space: &[u8; SPACE_ID_LEN]) -> bool { + match SpaceWalker::new(view, space) { + Ok(mut walker) => walker.next().is_some(), + Err(_) => false, + } +} + +/// The distinct spaces the log writes into, ascending. Small by construction: the log is what has +/// been written since the last checkpoint. +fn log_spaces(log: &[u8], out: &mut KVVec<[u8; SPACE_ID_LEN]>) -> Result<()> { + out.clear(); + let mut entries = match log_entries(log) { + Ok(e) => e, + Err(_) => return Ok(()), + }; + while let Some((space, _, _, _, _)) = entries.next() { + let mut s = [0u8; SPACE_ID_LEN]; + s.copy_from_slice(&space[..SPACE_ID_LEN]); + if !out.as_slice().contains(&s) { + out.push(s, GFP_KERNEL)?; + } + } + sort_spaces(out.as_mut_slice()); + Ok(()) +} + +/// Heapsort for a store's spaces. The log holds few, but `sort_unstable` is not used anywhere in +/// this module: it allocates about 3.5 KiB of kernel stack per level, which is how reading a +/// store once took the machine down. +fn sort_spaces(spaces: &mut [[u8; SPACE_ID_LEN]]) { + let len = spaces.len(); + if len < 2 { + return; + } + let mut start = len / 2; + while start > 0 { + start -= 1; + sift_down_spaces(spaces, start, len); + } + let mut end = len; + while end > 1 { + end -= 1; + spaces.swap(0, end); + sift_down_spaces(spaces, 0, end); + } +} + +fn sift_down_spaces(spaces: &mut [[u8; SPACE_ID_LEN]], mut root: usize, end: usize) { + loop { + let mut child = root * 2 + 1; + if child >= end { + return; + } + if child + 1 < end && spaces[child] < spaces[child + 1] { + child += 1; + } + if spaces[root] >= spaces[child] { + return; + } + spaces.swap(root, child); + root = child; + } +} + +/// Hand the caller a value: its length, or -ENOENT when there is nothing there. +/// +/// When the value does not fit in `len` the bytes are *not* copied and the length is returned +/// anyway, so the caller can size its buffer and ask again — a short read that silently truncated +/// would be worse than an error. +fn copy_out(value: &[u8], buf: *mut u8, len: usize) -> isize { + if value.len() > len { + return value.len() as isize; + } + if !value.is_empty() { + // SAFETY: the shim guarantees `buf` holds `len` bytes and `len >= value.len()`. + unsafe { + core::ptr::copy_nonoverlapping(value.as_ptr(), buf, value.len()); + } + } + value.len() as isize +} + +/// The next record of one space in an image walk, skipping the records of every other space. +fn next_in_space<'a>( + records: &mut Records<'a>, + wanted: &[u8; SPACE_ID_LEN], +) -> Option<(&'a [u8; RAW_KEY_LEN], &'a [u8])> { + while let Some((space, key, value)) = records.next() { + if space == wanted { + return Some((key, value)); + } + } + None +} + +/// Packs a walk's records into the caller's buffer, skipping what the cursor has covered. +struct Batch<'a> { + out: &'a mut [u8], + written: usize, + returned: u64, + seen: u64, + cursor: u64, + /// Set when a record did not fit: the size it needs, for the `-ERANGE` answer. + too_big: usize, +} + +impl Batch<'_> { + /// Offer one record. `false` means nothing more will fit in this batch. + fn offer(&mut self, key: &[u8], value: &[u8]) -> bool { + self.seen += 1; + if self.seen <= self.cursor { + return true; + } + match pack_record(key, value, self.out, self.written) { + Some(n) => { + self.written += n; + self.returned += 1; + true + } + None => { + self.too_big = RAW_KEY_LEN + 4 + value.len(); + false + } + } + } +} + /// `CUBE_OP_GET`: read a coordinate. /// /// Returns the record's length, or a negative errno. When the value does not fit in `len` the /// bytes are *not* copied and the length is returned anyway, so the caller can size its buffer /// and ask again — a short read that silently truncated would be worse than an error. /// +/// The log has the last word, because it holds what has been written since the last checkpoint; +/// only if it says nothing about the coordinate does the image answer. Nothing is copied out of +/// either: the value the caller gets is a slice of bytes already in memory. +/// /// # Safety /// `space` must point to 32 readable bytes; `buf` to `len` writable bytes. #[unsafe(no_mangle)] @@ -1273,58 +2017,32 @@ pub unsafe extern "C" fn cubelinux_kernel_get( len: usize, ) -> isize { let (sp, key) = unsafe { coord_key(space, x, y, z) }; - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as isize), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; // -EINVAL - } - }; - let window = log_window(&device, &layout); - let (mut merged, _applied) = match build_merged( - &live[..core::cmp::min(header.image_bytes.unwrap_or(live.len() as u64) as usize, live.len())], - window, - &header, - ) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; - } - }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); - // The last entry of the coordinate's group is the one that counts: newest write wins, and - // a removal means there is nothing there. - let mut found: Option<&Entry> = None; - for e in entries { - if e.space == sp && e.key == key { - found = Some(e); - } - } - match found { - None => -2, // -ENOENT - Some(e) if e.deleted => -2, - Some(e) => { - let value = merged.value(e); - if value.len() > len { - return value.len() as isize; - } - if !value.is_empty() { - // SAFETY: the shim guarantees `buf` holds `len` bytes and `len >= value.len()`. - unsafe { - core::ptr::copy_nonoverlapping(value.as_ptr(), buf, value.len()); - } - } - value.len() as isize + match log_effect(view.log(), &sp, &key) { + Some(Effect::Delete) => return -2, // -ENOENT + Some(Effect::Write(value)) => return copy_out(value, buf, len), + None => {} + } + + // Walk the records where they lie, starting where this space's records start: the image is in + // `(space, key)` order, so the record for a coordinate is passed exactly once, and a coordinate + // in a late space does not pay for every record before it. + let at = match view.space_start(&sp) { + Ok(Some(at)) => at, + Ok(None) => return -2, // -ENOENT: the space is not in the image, and the log already spoke + Err(_) => return -12, // -ENOMEM + }; + let mut records = Records::at(view.image(), &view.header, at); + while let Some((record_space, record_key, value)) = records.next() { + if record_space == &sp && record_key == &key { + return copy_out(value, buf, len); } } + -2 // -ENOENT } /// Pack one record the way the walk's uapi names it: `key(24) | value_len(u32, LE) | value`. @@ -1351,9 +2069,11 @@ fn pack_record(key: &[u8], value: &[u8], out: &mut [u8], at: usize) -> Option pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as i32), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; // -EINVAL - } + + // The live records of this space: the image's, with the log's edits applied as an overlay. + // A space is what a walk names, so only its own edits are collected — and the log is what a + // checkpoint has not yet folded, so there are few. + let mut walker = match SpaceWalker::new(&view, &wanted) { + Ok(w) => w, + Err(_) => return -12, // -ENOMEM }; - let window = log_window(&device, &layout); - let end = core::cmp::min( - header.image_bytes.unwrap_or(live.len() as u64) as usize, - live.len(), - ); - let (mut merged, _applied) = match build_merged(&live[..end], window, &header) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; - } - }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); + // SAFETY: the shim guarantees `cap` writable bytes at `buf`. let out = unsafe { core::slice::from_raw_parts_mut(buf, cap) }; - - let mut written = 0usize; - let mut returned: u64 = 0; - let mut seen: u64 = 0; - let mut first_too_big: usize = 0; - - let mut i = 0usize; - while i < entries.len() { - let head = &entries[i]; - // The winner of a coordinate is the LAST of its group: the newest write, or a removal. - let mut last = i; - while last + 1 < entries.len() - && entries[last + 1].space == head.space - && entries[last + 1].key == head.key - { - last += 1; - } - let winner = &entries[last]; - i = last + 1; - if winner.deleted || winner.space != wanted { - continue; - } - seen += 1; - if seen <= cursor { - continue; - } - match pack_record(&winner.key, merged.value(winner), out, written) { - Some(n) => { - written += n; - returned += 1; - } - None => { - first_too_big = RAW_KEY_LEN + 4 + winner.value_len as usize; - break; - } + let mut batch = Batch { + out, + written: 0, + returned: 0, + seen: 0, + cursor, + too_big: 0, + }; + while let Some((key, value)) = walker.next() { + if !batch.offer(key, value) { + break; } } // SAFETY: both out-pointers are writable under this function's contract. unsafe { - if first_too_big > 0 && written == 0 { + if batch.too_big > 0 && batch.written == 0 { // Not one whole record fits. Say how much it needs, the way a read does. - *out_len = first_too_big as u64; + *out_len = batch.too_big as u64; *out_cursor = cursor; return -34; // -ERANGE } - *out_len = written as u64; - *out_cursor = cursor + returned; + *out_len = batch.written as u64; + *out_cursor = cursor + batch.returned; } 0 } @@ -1464,60 +2147,67 @@ pub unsafe extern "C" fn cubelinux_kernel_enum( /// `space_out` must point to 32 writable bytes. #[unsafe(no_mangle)] pub unsafe extern "C" fn cubelinux_kernel_spaces(cursor: u64, space_out: *mut u8) -> i32 { - let (device, layout) = match device_and_layout() { - Ok(pair) => pair, + let view = match read_view() { + Ok(v) => v, Err(e) => return -(e.to_errno() as i32), }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: {}\n", what); - return -22; + + // Candidates, in ascending order: the spaces the image's records run through (a checkpoint + // writes them in key order, and a space is the first half of the key), and any space the log + // writes into that the image has never seen. One pass over the image collects the first kind; + // the second kind is whatever the log holds, which is small. + let mut log_only = KVVec::<[u8; SPACE_ID_LEN]>::new(); + if log_spaces(view.log(), &mut log_only).is_err() { + return -12; // -ENOMEM + } + // A space is only worth reporting if it still holds a record: one whose records the log has all + // removed is not a space anybody can list. + let emit = |space: &[u8; SPACE_ID_LEN], index: &mut u64| -> bool { + if !space_has_records(&view, space) { + return false; } - }; - let window = log_window(&device, &layout); - let end = core::cmp::min( - header.image_bytes.unwrap_or(live.len() as u64) as usize, - live.len(), - ); - let (mut merged, _applied) = match build_merged(&live[..end], window, &header) { - Ok(m) => m, - Err(what) => { - pr_err!("cubelinux: cannot build the store: {}\n", what); - return -22; + if *index == cursor { + // SAFETY: the caller guarantees 32 writable bytes at `space_out`. + unsafe { + core::ptr::copy_nonoverlapping(space.as_ptr(), space_out, SPACE_ID_LEN); + } + return true; } + *index += 1; + false + }; + + // The image's own spaces are the table of where they start — no pass over the records, because + // that pass already happened when the table was built. + let starts = match space_starts(&view) { + Ok(s) => s, + Err(_) => return -12, // -ENOMEM }; - merged.sort_entries(); - let entries = merged.entries.as_slice(); let mut index: u64 = 0; - let mut previous: Option<[u8; SPACE_ID_LEN]> = None; - let mut i = 0usize; - while i < entries.len() { - let head = &entries[i]; - let mut last = i; - while last + 1 < entries.len() - && entries[last + 1].space == head.space - && entries[last + 1].key == head.key - { - last += 1; - } - let winner = &entries[last]; - i = last + 1; - if winner.deleted { - continue; - } - if previous != Some(winner.space) { - if index == cursor { - // SAFETY: the caller guarantees 32 writable bytes at `space_out`. - unsafe { - core::ptr::copy_nonoverlapping(winner.space.as_ptr(), space_out, SPACE_ID_LEN); - } + let mut log_at = 0usize; + for (space, _) in starts.as_slice() { + let space: [u8; SPACE_ID_LEN] = *space; + while log_at < log_only.len() && log_only.as_slice()[log_at] < space { + let candidate = log_only.as_slice()[log_at]; + log_at += 1; + if emit(&candidate, &mut index) { return 0; } - index += 1; - previous = Some(winner.space); + } + // A space the log also writes into is the same space, not a second one. + if log_at < log_only.len() && log_only.as_slice()[log_at] == space { + log_at += 1; + } + if emit(&space, &mut index) { + return 0; + } + } + while log_at < log_only.len() { + let candidate = log_only.as_slice()[log_at]; + log_at += 1; + if emit(&candidate, &mut index) { + return 0; } } -2 // -ENOENT: no such space; the walk is finished @@ -1591,6 +2281,8 @@ struct CubeStoreModule { impl kernel::InPlaceModule for CubeStoreModule { fn init(_module: &'static ThisModule) -> impl PinInit { pr_info!("cubelinux: store reader registered at /dev/cubelinux\n"); + // SAFETY: called exactly once, in the module initializer, before anything can take it. + unsafe { SPACE_STARTS.init() }; try_pin_init!(Self { _miscdev <- MiscDeviceRegistration::register(MiscDeviceOptions { name: c_str!("cubelinux"),