cubelinux/cubelinux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CUBELinux.0.4: the kernel folds its own log

Browse Source 
Until now the kernel could append to a log but needed a *userspace* checkpoint to
reclaim it — the dependency the write-authority decision was meant to remove.
This is the fold.

A store device is no longer an image at offset 0. It is:

  [control copy A][control copy B][slot 0][slot 1][log]

with the control block saying which slot is live and how much log is in use, and
two copies each carrying a generation and a checksum. Two slots buy atomicity:
the folded image is written to the slot nothing is reading and flushed, and only
then does the control change — one small write to the copy that is *not* current.
At every instant there is either the old image with a log that still describes the
mutations since it, or the new image with an empty log. A crash in the middle of
the copy leaves the previous store intact, which is the entire reason for two
slots. Folding into the only copy of an image is not atomic, so `sync` refuses on
a bare image rather than pretending.

`sync` is a real operation, not a test hook: a caller that wants the log reclaimed
— a shutdown, a snapshot, a handover — is entitled to ask.

Gate (kernel/verify-kernel-checkpoint.sh): the kernel writes four mutations and
then folds its own log. The image it writes is compared with the one userspace
writes from the same mutations byte for byte, not by digest — and it is identical,
with the log empty afterwards.

Three bugs came out of building this, all caught by the gates rather than by
inspection:

- the refactor that extracted the merge left the image walk *duplicated* inside
  the digest, re-adding image records after the log's entries — which gave them a
  newer sequence number and quietly resurrected records the log had deleted;
- a v1 image has no extent, so it has no log either, and the new layout code was
  demanding both;
- an unwritten log region is empty, not broken, and the first append was refusing
  it. The append gate's harness was also counting a *refused* write as accepted,
  which is how the second one hid.

All six gates pass: three image reads (v1 curated, v1 snapshot at 35,318 records,
v2 store), the log replay, the append with its SIGKILL durability test, and the
checkpoint compared byte for byte.
This commit is contained in:
CUBELinux build
2026-09-18 21:01:41 -04:00
parent 93e2c8c39b
commit 451754c7ac
2 changed files with 584 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -9,7 +9,7 @@ NAME = CUBELinux
# release. The base version stays in VERSION/PATCHLEVEL/SUBLEVEL above (visible in
# `make kernelversion`), while `uname -r` and /lib/modules report the CUBELinux
# release, with any -dirty or SCM suffix still appended by setlocalversion.
CUBELINUX_VERSION = CUBELinux.0.3
CUBELINUX_VERSION = CUBELinux.0.4
# *DOCUMENTATION*
# To see a list of typical targets execute "make help"
drivers/cube/cubelinux_store.rs
+583 -224
View File
@@ -93,11 +93,30 @@ const WAL_HEADER_LEN: usize = 6;
const ENTRY_FIXED: usize = 1 + 4 + 32 + 24 + 4;
/// The log begins at the first 4 KiB boundary at or after the image. Fixed by geometry so
/// no superblock is needed to find it, and stated by the v2 header's extent.
///
/// That is the *bare* layout: an image on a device with a log after it. A device that is
/// meant to be checkpointed has a control block instead (see `Control`), because folding a
/// log into the only copy of an image is not atomic and a crash in the middle of it loses the
/// store.
const LOG_ALIGN: usize = 4096;
/// Control block magic, distinct from the image's `CUBE` and the log's `CUBW`.
const CTL_MAGIC: [u8; 4] = *b"CUBS";
const CTL_VERSION: u8 = 1;
/// One control copy, and where the two live.
const CTL_COPY_LEN: usize = 2048;
const CTL_COPY_A: usize = 0;
const CTL_COPY_B: usize = 2048;
/// Where the two image slots begin.
const CTL_DATA_OFF: u64 = 4096;
/// Bytes of a control copy covered by its checksum.
const CTL_SUMMED: usize = 44;
/// The one mutation this device accepts so far. The coordinate interface's opcodes will be
/// the language's verbs; this is the first of them.
const OP_PUT: u8 = 1;
/// Fold the log into the image.
const OP_SYNC: u8 = 3;
/// FNV-1a offset basis.
const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
@@ -261,7 +280,17 @@ fn parse_header(image: &[u8]) -> Result<Header, &'static str> {
fn digest(image: &[u8]) -> Line {
let mut line = Line::new();
let header = match parse_header(image) {
// Where the image is, and which log belongs to it: a store device says so in its control
// block, a bare image says so in its own header.
let layout = match resolve_layout(image) {
Ok(l) => l,
Err(what) => {
let _ = write!(line, "error={what}");
return line;
}
};
let live = &image[layout.image_off..];
let header = match parse_header(live) {
Ok(h) => h,
Err(what) => {
let _ = write!(line, "error={what}");
@@ -269,8 +298,8 @@ fn digest(image: &[u8]) -> Line {
}
};
let end = match header.image_bytes {
Some(n) => core::cmp::min(n as usize, image.len()),
None => image.len(),
Some(n) => core::cmp::min(n as usize, live.len()),
None => live.len(),
};
// With a v2 image, the store is the image *plus* whatever log follows it. The order the
@@ -278,7 +307,8 @@ fn digest(image: &[u8]) -> Line {
// later writes winning. Without a log there is nothing to merge, and the image's own
// order is the answer — which is also what keeps v1 images readable and unchanged.
if header.image_bytes.is_some() {
match merged_digest(image, end, &header) {
let log = log_window(image, &layout);
match merged_digest(&live[..end], log, &header) {
Ok(line) => return line,
// A log that exists but cannot be read is worth saying out loud: the digest
// alone would look like a shorter store.
@@ -295,12 +325,14 @@ fn digest(image: &[u8]) -> Line {
let mut count: u64 = 0;
let mut value_bytes: u64 = 0;
let mut errors: u64 = 0;
let mut off = if header.version == VERSION_V1 {
HEADER_LEN_V1
} else {
HEADER_LEN_V2
};
let mut off = layout.image_off
+ if header.version == VERSION_V1 {
HEADER_LEN_V1
} else {
HEADER_LEN_V2
};
let mut remaining = header.record_count;
let end = layout.image_off + end;
while off + RECORD_FIXED <= end {
if remaining == Some(0) {
@@ -365,6 +397,142 @@ fn digest(image: &[u8]) -> Line {
line
}
/// What a store device's control block says: where the two image slots and the log are,
/// which slot is live, and how much of the log is in use.
#[derive(Clone, Copy)]
struct Control {
curve: u8,
slot_bytes: u64,
log_capacity: u64,
log_used: u64,
generation: u64,
active_slot: u8,
}
impl Control {
/// Parse a control copy. `None` when it is not one, or its checksum does not hold — a
/// torn update in the middle of a checkpoint is exactly what that checksum is for.
fn decode(b: &[u8]) -> Option<Control> {
if b.len() < CTL_SUMMED + 4 || &b[0..4] != CTL_MAGIC {
return None;
}
let word = |at: usize| -> u64 {
let mut w = [0u8; 8];
w.copy_from_slice(&b[at..at + 8]);
u64::from_le_bytes(w)
};
let mut c = [0u8; 4];
c.copy_from_slice(&b[44..48]);
if u32::from_le_bytes(c) != crc32(&b[..CTL_SUMMED]) {
return None;
}
let control = Control {
curve: b[5],
slot_bytes: word(8),
log_capacity: word(16),
log_used: word(24),
generation: word(32),
active_slot: b[40],
};
if b[4] != CTL_VERSION || control.active_slot > 1 || control.slot_bytes == 0 {
return None;
}
Some(control)
}
/// Encode into a control copy, checksum included.
fn encode(&self, b: &mut [u8]) {
for byte in b.iter_mut() {
*byte = 0;
}
b[0..4].copy_from_slice(&CTL_MAGIC);
b[4] = CTL_VERSION;
b[5] = self.curve;
b[8..16].copy_from_slice(&self.slot_bytes.to_le_bytes());
b[16..24].copy_from_slice(&self.log_capacity.to_le_bytes());
b[24..32].copy_from_slice(&self.log_used.to_le_bytes());
b[32..40].copy_from_slice(&self.generation.to_le_bytes());
b[40] = self.active_slot;
let crc = crc32(&b[..CTL_SUMMED]);
b[44..48].copy_from_slice(&crc.to_le_bytes());
}
fn image_off(&self) -> u64 {
CTL_DATA_OFF + self.active_slot as u64 * self.slot_bytes
}
fn spare_off(&self) -> u64 {
CTL_DATA_OFF + (1 - self.active_slot) as u64 * self.slot_bytes
}
fn log_off(&self) -> u64 {
CTL_DATA_OFF + 2 * self.slot_bytes
}
}
/// The valid control block: the copy with the higher generation. A reader that finds one
/// copy torn uses the other, which is the entire reason there are two.
fn read_control(b: &[u8]) -> Option<Control> {
if b.len() < CTL_COPY_B + CTL_COPY_LEN {
return None;
}
let a = Control::decode(&b[CTL_COPY_A..CTL_COPY_A + CTL_COPY_LEN]);
let b_copy = Control::decode(&b[CTL_COPY_B..CTL_COPY_B + CTL_COPY_LEN]);
match (a, b_copy) {
(Some(x), Some(y)) => Some(if x.generation >= y.generation { x } else { y }),
(Some(x), None) => Some(x),
(None, Some(y)) => Some(y),
(None, None) => None,
}
}
/// Where the image is and what log belongs to it.
struct Layout {
/// Byte offset of the live image.
image_off: usize,
/// Byte offset of its log.
log_off: usize,
/// Bytes of log in use.
log_used: usize,
/// The control block, when this is a store device rather than a bare image.
control: Option<Control>,
}
/// Work out the layout: a store device when the control block is there, a bare image when it
/// is not. Both are readable; only the store layout can be checkpointed atomically, which is
/// why `sync` refuses on the other.
fn resolve_layout(b: &[u8]) -> Result<Layout, &'static str> {
if b.len() >= CTL_COPY_B + CTL_COPY_LEN && &b[0..4] == CTL_MAGIC {
let control = read_control(b).ok_or("control-block-corrupt")?;
let log_off = control.log_off() as usize;
let used = control.log_used as usize;
if log_off + used > b.len() || used > control.log_capacity as usize {
return Err("log-outside-device");
}
return Ok(Layout {
image_off: control.image_off() as usize,
log_off,
log_used: used,
control: Some(control),
});
}
// A bare image. A v1 image has no extent and therefore no log — it is the whole device
// as far as this module is concerned, and it is read the way it always was.
let header = parse_header(b)?;
let log_off = match header.image_bytes {
Some(n) => log_offset(n),
None => b.len(),
};
Ok(Layout {
image_off: 0,
log_off,
// A bare device records no log length, so the log is bounded by the buffer and by
// the entries that parse: `apply_log` stops at the first one that does not.
log_used: b.len().saturating_sub(log_off),
control: None,
})
}
/// One record on its way into the merged store. Fixed size, so a `KVVec` of these sorts
/// in place; values live in a pool beside them and are referenced by offset.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
@@ -443,13 +611,8 @@ impl Merged {
/// replayed from the start and the first one that is short, mis-framed or fails its
/// checksum ends the log. A fault in the middle cannot be told from a torn tail without a
/// second copy, so the log stops there and reports what it dropped instead of guessing.
fn apply_log(image: &[u8], image_bytes: usize, merged: &mut Merged) -> Result<Option<u64>, &'static str> {
let log_off = (image_bytes + LOG_ALIGN - 1) & !(LOG_ALIGN - 1);
if log_off + WAL_HEADER_LEN > image.len() {
return Ok(None);
}
let log = &image[log_off..];
if &log[0..4] != WAL_MAGIC {
fn apply_log(log: &[u8], merged: &mut Merged) -> Result<Option<u64>, &'static str> {
if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC {
return Ok(None);
}
if log[4] != WAL_VERSION {
@@ -506,19 +669,36 @@ fn crc32(bytes: &[u8]) -> u32 {
!crc
}
/// Digest the store the image and its log together describe.
///
/// Returns `Err("no-log")` when the device holds no log, so the caller can fall back to
/// reading the image alone.
fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result<Line, &'static str> {
/// The log window that belongs to the live image.
fn log_window<'a>(device: &'a [u8], layout: &Layout) -> &'a [u8] {
let end = core::cmp::min(
layout.log_off + WAL_HEADER_LEN + layout.log_used,
device.len(),
);
if layout.log_off <= end {
&device[layout.log_off..end]
} else {
&[]
}
}
/// The store the image and its log together describe: the image's records, then the log's
/// entries applied over them. This is what a checkpoint writes and what the digest is taken
/// over, so both go through here rather than each walking the bytes its own way.
fn build_merged(image: &[u8], log: &[u8], header: &Header) -> Result<(Merged, u64), &'static str> {
let mut merged = Merged::new().map_err(|_| "out-of-memory")?;
// The image's records first; the log overrides them where they collide.
let mut off = HEADER_LEN_V2;
let mut image_records: u64 = 0;
while off + RECORD_FIXED <= image_bytes {
let extent = header.image_bytes.ok_or("v1-has-no-extent")? as usize;
let extent = core::cmp::min(extent, image.len());
let mut off = if header.version == VERSION_V1 {
HEADER_LEN_V1
} else {
HEADER_LEN_V2
};
let mut seen: u64 = 0;
while off + RECORD_FIXED <= extent {
if let Some(c) = header.record_count {
if image_records >= c {
if seen >= c {
break;
}
}
@@ -528,42 +708,50 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result<Li
word.copy_from_slice(&image[off + SPACE_ID_LEN + RAW_KEY_LEN..off + RECORD_FIXED]);
let value_len = u64::from_le_bytes(word) as usize;
let value_at = off + RECORD_FIXED;
if value_at + value_len > image_bytes {
if value_at + value_len > extent {
return Err("truncated-image");
}
merged
.add(space, key, &image[value_at..value_at + value_len], false)
.map_err(|_| "out-of-memory")?;
image_records += 1;
seen += 1;
off = value_at + value_len;
}
let applied = match apply_log(image, image_bytes, &mut merged)? {
let applied = match apply_log(log, &mut merged)? {
Some(n) => n,
None => return Err("no-log"),
None => 0,
};
Ok((merged, applied))
}
/// Digest the store the image and its log together describe.
///
/// `image` is the live image's bytes alone; `log` is exactly the window of log that belongs
/// to it. Returns `Err("no-log")` when there is none, so the caller can fall back to reading
/// the image by itself.
fn merged_digest(image: &[u8], log: &[u8], header: &Header) -> Result<Line, &'static str> {
if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC {
return Err("no-log");
}
let (mut merged, applied) = build_merged(image, log, header)?;
let mut line = Line::new();
merged.sort_entries();
let survivors = merged.entries.as_slice();
let entries = merged.entries.as_slice();
let mut h = FNV_OFFSET;
let mut count: u64 = 0;
let mut value_bytes: u64 = 0;
// Sorting put every write of a coordinate together with the later one last, so a group
// is decided by its *final* entry — the newest write, or a removal. Keeping the first
// instead would resurrect records the log deleted, which is exactly what the gate caught
// the first time it ran.
let mut i = 0;
while i < survivors.len() {
let head = &survivors[i];
while i < entries.len() {
let head = &entries[i];
let mut last = i;
while last + 1 < survivors.len()
&& survivors[last + 1].space == head.space
&& survivors[last + 1].key == head.key
while last + 1 < entries.len()
&& entries[last + 1].space == head.space
&& entries[last + 1].key == head.key
{
last += 1;
}
let winner = &survivors[last];
let winner = &entries[last];
i = last + 1;
if winner.deleted {
continue;
@@ -580,6 +768,7 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result<Li
// `bytes` is the size a checkpoint of this store would produce — the same number
// userspace reports for the image it writes from the same records.
let folded = HEADER_LEN_V2 as u64 + count * RECORD_FIXED as u64 + value_bytes;
let mut line = Line::new();
// The digest line is field-for-field what `cube-image digest` prints, so the two are
// compared by diff. What the log contributed goes on its own line: a padded log region
// makes a byte count meaningless, an entry count does not.
@@ -592,36 +781,15 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result<Li
Ok(line)
}
/// Morton-encode a point into the store's 24-byte key — the same interleaving
/// `cube-core`'s `Curve for Morton` does, because a key written here has to be the key a
/// userspace reader decodes. Bit `i` of each axis lands at bit `3i` of the 192-bit key,
/// counting from the least significant bit, which lives in the *last* byte.
fn morton_encode(x: u64, y: u64, z: u64) -> [u8; 24] {
let mut k = [0u8; 24];
let put = |n: usize, v: bool, k: &mut [u8; 24]| {
if !v {
return;
}
let byte = 23 - (n / 8);
k[byte] |= 1 << (n % 8);
};
for i in 0..64 {
put(3 * i, (x >> i) & 1 != 0, &mut k);
put(3 * i + 1, (y >> i) & 1 != 0, &mut k);
put(3 * i + 2, (z >> i) & 1 != 0, &mut k);
}
k
}
/// Where the log region starts, given the image's declared extent.
/// Where a bare image's log starts, given the image's declared extent.
fn log_offset(image_bytes: u64) -> usize {
let n = image_bytes as usize;
(n + LOG_ALIGN - 1) & !(LOG_ALIGN - 1)
}
/// Length of the log's valid prefix: the header plus every entry that parses and passes its
/// checksum. This is the append position, and the reason a torn tail is overwritten rather
/// than appended to — the same rule the userspace log uses.
/// Length of a bare log's valid prefix: the header plus every entry that parses and passes
/// its checksum. This is the append position, and the reason a torn tail is overwritten
/// rather than appended to.
fn log_valid_len(log: &[u8]) -> Result<usize, &'static str> {
if log.len() < WAL_HEADER_LEN {
return Err("short-log");
@@ -656,6 +824,27 @@ fn log_valid_len(log: &[u8]) -> Result<usize, &'static str> {
Ok(off)
}
/// Morton-encode a point into the store's 24-byte key — the same interleaving
/// `cube-core`'s `Curve for Morton` does, because a key written here has to be the key a
/// userspace reader decodes. Bit `i` of each axis lands at bit `3i` of the 192-bit key,
/// counting from the least significant bit, which lives in the *last* byte.
fn morton_encode(x: u64, y: u64, z: u64) -> [u8; 24] {
let mut k = [0u8; 24];
let put = |n: usize, v: bool, k: &mut [u8; 24]| {
if !v {
return;
}
let byte = 23 - (n / 8);
k[byte] |= 1 << (n % 8);
};
for i in 0..64 {
put(3 * i, (x >> i) & 1 != 0, &mut k);
put(3 * i + 1, (y >> i) & 1 != 0, &mut k);
put(3 * i + 2, (z >> i) & 1 != 0, &mut k);
}
k
}
/// A mutation to append: the byte-plane write the store contract calls `put`.
struct Mutation {
space: [u8; 32],
@@ -663,14 +852,106 @@ struct Mutation {
value: KVVec<u8>,
}
/// Append a mutation to the log and make it durable.
/// Build a log entry: `op | crc32 | space | key | len | value`.
fn encode_entry(space: &[u8; 32], key: &[u8; 24], value: &[u8]) -> Result<KVVec<u8>, AllocError> {
let total = ENTRY_FIXED + value.len();
let mut entry = KVVec::<u8>::with_capacity(total, GFP_KERNEL)?;
entry.extend_from_slice(&[1u8], GFP_KERNEL)?;
entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?;
entry.extend_from_slice(space, GFP_KERNEL)?;
entry.extend_from_slice(key, GFP_KERNEL)?;
entry.extend_from_slice(&(value.len() as u32).to_le_bytes(), GFP_KERNEL)?;
entry.extend_from_slice(value, GFP_KERNEL)?;
// The checksum covers everything after the crc field.
let crc = crc32(&entry.as_slice()[5..]);
entry.as_mut_slice()[1..5].copy_from_slice(&crc.to_le_bytes());
Ok(entry)
}
/// Write `bytes` at `off` and flush, or say why not. The one place the module writes.
fn write_and_sync(file: *mut bindings::file, off: u64, bytes: &[u8]) -> Result<(), Error> {
let mut at: bindings::loff_t = off as bindings::loff_t;
// SAFETY: the caller passes a live `struct file *`; `bytes` outlives the call; `at` is a
// valid loff_t.
let wrote = unsafe {
bindings::kernel_write(
file,
bytes.as_ptr().cast::<core::ffi::c_void>(),
bytes.len(),
&mut at,
)
};
if wrote < 0 {
return Err(Error::from_errno(wrote as i32));
}
if wrote as usize != bytes.len() {
return Err(EIO);
}
// Durability before acknowledgement: a short write that was never flushed is a write that
// did not happen.
// SAFETY: the caller passes a live, writable file; datasync is 0 (a full sync).
let rc = unsafe { bindings::vfs_fsync(file, 0) };
if rc < 0 {
return Err(Error::from_errno(rc));
}
Ok(())
}
/// The control copy an update should write: the older of the two, so the current one stays
/// readable throughout.
fn older_copy(generation: u64) -> usize {
if generation % 2 == 0 {
CTL_COPY_A
} else {
CTL_COPY_B
}
}
/// Append a mutation to the log and make it durable, returning the updated control block
/// when the layout has one.
///
/// The order is the durability contract: the entry is written and flushed to the device
/// *before* the caller is told it happened. A mutation this function has returned `Ok` for
/// survives a power cut; one it has not may or may not, which is exactly the boundary an
/// acknowledgement is supposed to mark.
fn append(image_bytes: u64, m: &Mutation) -> Result<(), Error> {
let log_off = log_offset(image_bytes);
///
/// On a store device the entry goes after the log bytes already in use, and the control
/// block's count is raised afterwards — entry first, count second, so a crash between them
/// loses an *unacknowledged* mutation rather than counting one that is not there. A count
/// that ran ahead would make replay read past valid data; a count that lags only forgets.
fn append(image: &[u8], layout: &Layout, m: &Mutation) -> Result<Option<Control>, Error> {
let entry = encode_entry(&m.space, &m.key, m.value.as_slice())?;
// Where it goes: after the bytes in use on a store device, after the valid prefix on a
// bare image, where nothing records the length.
let region_at = core::cmp::min(layout.log_off, image.len());
let region = &image[region_at..];
let (used, capacity) = match layout.control {
// `log_used` counts entry bytes *after* the log's header, so the header is not
// something an append has to account for in the count.
Some(c) => (c.log_used as usize, c.log_capacity as usize),
None => {
// A region with no header is a log that has never been written; it is empty, not
// broken, and the header goes down before the first entry.
let used = match log_valid_len(region) {
Ok(valid) => valid - WAL_HEADER_LEN,
Err("no-log") | Err("short-log") => 0,
Err(what) => {
pr_err!("cubelinux: log region unreadable: {}\n", what);
return Err(EINVAL);
}
};
(used, region.len())
}
};
if WAL_HEADER_LEN + used + entry.len() > capacity {
pr_err!(
"cubelinux: the log is full ({} of {} bytes); checkpoint before appending\n",
used,
capacity
);
return Err(ENOSPC);
}
// SAFETY: STORE_DEVICE is a NUL-terminated literal; filp_open returns a valid file or an
// error pointer, which is checked. O_RDWR is 2.
@@ -680,108 +961,137 @@ fn append(image_bytes: u64, m: &Mutation) -> Result<(), Error> {
return Err(EINVAL);
}
let mut result = Ok(());
// The region's header and existing entries, so the append lands after the valid prefix.
let mut head = [0u8; 4096];
let mut pos: bindings::loff_t = log_off as bindings::loff_t;
// SAFETY: `file` is live; `head` is a 4 KiB buffer we own; `pos` is a valid loff_t.
let n = unsafe {
bindings::kernel_read(
let mut result: Result<(), Error> = Ok(());
// A log region that has never been written is zeros, not a log: give it a header, the
// same thing the userspace log does when its file does not exist.
// A log region with no header is not a log: give it one, whether this is a bare image or
// a store device, so a reader always finds the framing it expects.
if region.len() < WAL_HEADER_LEN || &region[0..4] != WAL_MAGIC {
let mut hdr = [0u8; WAL_HEADER_LEN];
hdr[0..4].copy_from_slice(&WAL_MAGIC);
hdr[4] = WAL_VERSION;
hdr[5] = 0; // morton
result = write_and_sync(file, layout.log_off as u64, &hdr);
}
if result.is_ok() {
result = write_and_sync(
file,
head.as_mut_ptr().cast::<core::ffi::c_void>(),
head.len(),
&mut pos,
)
};
if n < 0 {
result = Err(Error::from_errno(n as i32));
(region_at + WAL_HEADER_LEN + used) as u64,
entry.as_slice(),
);
}
// The count second, so replay can never be told about an entry that is not there.
let mut updated = None;
if result.is_ok() {
// A log region that has never been written is zeros, not a log. Give it a header
// first — the same thing the userspace log does when its file does not exist.
let region = &head[..n as usize];
let valid = if region.len() < WAL_HEADER_LEN || &region[0..4] != WAL_MAGIC {
let mut hdr = [0u8; WAL_HEADER_LEN];
hdr[0..4].copy_from_slice(&WAL_MAGIC);
hdr[4] = WAL_VERSION;
hdr[5] = 0; // morton
let mut at: bindings::loff_t = log_off as bindings::loff_t;
// SAFETY: `file` is live and writable; `hdr` outlives the call; `at` is valid.
let wrote = unsafe {
bindings::kernel_write(
file,
hdr.as_ptr().cast::<core::ffi::c_void>(),
hdr.len(),
&mut at,
)
if let Some(c) = layout.control {
let next = Control {
log_used: c.log_used + entry.len() as u64,
generation: c.generation + 1,
..c
};
if wrote < 0 {
result = Err(Error::from_errno(wrote as i32));
let mut buf = [0u8; CTL_COPY_LEN];
next.encode(&mut buf);
result = write_and_sync(file, older_copy(c.generation) as u64, &buf);
if result.is_ok() {
updated = Some(next);
}
WAL_HEADER_LEN
} else {
match log_valid_len(region) {
Ok(v) => v,
Err(what) => {
pr_err!("cubelinux: log region unreadable: {}\n", what);
result = Err(EINVAL);
WAL_HEADER_LEN
}
}
};
if result.is_ok() {
match Ok::<usize, &'static str>(valid) {
Ok(valid) => {
// Build the entry: op | crc | space | key | len | value.
let total = ENTRY_FIXED + m.value.len();
let mut entry = KVVec::<u8>::with_capacity(total, GFP_KERNEL)?;
entry.extend_from_slice(&[1u8], GFP_KERNEL)?;
entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?;
entry.extend_from_slice(&m.space, GFP_KERNEL)?;
entry.extend_from_slice(&m.key, GFP_KERNEL)?;
entry.extend_from_slice(&(m.value.len() as u32).to_le_bytes(), GFP_KERNEL)?;
entry.extend_from_slice(m.value.as_slice(), GFP_KERNEL)?;
// The checksum covers everything after the crc field.
let crc = crc32(&entry.as_slice()[5..]);
entry.as_mut_slice()[1..5].copy_from_slice(&crc.to_le_bytes());
let mut at: bindings::loff_t = (log_off + valid) as bindings::loff_t;
// SAFETY: `file` is live and opened for writing; `entry` is a kernel buffer
// that outlives the call; `at` is a valid loff_t.
let written = unsafe {
bindings::kernel_write(
file,
entry.as_slice().as_ptr().cast::<core::ffi::c_void>(),
entry.len(),
&mut at,
)
};
if written < 0 {
result = Err(Error::from_errno(written as i32));
} else if written as usize != entry.len() {
result = Err(EIO);
} else {
// Durability before acknowledgement: a short write that was never
// flushed is a write that did not happen.
// SAFETY: `file` is live and writable; datasync is 0 (full sync).
let rc = unsafe { bindings::vfs_fsync(file, 0) };
if rc < 0 {
result = Err(Error::from_errno(rc));
}
}
}
Err(what) => {
pr_err!("cubelinux: log region unreadable: {}\n", what);
result = Err(EINVAL);
}
}
}
}
// SAFETY: `file` came from filp_open and has not been closed.
unsafe { bindings::filp_close(file, core::ptr::null_mut()) };
result
result.map(|_| updated)
}
/// Serialize the merged store into the pinned v2 image.
///
/// The winners are collected first because the header states the image's extent and its
/// record count, and those are only known once the coalescing is done.
fn serialize_image(merged: &mut Merged, curve: u8) -> Result<KVVec<u8>, AllocError> {
merged.sort_entries();
let entries = merged.entries.as_slice();
let mut winners = KVVec::<u32>::new();
let mut values_len: u64 = 0;
let mut i = 0;
while i < entries.len() {
let head = &entries[i];
let mut last = i;
while last + 1 < entries.len()
&& entries[last + 1].space == head.space
&& entries[last + 1].key == head.key
{
last += 1;
}
if !entries[last].deleted {
winners.push(last as u32, GFP_KERNEL)?;
values_len += entries[last].value_len as u64;
}
i = last + 1;
}
let image_bytes = HEADER_LEN_V2 as u64 + winners.len() as u64 * RECORD_FIXED as u64 + values_len;
let mut out = KVVec::<u8>::with_capacity(image_bytes as usize, GFP_KERNEL)?;
out.extend_from_slice(MAGIC, GFP_KERNEL)?;
out.extend_from_slice(&[VERSION, curve], GFP_KERNEL)?;
out.extend_from_slice(&image_bytes.to_le_bytes(), GFP_KERNEL)?;
out.extend_from_slice(&(winners.len() as u64).to_le_bytes(), GFP_KERNEL)?;
for w in winners.as_slice() {
let e = &merged.entries.as_slice()[*w as usize];
let value = merged.value(e);
out.extend_from_slice(&e.space, GFP_KERNEL)?;
out.extend_from_slice(&e.key, GFP_KERNEL)?;
out.extend_from_slice(&(value.len() as u64).to_le_bytes(), GFP_KERNEL)?;
out.extend_from_slice(value, GFP_KERNEL)?;
}
Ok(out)
}
/// Fold the log into the image: write the merged store into the spare slot, then make it
/// live.
///
/// The order is what makes this atomic. The new image goes to the slot nothing is reading and
/// is flushed; only then does the control block change, and that change is one small write to
/// the control copy that is *not* current. At every instant there is either the old image with
/// a log that still describes the mutations since it, or the new image with an empty log. A
/// crash in the middle of the copy leaves the previous store intact, which is the entire
/// reason for two slots.
fn checkpoint(ctl: &Control, merged: &mut Merged) -> Result<u64, Error> {
let new_image = serialize_image(merged, ctl.curve)?;
if new_image.len() as u64 > ctl.slot_bytes {
pr_err!(
"cubelinux: checkpoint needs {} bytes but a slot holds {}\n",
new_image.len(),
ctl.slot_bytes
);
return Err(ENOSPC);
}
// SAFETY: as in `append`.
let file = unsafe { bindings::filp_open(STORE_DEVICE.as_ptr().cast::<u8>(), 2, 0) };
let file = kernel::error::from_err_ptr(file)?;
if file.is_null() {
return Err(EINVAL);
}
let mut result = write_and_sync(file, ctl.spare_off(), new_image.as_slice());
if result.is_ok() {
let next = Control {
log_used: 0,
generation: ctl.generation + 1,
active_slot: 1 - ctl.active_slot,
..*ctl
};
let mut buf = [0u8; CTL_COPY_LEN];
next.encode(&mut buf);
result = write_and_sync(file, older_copy(ctl.generation) as u64, &buf);
}
// SAFETY: `file` came from filp_open and has not been closed.
unsafe { bindings::filp_close(file, core::ptr::null_mut()) };
result.map(|_| new_image.len() as u64)
}
/// The module's registration; holds the misc device for as long as the module lives.
@@ -820,78 +1130,127 @@ impl MiscDevice for CubeStore {
)
}
/// Write one mutation, framed as the argument block the coordinate interface would
/// pass: `op(1) | space(32) | x(8) | y(8) | z(8) | len(4) | value[len]`, little-endian.
/// Write one operation, framed as the argument block the coordinate interface will pass.
///
/// This is the byte plane — a `put` of bytes at a coordinate, with no header written
/// beside them — which is what the store contract defines and what the differential
/// gate compares against userspace's `cell put`. The header tier sits above this.
/// ```text
/// put op=1 | space(32) | x(8) | y(8) | z(8) | len(4) | value[len]
/// sync op=3
/// ```
///
/// `put` is the byte plane — bytes at a coordinate, with no header written beside them —
/// which is what the store contract defines and what the differential gate compares
/// against userspace's `cell put`. The header tier sits above this.
///
/// `sync` folds the log into the image. It is a real operation rather than a test hook: a
/// caller that wants the log reclaimed — a shutdown, a snapshot, a handover — is entitled
/// to ask.
fn write_iter(kiocb: Kiocb<'_, Self::Ptr>, iov: &mut IovIterSource<'_>) -> Result<usize> {
let mut buf = KVVec::<u8>::new();
let len = iov.copy_from_iter_vec(&mut buf, GFP_KERNEL)?;
let bytes = buf.as_slice();
if bytes.len() < 1 + 32 + 24 + 4 {
if bytes.is_empty() {
return Err(EINVAL);
}
if bytes[0] != OP_PUT {
return Err(EINVAL);
}
let mut space = [0u8; 32];
space.copy_from_slice(&bytes[1..33]);
let axis = |at: usize| -> u64 {
let mut w = [0u8; 8];
w.copy_from_slice(&bytes[at..at + 8]);
u64::from_le_bytes(w)
};
let (x, y, z) = (axis(33), axis(41), axis(49));
let mut w = [0u8; 4];
w.copy_from_slice(&bytes[57..61]);
let value_len = u32::from_le_bytes(w) as usize;
if bytes.len() < 61 + value_len {
return Err(EINVAL);
}
let mut value = KVVec::<u8>::new();
value.extend_from_slice(&bytes[61..61 + value_len], GFP_KERNEL)?;
let mutation = Mutation {
space,
key: morton_encode(x, y, z),
value,
};
// The image's extent says where the log begins. Reading it is not optional: the log
// has no fixed place of its own, on purpose — no superblock to keep in step.
let mut head = KVVec::<u8>::new();
read_image(&mut head)?;
let header = match parse_header(&head) {
Ok(h) => h,
Err(what) => {
pr_err!("cubelinux: cannot append: {}\n", what);
return Err(EINVAL);
}
};
let image_bytes = match header.image_bytes {
Some(n) => n,
None => {
pr_err!("cubelinux: cannot append to a v1 store (no extent for the log)\n");
return Err(EINVAL);
}
};
let me = kiocb.file();
match append(image_bytes, &mutation) {
Ok(()) => {
dev_info!(
me.dev,
"cubelinux: appended {} bytes at the log ({} value bytes)\n",
ENTRY_FIXED + value_len,
value_len
);
Ok(len)
// Both operations need the device: its layout says where the image and log are.
let mut device = KVVec::<u8>::new();
read_image(&mut device)?;
let layout = match resolve_layout(&device) {
Ok(l) => l,
Err(what) => {
dev_err!(me.dev, "cubelinux: cannot resolve the store layout: {}\n", what);
return Err(EINVAL);
}
Err(e) => {
dev_err!(me.dev, "cubelinux: append failed: {:?}\n", e);
Err(e)
};
match bytes[0] {
OP_PUT => {
if bytes.len() < 1 + 32 + 24 + 4 {
return Err(EINVAL);
}
let mut space = [0u8; 32];
space.copy_from_slice(&bytes[1..33]);
let axis = |at: usize| -> u64 {
let mut w = [0u8; 8];
w.copy_from_slice(&bytes[at..at + 8]);
u64::from_le_bytes(w)
};
let (x, y, z) = (axis(33), axis(41), axis(49));
let mut w = [0u8; 4];
w.copy_from_slice(&bytes[57..61]);
let value_len = u32::from_le_bytes(w) as usize;
if bytes.len() < 61 + value_len {
return Err(EINVAL);
}
let mut value = KVVec::<u8>::new();
value.extend_from_slice(&bytes[61..61 + value_len], GFP_KERNEL)?;
let mutation = Mutation {
space,
key: morton_encode(x, y, z),
value,
};
match append(&device, &layout, &mutation) {
Ok(_) => {
dev_info!(
me.dev,
"cubelinux: appended {} bytes for a {} byte value\n",
ENTRY_FIXED + value_len,
value_len
);
Ok(len)
}
Err(e) => {
dev_err!(me.dev, "cubelinux: append failed: {:?}\n", e);
Err(e)
}
}
}
OP_SYNC => {
let ctl = match layout.control {
Some(c) => c,
None => {
// A bare image has one slot, so folding into it is not atomic and a
// crash in the middle would lose the store. Refusing is the honest
// answer; the fix is to format the device as a store.
dev_err!(
me.dev,
"cubelinux: cannot checkpoint a bare image (no control block)\n"
);
return Err(EINVAL);
}
};
// The store the image and its log describe, ready to be written out.
let live = &device[layout.image_off..];
let header = match parse_header(live) {
Ok(h) => h,
Err(what) => {
dev_err!(me.dev, "cubelinux: {}\n", what);
return Err(EINVAL);
}
};
let window = log_window(&device, &layout);
let mut merged = match build_merged(live, window, &header) {
Ok(m) => m,
Err(what) => {
dev_err!(me.dev, "cubelinux: cannot build the store: {}\n", what);
return Err(EINVAL);
}
};
match checkpoint(&ctl, &mut merged.0) {
Ok(bytes) => {
dev_info!(me.dev, "cubelinux: checkpointed {} bytes\n", bytes);
Ok(len)
}
Err(e) => {
dev_err!(me.dev, "cubelinux: checkpoint failed: {:?}\n", e);
Err(e)
}
}
}
other => {
dev_err!(me.dev, "cubelinux: unknown operation {}\n", other);
Err(EINVAL)
}
}
}