diff --git a/Makefile b/Makefile index de0295a1e..4359725d0 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ NAME = CUBELinux # release. The base version stays in VERSION/PATCHLEVEL/SUBLEVEL above (visible in # `make kernelversion`), while `uname -r` and /lib/modules report the CUBELinux # release, with any -dirty or SCM suffix still appended by setlocalversion. -CUBELINUX_VERSION = CUBELinux.0.3 +CUBELINUX_VERSION = CUBELinux.0.4 # *DOCUMENTATION* # To see a list of typical targets execute "make help" diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index db5480735..7a5690e35 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -93,11 +93,30 @@ const WAL_HEADER_LEN: usize = 6; const ENTRY_FIXED: usize = 1 + 4 + 32 + 24 + 4; /// The log begins at the first 4 KiB boundary at or after the image. Fixed by geometry so /// no superblock is needed to find it, and stated by the v2 header's extent. +/// +/// That is the *bare* layout: an image on a device with a log after it. A device that is +/// meant to be checkpointed has a control block instead (see `Control`), because folding a +/// log into the only copy of an image is not atomic and a crash in the middle of it loses the +/// store. const LOG_ALIGN: usize = 4096; +/// Control block magic, distinct from the image's `CUBE` and the log's `CUBW`. +const CTL_MAGIC: [u8; 4] = *b"CUBS"; +const CTL_VERSION: u8 = 1; +/// One control copy, and where the two live. +const CTL_COPY_LEN: usize = 2048; +const CTL_COPY_A: usize = 0; +const CTL_COPY_B: usize = 2048; +/// Where the two image slots begin. +const CTL_DATA_OFF: u64 = 4096; +/// Bytes of a control copy covered by its checksum. +const CTL_SUMMED: usize = 44; + /// The one mutation this device accepts so far. The coordinate interface's opcodes will be /// the language's verbs; this is the first of them. const OP_PUT: u8 = 1; +/// Fold the log into the image. +const OP_SYNC: u8 = 3; /// FNV-1a offset basis. const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; @@ -261,7 +280,17 @@ fn parse_header(image: &[u8]) -> Result { fn digest(image: &[u8]) -> Line { let mut line = Line::new(); - let header = match parse_header(image) { + // Where the image is, and which log belongs to it: a store device says so in its control + // block, a bare image says so in its own header. + let layout = match resolve_layout(image) { + Ok(l) => l, + Err(what) => { + let _ = write!(line, "error={what}"); + return line; + } + }; + let live = &image[layout.image_off..]; + let header = match parse_header(live) { Ok(h) => h, Err(what) => { let _ = write!(line, "error={what}"); @@ -269,8 +298,8 @@ fn digest(image: &[u8]) -> Line { } }; let end = match header.image_bytes { - Some(n) => core::cmp::min(n as usize, image.len()), - None => image.len(), + Some(n) => core::cmp::min(n as usize, live.len()), + None => live.len(), }; // With a v2 image, the store is the image *plus* whatever log follows it. The order the @@ -278,7 +307,8 @@ fn digest(image: &[u8]) -> Line { // later writes winning. Without a log there is nothing to merge, and the image's own // order is the answer — which is also what keeps v1 images readable and unchanged. if header.image_bytes.is_some() { - match merged_digest(image, end, &header) { + let log = log_window(image, &layout); + match merged_digest(&live[..end], log, &header) { Ok(line) => return line, // A log that exists but cannot be read is worth saying out loud: the digest // alone would look like a shorter store. @@ -295,12 +325,14 @@ fn digest(image: &[u8]) -> Line { let mut count: u64 = 0; let mut value_bytes: u64 = 0; let mut errors: u64 = 0; - let mut off = if header.version == VERSION_V1 { - HEADER_LEN_V1 - } else { - HEADER_LEN_V2 - }; + let mut off = layout.image_off + + if header.version == VERSION_V1 { + HEADER_LEN_V1 + } else { + HEADER_LEN_V2 + }; let mut remaining = header.record_count; + let end = layout.image_off + end; while off + RECORD_FIXED <= end { if remaining == Some(0) { @@ -365,6 +397,142 @@ fn digest(image: &[u8]) -> Line { line } +/// What a store device's control block says: where the two image slots and the log are, +/// which slot is live, and how much of the log is in use. +#[derive(Clone, Copy)] +struct Control { + curve: u8, + slot_bytes: u64, + log_capacity: u64, + log_used: u64, + generation: u64, + active_slot: u8, +} + +impl Control { + /// Parse a control copy. `None` when it is not one, or its checksum does not hold — a + /// torn update in the middle of a checkpoint is exactly what that checksum is for. + fn decode(b: &[u8]) -> Option { + if b.len() < CTL_SUMMED + 4 || &b[0..4] != CTL_MAGIC { + return None; + } + let word = |at: usize| -> u64 { + let mut w = [0u8; 8]; + w.copy_from_slice(&b[at..at + 8]); + u64::from_le_bytes(w) + }; + let mut c = [0u8; 4]; + c.copy_from_slice(&b[44..48]); + if u32::from_le_bytes(c) != crc32(&b[..CTL_SUMMED]) { + return None; + } + let control = Control { + curve: b[5], + slot_bytes: word(8), + log_capacity: word(16), + log_used: word(24), + generation: word(32), + active_slot: b[40], + }; + if b[4] != CTL_VERSION || control.active_slot > 1 || control.slot_bytes == 0 { + return None; + } + Some(control) + } + + /// Encode into a control copy, checksum included. + fn encode(&self, b: &mut [u8]) { + for byte in b.iter_mut() { + *byte = 0; + } + b[0..4].copy_from_slice(&CTL_MAGIC); + b[4] = CTL_VERSION; + b[5] = self.curve; + b[8..16].copy_from_slice(&self.slot_bytes.to_le_bytes()); + b[16..24].copy_from_slice(&self.log_capacity.to_le_bytes()); + b[24..32].copy_from_slice(&self.log_used.to_le_bytes()); + b[32..40].copy_from_slice(&self.generation.to_le_bytes()); + b[40] = self.active_slot; + let crc = crc32(&b[..CTL_SUMMED]); + b[44..48].copy_from_slice(&crc.to_le_bytes()); + } + + fn image_off(&self) -> u64 { + CTL_DATA_OFF + self.active_slot as u64 * self.slot_bytes + } + + fn spare_off(&self) -> u64 { + CTL_DATA_OFF + (1 - self.active_slot) as u64 * self.slot_bytes + } + + fn log_off(&self) -> u64 { + CTL_DATA_OFF + 2 * self.slot_bytes + } +} + +/// The valid control block: the copy with the higher generation. A reader that finds one +/// copy torn uses the other, which is the entire reason there are two. +fn read_control(b: &[u8]) -> Option { + if b.len() < CTL_COPY_B + CTL_COPY_LEN { + return None; + } + let a = Control::decode(&b[CTL_COPY_A..CTL_COPY_A + CTL_COPY_LEN]); + let b_copy = Control::decode(&b[CTL_COPY_B..CTL_COPY_B + CTL_COPY_LEN]); + match (a, b_copy) { + (Some(x), Some(y)) => Some(if x.generation >= y.generation { x } else { y }), + (Some(x), None) => Some(x), + (None, Some(y)) => Some(y), + (None, None) => None, + } +} + +/// Where the image is and what log belongs to it. +struct Layout { + /// Byte offset of the live image. + image_off: usize, + /// Byte offset of its log. + log_off: usize, + /// Bytes of log in use. + log_used: usize, + /// The control block, when this is a store device rather than a bare image. + control: Option, +} + +/// Work out the layout: a store device when the control block is there, a bare image when it +/// is not. Both are readable; only the store layout can be checkpointed atomically, which is +/// why `sync` refuses on the other. +fn resolve_layout(b: &[u8]) -> Result { + if b.len() >= CTL_COPY_B + CTL_COPY_LEN && &b[0..4] == CTL_MAGIC { + let control = read_control(b).ok_or("control-block-corrupt")?; + let log_off = control.log_off() as usize; + let used = control.log_used as usize; + if log_off + used > b.len() || used > control.log_capacity as usize { + return Err("log-outside-device"); + } + return Ok(Layout { + image_off: control.image_off() as usize, + log_off, + log_used: used, + control: Some(control), + }); + } + // A bare image. A v1 image has no extent and therefore no log — it is the whole device + // as far as this module is concerned, and it is read the way it always was. + let header = parse_header(b)?; + let log_off = match header.image_bytes { + Some(n) => log_offset(n), + None => b.len(), + }; + Ok(Layout { + image_off: 0, + log_off, + // A bare device records no log length, so the log is bounded by the buffer and by + // the entries that parse: `apply_log` stops at the first one that does not. + log_used: b.len().saturating_sub(log_off), + control: None, + }) +} + /// One record on its way into the merged store. Fixed size, so a `KVVec` of these sorts /// in place; values live in a pool beside them and are referenced by offset. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -443,13 +611,8 @@ impl Merged { /// replayed from the start and the first one that is short, mis-framed or fails its /// checksum ends the log. A fault in the middle cannot be told from a torn tail without a /// second copy, so the log stops there and reports what it dropped instead of guessing. -fn apply_log(image: &[u8], image_bytes: usize, merged: &mut Merged) -> Result, &'static str> { - let log_off = (image_bytes + LOG_ALIGN - 1) & !(LOG_ALIGN - 1); - if log_off + WAL_HEADER_LEN > image.len() { - return Ok(None); - } - let log = &image[log_off..]; - if &log[0..4] != WAL_MAGIC { +fn apply_log(log: &[u8], merged: &mut Merged) -> Result, &'static str> { + if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { return Ok(None); } if log[4] != WAL_VERSION { @@ -506,19 +669,36 @@ fn crc32(bytes: &[u8]) -> u32 { !crc } -/// Digest the store the image and its log together describe. -/// -/// Returns `Err("no-log")` when the device holds no log, so the caller can fall back to -/// reading the image alone. -fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result { +/// The log window that belongs to the live image. +fn log_window<'a>(device: &'a [u8], layout: &Layout) -> &'a [u8] { + let end = core::cmp::min( + layout.log_off + WAL_HEADER_LEN + layout.log_used, + device.len(), + ); + if layout.log_off <= end { + &device[layout.log_off..end] + } else { + &[] + } +} + +/// The store the image and its log together describe: the image's records, then the log's +/// entries applied over them. This is what a checkpoint writes and what the digest is taken +/// over, so both go through here rather than each walking the bytes its own way. +fn build_merged(image: &[u8], log: &[u8], header: &Header) -> Result<(Merged, u64), &'static str> { let mut merged = Merged::new().map_err(|_| "out-of-memory")?; - // The image's records first; the log overrides them where they collide. - let mut off = HEADER_LEN_V2; - let mut image_records: u64 = 0; - while off + RECORD_FIXED <= image_bytes { + let extent = header.image_bytes.ok_or("v1-has-no-extent")? as usize; + let extent = core::cmp::min(extent, image.len()); + let mut off = if header.version == VERSION_V1 { + HEADER_LEN_V1 + } else { + HEADER_LEN_V2 + }; + let mut seen: u64 = 0; + while off + RECORD_FIXED <= extent { if let Some(c) = header.record_count { - if image_records >= c { + if seen >= c { break; } } @@ -528,42 +708,50 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result
  • image_bytes { + if value_at + value_len > extent { return Err("truncated-image"); } merged .add(space, key, &image[value_at..value_at + value_len], false) .map_err(|_| "out-of-memory")?; - image_records += 1; + seen += 1; off = value_at + value_len; } - let applied = match apply_log(image, image_bytes, &mut merged)? { + let applied = match apply_log(log, &mut merged)? { Some(n) => n, - None => return Err("no-log"), + None => 0, }; + Ok((merged, applied)) +} + +/// Digest the store the image and its log together describe. +/// +/// `image` is the live image's bytes alone; `log` is exactly the window of log that belongs +/// to it. Returns `Err("no-log")` when there is none, so the caller can fall back to reading +/// the image by itself. +fn merged_digest(image: &[u8], log: &[u8], header: &Header) -> Result { + if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { + return Err("no-log"); + } + let (mut merged, applied) = build_merged(image, log, header)?; - let mut line = Line::new(); merged.sort_entries(); - let survivors = merged.entries.as_slice(); + let entries = merged.entries.as_slice(); let mut h = FNV_OFFSET; let mut count: u64 = 0; let mut value_bytes: u64 = 0; - // Sorting put every write of a coordinate together with the later one last, so a group - // is decided by its *final* entry — the newest write, or a removal. Keeping the first - // instead would resurrect records the log deleted, which is exactly what the gate caught - // the first time it ran. let mut i = 0; - while i < survivors.len() { - let head = &survivors[i]; + while i < entries.len() { + let head = &entries[i]; let mut last = i; - while last + 1 < survivors.len() - && survivors[last + 1].space == head.space - && survivors[last + 1].key == head.key + while last + 1 < entries.len() + && entries[last + 1].space == head.space + && entries[last + 1].key == head.key { last += 1; } - let winner = &survivors[last]; + let winner = &entries[last]; i = last + 1; if winner.deleted { continue; @@ -580,6 +768,7 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result
  • Result
  • [u8; 24] { - let mut k = [0u8; 24]; - let put = |n: usize, v: bool, k: &mut [u8; 24]| { - if !v { - return; - } - let byte = 23 - (n / 8); - k[byte] |= 1 << (n % 8); - }; - for i in 0..64 { - put(3 * i, (x >> i) & 1 != 0, &mut k); - put(3 * i + 1, (y >> i) & 1 != 0, &mut k); - put(3 * i + 2, (z >> i) & 1 != 0, &mut k); - } - k -} - -/// Where the log region starts, given the image's declared extent. +/// Where a bare image's log starts, given the image's declared extent. fn log_offset(image_bytes: u64) -> usize { let n = image_bytes as usize; (n + LOG_ALIGN - 1) & !(LOG_ALIGN - 1) } -/// Length of the log's valid prefix: the header plus every entry that parses and passes its -/// checksum. This is the append position, and the reason a torn tail is overwritten rather -/// than appended to — the same rule the userspace log uses. +/// Length of a bare log's valid prefix: the header plus every entry that parses and passes +/// its checksum. This is the append position, and the reason a torn tail is overwritten +/// rather than appended to. fn log_valid_len(log: &[u8]) -> Result { if log.len() < WAL_HEADER_LEN { return Err("short-log"); @@ -656,6 +824,27 @@ fn log_valid_len(log: &[u8]) -> Result { Ok(off) } +/// Morton-encode a point into the store's 24-byte key — the same interleaving +/// `cube-core`'s `Curve for Morton` does, because a key written here has to be the key a +/// userspace reader decodes. Bit `i` of each axis lands at bit `3i` of the 192-bit key, +/// counting from the least significant bit, which lives in the *last* byte. +fn morton_encode(x: u64, y: u64, z: u64) -> [u8; 24] { + let mut k = [0u8; 24]; + let put = |n: usize, v: bool, k: &mut [u8; 24]| { + if !v { + return; + } + let byte = 23 - (n / 8); + k[byte] |= 1 << (n % 8); + }; + for i in 0..64 { + put(3 * i, (x >> i) & 1 != 0, &mut k); + put(3 * i + 1, (y >> i) & 1 != 0, &mut k); + put(3 * i + 2, (z >> i) & 1 != 0, &mut k); + } + k +} + /// A mutation to append: the byte-plane write the store contract calls `put`. struct Mutation { space: [u8; 32], @@ -663,14 +852,106 @@ struct Mutation { value: KVVec, } -/// Append a mutation to the log and make it durable. +/// Build a log entry: `op | crc32 | space | key | len | value`. +fn encode_entry(space: &[u8; 32], key: &[u8; 24], value: &[u8]) -> Result, AllocError> { + let total = ENTRY_FIXED + value.len(); + let mut entry = KVVec::::with_capacity(total, GFP_KERNEL)?; + entry.extend_from_slice(&[1u8], GFP_KERNEL)?; + entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?; + entry.extend_from_slice(space, GFP_KERNEL)?; + entry.extend_from_slice(key, GFP_KERNEL)?; + entry.extend_from_slice(&(value.len() as u32).to_le_bytes(), GFP_KERNEL)?; + entry.extend_from_slice(value, GFP_KERNEL)?; + // The checksum covers everything after the crc field. + let crc = crc32(&entry.as_slice()[5..]); + entry.as_mut_slice()[1..5].copy_from_slice(&crc.to_le_bytes()); + Ok(entry) +} + +/// Write `bytes` at `off` and flush, or say why not. The one place the module writes. +fn write_and_sync(file: *mut bindings::file, off: u64, bytes: &[u8]) -> Result<(), Error> { + let mut at: bindings::loff_t = off as bindings::loff_t; + // SAFETY: the caller passes a live `struct file *`; `bytes` outlives the call; `at` is a + // valid loff_t. + let wrote = unsafe { + bindings::kernel_write( + file, + bytes.as_ptr().cast::(), + bytes.len(), + &mut at, + ) + }; + if wrote < 0 { + return Err(Error::from_errno(wrote as i32)); + } + if wrote as usize != bytes.len() { + return Err(EIO); + } + // Durability before acknowledgement: a short write that was never flushed is a write that + // did not happen. + // SAFETY: the caller passes a live, writable file; datasync is 0 (a full sync). + let rc = unsafe { bindings::vfs_fsync(file, 0) }; + if rc < 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// The control copy an update should write: the older of the two, so the current one stays +/// readable throughout. +fn older_copy(generation: u64) -> usize { + if generation % 2 == 0 { + CTL_COPY_A + } else { + CTL_COPY_B + } +} + +/// Append a mutation to the log and make it durable, returning the updated control block +/// when the layout has one. /// /// The order is the durability contract: the entry is written and flushed to the device /// *before* the caller is told it happened. A mutation this function has returned `Ok` for /// survives a power cut; one it has not may or may not, which is exactly the boundary an /// acknowledgement is supposed to mark. -fn append(image_bytes: u64, m: &Mutation) -> Result<(), Error> { - let log_off = log_offset(image_bytes); +/// +/// On a store device the entry goes after the log bytes already in use, and the control +/// block's count is raised afterwards — entry first, count second, so a crash between them +/// loses an *unacknowledged* mutation rather than counting one that is not there. A count +/// that ran ahead would make replay read past valid data; a count that lags only forgets. +fn append(image: &[u8], layout: &Layout, m: &Mutation) -> Result, Error> { + let entry = encode_entry(&m.space, &m.key, m.value.as_slice())?; + + // Where it goes: after the bytes in use on a store device, after the valid prefix on a + // bare image, where nothing records the length. + let region_at = core::cmp::min(layout.log_off, image.len()); + let region = &image[region_at..]; + let (used, capacity) = match layout.control { + // `log_used` counts entry bytes *after* the log's header, so the header is not + // something an append has to account for in the count. + Some(c) => (c.log_used as usize, c.log_capacity as usize), + None => { + // A region with no header is a log that has never been written; it is empty, not + // broken, and the header goes down before the first entry. + let used = match log_valid_len(region) { + Ok(valid) => valid - WAL_HEADER_LEN, + Err("no-log") | Err("short-log") => 0, + Err(what) => { + pr_err!("cubelinux: log region unreadable: {}\n", what); + return Err(EINVAL); + } + }; + (used, region.len()) + } + }; + if WAL_HEADER_LEN + used + entry.len() > capacity { + pr_err!( + "cubelinux: the log is full ({} of {} bytes); checkpoint before appending\n", + used, + capacity + ); + return Err(ENOSPC); + } // SAFETY: STORE_DEVICE is a NUL-terminated literal; filp_open returns a valid file or an // error pointer, which is checked. O_RDWR is 2. @@ -680,108 +961,137 @@ fn append(image_bytes: u64, m: &Mutation) -> Result<(), Error> { return Err(EINVAL); } - let mut result = Ok(()); - // The region's header and existing entries, so the append lands after the valid prefix. - let mut head = [0u8; 4096]; - let mut pos: bindings::loff_t = log_off as bindings::loff_t; - // SAFETY: `file` is live; `head` is a 4 KiB buffer we own; `pos` is a valid loff_t. - let n = unsafe { - bindings::kernel_read( + let mut result: Result<(), Error> = Ok(()); + // A log region that has never been written is zeros, not a log: give it a header, the + // same thing the userspace log does when its file does not exist. + // A log region with no header is not a log: give it one, whether this is a bare image or + // a store device, so a reader always finds the framing it expects. + if region.len() < WAL_HEADER_LEN || ®ion[0..4] != WAL_MAGIC { + let mut hdr = [0u8; WAL_HEADER_LEN]; + hdr[0..4].copy_from_slice(&WAL_MAGIC); + hdr[4] = WAL_VERSION; + hdr[5] = 0; // morton + result = write_and_sync(file, layout.log_off as u64, &hdr); + } + if result.is_ok() { + result = write_and_sync( file, - head.as_mut_ptr().cast::(), - head.len(), - &mut pos, - ) - }; - if n < 0 { - result = Err(Error::from_errno(n as i32)); + (region_at + WAL_HEADER_LEN + used) as u64, + entry.as_slice(), + ); } + // The count second, so replay can never be told about an entry that is not there. + let mut updated = None; if result.is_ok() { - // A log region that has never been written is zeros, not a log. Give it a header - // first — the same thing the userspace log does when its file does not exist. - let region = &head[..n as usize]; - let valid = if region.len() < WAL_HEADER_LEN || ®ion[0..4] != WAL_MAGIC { - let mut hdr = [0u8; WAL_HEADER_LEN]; - hdr[0..4].copy_from_slice(&WAL_MAGIC); - hdr[4] = WAL_VERSION; - hdr[5] = 0; // morton - let mut at: bindings::loff_t = log_off as bindings::loff_t; - // SAFETY: `file` is live and writable; `hdr` outlives the call; `at` is valid. - let wrote = unsafe { - bindings::kernel_write( - file, - hdr.as_ptr().cast::(), - hdr.len(), - &mut at, - ) + if let Some(c) = layout.control { + let next = Control { + log_used: c.log_used + entry.len() as u64, + generation: c.generation + 1, + ..c }; - if wrote < 0 { - result = Err(Error::from_errno(wrote as i32)); + let mut buf = [0u8; CTL_COPY_LEN]; + next.encode(&mut buf); + result = write_and_sync(file, older_copy(c.generation) as u64, &buf); + if result.is_ok() { + updated = Some(next); } - WAL_HEADER_LEN - } else { - match log_valid_len(region) { - Ok(v) => v, - Err(what) => { - pr_err!("cubelinux: log region unreadable: {}\n", what); - result = Err(EINVAL); - WAL_HEADER_LEN - } - } - }; - if result.is_ok() { - match Ok::(valid) { - Ok(valid) => { - // Build the entry: op | crc | space | key | len | value. - let total = ENTRY_FIXED + m.value.len(); - let mut entry = KVVec::::with_capacity(total, GFP_KERNEL)?; - entry.extend_from_slice(&[1u8], GFP_KERNEL)?; - entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?; - entry.extend_from_slice(&m.space, GFP_KERNEL)?; - entry.extend_from_slice(&m.key, GFP_KERNEL)?; - entry.extend_from_slice(&(m.value.len() as u32).to_le_bytes(), GFP_KERNEL)?; - entry.extend_from_slice(m.value.as_slice(), GFP_KERNEL)?; - // The checksum covers everything after the crc field. - let crc = crc32(&entry.as_slice()[5..]); - entry.as_mut_slice()[1..5].copy_from_slice(&crc.to_le_bytes()); - - let mut at: bindings::loff_t = (log_off + valid) as bindings::loff_t; - // SAFETY: `file` is live and opened for writing; `entry` is a kernel buffer - // that outlives the call; `at` is a valid loff_t. - let written = unsafe { - bindings::kernel_write( - file, - entry.as_slice().as_ptr().cast::(), - entry.len(), - &mut at, - ) - }; - if written < 0 { - result = Err(Error::from_errno(written as i32)); - } else if written as usize != entry.len() { - result = Err(EIO); - } else { - // Durability before acknowledgement: a short write that was never - // flushed is a write that did not happen. - // SAFETY: `file` is live and writable; datasync is 0 (full sync). - let rc = unsafe { bindings::vfs_fsync(file, 0) }; - if rc < 0 { - result = Err(Error::from_errno(rc)); - } - } - } - Err(what) => { - pr_err!("cubelinux: log region unreadable: {}\n", what); - result = Err(EINVAL); - } - } } } // SAFETY: `file` came from filp_open and has not been closed. unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; - result + result.map(|_| updated) +} + +/// Serialize the merged store into the pinned v2 image. +/// +/// The winners are collected first because the header states the image's extent and its +/// record count, and those are only known once the coalescing is done. +fn serialize_image(merged: &mut Merged, curve: u8) -> Result, AllocError> { + merged.sort_entries(); + let entries = merged.entries.as_slice(); + + let mut winners = KVVec::::new(); + let mut values_len: u64 = 0; + let mut i = 0; + while i < entries.len() { + let head = &entries[i]; + let mut last = i; + while last + 1 < entries.len() + && entries[last + 1].space == head.space + && entries[last + 1].key == head.key + { + last += 1; + } + if !entries[last].deleted { + winners.push(last as u32, GFP_KERNEL)?; + values_len += entries[last].value_len as u64; + } + i = last + 1; + } + + let image_bytes = HEADER_LEN_V2 as u64 + winners.len() as u64 * RECORD_FIXED as u64 + values_len; + let mut out = KVVec::::with_capacity(image_bytes as usize, GFP_KERNEL)?; + out.extend_from_slice(MAGIC, GFP_KERNEL)?; + out.extend_from_slice(&[VERSION, curve], GFP_KERNEL)?; + out.extend_from_slice(&image_bytes.to_le_bytes(), GFP_KERNEL)?; + out.extend_from_slice(&(winners.len() as u64).to_le_bytes(), GFP_KERNEL)?; + + for w in winners.as_slice() { + let e = &merged.entries.as_slice()[*w as usize]; + let value = merged.value(e); + out.extend_from_slice(&e.space, GFP_KERNEL)?; + out.extend_from_slice(&e.key, GFP_KERNEL)?; + out.extend_from_slice(&(value.len() as u64).to_le_bytes(), GFP_KERNEL)?; + out.extend_from_slice(value, GFP_KERNEL)?; + } + Ok(out) +} + +/// Fold the log into the image: write the merged store into the spare slot, then make it +/// live. +/// +/// The order is what makes this atomic. The new image goes to the slot nothing is reading and +/// is flushed; only then does the control block change, and that change is one small write to +/// the control copy that is *not* current. At every instant there is either the old image with +/// a log that still describes the mutations since it, or the new image with an empty log. A +/// crash in the middle of the copy leaves the previous store intact, which is the entire +/// reason for two slots. +fn checkpoint(ctl: &Control, merged: &mut Merged) -> Result { + let new_image = serialize_image(merged, ctl.curve)?; + if new_image.len() as u64 > ctl.slot_bytes { + pr_err!( + "cubelinux: checkpoint needs {} bytes but a slot holds {}\n", + new_image.len(), + ctl.slot_bytes + ); + return Err(ENOSPC); + } + + // SAFETY: as in `append`. + let file = unsafe { bindings::filp_open(STORE_DEVICE.as_ptr().cast::(), 2, 0) }; + let file = kernel::error::from_err_ptr(file)?; + if file.is_null() { + return Err(EINVAL); + } + + let mut result = write_and_sync(file, ctl.spare_off(), new_image.as_slice()); + if result.is_ok() { + let next = Control { + log_used: 0, + generation: ctl.generation + 1, + active_slot: 1 - ctl.active_slot, + ..*ctl + }; + let mut buf = [0u8; CTL_COPY_LEN]; + next.encode(&mut buf); + result = write_and_sync(file, older_copy(ctl.generation) as u64, &buf); + } + + // SAFETY: `file` came from filp_open and has not been closed. + unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; + result.map(|_| new_image.len() as u64) } /// The module's registration; holds the misc device for as long as the module lives. @@ -820,78 +1130,127 @@ impl MiscDevice for CubeStore { ) } - /// Write one mutation, framed as the argument block the coordinate interface would - /// pass: `op(1) | space(32) | x(8) | y(8) | z(8) | len(4) | value[len]`, little-endian. + /// Write one operation, framed as the argument block the coordinate interface will pass. /// - /// This is the byte plane — a `put` of bytes at a coordinate, with no header written - /// beside them — which is what the store contract defines and what the differential - /// gate compares against userspace's `cell put`. The header tier sits above this. + /// ```text + /// put op=1 | space(32) | x(8) | y(8) | z(8) | len(4) | value[len] + /// sync op=3 + /// ``` + /// + /// `put` is the byte plane — bytes at a coordinate, with no header written beside them — + /// which is what the store contract defines and what the differential gate compares + /// against userspace's `cell put`. The header tier sits above this. + /// + /// `sync` folds the log into the image. It is a real operation rather than a test hook: a + /// caller that wants the log reclaimed — a shutdown, a snapshot, a handover — is entitled + /// to ask. fn write_iter(kiocb: Kiocb<'_, Self::Ptr>, iov: &mut IovIterSource<'_>) -> Result { let mut buf = KVVec::::new(); let len = iov.copy_from_iter_vec(&mut buf, GFP_KERNEL)?; let bytes = buf.as_slice(); - if bytes.len() < 1 + 32 + 24 + 4 { + if bytes.is_empty() { return Err(EINVAL); } - if bytes[0] != OP_PUT { - return Err(EINVAL); - } - let mut space = [0u8; 32]; - space.copy_from_slice(&bytes[1..33]); - let axis = |at: usize| -> u64 { - let mut w = [0u8; 8]; - w.copy_from_slice(&bytes[at..at + 8]); - u64::from_le_bytes(w) - }; - let (x, y, z) = (axis(33), axis(41), axis(49)); - let mut w = [0u8; 4]; - w.copy_from_slice(&bytes[57..61]); - let value_len = u32::from_le_bytes(w) as usize; - if bytes.len() < 61 + value_len { - return Err(EINVAL); - } - - let mut value = KVVec::::new(); - value.extend_from_slice(&bytes[61..61 + value_len], GFP_KERNEL)?; - let mutation = Mutation { - space, - key: morton_encode(x, y, z), - value, - }; - - // The image's extent says where the log begins. Reading it is not optional: the log - // has no fixed place of its own, on purpose — no superblock to keep in step. - let mut head = KVVec::::new(); - read_image(&mut head)?; - let header = match parse_header(&head) { - Ok(h) => h, - Err(what) => { - pr_err!("cubelinux: cannot append: {}\n", what); - return Err(EINVAL); - } - }; - let image_bytes = match header.image_bytes { - Some(n) => n, - None => { - pr_err!("cubelinux: cannot append to a v1 store (no extent for the log)\n"); - return Err(EINVAL); - } - }; - let me = kiocb.file(); - match append(image_bytes, &mutation) { - Ok(()) => { - dev_info!( - me.dev, - "cubelinux: appended {} bytes at the log ({} value bytes)\n", - ENTRY_FIXED + value_len, - value_len - ); - Ok(len) + + // Both operations need the device: its layout says where the image and log are. + let mut device = KVVec::::new(); + read_image(&mut device)?; + let layout = match resolve_layout(&device) { + Ok(l) => l, + Err(what) => { + dev_err!(me.dev, "cubelinux: cannot resolve the store layout: {}\n", what); + return Err(EINVAL); } - Err(e) => { - dev_err!(me.dev, "cubelinux: append failed: {:?}\n", e); - Err(e) + }; + + match bytes[0] { + OP_PUT => { + if bytes.len() < 1 + 32 + 24 + 4 { + return Err(EINVAL); + } + let mut space = [0u8; 32]; + space.copy_from_slice(&bytes[1..33]); + let axis = |at: usize| -> u64 { + let mut w = [0u8; 8]; + w.copy_from_slice(&bytes[at..at + 8]); + u64::from_le_bytes(w) + }; + let (x, y, z) = (axis(33), axis(41), axis(49)); + let mut w = [0u8; 4]; + w.copy_from_slice(&bytes[57..61]); + let value_len = u32::from_le_bytes(w) as usize; + if bytes.len() < 61 + value_len { + return Err(EINVAL); + } + let mut value = KVVec::::new(); + value.extend_from_slice(&bytes[61..61 + value_len], GFP_KERNEL)?; + let mutation = Mutation { + space, + key: morton_encode(x, y, z), + value, + }; + match append(&device, &layout, &mutation) { + Ok(_) => { + dev_info!( + me.dev, + "cubelinux: appended {} bytes for a {} byte value\n", + ENTRY_FIXED + value_len, + value_len + ); + Ok(len) + } + Err(e) => { + dev_err!(me.dev, "cubelinux: append failed: {:?}\n", e); + Err(e) + } + } + } + OP_SYNC => { + let ctl = match layout.control { + Some(c) => c, + None => { + // A bare image has one slot, so folding into it is not atomic and a + // crash in the middle would lose the store. Refusing is the honest + // answer; the fix is to format the device as a store. + dev_err!( + me.dev, + "cubelinux: cannot checkpoint a bare image (no control block)\n" + ); + return Err(EINVAL); + } + }; + // The store the image and its log describe, ready to be written out. + let live = &device[layout.image_off..]; + let header = match parse_header(live) { + Ok(h) => h, + Err(what) => { + dev_err!(me.dev, "cubelinux: {}\n", what); + return Err(EINVAL); + } + }; + let window = log_window(&device, &layout); + let mut merged = match build_merged(live, window, &header) { + Ok(m) => m, + Err(what) => { + dev_err!(me.dev, "cubelinux: cannot build the store: {}\n", what); + return Err(EINVAL); + } + }; + match checkpoint(&ctl, &mut merged.0) { + Ok(bytes) => { + dev_info!(me.dev, "cubelinux: checkpointed {} bytes\n", bytes); + Ok(len) + } + Err(e) => { + dev_err!(me.dev, "cubelinux: checkpoint failed: {:?}\n", e); + Err(e) + } + } + } + other => { + dev_err!(me.dev, "cubelinux: unknown operation {}\n", other); + Err(EINVAL) } } }