// The CUBE store's format, in one file, used by both sides. // // `cube-store-raw` (and through it `cube-cli`, `cube-image`) and this kernel's driver have to agree // about the store's bytes to the byte: the digest line is diffed between a kernel-written image and // a userspace-written one, a listing taken through `cube(2)` is compared with one taken by // userspace, and a store the kernel folds has to be a store userspace can read. They were two // implementations of one format, and when v3 landed the kernel moved and userspace did not — an // hour of "unsupported-version" and a README's worth of careful duplication. // // Two build systems cannot share a crate: the kernel's Rust build compiles what is in its own // module tree, and a cargo crate is not that. So they share a *file*, instead: this one, which the // driver declares as a module and `crates/cube-format` includes by path. There is no copy to drift // from, which is the only arrangement that cannot go stale. // // Everything here is a pure function over slices: no allocation, no I/O, no `std`, no logging. The // kernel reads into buffers with its own allocator and userspace already has the whole image; what // they must agree on is the layout, and that is what lives here. // // The constants are `pub` because both sides need them (a reader computing a stride, a writer // sizing an image), and `#[allow(dead_code)]` is applied per item where one side does not use it — // an unused `pub` constant in a kernel module is a warning, and a warning-free build is how the // gates stay readable. /// Every image starts with these four bytes. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const MAGIC: &[u8; 4] = b"CUBE"; /// The original format: magic, version, curve, then records until zero padding. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const VERSION_V1: u8 = 1; /// The packed format: the same, plus the image's byte extent and record count. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const VERSION_V2: u8 = 2; /// The addressed format: the same, plus a space table, a fixed-size index, and packed values. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const VERSION_V3: u8 = 3; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const HEADER_LEN_V1: usize = 6; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const HEADER_LEN_V2: usize = 6 + 8 + 8; /// magic(4) version(1) curve(1) image_bytes(8) record_count(8) space_count(8) index_off(8) /// values_off(8). #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const HEADER_LEN_V3: usize = 4 + 1 + 1 + 8 + 8 + 8 + 8 + 8; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const SPACE_ID_LEN: usize = 32; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const RAW_KEY_LEN: usize = 24; /// A packed record's fixed part: `space | key | value_len(u64)`, with the value behind it. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const RECORD_FIXED: usize = SPACE_ID_LEN + RAW_KEY_LEN + 8; /// A v3 index entry: `key | value_off(u64) | value_len(u64)`. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const INDEX_ENTRY: usize = RAW_KEY_LEN + 8 + 8; /// A v3 space-table row: `space | first index(u64) | records(u64)`. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const SPACE_ENTRY: usize = SPACE_ID_LEN + 8 + 8; /// The log's framing, which the fold and the readers both parse. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const WAL_MAGIC: &[u8; 4] = b"CUBW"; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const WAL_VERSION: u8 = 1; #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const WAL_HEADER_LEN: usize = 6; /// `op(1) | crc(4) | space(32) | key(24) | len(4)`, with the value behind it. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const ENTRY_FIXED: usize = 1 + 4 + SPACE_ID_LEN + RAW_KEY_LEN + 4; /// `CUBE_OP_PUT`: an entry that stores a value. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const WAL_OP_WRITE: u8 = 1; /// `CUBE_OP_DEL`: an entry that removes one. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const WAL_OP_DELETE: u8 = 2; /// What an image's header says, in any version. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct Header { pub version: u8, pub curve: u8, /// The image's byte length, header included. `None` in v1, which states no extent. pub image_bytes: Option, /// How many records follow the header. `None` in v1. pub record_count: Option, } impl Header { /// Where the records start. pub fn records_off(&self) -> usize { if self.version == VERSION_V1 { HEADER_LEN_V1 } else if self.version == VERSION_V3 { HEADER_LEN_V3 } else { HEADER_LEN_V2 } } /// The image's extent within `bytes`: what the header promises, or all of it for v1. pub fn extent(&self, bytes_len: usize) -> usize { match self.image_bytes { Some(n) => core::cmp::min(n as usize, bytes_len), None => bytes_len, } } } /// Why an image was refused. A reader that guesses at an extent can read somebody else's bytes, so /// these are all refusals rather than defaults. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub enum Bad { /// Not a CUBE image at all. Magic, /// A version this build does not know. Version(u8), /// A header that does not fit, or an extent that does not cover its own header. Extent, /// A v3 table that does not sit where the header says it does. Tables, } /// Read and validate an image header. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn parse_header(bytes: &[u8]) -> Result { if bytes.len() < 5 || &bytes[0..4] != MAGIC { return Err(Bad::Magic); } let version = bytes[4]; let curve = bytes[5]; match version { VERSION_V1 => Ok(Header { version, curve, image_bytes: None, record_count: None, }), VERSION_V2 => { if bytes.len() < HEADER_LEN_V2 { return Err(Bad::Magic); } let image_bytes = le_u64(bytes, 6); let record_count = le_u64(bytes, 14); if (image_bytes as usize) < HEADER_LEN_V2 { return Err(Bad::Extent); } Ok(Header { version, curve, image_bytes: Some(image_bytes), record_count: Some(record_count), }) } VERSION_V3 => { if bytes.len() < HEADER_LEN_V3 { return Err(Bad::Magic); } let geometry = V3::decode(bytes)?; Ok(Header { version, curve, image_bytes: Some(le_u64(bytes, 6)), record_count: Some(geometry.record_count), }) } other => Err(Bad::Version(other)), } } /// The v3 tables' geometry: where the index is, where the values start, and how many of each. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct V3 { pub record_count: u64, pub space_count: u64, pub index_off: u64, pub values_off: u64, } impl V3 { /// Read the tables' geometry, refusing anything that does not add up. /// /// The three equalities here are the whole point of the format: a reader computes a record's /// place as `index_off + i * INDEX_ENTRY`, and that is only a place if the index really starts /// there and really is that wide. pub fn decode(bytes: &[u8]) -> Result { if bytes.len() < HEADER_LEN_V3 || &bytes[0..4] != MAGIC || bytes[4] != VERSION_V3 { return Err(Bad::Magic); } let geometry = V3 { record_count: le_u64(bytes, 14), space_count: le_u64(bytes, 22), index_off: le_u64(bytes, 30), values_off: le_u64(bytes, 38), }; let image_bytes = le_u64(bytes, 6); if image_bytes < HEADER_LEN_V3 as u64 { return Err(Bad::Extent); } let table_end = HEADER_LEN_V3 as u64 + geometry.space_count * SPACE_ENTRY as u64; let index_end = geometry.index_off + geometry.record_count * INDEX_ENTRY as u64; if geometry.index_off != table_end || geometry.values_off != index_end || geometry.values_off > image_bytes { return Err(Bad::Tables); } Ok(geometry) } /// Write the header this geometry describes. `image_bytes` is filled in by the caller's /// arithmetic, since only it knows how long the values are. pub fn encode(&self, out: &mut [u8], curve: u8, image_bytes: u64) -> Result { if out.len() < HEADER_LEN_V3 { return Err(Bad::Extent); } out[0..4].copy_from_slice(MAGIC); out[4] = VERSION_V3; out[5] = curve; out[6..14].copy_from_slice(&image_bytes.to_le_bytes()); out[14..22].copy_from_slice(&self.record_count.to_le_bytes()); out[22..30].copy_from_slice(&self.space_count.to_le_bytes()); out[30..38].copy_from_slice(&self.index_off.to_le_bytes()); out[38..46].copy_from_slice(&self.values_off.to_le_bytes()); Ok(HEADER_LEN_V3) } /// One space-table row: the space, where its records start in the index, and how many. pub fn space_row<'a>(&self, bytes: &'a [u8], row: u64) -> Option<(&'a [u8; SPACE_ID_LEN], u64, u64)> { if row >= self.space_count { return None; } let at = HEADER_LEN_V3 + row as usize * SPACE_ENTRY; if at + SPACE_ENTRY > bytes.len() { return None; } let space: &[u8; SPACE_ID_LEN] = bytes[at..at + SPACE_ID_LEN].try_into().ok()?; Some(( space, le_u64(bytes, at + SPACE_ID_LEN), le_u64(bytes, at + SPACE_ID_LEN + 8), )) } /// One index entry, by its position in the index. pub fn index_entry<'a>(&self, bytes: &'a [u8], at: u64) -> Option> { if at >= self.record_count { return None; } let off = self.index_off as usize + at as usize * INDEX_ENTRY; if off + INDEX_ENTRY > bytes.len() { return None; } let entry = IndexEntry { key: bytes[off..off + RAW_KEY_LEN].try_into().ok()?, value_off: le_u64(bytes, off + RAW_KEY_LEN), value_len: le_u64(bytes, off + RAW_KEY_LEN + 8), }; // A value that is not inside the image is a truncated image, not an empty one. if entry.value_off < self.values_off || entry.value_off + entry.value_len > bytes.len() as u64 { return None; } Some(entry) } /// The value an index entry points at. pub fn value<'a>(&self, bytes: &'a [u8], entry: &IndexEntry<'_>) -> Option<&'a [u8]> { let at = entry.value_off as usize; let len = entry.value_len as usize; if at < self.values_off as usize || at + len > bytes.len() { return None; } Some(&bytes[at..at + len]) } } /// A v3 index entry: the key, and where its value lies. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct IndexEntry<'a> { pub key: &'a [u8; RAW_KEY_LEN], pub value_off: u64, pub value_len: u64, } /// One packed record, and where the next one starts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct PackedRecord<'a> { pub space: &'a [u8; SPACE_ID_LEN], pub key: &'a [u8; RAW_KEY_LEN], pub value: &'a [u8], /// The offset of the record after this one. pub next: usize, } /// Read one packed record at `off`, if there is a whole one before `end`. /// /// This is the v1/v2 layout's only parsing rule, and it is shared for the same reason the constants /// are: a reader that disagrees about a frame's length reads every following record wrong. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn packed_record(bytes: &[u8], off: usize, end: usize) -> Option> { if off + RECORD_FIXED > end || end > bytes.len() { return None; } let frame = &bytes[off..off + RECORD_FIXED]; let value_len = le_u64(frame, SPACE_ID_LEN + RAW_KEY_LEN) as usize; let value_at = off + RECORD_FIXED; if value_at + value_len > end { return None; } Some(PackedRecord { space: frame[..SPACE_ID_LEN].try_into().ok()?, key: frame[SPACE_ID_LEN..SPACE_ID_LEN + RAW_KEY_LEN].try_into().ok()?, value: &bytes[value_at..value_at + value_len], next: value_at + value_len, }) } /// One log entry, and where the next one starts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct WalEntry<'a> { pub space: &'a [u8; SPACE_ID_LEN], pub key: &'a [u8; RAW_KEY_LEN], pub op: u8, pub value: &'a [u8], pub next: usize, } /// Read one log entry at `off`. /// /// The checksum covers space, key, length and value, so a torn tail is stopped at rather than /// applied — the rule the fold and every reader share. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn wal_entry(log: &[u8], off: usize) -> Option> { if off + ENTRY_FIXED > log.len() { return None; } let op = log[off]; if op != WAL_OP_WRITE && op != WAL_OP_DELETE { return None; } let crc = le_u32(log, off + 1); let len = le_u32(log, off + 61) as usize; let frame_end = off + ENTRY_FIXED + len; if frame_end > log.len() { return None; } if crc32(&log[off + 5..frame_end]) != crc { return None; } Some(WalEntry { space: log[off + 5..off + 37].try_into().ok()?, key: log[off + 37..off + 61].try_into().ok()?, op, value: &log[off + ENTRY_FIXED..frame_end], next: frame_end, }) } /// A little-endian `u64` at `at`, or zero if it does not fit — callers bound-check first, and the /// alternative is a panic in a reader that exists to refuse rather than guess. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn le_u64(bytes: &[u8], at: usize) -> u64 { let mut w = [0u8; 8]; if at + 8 <= bytes.len() { w.copy_from_slice(&bytes[at..at + 8]); } u64::from_le_bytes(w) } /// A little-endian `u32` at `at`, with the same rule. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn le_u32(bytes: &[u8], at: usize) -> u32 { let mut w = [0u8; 4]; if at + 4 <= bytes.len() { w.copy_from_slice(&bytes[at..at + 4]); } u32::from_le_bytes(w) } /// CRC-32 (IEEE 802.3), bitwise. /// /// A corruption check, not a security check: it catches a torn write or a flipped bit, and says /// nothing about whether anyone tampered with the bytes — that is `cube-crypt`'s job, and it belongs /// on the record rather than on the framing. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn crc32(bytes: &[u8]) -> u32 { let mut crc = 0xFFFF_FFFFu32; for byte in bytes { crc ^= *byte as u32; for _ in 0..8 { let mask = (crc & 1).wrapping_neg(); crc = (crc >> 1) ^ (0xEDB8_8320 & mask); } } !crc } /// The store's digest hash: FNV-1a, 64-bit. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; /// Fold more bytes into an FNV-1a hash. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub fn fnv1a64(bytes: &[u8], mut h: u64) -> u64 { for byte in bytes { h ^= *byte as u64; h = h.wrapping_mul(0x0000_0100_0000_01b3); } h } /// The digest line, as both sides print it. /// /// `bytes` is the records' logical size — the same number whatever layout holds them — because this /// line is what a kernel-written store and a userspace-written one are compared by, and a figure /// that changed with the layout would make one store look like two. #[allow(dead_code)] // one build uses it and the other does not: neither subset is dead pub struct Digest { pub version: u8, pub curve: u8, pub records: u64, pub value_bytes: u64, pub hash: u64, pub errors: u64, } impl Digest { /// The logical size of the store these records describe. pub fn logical_bytes(records: u64, value_bytes: u64) -> u64 { records * RECORD_FIXED as u64 + value_bytes } /// Fold one record in. pub fn add(&mut self, space: &[u8], key: &[u8], value: &[u8]) { self.hash = fnv1a64(space, self.hash); self.hash = fnv1a64(key, self.hash); self.hash = fnv1a64(&(value.len() as u64).to_le_bytes(), self.hash); self.hash = fnv1a64(value, self.hash); self.records += 1; self.value_bytes += value.len() as u64; } pub fn new(version: u8, curve: u8) -> Self { Digest { version, curve, records: 0, value_bytes: 0, hash: FNV_OFFSET, errors: 0, } } }