diff --git a/drivers/cube/cube_format.rs b/drivers/cube/cube_format.rs index 4afb66a96..7e2cbd80a 100644 --- a/drivers/cube/cube_format.rs +++ b/drivers/cube/cube_format.rs @@ -30,6 +30,10 @@ pub const VERSION_V1: u8 = 1; pub const VERSION_V2: u8 = 2; /// The addressed format: the same, plus a space table, a fixed-size index, and packed values. pub const VERSION_V3: u8 = 3; +/// v3, plus a 16-bit class mask in each index entry, so a scan by flag is a seek rather than a +/// walk. The mask is the flag substrate (DESIGN-flag-vocabularies.md): a raw `u16` whose bits are a +/// vocabulary's business, written at put time and read by `CUBE_OP_FLAG_SCAN`. +pub const VERSION_V4: u8 = 4; pub const HEADER_LEN_V1: usize = 6; pub const HEADER_LEN_V2: usize = 6 + 8 + 8; @@ -39,19 +43,28 @@ pub const HEADER_LEN_V3: usize = 4 + 1 + 1 + 8 + 8 + 8 + 8 + 8; pub const SPACE_ID_LEN: usize = 32; pub const RAW_KEY_LEN: usize = 24; +/// The class mask's width: one `u16` per record. +pub const FLAGS_LEN: usize = 2; /// A packed record's fixed part: `space | key | value_len(u64)`, with the value behind it. pub const RECORD_FIXED: usize = SPACE_ID_LEN + RAW_KEY_LEN + 8; /// A v3 index entry: `key | value_off(u64) | value_len(u64)`. pub const INDEX_ENTRY: usize = RAW_KEY_LEN + 8 + 8; +/// A v4 index entry: `key | flags(u16) | value_off(u64) | value_len(u64)`. +pub const INDEX_ENTRY_V4: usize = RAW_KEY_LEN + FLAGS_LEN + 8 + 8; /// A v3 space-table row: `space | first index(u64) | records(u64)`. pub const SPACE_ENTRY: usize = SPACE_ID_LEN + 8 + 8; /// The log's framing, which the fold and the readers both parse. pub const WAL_MAGIC: &[u8; 4] = b"CUBW"; +/// The original log entry: no class mask. pub const WAL_VERSION: u8 = 1; +/// The flagged log entry: the entry carries a `u16` class mask before its length. +pub const WAL_VERSION_V2: u8 = 2; pub const WAL_HEADER_LEN: usize = 6; /// `op(1) | crc(4) | space(32) | key(24) | len(4)`, with the value behind it. pub const ENTRY_FIXED: usize = 1 + 4 + SPACE_ID_LEN + RAW_KEY_LEN + 4; +/// v2's entry: the same, with a `flags(2)` field before the length. +pub const ENTRY_FIXED_V2: usize = 1 + 4 + SPACE_ID_LEN + RAW_KEY_LEN + FLAGS_LEN + 4; /// `CUBE_OP_PUT`: an entry that stores a value. pub const WAL_OP_WRITE: u8 = 1; @@ -74,7 +87,7 @@ impl Header { pub fn records_off(&self) -> usize { if self.version == VERSION_V1 { HEADER_LEN_V1 - } else if self.version == VERSION_V3 { + } else if self.version == VERSION_V3 || self.version == VERSION_V4 { HEADER_LEN_V3 } else { HEADER_LEN_V2 @@ -134,7 +147,7 @@ pub fn parse_header(bytes: &[u8]) -> Result { record_count: Some(record_count), }) } - VERSION_V3 => { + VERSION_V3 | VERSION_V4 => { if bytes.len() < HEADER_LEN_V3 { return Err(Bad::Magic); } @@ -153,6 +166,9 @@ pub fn parse_header(bytes: &[u8]) -> Result { /// The v3 tables' geometry: where the index is, where the values start, and how many of each. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct V3 { + /// The version this geometry belongs to: `VERSION_V3` or `VERSION_V4`, which differ only in + /// the index entry's stride (the class mask adds two bytes). + pub version: u8, pub record_count: u64, pub space_count: u64, pub index_off: u64, @@ -160,16 +176,27 @@ pub struct V3 { } impl V3 { + /// The width of one index entry for this geometry's version. + pub fn index_stride(&self) -> usize { + if self.version == VERSION_V4 { + INDEX_ENTRY_V4 + } else { + INDEX_ENTRY + } + } + /// Read the tables' geometry, refusing anything that does not add up. /// /// The three equalities here are the whole point of the format: a reader computes a record's - /// place as `index_off + i * INDEX_ENTRY`, and that is only a place if the index really starts + /// place as `index_off + i * stride`, and that is only a place if the index really starts /// there and really is that wide. pub fn decode(bytes: &[u8]) -> Result { - if bytes.len() < HEADER_LEN_V3 || &bytes[0..4] != MAGIC || bytes[4] != VERSION_V3 { + let version = bytes.get(4).copied().ok_or(Bad::Magic)?; + if bytes.len() < HEADER_LEN_V3 || &bytes[0..4] != MAGIC || (version != VERSION_V3 && version != VERSION_V4) { return Err(Bad::Magic); } let geometry = V3 { + version, record_count: le_u64(bytes, 14), space_count: le_u64(bytes, 22), index_off: le_u64(bytes, 30), @@ -180,7 +207,7 @@ impl V3 { return Err(Bad::Extent); } let table_end = HEADER_LEN_V3 as u64 + geometry.space_count * SPACE_ENTRY as u64; - let index_end = geometry.index_off + geometry.record_count * INDEX_ENTRY as u64; + let index_end = geometry.index_off + geometry.record_count * geometry.index_stride() as u64; if geometry.index_off != table_end || geometry.values_off != index_end || geometry.values_off > image_bytes @@ -229,14 +256,24 @@ impl V3 { if at >= self.record_count { return None; } - let off = self.index_off as usize + at as usize * INDEX_ENTRY; - if off + INDEX_ENTRY > bytes.len() { + let stride = self.index_stride(); + let off = self.index_off as usize + at as usize * stride; + if off + stride > bytes.len() { return None; } + // The flag field exists only in v4; a v3 entry reads as a zero mask, which is honest — + // "no class" — and matches nothing in a scan. + let flags = if self.version == VERSION_V4 { + le_u16(bytes, off + RAW_KEY_LEN) + } else { + 0 + }; + let vo = off + RAW_KEY_LEN + if self.version == VERSION_V4 { FLAGS_LEN } else { 0 }; let entry = IndexEntry { key: bytes[off..off + RAW_KEY_LEN].try_into().ok()?, - value_off: le_u64(bytes, off + RAW_KEY_LEN), - value_len: le_u64(bytes, off + RAW_KEY_LEN + 8), + flags, + value_off: le_u64(bytes, vo), + value_len: le_u64(bytes, vo + 8), }; // A value that is not inside the image is a truncated image, not an empty one. if entry.value_off < self.values_off @@ -258,10 +295,12 @@ impl V3 { } } -/// A v3 index entry: the key, and where its value lies. +/// An addressed index entry: the key, its class mask, and where its value lies. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct IndexEntry<'a> { pub key: &'a [u8; RAW_KEY_LEN], + /// The class mask (v4), or zero (v3, where no mask was written). + pub flags: u16, pub value_off: u64, pub value_len: u64, } @@ -304,16 +343,25 @@ pub struct WalEntry<'a> { pub space: &'a [u8; SPACE_ID_LEN], pub key: &'a [u8; RAW_KEY_LEN], pub op: u8, + /// The class mask (WAL version 2), or zero (version 1, where no mask was written). + pub flags: u16, pub value: &'a [u8], pub next: usize, } -/// Read one log entry at `off`. +/// Read one log entry at `off`. The log slice includes its header, so the version at `log[4]` +/// decides the entry's stride: version 2 carries a two-byte class mask before the length. /// -/// The checksum covers space, key, length and value, so a torn tail is stopped at rather than -/// applied — the rule the fold and every reader share. +/// The checksum covers space, key, the mask, length and value, so a torn tail is stopped at rather +/// than applied — the rule the fold and every reader share. pub fn wal_entry(log: &[u8], off: usize) -> Option> { - if off + ENTRY_FIXED > log.len() { + let version = log.get(4).copied().unwrap_or(WAL_VERSION); + let (fixed, len_at) = if version == WAL_VERSION_V2 { + (ENTRY_FIXED_V2, RAW_KEY_LEN + FLAGS_LEN + SPACE_ID_LEN + 5) + } else { + (ENTRY_FIXED, 61) + }; + if off + fixed > log.len() { return None; } let op = log[off]; @@ -321,19 +369,25 @@ pub fn wal_entry(log: &[u8], off: usize) -> Option> { return None; } let crc = le_u32(log, off + 1); - let len = le_u32(log, off + 61) as usize; - let frame_end = off + ENTRY_FIXED + len; + let len = le_u32(log, off + len_at) as usize; + let frame_end = off + fixed + len; if frame_end > log.len() { return None; } if crc32(&log[off + 5..frame_end]) != crc { return None; } + let flags = if version == WAL_VERSION_V2 { + le_u16(log, off + SPACE_ID_LEN + RAW_KEY_LEN + 5) + } else { + 0 + }; Some(WalEntry { space: log[off + 5..off + 37].try_into().ok()?, key: log[off + 37..off + 61].try_into().ok()?, op, - value: &log[off + ENTRY_FIXED..frame_end], + flags, + value: &log[off + fixed..frame_end], next: frame_end, }) } @@ -357,6 +411,15 @@ pub fn le_u32(bytes: &[u8], at: usize) -> u32 { u32::from_le_bytes(w) } +/// A little-endian `u16` at `at`, with the same rule. +pub fn le_u16(bytes: &[u8], at: usize) -> u16 { + let mut w = [0u8; 2]; + if at + 2 <= bytes.len() { + w.copy_from_slice(&bytes[at..at + 2]); + } + u16::from_le_bytes(w) +} + /// CRC-32 (IEEE 802.3), bitwise. /// /// A corruption check, not a security check: it catches a torn write or a flipped bit, and says