diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index bb9106a4d..74e2ad6f3 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -102,6 +102,8 @@ const VERSION_V1: u8 = cube_format::VERSION_V1; /// asking which spaces exist is the space table, and a spatial range is a contiguous run of index /// entries. An index entry is 40 bytes against v2's 64-byte frame, so the image also gets smaller. const VERSION_V3: u8 = cube_format::VERSION_V3; +/// v3, plus a 16-bit class mask in each index entry. +const VERSION_V4: u8 = cube_format::VERSION_V4; const HEADER_LEN_V1: usize = cube_format::HEADER_LEN_V1; const HEADER_LEN_V2: usize = cube_format::HEADER_LEN_V2; /// magic(4) version(1) curve(1) image_bytes(8) record_count(8) space_count(8) index_off(8) @@ -109,6 +111,9 @@ const HEADER_LEN_V2: usize = cube_format::HEADER_LEN_V2; const HEADER_LEN_V3: usize = cube_format::HEADER_LEN_V3; /// `key(24) | value_off(8) | value_len(8)`. const INDEX_ENTRY: usize = cube_format::INDEX_ENTRY; +/// `key(24) | flags(2) | value_off(8) | value_len(8)`. +const INDEX_ENTRY_V4: usize = cube_format::INDEX_ENTRY_V4; +const FLAGS_LEN: usize = cube_format::FLAGS_LEN; /// `space(32) | first index(8) | records(8)`. const SPACE_ENTRY: usize = cube_format::SPACE_ENTRY; const SPACE_ID_LEN: usize = cube_format::SPACE_ID_LEN; @@ -165,10 +170,24 @@ const MAX_BYTES: usize = 128 * 1024 * 1024; /// cannot mistake it for the other. const WAL_MAGIC: [u8; 4] = *cube_format::WAL_MAGIC; const WAL_VERSION: u8 = cube_format::WAL_VERSION; +/// The flagged log entry: the same, with a two-byte class mask before the length. +const WAL_VERSION_V2: u8 = cube_format::WAL_VERSION_V2; /// Log header: magic(4) + version(1) + curve tag(1). const WAL_HEADER_LEN: usize = cube_format::WAL_HEADER_LEN; /// Log entry, before the value: op(1) + crc32(4) + space(32) + key(24) + len(4). const ENTRY_FIXED: usize = cube_format::ENTRY_FIXED; +/// The flagged entry: the same, with `flags(2)` before the length. +const ENTRY_FIXED_V2: usize = cube_format::ENTRY_FIXED_V2; + +/// One log entry's fixed size, and the offset of its length, for a log version. Version 1 has no +/// class mask; version 2 carries a two-byte one before the length. +fn wal_frame(version: u8) -> (usize, usize) { + if version == WAL_VERSION_V2 { + (ENTRY_FIXED_V2, RAW_KEY_LEN + FLAGS_LEN + SPACE_ID_LEN + 5) + } else { + (ENTRY_FIXED, 61) + } +} /// The log begins at the first 4 KiB boundary at or after the image. Fixed by geometry so /// no superblock is needed to find it, and stated by the v2 header's extent. /// @@ -236,6 +255,11 @@ const OP_SYNC: u8 = 3; /// ``` const BOOT_SPACE: [u8; SPACE_ID_LEN] = [0xFC; SPACE_ID_LEN]; +/// The class mask the boot record stamps: `EventFlags::BOOT`, bit 7. The kernel cannot link +/// `cube-core`, so the bit is spelled out here and on the userspace side, and the agreement is held +/// by `verify-boot-record` — the same shape as the `0xFC` space id, for the same reason. +const BOOT_FLAGS: u16 = 1 << 7; + /// The coordinate the record lives at. One record, the current boot: each boot overwrites the last, /// which is the same shape as the userspace boot marker it replaces and is what "which boot is this" /// needs. History would be a second record per boot, and nobody has asked for one. @@ -296,16 +320,58 @@ fn fnv1a64(bytes: &[u8], mut h: u64) -> u64 { /// version exposes to Rust, it goes through the page cache the way any other read does, and /// it needs no C helper. If the store ever has to be read before the VFS is up (a root /// filesystem, say), that is the moment to reach for the block layer directly. +/// The store device, opened once and kept for the whole boot. +/// +/// Every operation used to open the device, read it, and close it again — the open is the dominant +/// cost of a `cube(2)` call, and it was paid on every single one. The path is fixed at boot +/// (`cube_store=` is parsed once), so there is nothing to invalidate: the handle is opened lazily +/// on first use and reused. +/// +/// Lazy rather than at module init, for the reason the driver does nothing at init: the block +/// driver that provides the device may not be up yet, and an open at init would reintroduce the +/// ordering problem the driver's own doc says it avoided. A caller whose filesystem view cannot +/// reach the path (a chroot before its `/dev` is mounted) gets the error that open returned; every +/// later caller gets the cached handle regardless of its own root. +/// A cached store-device file. The raw pointer is kept for the whole boot — the path is fixed at +/// boot (`cube_store=` is parsed once), so there is nothing to invalidate, and the device is meant +/// to stay open. The wrapper carries the `Send`/`Sync` the raw pointer lacks, and the global lock +/// serializes access, so no two threads touch the pointer unsynchronized. +struct StoreFile(*mut bindings::file); + +// SAFETY: the pointer is valid for the whole boot (the path is fixed and the block driver does not +// unbind), and every use is through the global lock below. +unsafe impl Send for StoreFile {} +unsafe impl Sync for StoreFile {} + +kernel::sync::global_lock! { + /// SAFETY: Initialized (to None) before first use. + unsafe(uninit) static STORE_FILE: Mutex> = None; +} + +/// The store device's file, opened on first use and cached for the boot. O_RDWR, because the one +/// handle serves both the reads and the log-append / checkpoint writes. +fn store_file() -> Result<*mut bindings::file> { + let mut guard = STORE_FILE.lock(); + if let Some(file) = guard.as_ref() { + return Ok(file.0); + } + // O_RDWR is 2. SAFETY: store_device() is a NUL-terminated C string the module parameter filled + // at boot; filp_open returns a valid `struct file *` or an error pointer, checked below. + let filp = unsafe { bindings::filp_open(store_device(), 2, 0) }; + let filp = kernel::error::from_err_ptr(filp)?; + if filp.is_null() { + return Err(EINVAL); + } + *guard = Some(StoreFile(filp)); + Ok(filp) +} + fn read_image(image: &mut KVVec) -> Result<()> { // O_RDONLY is 0 in Linux; filp_open takes the raw flags word. // SAFETY: store_device() is a NUL-terminated C string that the module parameter filled at // boot, and filp_open either returns a valid `struct file *` or an error pointer, which is // checked below. - let file = unsafe { bindings::filp_open(store_device(), 0, 0) }; - let file = kernel::error::from_err_ptr(file)?; - if file.is_null() { - return Err(EINVAL); - } + let file = store_file()?; let mut pos: bindings::loff_t = 0; let mut chunk = KVVec::::with_capacity(4096, GFP_KERNEL)?; @@ -341,9 +407,6 @@ fn read_image(image: &mut KVVec) -> Result<()> { } } - // SAFETY: `file` came from filp_open and has not been closed; the owner argument is - // only meaningful for locks that no one holds here. - unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; // Less than the shortest header is not an image at all; `digest` reports the rest. if result.is_ok() && image.len() < HEADER_LEN_V1 { @@ -378,6 +441,9 @@ fn parse_header(image: &[u8]) -> Result { /// The v3 header: what a reader needs to address the image by arithmetic. struct HeaderV3 { + /// `VERSION_V3` or `VERSION_V4` — they share the geometry and differ only in the index + /// entry's stride. + version: u8, image_bytes: u64, record_count: u64, space_count: u64, @@ -386,9 +452,18 @@ struct HeaderV3 { } impl HeaderV3 { + /// The width of one index entry for this header's version. + fn stride(&self) -> usize { + if self.version == VERSION_V4 { + INDEX_ENTRY_V4 + } else { + INDEX_ENTRY + } + } + fn encode(&self, out: &mut KVVec, curve: u8) -> Result<(), AllocError> { out.extend_from_slice(MAGIC, GFP_KERNEL)?; - out.extend_from_slice(&[VERSION_V3, curve], GFP_KERNEL)?; + out.extend_from_slice(&[self.version, curve], GFP_KERNEL)?; out.extend_from_slice(&self.image_bytes.to_le_bytes(), GFP_KERNEL)?; out.extend_from_slice(&self.record_count.to_le_bytes(), GFP_KERNEL)?; out.extend_from_slice(&self.space_count.to_le_bytes(), GFP_KERNEL)?; @@ -398,7 +473,7 @@ impl HeaderV3 { } fn decode(image: &[u8]) -> Result { - // The shared file owns this: the offsets a reader computes (`index_off + i * INDEX_ENTRY`) + // The shared file owns this: the offsets a reader computes (`index_off + i * stride`) // are only places if the index really starts there and really is that wide, and that is a // statement about the format rather than about this driver. let geometry = cube_format::V3::decode(image).map_err(|bad| match bad { @@ -407,6 +482,7 @@ impl HeaderV3 { })?; let header = parse_header(image)?; Ok(HeaderV3 { + version: geometry.version, image_bytes: header.image_bytes.unwrap_or(0), record_count: geometry.record_count, space_count: geometry.space_count, @@ -685,6 +761,8 @@ fn resolve_layout(b: &[u8]) -> Result { struct Entry { space: [u8; 32], key: [u8; 24], + /// The class mask, carried from the log or the image so a fold does not lose it. + flags: u16, /// Order in which this entry arrived. Breaks ties between the same coordinate, so the /// later write wins after sorting. seq: u32, @@ -715,7 +793,7 @@ impl Merged { /// Add a record. `deleted` marks a removal, which is kept in the list so it can /// override an older value for the same coordinate. - fn add(&mut self, space: &[u8], key: &[u8], value: &[u8], deleted: bool) -> Result<(), AllocError> { + fn add(&mut self, space: &[u8], key: &[u8], flags: u16, value: &[u8], deleted: bool) -> Result<(), AllocError> { let mut sp = [0u8; 32]; sp.copy_from_slice(&space[..32]); let mut k = [0u8; 24]; @@ -728,6 +806,7 @@ impl Merged { Entry { space: sp, key: k, + flags, seq, value_off, value_len: value.len() as u32, @@ -813,13 +892,14 @@ fn apply_log(log: &[u8], merged: &mut Merged) -> Result, &'static st if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { return Ok(None); } - if log[4] != WAL_VERSION { + if log[4] != WAL_VERSION && log[4] != WAL_VERSION_V2 { return Err("unsupported-log-version"); } + let (fixed, len_at) = wal_frame(log[4]); let mut applied: u64 = 0; let mut off = WAL_HEADER_LEN; - while off + ENTRY_FIXED <= log.len() { + while off + fixed <= log.len() { let start = off; let op = log[off]; if op != 1 && op != 2 { @@ -830,21 +910,26 @@ fn apply_log(log: &[u8], merged: &mut Merged) -> Result, &'static st let crc = u32::from_le_bytes(word); let space = &log[off + 5..off + 37]; let key = &log[off + 37..off + 61]; - word.copy_from_slice(&log[off + 61..off + 65]); + let flags = if log[4] == WAL_VERSION_V2 { + cube_format::le_u16(log, off + 61) + } else { + 0 + }; + word.copy_from_slice(&log[off + len_at..off + len_at + 4]); let len = u32::from_le_bytes(word) as usize; - let frame_end = start + ENTRY_FIXED + len; + let frame_end = start + fixed + len; if frame_end > log.len() { break; } - // The checksum covers space, key, length and value, so a corrupted entry is + // The checksum covers space, key, the mask, length and value, so a corrupted entry is // stopped at rather than applied. if crc32(&log[start + 5..frame_end]) != crc { break; } - let value = &log[start + ENTRY_FIXED..frame_end]; + let value = &log[start + fixed..frame_end]; off = frame_end; merged - .add(space, key, value, op == 2) + .add(space, key, flags, value, op == 2) .map_err(|_| "out-of-memory")?; applied += 1; } @@ -892,7 +977,7 @@ fn build_merged(image: &[u8], log: &[u8], header: &Header) -> Result<(Merged, u6 // addressed one, and every fold after that reads an addressed one — so a reader that only knew // the packed layout could fold a store exactly once. Found on the box, where that is not a // hypothesis: the live store's first fold succeeded and its second answered -EINVAL. - if header.version == VERSION_V3 { + if header.version == VERSION_V3 || header.version == VERSION_V4 { let geometry = cube_format::V3::decode(image).map_err(|_| "bad-tables")?; let mut row = 0u64; while row < geometry.space_count { @@ -906,7 +991,7 @@ fn build_merged(image: &[u8], log: &[u8], header: &Header) -> Result<(Merged, u6 .value(image, &entry) .ok_or("truncated-image")?; merged - .add(space, entry.key, value, false) + .add(space, entry.key, entry.flags, value, false) .map_err(|_| "out-of-memory")?; i += 1; } @@ -943,7 +1028,7 @@ fn build_merged(image: &[u8], log: &[u8], header: &Header) -> Result<(Merged, u6 return Err("truncated-image"); } merged - .add(space, key, &image[value_at..value_at + value_len], false) + .add(space, key, 0, &image[value_at..value_at + value_len], false) .map_err(|_| "out-of-memory")?; seen += 1; off = value_at + value_len; @@ -1029,11 +1114,12 @@ fn log_valid_len(log: &[u8]) -> Result { if &log[0..4] != WAL_MAGIC { return Err("no-log"); } - if log[4] != WAL_VERSION { + if log[4] != WAL_VERSION && log[4] != WAL_VERSION_V2 { return Err("unsupported-log-version"); } + let (fixed, len_at) = wal_frame(log[4]); let mut off = WAL_HEADER_LEN; - while off + ENTRY_FIXED <= log.len() { + while off + fixed <= log.len() { let start = off; let op = log[off]; if op != 1 && op != 2 { @@ -1042,12 +1128,12 @@ fn log_valid_len(log: &[u8]) -> Result { let mut word = [0u8; 4]; word.copy_from_slice(&log[off + 1..off + 5]); let crc = u32::from_le_bytes(word); - word.copy_from_slice(&log[off + 61..off + 65]); + word.copy_from_slice(&log[off + len_at..off + len_at + 4]); let len = u32::from_le_bytes(word) as usize; // Only a validated entry moves the append point. Advancing first and checking after // counts a torn entry as part of the prefix, so the next append lands *after* the // corruption and buries it — the opposite of the rule that a torn tail is overwritten. - let frame_end = start + ENTRY_FIXED + len; + let frame_end = start + fixed + len; if frame_end > log.len() { break; } @@ -1072,21 +1158,30 @@ fn morton_encode(x: u64, y: u64, z: u64) -> [u8; 24] { cube_format::morton_key(x, y, z) } -/// A mutation to append: the byte-plane write the store contract calls `put`. +/// A mutation to append: the byte-plane write the store contract calls `put`, plus the class +/// mask the caller stamps at write time — the moment the event's class is known for certain. struct Mutation { space: [u8; 32], key: [u8; 24], + flags: u16, value: KVVec, } -/// Build a log entry: `op | crc32 | space | key | len | value`. -fn encode_entry(op: u8, space: &[u8; 32], key: &[u8; 24], value: &[u8]) -> Result, AllocError> { - let total = ENTRY_FIXED + value.len(); +/// Build a log entry: `op | crc32 | space | key | flags | len | value`. +fn encode_entry( + op: u8, + space: &[u8; 32], + key: &[u8; 24], + flags: u16, + value: &[u8], +) -> Result, AllocError> { + let total = ENTRY_FIXED_V2 + value.len(); let mut entry = KVVec::::with_capacity(total, GFP_KERNEL)?; entry.extend_from_slice(&[op], GFP_KERNEL)?; entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?; entry.extend_from_slice(space, GFP_KERNEL)?; entry.extend_from_slice(key, GFP_KERNEL)?; + entry.extend_from_slice(&flags.to_le_bytes(), GFP_KERNEL)?; entry.extend_from_slice(&(value.len() as u32).to_le_bytes(), GFP_KERNEL)?; entry.extend_from_slice(value, GFP_KERNEL)?; // The checksum covers everything after the crc field. @@ -1152,7 +1247,7 @@ fn append( m: &Mutation, op: u8, ) -> Result, Error> { - let entry = encode_entry(op, &m.space, &m.key, m.value.as_slice())?; + let entry = encode_entry(op, &m.space, &m.key, m.flags, m.value.as_slice())?; // Where it goes: after the bytes in use on a store device, after the valid prefix on a // bare image, where nothing records the length. @@ -1176,6 +1271,22 @@ fn append( (used, region.len()) } }; + // The entries this function writes are v2, so a log whose header is v1 cannot take one: + // the reader frames every entry by the header's version, and a v1 header would mis-frame + // the v2 entries (and the v1 entries already there, if a header were simply rewritten). + // A v1 log that still holds entries is folded into the image first — that is the v4 + // migration — and the entry then lands in a fresh v2 log. A v1 log that is empty only + // needs its header upgraded. + if region.len() >= WAL_HEADER_LEN && ®ion[0..4] == WAL_MAGIC && region[4] == WAL_VERSION && used > 0 { + if layout.control.is_some() { + fold_now(image, layout)?; + let (device, layout) = device_and_layout()?; + return append(&device, &layout, m, op); + } + pr_err!("cubelinux: a bare image cannot migrate its v1 log; refold it with the userspace tools\n"); + return Err(EINVAL); + } + if WAL_HEADER_LEN + used + entry.len() > capacity { pr_err!( "cubelinux: the log is full ({} of {} bytes); checkpoint before appending\n", @@ -1187,21 +1298,17 @@ fn append( // SAFETY: store_device() is a NUL-terminated C string filled at boot; filp_open returns a // valid file or an error pointer, which is checked. O_RDWR is 2. - let file = unsafe { bindings::filp_open(store_device(), 2, 0) }; - let file = kernel::error::from_err_ptr(file)?; - if file.is_null() { - return Err(EINVAL); - } + let file = store_file()?; let mut result: Result<(), Error> = Ok(()); // A log region that has never been written is zeros, not a log: give it a header, the - // same thing the userspace log does when its file does not exist. - // A log region with no header is not a log: give it one, whether this is a bare image or - // a store device, so a reader always finds the framing it expects. - if region.len() < WAL_HEADER_LEN || ®ion[0..4] != WAL_MAGIC { + // same thing the userspace log does when its file does not exist. A log region whose + // header is not v2 is upgraded, because the entry being written is v2 and the header is + // what a reader frames it by. + if region.len() < WAL_HEADER_LEN || ®ion[0..4] != WAL_MAGIC || region[4] != WAL_VERSION_V2 { let mut hdr = [0u8; WAL_HEADER_LEN]; hdr[0..4].copy_from_slice(&WAL_MAGIC); - hdr[4] = WAL_VERSION; + hdr[4] = WAL_VERSION_V2; hdr[5] = 0; // morton result = write_and_sync(file, layout.log_off as u64, &hdr); } @@ -1231,8 +1338,6 @@ fn append( } } - // SAFETY: `file` came from filp_open and has not been closed. - unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; result.map(|_| updated) } @@ -1276,10 +1381,11 @@ fn serialize_image(merged: &mut Merged, curve: u8) -> Result, AllocErr } let header = HeaderV3 { + version: VERSION_V4, record_count: winners.len() as u64, space_count: spaces.len() as u64, index_off: (HEADER_LEN_V3 + spaces.len() * SPACE_ENTRY) as u64, - values_off: (HEADER_LEN_V3 + spaces.len() * SPACE_ENTRY + winners.len() * INDEX_ENTRY) as u64, + values_off: (HEADER_LEN_V3 + spaces.len() * SPACE_ENTRY + winners.len() * INDEX_ENTRY_V4) as u64, image_bytes: 0, }; let image_bytes = header.values_off + values_len; @@ -1296,16 +1402,18 @@ fn serialize_image(merged: &mut Merged, curve: u8) -> Result, AllocErr } // The index, whose entries are the addresses: a fixed stride is what makes `index_off + - // i * INDEX_ENTRY` a place a reader can go to without reading anything before it. + // i * stride` a place a reader can go to without reading anything before it. v4's stride is + // two wider, carrying the class mask beside the address — never inside the value. let mut value_at = header.values_off; for w in winners.as_slice() { let e = &merged.entries.as_slice()[*w as usize]; out.extend_from_slice(&e.key, GFP_KERNEL)?; + out.extend_from_slice(&e.flags.to_le_bytes(), GFP_KERNEL)?; out.extend_from_slice(&value_at.to_le_bytes(), GFP_KERNEL)?; // The length is written as 8 bytes even though `Entry` carries it in 4: an index entry - // is 40 bytes and the header's `values_off` is computed from that, so a 4-byte field here - // shortens every entry by four and leaves the last ones overlapping the values. Both - // readers assume the stride the header declares, which is the point of a fixed stride. + // has a fixed stride and the header's `values_off` is computed from that, so a shorter + // field here would leave the last entries overlapping the values. Both readers assume the + // stride the header declares, which is the point of a fixed stride. out.extend_from_slice(&(e.value_len as u64).to_le_bytes(), GFP_KERNEL)?; value_at += e.value_len as u64; } @@ -1340,11 +1448,7 @@ fn checkpoint(ctl: &Control, merged: &mut Merged) -> Result { } // SAFETY: as in `append`. - let file = unsafe { bindings::filp_open(store_device(), 2, 0) }; - let file = kernel::error::from_err_ptr(file)?; - if file.is_null() { - return Err(EINVAL); - } + let file = store_file()?; let mut result = write_and_sync(file, ctl.spare_off(), new_image.as_slice()); if result.is_ok() { @@ -1359,11 +1463,35 @@ fn checkpoint(ctl: &Control, merged: &mut Merged) -> Result { result = write_and_sync(file, older_copy(ctl.generation) as u64, &buf); } - // SAFETY: `file` came from filp_open and has not been closed. - unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; result.map(|_| new_image.len() as u64) } +/// Fold the current log into the image — the shared body of `sync` and the append-time +/// migration. After it, the image is the pinned v4 shape and the log is empty, so the next +/// append starts a fresh v2 log. +fn fold_now(image: &[u8], layout: &Layout) -> Result<(), Error> { + let ctl = match layout.control { + Some(c) => c, + None => return Err(EINVAL), // a bare image has no spare slot to fold into + }; + let live = &image[layout.image_off..]; + let header = match parse_header(live) { + Ok(h) => h, + Err(_) => return Err(EINVAL), + }; + let window = log_window(image, layout); + let extent = core::cmp::min( + header.image_bytes.unwrap_or(live.len() as u64) as usize, + live.len(), + ); + let mut merged = match build_merged(&live[..extent], window, &header) { + Ok((m, _)) => m, + Err(_) => return Err(EINVAL), + }; + checkpoint(&ctl, &mut merged)?; + Ok(()) +} + /// Everything a syscall needs to reach the store: read the device, resolve its layout, and /// hand back what was asked for. /// @@ -1509,6 +1637,7 @@ fn ensure_boot_record() { let mutation = Mutation { space: BOOT_SPACE, key: morton_encode(BOOT_POINT.0, BOOT_POINT.1, BOOT_POINT.2), + flags: BOOT_FLAGS, value, }; match append(&device, &layout, &mutation, 1) { @@ -1546,6 +1675,7 @@ pub unsafe extern "C" fn cubelinux_kernel_put( let mutation = Mutation { space: sp, key, + flags: 0, value: bytes, }; let (device, layout) = match device_and_layout() { @@ -1735,18 +1865,10 @@ fn read_view() -> Result { // O_RDONLY is 0 in Linux; filp_open takes the raw flags word. // SAFETY: `store_device()` is a NUL-terminated C string the module parameter filled at boot, // and filp_open returns a valid `struct file *` or an error pointer, checked here. - let file = unsafe { bindings::filp_open(store_device(), 0, 0) }; - let file = kernel::error::from_err_ptr(file)?; - if file.is_null() { - return Err(EINVAL); - } + let file = store_file()?; let view = read_view_from(file); - // SAFETY: `file` came from filp_open and has not been closed; the owner argument is only - // meaningful for locks that nobody holds here. - unsafe { bindings::filp_close(file, core::ptr::null_mut()) }; - let view = view?; if view.image.len() < HEADER_LEN_V1 { return Err(EINVAL); @@ -1944,6 +2066,9 @@ fn log_effect<'a>(log: &'a [u8], space: &[u8; SPACE_ID_LEN], key: &[u8; RAW_KEY_ struct LogEntries<'a> { log: &'a [u8], off: usize, + /// The entry stride for this log's version, and where its length sits. + fixed: usize, + len_at: usize, } impl<'a> LogEntries<'a> { @@ -1952,7 +2077,7 @@ impl<'a> LogEntries<'a> { /// The value's *position* rather than a slice, because a caller collecting edits keeps them in /// a fixed-size vector and a borrow of the log would tie that vector's type to the log's life. fn next(&mut self) -> Option<(&'a [u8], &'a [u8], u8, usize, usize)> { - if self.off + ENTRY_FIXED > self.log.len() { + if self.off + self.fixed > self.log.len() { return None; } let start = self.off; @@ -1965,19 +2090,19 @@ impl<'a> LogEntries<'a> { let crc = u32::from_le_bytes(word); let space = &self.log[start + 5..start + 37]; let key = &self.log[start + 37..start + 61]; - word.copy_from_slice(&self.log[start + 61..start + 65]); + word.copy_from_slice(&self.log[start + self.len_at..start + self.len_at + 4]); let len = u32::from_le_bytes(word) as usize; - let frame_end = start + ENTRY_FIXED + len; + let frame_end = start + self.fixed + len; if frame_end > self.log.len() { return None; } - // The checksum covers space, key, length and value, so a torn tail is stopped at rather - // than applied — the same rule the fold uses. + // The checksum covers space, key, the mask, length and value, so a torn tail is stopped + // at rather than applied — the same rule the fold uses. if crc32(&self.log[start + 5..frame_end]) != crc { return None; } self.off = frame_end; - Some((space, key, op, start + ENTRY_FIXED, len)) + Some((space, key, op, start + self.fixed, len)) } } @@ -1986,12 +2111,15 @@ fn log_entries(log: &[u8]) -> Result, &'static str> { if log.len() < WAL_HEADER_LEN || &log[0..4] != WAL_MAGIC { return Err("no-log"); } - if log[4] != WAL_VERSION { + if log[4] != WAL_VERSION && log[4] != WAL_VERSION_V2 { return Err("unsupported-log-version"); } + let (fixed, len_at) = wal_frame(log[4]); Ok(LogEntries { log, off: WAL_HEADER_LEN, + fixed, + len_at, }) } @@ -2394,17 +2522,12 @@ struct Addressed { impl Addressed { /// Open the store and read its control block, v3 header, and log window — not its records. fn open() -> Result> { - // SAFETY: `store_device()` is a NUL-terminated C string the module parameter filled at - // boot; filp_open returns a valid `struct file *` or an error pointer, checked below. - let file = unsafe { bindings::filp_open(store_device(), 0, 0) }; - let file = kernel::error::from_err_ptr(file)?; - if file.is_null() { - return Err(EINVAL); - } + let file = store_file()?; let mut opened = Addressed { file, image_off: 0, header: HeaderV3 { + version: VERSION_V3, image_bytes: 0, record_count: 0, space_count: 0, @@ -2427,7 +2550,7 @@ impl Addressed { let mut raw = KVVec::::with_capacity(HEADER_LEN_V3, GFP_KERNEL)?; read_exact_at(file, opened.image_off, HEADER_LEN_V3, &mut raw, &mut opened.scratch)?; - if raw.as_slice()[4] != VERSION_V3 { + if raw.as_slice()[4] != VERSION_V3 && raw.as_slice()[4] != VERSION_V4 { return Ok(None); // v1 or v2: the walking reader } opened.header = HeaderV3::decode(raw.as_slice()).map_err(|what| { @@ -2481,13 +2604,19 @@ impl Addressed { /// One index entry, by its position in the index. fn index_entry(&mut self, at: u64, buf: &mut KVVec) -> Result<(u64, u64)> { - let off = self.header.index_off + at * INDEX_ENTRY as u64; - self.read_image_at(off, INDEX_ENTRY, buf)?; + let stride = self.header.stride(); + let off = self.header.index_off + at * stride as u64; + self.read_image_at(off, stride, buf)?; let entry = buf.as_slice(); + let vo = if self.header.version == VERSION_V4 { + RAW_KEY_LEN + FLAGS_LEN + } else { + RAW_KEY_LEN + }; let mut w = [0u8; 8]; - w.copy_from_slice(&entry[RAW_KEY_LEN..RAW_KEY_LEN + 8]); + w.copy_from_slice(&entry[vo..vo + 8]); let value_off = u64::from_le_bytes(w); - w.copy_from_slice(&entry[RAW_KEY_LEN + 8..INDEX_ENTRY]); + w.copy_from_slice(&entry[vo + 8..vo + 16]); Ok((value_off, u64::from_le_bytes(w))) } @@ -2498,8 +2627,9 @@ impl Addressed { let mut hi = records; while lo < hi { let mid = lo + (hi - lo) / 2; - let off = self.header.index_off + (first + mid) * INDEX_ENTRY as u64; - self.read_image_at(off, INDEX_ENTRY, &mut buf)?; + let stride = self.header.stride(); + let off = self.header.index_off + (first + mid) * stride as u64; + self.read_image_at(off, stride, &mut buf)?; let found = &buf.as_slice()[..RAW_KEY_LEN]; if found < &key[..] { lo = mid + 1; @@ -2530,8 +2660,9 @@ impl Addressed { let mut hi = records; while lo < hi { let mid = lo + (hi - lo) / 2; - let off = self.header.index_off + (first + mid) * INDEX_ENTRY as u64; - self.read_image_at(off, INDEX_ENTRY, &mut buf)?; + let stride = self.header.stride(); + let off = self.header.index_off + (first + mid) * stride as u64; + self.read_image_at(off, stride, &mut buf)?; if &buf.as_slice()[..RAW_KEY_LEN] < &key[..] { lo = mid + 1; } else { @@ -2554,14 +2685,6 @@ impl Addressed { } } -impl Drop for Addressed { - fn drop(&mut self) { - // SAFETY: `self.file` came from filp_open and is closed exactly once, here; the owner - // argument is only meaningful for locks that nobody holds. - unsafe { bindings::filp_close(self.file, core::ptr::null_mut()) }; - } -} - /// `CUBE_OP_GET` against a v3 image: the log's newest word, else a binary search. /// /// # Safety @@ -3340,6 +3463,7 @@ pub unsafe extern "C" fn cubelinux_kernel_del(space: *const u8, x: u64, y: u64, let mutation = Mutation { space: sp, key, + flags: 0, value: KVVec::new(), }; let (device, layout) = match device_and_layout() { @@ -3362,26 +3486,8 @@ pub extern "C" fn cubelinux_kernel_sync() -> i32 { Ok(pair) => pair, Err(e) => return -(e.to_errno() as i32), }; - let ctl = match layout.control { - Some(c) => c, - None => return -22, // -EINVAL: a bare image has no spare slot to fold into - }; - let live = &device[layout.image_off..]; - let header = match parse_header(live) { - Ok(h) => h, - Err(_) => return -22, - }; - let window = log_window(&device, &layout); - let extent = core::cmp::min( - header.image_bytes.unwrap_or(live.len() as u64) as usize, - live.len(), - ); - let mut merged = match build_merged(&live[..extent], window, &header) { - Ok((m, _)) => m, - Err(_) => return -22, - }; - match checkpoint(&ctl, &mut merged) { - Ok(_) => 0, + match fold_now(&device, &layout) { + Ok(()) => 0, Err(e) => -(e.to_errno() as i32), } } @@ -3485,6 +3591,7 @@ impl MiscDevice for CubeStore { let mutation = Mutation { space, key: morton_encode(x, y, z), + flags: 0, value, }; match append(&device, &layout, &mutation, 1) {