diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 9ed57e753..6315561e8 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -390,6 +390,29 @@ kernel::sync::global_lock! { unsafe(uninit) static STORE_FILE: Mutex> = None; } +/// The image's space table, held across calls. +/// +/// Every coordinate read binary-searches this table for the space's first record, and every batch of +/// a listing reads a row of it — from the device, per probe, for data that changes only when the +/// store is folded. Measured, a coordinate read costs the same in a store forty times larger while +/// the index search does nine probe reads against fifteen, so the per-call cost is setup rather than +/// device reads; a read per probe is the clearest piece of that setup left. +/// +/// Keyed on the **control block's generation**, which the fold raises along with everything else it +/// writes, and on the image offset — a fold flips the slot, so both change together and a stale table +/// cannot outlive the image it describes. Nothing else has to invalidate it: a writer that bumps the +/// generation invalidates it automatically, including a writer this driver never sees. +struct TableCache { + generation: u64, + image_off: u64, + table: KVVec, +} + +kernel::sync::global_lock! { + /// SAFETY: Initialized (to None) before first use. + unsafe(uninit) static TABLE_CACHE: Mutex> = None; +} + kernel::sync::global_lock! { /// Serialises the store's read-modify-write operations. /// @@ -2875,6 +2898,8 @@ struct Addressed { header: HeaderV3, /// The log window, which is what can override the image and is small. log: KVVec, + /// The image's space table, as it lies — filled from the cache, not from the device. + spaces: KVVec, /// Reused for every span read, so a lookup does not allocate per probe. scratch: KVVec, } @@ -2895,6 +2920,7 @@ impl Addressed { values_off: 0, }, log: KVVec::new(), + spaces: KVVec::new(), scratch: KVVec::new(), }; opened.scratch = KVVec::::with_capacity(4096, GFP_KERNEL)?; @@ -2918,6 +2944,37 @@ impl Addressed { EINVAL })?; + // The space table: the cache's copy when the store has not moved, a fresh read when it has. + let bytes = opened.header.space_count as usize * SPACE_ENTRY; + if bytes > MAX_BYTES { + return Err(EINVAL); + } + { + let mut guard = TABLE_CACHE.lock(); + if guard.is_none() { + *guard = Some(TableCache { + generation: u64::MAX, + image_off: 0, + table: KVVec::new(), + }); + } + let cache = guard.as_mut().unwrap(); + if cache.generation != control.generation || cache.image_off != opened.image_off { + read_exact_at( + file, + opened.image_off + HEADER_LEN_V3 as u64, + bytes, + &mut cache.table, + &mut opened.scratch, + )?; + cache.generation = control.generation; + cache.image_off = opened.image_off; + } + // Copied rather than borrowed: the table is `space_count * 48` bytes — a few hundred here, + // and the copy is cheaper than the reads it replaces. + opened.spaces.extend_from_slice(cache.table.as_slice(), GFP_KERNEL)?; + } + if control.log_used > 0 { let len = core::cmp::min(WAL_HEADER_LEN + control.log_used as usize, MAX_BYTES); read_exact_at(file, control.log_off() as u64, len, &mut opened.log, &mut opened.scratch)?; @@ -2946,9 +3003,8 @@ impl Addressed { let mut hi = self.header.space_count; while lo < hi { let mid = lo + (hi - lo) / 2; - let at = HEADER_LEN_V3 as u64 + mid * SPACE_ENTRY as u64; - self.read_image_at(at, SPACE_ENTRY, &mut buf)?; - let entry = buf.as_slice(); + let from = mid as usize * SPACE_ENTRY; + let entry = &self.spaces.as_slice()[from..from + SPACE_ENTRY]; let found = &entry[..SPACE_ID_LEN]; if found < &space[..] { lo = mid + 1; @@ -2975,9 +3031,8 @@ impl Addressed { if row >= self.header.space_count { return Ok(None); } - let mut buf = KVVec::::new(); - self.read_image_at(HEADER_LEN_V3 as u64 + row * SPACE_ENTRY as u64, SPACE_ENTRY, &mut buf)?; - let entry = buf.as_slice(); + let from = row as usize * SPACE_ENTRY; + let entry = &self.spaces.as_slice()[from..from + SPACE_ENTRY]; let space: [u8; SPACE_ID_LEN] = match entry[..SPACE_ID_LEN].try_into() { Ok(s) => s, Err(_) => return Err(EINVAL), @@ -4478,6 +4533,8 @@ impl kernel::InPlaceModule for CubeStoreModule { unsafe { STORE_FILE.init() }; // SAFETY: called exactly once, in the module initializer, before anything can take it. unsafe { STORE_OP.init() }; + // SAFETY: as above. + unsafe { TABLE_CACHE.init() }; try_pin_init!(Self { _miscdev <- MiscDeviceRegistration::register(MiscDeviceOptions { name: c_str!("cubelinux"),