diff --git a/drivers/cube/cubelinux_store.rs b/drivers/cube/cubelinux_store.rs index 6315561e8..3ff1e1f1f 100644 --- a/drivers/cube/cubelinux_store.rs +++ b/drivers/cube/cubelinux_store.rs @@ -405,7 +405,18 @@ kernel::sync::global_lock! { struct TableCache { generation: u64, image_off: u64, + /// The v3/v4 header, so a call that finds the store unmoved does not read it again. + header: HeaderV3, + /// Where the log is and how much of it is in use — from the same control block the generation + /// came from, so a cache hit needs no second read to find the log. + log_off: u64, + log_used: u64, table: KVVec, + /// The two control-block heads, and a scratch to read them with: both persist so that validating + /// a call costs no allocation. A 4096-byte scratch here is what the whole block used to need; + /// 96 bytes of head is all the validation ever wanted. + heads: KVVec, + scratch: KVVec, } kernel::sync::global_lock! { @@ -528,6 +539,11 @@ fn parse_header(image: &[u8]) -> Result { } /// The v3 header: what a reader needs to address the image by arithmetic. +/// +/// `Copy` because it is six plain numbers and every reader of it wants a value, not a borrow — a +/// cached header has to be handed out by value, and nothing here is big enough to be worth +/// borrowing. +#[derive(Clone, Copy)] struct HeaderV3 { /// `VERSION_V3` or `VERSION_V4` — they share the geometry and differ only in the index /// entry's stride. @@ -2905,20 +2921,65 @@ struct Addressed { } impl Addressed { - /// Open the store and read its control block, v3 header, and log window — not its records. + /// Open the store: validate it in 96 bytes, and read the rest only when it has moved. + /// + /// The control block is 4096 bytes and the question every call asks of it is one number — has + /// this store changed since I last looked. That number, and the fields beside it, are in the + /// first `CTL_SUMMED + 4` bytes of each copy, so a call that finds the store unmoved reads two + /// heads and nothing else: the header, the space table and the layout come from the cache. + /// Measured, the per-call cost of a read is setup rather than device reads, and this is the + /// largest single piece of that setup. + /// + /// **Both heads are read**, and that is the whole safety of it: a write raises one copy's + /// generation and leaves the other at the old one, so reading only one copy and finding it + /// unchanged would call a moved store unmoved. The pair is what makes the answer true. fn open() -> Result> { let file = store_file()?; + + let mut guard = TABLE_CACHE.lock(); + if guard.is_none() { + *guard = Some(TableCache { + generation: u64::MAX, + image_off: 0, + header: HeaderV3 { + version: VERSION_V3, + image_bytes: 0, + record_count: 0, + space_count: 0, + index_off: 0, + values_off: 0, + }, + log_off: 0, + log_used: 0, + table: KVVec::new(), + heads: KVVec::new(), + scratch: KVVec::new(), + }); + } + let cache = guard.as_mut().unwrap(); + if cache.scratch.len() == 0 { + cache.scratch = KVVec::::with_capacity(4096, GFP_KERNEL)?; + cache.scratch.resize(4096, 0, GFP_KERNEL)?; + } + + let head_len = CTL_SUMMED + 4; + read_exact_at(file, CTL_COPY_A as u64, head_len, &mut cache.heads, &mut cache.scratch)?; + let gen_a = Control::decode(cache.heads.as_slice()).map(|c| c.generation); + read_exact_at(file, CTL_COPY_B as u64, head_len, &mut cache.heads, &mut cache.scratch)?; + let gen_b = Control::decode(cache.heads.as_slice()).map(|c| c.generation); + let generation = match (gen_a, gen_b) { + (Some(x), Some(y)) => core::cmp::max(x, y), + (Some(x), None) => x, + (None, Some(y)) => y, + // Neither copy decodes: there is no control block, so this is a bare image and the + // walking reader handles it. + (None, None) => return Ok(None), + } as u64; + let mut opened = Addressed { file, - image_off: 0, - header: HeaderV3 { - version: VERSION_V3, - image_bytes: 0, - record_count: 0, - space_count: 0, - index_off: 0, - values_off: 0, - }, + image_off: cache.image_off, + header: cache.header, log: KVVec::new(), spaces: KVVec::new(), scratch: KVVec::new(), @@ -2926,15 +2987,25 @@ impl Addressed { opened.scratch = KVVec::::with_capacity(4096, GFP_KERNEL)?; opened.scratch.resize(4096, 0, GFP_KERNEL)?; - let mut head = KVVec::::with_capacity(CTL_COPY_B + CTL_COPY_LEN, GFP_KERNEL)?; - read_exact_at(file, 0, CTL_COPY_B + CTL_COPY_LEN, &mut head, &mut opened.scratch)?; - if &head.as_slice()[0..4] != CTL_MAGIC.as_slice() { - return Ok(None); // a bare image, which the walking reader handles + if cache.generation == generation { + // The store has not moved: what describes it is what we already hold. The log is still + // read, because it is the one thing that changes without a fold — and a write raises the + // generation, so a log newer than this cache is a log this branch does not see. + opened.spaces.extend_from_slice(cache.table.as_slice(), GFP_KERNEL)?; + if cache.log_used > 0 { + let len = core::cmp::min(WAL_HEADER_LEN + cache.log_used as usize, MAX_BYTES); + read_exact_at(file, cache.log_off, len, &mut opened.log, &mut opened.scratch)?; + } + return Ok(Some(opened)); } + + // The store moved, or this is the first call: read what describes it, once, and keep it. + let mut head = KVVec::::new(); + read_exact_at(file, 0, CTL_COPY_B + CTL_COPY_LEN, &mut head, &mut opened.scratch)?; let control = read_control(head.as_slice()).ok_or(EINVAL)?; opened.image_off = control.image_off() as u64; - let mut raw = KVVec::::with_capacity(HEADER_LEN_V3, GFP_KERNEL)?; + let mut raw = KVVec::::new(); read_exact_at(file, opened.image_off, HEADER_LEN_V3, &mut raw, &mut opened.scratch)?; if raw.as_slice()[4] != VERSION_V3 && raw.as_slice()[4] != VERSION_V4 { return Ok(None); // v1 or v2: the walking reader @@ -2944,41 +3015,31 @@ impl Addressed { EINVAL })?; - // The space table: the cache's copy when the store has not moved, a fresh read when it has. let bytes = opened.header.space_count as usize * SPACE_ENTRY; if bytes > MAX_BYTES { return Err(EINVAL); } - { - let mut guard = TABLE_CACHE.lock(); - if guard.is_none() { - *guard = Some(TableCache { - generation: u64::MAX, - image_off: 0, - table: KVVec::new(), - }); - } - let cache = guard.as_mut().unwrap(); - if cache.generation != control.generation || cache.image_off != opened.image_off { - read_exact_at( - file, - opened.image_off + HEADER_LEN_V3 as u64, - bytes, - &mut cache.table, - &mut opened.scratch, - )?; - cache.generation = control.generation; - cache.image_off = opened.image_off; - } - // Copied rather than borrowed: the table is `space_count * 48` bytes — a few hundred here, - // and the copy is cheaper than the reads it replaces. - opened.spaces.extend_from_slice(cache.table.as_slice(), GFP_KERNEL)?; - } + read_exact_at( + file, + opened.image_off + HEADER_LEN_V3 as u64, + bytes, + &mut cache.table, + &mut opened.scratch, + )?; + // Copied rather than borrowed: the table is `space_count * 48` bytes — a few hundred here, + // and the copy is cheaper than the reads it replaces. + opened.spaces.extend_from_slice(cache.table.as_slice(), GFP_KERNEL)?; if control.log_used > 0 { let len = core::cmp::min(WAL_HEADER_LEN + control.log_used as usize, MAX_BYTES); read_exact_at(file, control.log_off() as u64, len, &mut opened.log, &mut opened.scratch)?; } + + cache.generation = generation; + cache.image_off = opened.image_off; + cache.header = opened.header; + cache.log_off = control.log_off(); + cache.log_used = control.log_used; Ok(Some(opened)) } @@ -2995,10 +3056,8 @@ impl Addressed { /// The table is sorted by space and fixed-size, so this is a binary search over arithmetic /// addresses — the same reason the index works. fn space_entry(&mut self, space: &[u8; SPACE_ID_LEN]) -> Result> { - // One buffer for the whole search, not one per probe: a binary search over a thousand - // spaces allocates a thousand times otherwise, and an allocator stall is a latency spike - // with no bound on it — the opposite of what a control loop is allowed to see. - let mut buf = KVVec::::new(); + // No buffer and no read: the table is held across calls, so this search is arithmetic over + // memory. It used to allocate once per search and read the device per probe. let mut lo = 0u64; let mut hi = self.header.space_count; while lo < hi {