CUBELinux.0.3: the kernel writes, and is compared while writing

0.2 read the store. This appends to it: a mutation through /dev/cubelinux is
written to the log and fsynced *before* the write is accepted, which is the
kernel's version of the contract cube_duratest.py proves for the daemon.

- the mutation comes in as the argument block the coordinate interface will
  pass: op(1) | space(32) | x(8) | y(8) | z(8) | len(4) | value[len]. The kernel
  Morton-encodes the point itself — a key written here has to be the key a
  userspace reader decodes, and that is a thing the gate would catch if it were
  merely similar;
- the append lands after the log's valid prefix, so a torn tail is overwritten
  rather than appended to, exactly as the userspace log does it;
- a log region that has never been written is zeros, not a log: the header is
  written first, the way the userspace log creates its file;
- the entry's CRC covers space, key, length and value, computed the same way.

The write is the byte plane — bytes at a coordinate, no header written beside
them — because that is what a differential comparison against userspace's
`cell put` can be exact about. The header tier sits above this.

Gate (kernel/verify-kernel-append.sh), four mutations including an empty value
and a record in a second space:

  reference : bytes=500 records=6 value_bytes=94 fnv1a64=6b679d39597a62b3
  kernel    : bytes=500 records=6 value_bytes=94 fnv1a64=6b679d39597a62b3
  folded    : bytes=500 records=6 value_bytes=94 fnv1a64=6b679d39597a62b3
  survived  : bytes=500 records=6 value_bytes=94 fnv1a64=6b679d39597a62b3

The third line is the one that matters: userspace *reads the log the kernel
wrote*, folds it, and lands on the same store. Without it the first two would
only show the kernel agreeing with itself. The fourth is a SIGKILL of the VM
with no shutdown and therefore no flush for us.

Not yet: the kernel folds nothing itself, so it still depends on a userspace
checkpoint to reclaim its log. That is the next step, and it has its own gate.

drivers/cube/cubelinux_store.rs

@@ -47,7 +47,7 @@ use kernel::{
     bindings, c_str,
     device::Device,
     fs::{File, Kiocb},
-    iov::IovIterDest,
+    iov::{IovIterDest, IovIterSource},
     miscdevice::{MiscDevice, MiscDeviceOptions, MiscDeviceRegistration},
     prelude::*,
     sync::aref::ARef,
@@ -95,6 +95,10 @@ const ENTRY_FIXED: usize = 1 + 4 + 32 + 24 + 4;
 /// no superblock is needed to find it, and stated by the v2 header's extent.
 const LOG_ALIGN: usize = 4096;
 
+/// The one mutation this device accepts so far. The coordinate interface's opcodes will be
+/// the language's verbs; this is the first of them.
+const OP_PUT: u8 = 1;
+
 /// FNV-1a offset basis.
 const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
 
@@ -588,6 +592,198 @@ fn merged_digest(image: &[u8], image_bytes: usize, header: &Header) -> Result<Li
     Ok(line)
 }
 
+/// Morton-encode a point into the store's 24-byte key — the same interleaving
+/// `cube-core`'s `Curve for Morton` does, because a key written here has to be the key a
+/// userspace reader decodes. Bit `i` of each axis lands at bit `3i` of the 192-bit key,
+/// counting from the least significant bit, which lives in the *last* byte.
+fn morton_encode(x: u64, y: u64, z: u64) -> [u8; 24] {
+    let mut k = [0u8; 24];
+    let put = |n: usize, v: bool, k: &mut [u8; 24]| {
+        if !v {
+            return;
+        }
+        let byte = 23 - (n / 8);
+        k[byte] |= 1 << (n % 8);
+    };
+    for i in 0..64 {
+        put(3 * i, (x >> i) & 1 != 0, &mut k);
+        put(3 * i + 1, (y >> i) & 1 != 0, &mut k);
+        put(3 * i + 2, (z >> i) & 1 != 0, &mut k);
+    }
+    k
+}
+
+/// Where the log region starts, given the image's declared extent.
+fn log_offset(image_bytes: u64) -> usize {
+    let n = image_bytes as usize;
+    (n + LOG_ALIGN - 1) & !(LOG_ALIGN - 1)
+}
+
+/// Length of the log's valid prefix: the header plus every entry that parses and passes its
+/// checksum. This is the append position, and the reason a torn tail is overwritten rather
+/// than appended to — the same rule the userspace log uses.
+fn log_valid_len(log: &[u8]) -> Result<usize, &'static str> {
+    if log.len() < WAL_HEADER_LEN {
+        return Err("short-log");
+    }
+    if &log[0..4] != WAL_MAGIC {
+        return Err("no-log");
+    }
+    if log[4] != WAL_VERSION {
+        return Err("unsupported-log-version");
+    }
+    let mut off = WAL_HEADER_LEN;
+    while off + ENTRY_FIXED <= log.len() {
+        let start = off;
+        let op = log[off];
+        if op != 1 && op != 2 {
+            break;
+        }
+        let mut word = [0u8; 4];
+        word.copy_from_slice(&log[off + 1..off + 5]);
+        let crc = u32::from_le_bytes(word);
+        word.copy_from_slice(&log[off + 61..off + 65]);
+        let len = u32::from_le_bytes(word) as usize;
+        off += ENTRY_FIXED;
+        if off + len > log.len() {
+            break;
+        }
+        if crc32(&log[start + 5..off + len]) != crc {
+            break;
+        }
+        off += len;
+    }
+    Ok(off)
+}
+
+/// A mutation to append: the byte-plane write the store contract calls `put`.
+struct Mutation {
+    space: [u8; 32],
+    key: [u8; 24],
+    value: KVVec<u8>,
+}
+
+/// Append a mutation to the log and make it durable.
+///
+/// The order is the durability contract: the entry is written and flushed to the device
+/// *before* the caller is told it happened. A mutation this function has returned `Ok` for
+/// survives a power cut; one it has not may or may not, which is exactly the boundary an
+/// acknowledgement is supposed to mark.
+fn append(image_bytes: u64, m: &Mutation) -> Result<(), Error> {
+    let log_off = log_offset(image_bytes);
+
+    // SAFETY: STORE_DEVICE is a NUL-terminated literal; filp_open returns a valid file or an
+    // error pointer, which is checked. O_RDWR is 2.
+    let file = unsafe { bindings::filp_open(STORE_DEVICE.as_ptr().cast::<u8>(), 2, 0) };
+    let file = kernel::error::from_err_ptr(file)?;
+    if file.is_null() {
+        return Err(EINVAL);
+    }
+
+    let mut result = Ok(());
+    // The region's header and existing entries, so the append lands after the valid prefix.
+    let mut head = [0u8; 4096];
+    let mut pos: bindings::loff_t = log_off as bindings::loff_t;
+    // SAFETY: `file` is live; `head` is a 4 KiB buffer we own; `pos` is a valid loff_t.
+    let n = unsafe {
+        bindings::kernel_read(
+            file,
+            head.as_mut_ptr().cast::<core::ffi::c_void>(),
+            head.len(),
+            &mut pos,
+        )
+    };
+    if n < 0 {
+        result = Err(Error::from_errno(n as i32));
+    }
+
+    if result.is_ok() {
+        // A log region that has never been written is zeros, not a log. Give it a header
+        // first — the same thing the userspace log does when its file does not exist.
+        let region = &head[..n as usize];
+        let valid = if region.len() < WAL_HEADER_LEN || &region[0..4] != WAL_MAGIC {
+            let mut hdr = [0u8; WAL_HEADER_LEN];
+            hdr[0..4].copy_from_slice(&WAL_MAGIC);
+            hdr[4] = WAL_VERSION;
+            hdr[5] = 0; // morton
+            let mut at: bindings::loff_t = log_off as bindings::loff_t;
+            // SAFETY: `file` is live and writable; `hdr` outlives the call; `at` is valid.
+            let wrote = unsafe {
+                bindings::kernel_write(
+                    file,
+                    hdr.as_ptr().cast::<core::ffi::c_void>(),
+                    hdr.len(),
+                    &mut at,
+                )
+            };
+            if wrote < 0 {
+                result = Err(Error::from_errno(wrote as i32));
+            }
+            WAL_HEADER_LEN
+        } else {
+            match log_valid_len(region) {
+                Ok(v) => v,
+                Err(what) => {
+                    pr_err!("cubelinux: log region unreadable: {}\n", what);
+                    result = Err(EINVAL);
+                    WAL_HEADER_LEN
+                }
+            }
+        };
+        if result.is_ok() {
+        match Ok::<usize, &'static str>(valid) {
+            Ok(valid) => {
+                // Build the entry: op | crc | space | key | len | value.
+                let total = ENTRY_FIXED + m.value.len();
+                let mut entry = KVVec::<u8>::with_capacity(total, GFP_KERNEL)?;
+                entry.extend_from_slice(&[1u8], GFP_KERNEL)?;
+                entry.extend_from_slice(&[0u8; 4][..], GFP_KERNEL)?;
+                entry.extend_from_slice(&m.space, GFP_KERNEL)?;
+                entry.extend_from_slice(&m.key, GFP_KERNEL)?;
+                entry.extend_from_slice(&(m.value.len() as u32).to_le_bytes(), GFP_KERNEL)?;
+                entry.extend_from_slice(m.value.as_slice(), GFP_KERNEL)?;
+                // The checksum covers everything after the crc field.
+                let crc = crc32(&entry.as_slice()[5..]);
+                entry.as_mut_slice()[1..5].copy_from_slice(&crc.to_le_bytes());
+
+                let mut at: bindings::loff_t = (log_off + valid) as bindings::loff_t;
+                // SAFETY: `file` is live and opened for writing; `entry` is a kernel buffer
+                // that outlives the call; `at` is a valid loff_t.
+                let written = unsafe {
+                    bindings::kernel_write(
+                        file,
+                        entry.as_slice().as_ptr().cast::<core::ffi::c_void>(),
+                        entry.len(),
+                        &mut at,
+                    )
+                };
+                if written < 0 {
+                    result = Err(Error::from_errno(written as i32));
+                } else if written as usize != entry.len() {
+                    result = Err(EIO);
+                } else {
+                    // Durability before acknowledgement: a short write that was never
+                    // flushed is a write that did not happen.
+                    // SAFETY: `file` is live and writable; datasync is 0 (full sync).
+                    let rc = unsafe { bindings::vfs_fsync(file, 0) };
+                    if rc < 0 {
+                        result = Err(Error::from_errno(rc));
+                    }
+                }
+            }
+            Err(what) => {
+                pr_err!("cubelinux: log region unreadable: {}\n", what);
+                result = Err(EINVAL);
+            }
+        }
+        }
+    }
+
+    // SAFETY: `file` came from filp_open and has not been closed.
+    unsafe { bindings::filp_close(file, core::ptr::null_mut()) };
+    result
+}
+
 /// The module's registration; holds the misc device for as long as the module lives.
 #[pin_data]
 struct CubeStoreModule {
@@ -624,6 +820,82 @@ impl MiscDevice for CubeStore {
         )
     }
 
+    /// Write one mutation, framed as the argument block the coordinate interface would
+    /// pass: `op(1) | space(32) | x(8) | y(8) | z(8) | len(4) | value[len]`, little-endian.
+    ///
+    /// This is the byte plane — a `put` of bytes at a coordinate, with no header written
+    /// beside them — which is what the store contract defines and what the differential
+    /// gate compares against userspace's `cell put`. The header tier sits above this.
+    fn write_iter(kiocb: Kiocb<'_, Self::Ptr>, iov: &mut IovIterSource<'_>) -> Result<usize> {
+        let mut buf = KVVec::<u8>::new();
+        let len = iov.copy_from_iter_vec(&mut buf, GFP_KERNEL)?;
+        let bytes = buf.as_slice();
+        if bytes.len() < 1 + 32 + 24 + 4 {
+            return Err(EINVAL);
+        }
+        if bytes[0] != OP_PUT {
+            return Err(EINVAL);
+        }
+        let mut space = [0u8; 32];
+        space.copy_from_slice(&bytes[1..33]);
+        let axis = |at: usize| -> u64 {
+            let mut w = [0u8; 8];
+            w.copy_from_slice(&bytes[at..at + 8]);
+            u64::from_le_bytes(w)
+        };
+        let (x, y, z) = (axis(33), axis(41), axis(49));
+        let mut w = [0u8; 4];
+        w.copy_from_slice(&bytes[57..61]);
+        let value_len = u32::from_le_bytes(w) as usize;
+        if bytes.len() < 61 + value_len {
+            return Err(EINVAL);
+        }
+
+        let mut value = KVVec::<u8>::new();
+        value.extend_from_slice(&bytes[61..61 + value_len], GFP_KERNEL)?;
+        let mutation = Mutation {
+            space,
+            key: morton_encode(x, y, z),
+            value,
+        };
+
+        // The image's extent says where the log begins. Reading it is not optional: the log
+        // has no fixed place of its own, on purpose — no superblock to keep in step.
+        let mut head = KVVec::<u8>::new();
+        read_image(&mut head)?;
+        let header = match parse_header(&head) {
+            Ok(h) => h,
+            Err(what) => {
+                pr_err!("cubelinux: cannot append: {}\n", what);
+                return Err(EINVAL);
+            }
+        };
+        let image_bytes = match header.image_bytes {
+            Some(n) => n,
+            None => {
+                pr_err!("cubelinux: cannot append to a v1 store (no extent for the log)\n");
+                return Err(EINVAL);
+            }
+        };
+
+        let me = kiocb.file();
+        match append(image_bytes, &mutation) {
+            Ok(()) => {
+                dev_info!(
+                    me.dev,
+                    "cubelinux: appended {} bytes at the log ({} value bytes)\n",
+                    ENTRY_FIXED + value_len,
+                    value_len
+                );
+                Ok(len)
+            }
+            Err(e) => {
+                dev_err!(me.dev, "cubelinux: append failed: {:?}\n", e);
+                Err(e)
+            }
+        }
+    }
+
     fn read_iter(mut kiocb: Kiocb<'_, Self::Ptr>, iov: &mut IovIterDest<'_>) -> Result<usize> {
         // The work happens here, on demand, rather than at init: by the time anything can
         // open this device, the block driver that provides the store is certainly up.