CUBELinux-2: WordFlags/tri-channel 16-bit flag field, tag-14 + scan_by_word_flag, cubetrace capture→replay→golden→lineage, CUBE VP-tree balling index (cubecode::ballindex), cubefs-visible WordFlags, and cube-mvw (sharded multi-writer MVCC store with per-shard WALs + group-commit + compaction, impl CubeBackend => CubeStore<MvwBackend> DB). Adds show-flags/scan-word-flags/trace-*/golden-capture CLI; ~ tests green.

2026-09-06 21:20:04 -04:00
parent 3f007a1f38
commit cc896fc336
22 changed files with 1695 additions and 7 deletions
@@ -477,6 +477,158 @@ impl Session {
                    ))
                }
            }
+            "scan-word-flags" => {
+                let a = it
+                    .next()
+                    .ok_or_else(|| "scan-word-flags needs <flag> (name or 0xbits)".to_string())?;
+                let bits = parse_word_flag(a)?;
+                let coords = store.scan_by_word_flag(bits);
+                if coords.is_empty() {
+                    Ok(format!("scan-word-flags {a} (0x{bits:04x}) -> (no matches)"))
+                } else {
+                    let names: Vec<String> =
+                        coords.iter().map(|c| c.pack_u32().to_string()).collect();
+                    Ok(format!(
+                        "scan-word-flags {a} (0x{bits:04x}) -> {} matches:\n  {}",
+                        coords.len(),
+                        names.join("\n  ")
+                    ))
+                }
+            }
+            "show-flags" => {
+                let cs = it.next().ok_or_else(|| "show-flags needs <C.Z.Y.X>".to_string())?;
+                let coord = parse_coord(cs).ok_or_else(|| format!("bad coordinate '{cs}' (want C.Z.Y.X)"))?;
+                match store.get_record(&coord) {
+                    None => Ok(format!("show-flags {cs} -> (no record)")),
+                    Some((h, body)) => Ok(format!(
+                        "show-flags {cs}\n  header_flags: 0x{:04x} {}\n  word_flags:   0x{:04x} {}\n  size: {} bytes",
+                        h.flags.bits(), header_flag_names(h.flags.bits()),
+                        h.word_flags.bits(), word_flag_names(h.word_flags.bits()),
+                        body.len()))
+                }
+            }
+            "trace-capture" => {
+                let src_s = it.next().ok_or_else(|| "trace-capture needs <source C.Z.Y.X> <target C.Z.Y.X>".to_string())?;
+                let tgt_s = it.next().ok_or_else(|| "trace-capture needs <target C.Z.Y.X>".to_string())?;
+                let src = parse_coord(src_s).ok_or_else(|| format!("bad source coord '{src_s}'"))?;
+                let tgt = parse_coord(tgt_s).ok_or_else(|| format!("bad target coord '{tgt_s}'"))?;
+                let snap = store.read_snapshot();
+                let mut vm = cubecode::Vm::new(snap);
+                let (result, trace) = vm.run_captured(src);
+                let out = vm.output().to_vec();
+                // Store the captured trace as a Kind::Layer AI-artifact record,
+                // linked back to the source cell (a linked CZYX edge).
+                let mut h = CubeHeader::new();
+                h.title = Some(format!("trace:{}", src.pack_u32()));
+                h.doc_type = Some(cubecode::Kind::Layer.as_str().to_string());
+                h.linked_records = vec![src];
+                h.refresh_flags();
+                store.put_record(tgt, &h, &cubecode::encode(&trace));
+                Ok(format!(
+                    "trace-capture {src_s} -> {tgt_s} (layer, {} ops, {:?}) output {} bytes",
+                    trace.len(),
+                    result,
+                    out.len()
+                ))
+            }
+            "trace-replay" => {
+                let tgt_s = it.next().ok_or_else(|| "trace-replay needs <C.Z.Y.X>".to_string())?;
+                let tgt = parse_coord(tgt_s).ok_or_else(|| format!("bad coord '{tgt_s}'"))?;
+                let snap = store.read_snapshot();
+                match cubecode::replay_trace(&snap, tgt) {
+                    Some((result, out)) => Ok(format!(
+                        "trace-replay {tgt_s}: {:?}\n  output ({} bytes): {}",
+                        result,
+                        out.len(),
+                        String::from_utf8_lossy(&out)
+                    )),
+                    None => Err(format!("trace-replay {tgt_s}: no valid trace record at that coord")),
+                }
+            }
+            "trace-verify" => {
+                let src_s = it.next().ok_or_else(|| "trace-verify needs <source> <trace> <golden>".to_string())?;
+                let tr_s = it.next().ok_or_else(|| "trace-verify needs <trace>".to_string())?;
+                let go_s = it.next().ok_or_else(|| "trace-verify needs <golden>".to_string())?;
+                let src = parse_coord(src_s).ok_or_else(|| format!("bad coord '{src_s}'"))?;
+                let tr = parse_coord(tr_s).ok_or_else(|| format!("bad coord '{tr_s}'"))?;
+                let go = parse_coord(go_s).ok_or_else(|| format!("bad coord '{go_s}'"))?;
+                let snap = store.read_snapshot();
+                match cubecode::verify_golden(&snap, src, tr, go) {
+                    Ok(()) => Ok(format!(
+                        "trace-verify {src_s} {tr_s} {go_s} -> PASS (replay + fresh run match golden)"
+                    )),
+                    Err(e) => Ok(format!("trace-verify {src_s} {tr_s} {go_s} -> FAIL: {e}")),
+                }
+            }
+            "trace-list" => {
+                let snap = store.read_snapshot();
+                let mut rows = Vec::new();
+                for k in snap.keys() {
+                    if let Some((h, _)) = snap.get_record(&k) {
+                        let dt = h.doc_type.clone().unwrap_or_default();
+                        let is_artifact = matches!(dt.as_str(), "layer" | "golden" | "checkpoint" | "variant")
+                            || h.title.as_deref().map_or(false, |t| t.starts_with("trace:") || t.starts_with("golden:"));
+                        if is_artifact {
+                            let links = h.linked_records
+                                .iter().map(|c| c.pack_u32().to_string()).collect::<Vec<_>>().join(",");
+                            rows.push(format!("{}  [{}]  {}  -> links: {links}",
+                                k.pack_u32(), dt, h.title.as_deref().unwrap_or("")));
+                        }
+                    }
+                }
+                if rows.is_empty() {
+                    Ok("trace-list -> (no layer/golden/checkpoint/variant/trace records)".to_string())
+                } else {
+                    Ok(format!("trace-list -> {} records:\n  {}", rows.len(), rows.join("\n  ")))
+                }
+            }
+            "trace-links" => {
+                let cs = it.next().ok_or_else(|| "trace-links needs <C.Z.Y.X> [depth]".to_string())?;
+                let coord = parse_coord(cs).ok_or_else(|| format!("bad coord '{cs}'"))?;
+                let depth = it.next().and_then(|d| d.parse::<usize>().ok()).unwrap_or(3);
+                let snap = store.read_snapshot();
+                let hops = cubecode::lineage(&snap, coord, depth);
+                if hops.is_empty() {
+                    return Ok(format!("trace-links {cs} -> (no record)"));
+                }
+                let mut lines = Vec::new();
+                for h in &hops {
+                    let links = h.links.iter().map(|c| c.pack_u32().to_string()).collect::<Vec<_>>().join(" ");
+                    lines.push(format!("  {}  [{}]  {}  -> {links}", h.coord.pack_u32(), h.doc_type, h.title));
+                }
+                Ok(format!("trace-links {cs} (depth {depth}) -> {} hops:\n{}", hops.len(), lines.join("\n")))
+            }
+            "golden-capture" => {
+                let src_s = it.next().ok_or_else(|| "golden-capture needs <source> <trace> <golden>".to_string())?;
+                let tr_s = it.next().ok_or_else(|| "golden-capture needs <trace>".to_string())?;
+                let go_s = it.next().ok_or_else(|| "golden-capture needs <golden>".to_string())?;
+                let name = it.next().unwrap_or("golden").to_string();
+                let src = parse_coord(src_s).ok_or_else(|| format!("bad coord '{src_s}'"))?;
+                let tr = parse_coord(tr_s).ok_or_else(|| format!("bad coord '{tr_s}'"))?;
+                let go = parse_coord(go_s).ok_or_else(|| format!("bad coord '{go_s}'"))?;
+                let snap = store.read_snapshot();
+                let mut vm = cubecode::Vm::new(snap);
+                let (_r, trace) = vm.run_captured(src);
+                let out = vm.output().to_vec();
+                // Trace record: Kind::Layer, linked to the source.
+                let mut h = CubeHeader::new();
+                h.title = Some(format!("trace:{name}"));
+                h.doc_type = Some(cubecode::Kind::Layer.as_str().to_string());
+                h.linked_records = vec![src];
+                h.refresh_flags();
+                store.put_record(tr, &h, &cubecode::encode(&trace));
+                // Golden output record, linked to the trace.
+                let mut hg = CubeHeader::new();
+                hg.title = Some(format!("golden:{name}"));
+                hg.doc_type = Some("golden".into());
+                hg.linked_records = vec![tr];
+                hg.refresh_flags();
+                store.put_record(go, &hg, &out);
+                Ok(format!(
+                    "golden-capture {src_s} -> {tr_s} (layer) + {go_s} (golden), output {} bytes",
+                    out.len()
+                ))
+            }
            "prog" => {
                let path = it.next().ok_or_else(|| "prog needs <path>".to_string())?;
                let mut ops: Vec<Op> = Vec::new();
@@ -1228,8 +1380,15 @@ impl Session {
                    .getattr(path)
                    .map_err(|e| format!("stat {path}: {e:?}"))?;
                Ok(format!(
-                    "stat {path} -> ino={} kind={:?} size={} mode={:o}",
-                    a.ino, a.kind, a.size, a.mode
+                    "stat {path} -> ino={} kind={:?} size={} mode={:o} word_flags=0x{:04x} {} header_flags=0x{:04x} {}",
+                    a.ino,
+                    a.kind,
+                    a.size,
+                    a.mode,
+                    a.word_flags,
+                    word_flag_names(a.word_flags),
+                    a.header_flags,
+                    header_flag_names(a.header_flags)
                ))
            }
            "keyinit" => {
@@ -1417,6 +1576,8 @@ fn header_for_code(
    h.doc_type = Some(kind.as_str().to_string());
    h.linked_records = Vec::new();
    h.owner_local_user = owner.map(|o| o.to_string());
+    // OS-created records carry per-word flags (tag 14).
+    h.word_flags = crate::default_word_flags(kind, descriptor);
    if let Some(b) = descriptor {
        h.flags.0 |= b.to_flags();
    }
@@ -1447,6 +1608,72 @@ pub fn parse_coord(s: &str) -> Option<cubecoords::Czyx> {
 }

 /// Parse a single `u8` axis token: decimal (`7`) or hex (`0x07`).
+
+fn header_flag_names(bits: u16) -> String {
+    let mut v: Vec<&str> = Vec::new();
+    if bits & cubecoords::HeaderFlags::TITLE != 0 { v.push("title"); }
+    if bits & cubecoords::HeaderFlags::DOC_TYPE != 0 { v.push("doc_type"); }
+    if bits & cubecoords::HeaderFlags::CREATED_AT != 0 { v.push("created_at"); }
+    if bits & cubecoords::HeaderFlags::SIZE_BYTES != 0 { v.push("size_bytes"); }
+    if bits & cubecoords::HeaderFlags::PERM_ROOT_ONLY != 0 { v.push("root_only"); }
+    if bits & cubecoords::HeaderFlags::PERM_LOCAL_USER != 0 { v.push("local_user"); }
+    if bits & cubecoords::HeaderFlags::PERM_REMOTE_USER != 0 { v.push("remote_user"); }
+    if bits & cubecoords::HeaderFlags::HAS_ASSOCIATIONS != 0 { v.push("associations"); }
+    if bits & cubecoords::HeaderFlags::HAS_PATH != 0 { v.push("path"); }
+    if v.is_empty() { String::new() } else { format!("({})", v.join("|")) }
+}
+
+fn word_flag_names(bits: u16) -> String {
+    let mut v: Vec<&str> = Vec::new();
+    let arr = bits & cubecoords::WordFlags::ARR_MASK;
+    if arr == 0 { v.push("tri6"); } else if arr == 1 { v.push("pair"); }
+    else if arr == 2 { v.push("triad"); } else { v.push("quad"); }
+    if bits & cubecoords::WordFlags::START_RECORD != 0 { v.push("start"); }
+    if bits & cubecoords::WordFlags::END_RECORD != 0 { v.push("end"); }
+    if bits & cubecoords::WordFlags::IS_HEADER != 0 { v.push("header"); }
+    if bits & cubecoords::WordFlags::CONTINUATION != 0 { v.push("cont"); }
+    let ty = (bits & cubecoords::WordFlags::TYPE_MASK) >> 6;
+    match ty { 0 => v.push("type=text"), 1 => v.push("type=binary"), 2 => v.push("type=code"), _ => v.push("type=ai") }
+    if bits & cubecoords::WordFlags::PERM_ROOT != 0 { v.push("root"); }
+    if bits & cubecoords::WordFlags::PERM_USER != 0 { v.push("user"); }
+    if bits & cubecoords::WordFlags::ASSOC_EDGE != 0 { v.push("assoc"); }
+    if bits & cubecoords::WordFlags::ENCRYPTED != 0 { v.push("encrypted"); }
+    if bits & cubecoords::WordFlags::COMPRESSED != 0 { v.push("compressed"); }
+    if bits & cubecoords::WordFlags::STEGO != 0 { v.push("stego"); }
+    if bits & cubecoords::WordFlags::NULL_LOOKUP != 0 { v.push("null"); }
+    if bits & cubecoords::WordFlags::CHECKSUM != 0 { v.push("checksum"); }
+    format!("({})", v.join("|"))
+}
+
+fn parse_word_flag(a: &str) -> Result<u16, String> {
+    let lower = a.to_lowercase();
+    let bits = match lower.as_str() {
+        "start" => cubecoords::WordFlags::START_RECORD,
+        "end" => cubecoords::WordFlags::END_RECORD,
+        "header" => cubecoords::WordFlags::IS_HEADER,
+        "cont" | "continuation" => cubecoords::WordFlags::CONTINUATION,
+        "text" | "type-text" => 0 << 6,
+        "binary" | "type-binary" => 1 << 6,
+        "code" | "type-code" => 2 << 6,
+        "ai" | "type-ai" => 3 << 6,
+        "root" => cubecoords::WordFlags::PERM_ROOT,
+        "user" => cubecoords::WordFlags::PERM_USER,
+        "assoc" | "assoc-edge" => cubecoords::WordFlags::ASSOC_EDGE,
+        "encrypted" | "enc" => cubecoords::WordFlags::ENCRYPTED,
+        "compressed" | "comp" => cubecoords::WordFlags::COMPRESSED,
+        "stego" => cubecoords::WordFlags::STEGO,
+        "null" | "null-lookup" => cubecoords::WordFlags::NULL_LOOKUP,
+        "checksum" => cubecoords::WordFlags::CHECKSUM,
+        other => match other.strip_prefix("0x") {
+            Some(hex) => return u16::from_str_radix(hex, 16)
+                .map_err(|_| format!("invalid hex word flag '{a}'")),
+            None => return other.parse::<u16>()
+                .map_err(|_| format!("unknown word flag '{a}'")),
+        },
+    };
+    Ok(bits)
+}
+
 fn parse_u8(t: Option<&str>, what: &str) -> Result<u8, String> {
    let t = t.ok_or_else(|| what.to_string())?;
    t.parse::<u8>()
@@ -2334,4 +2561,25 @@ mod tests {
        );
        let _ = std::fs::remove_dir_all(&dir);
    }
+
+    #[test]
+    fn header_for_code_sets_word_flags() {
+        // Fn cell with a security-sensitive descriptor -> per-word flags set.
+        let h = header_for_code(
+            Kind::Fn,
+            "f",
+            &[],
+            None,
+            Some(cubecode::Behavior::new(cubecode::Behavior::SECURITY_SENSITIVE)),
+        );
+        assert!(h.word_flags.has(cubecoords::WordFlags::START_RECORD));
+        assert!(h.word_flags.has(cubecoords::WordFlags::END_RECORD));
+        assert!(h.word_flags.has(cubecoords::WordFlags::IS_HEADER));
+        assert!(h.word_flags.has(cubecoords::WordFlags::ENCRYPTED), "sec descriptor -> ENCRYPTED");
+        assert_eq!((h.word_flags.bits() & cubecoords::WordFlags::TYPE_MASK), 0b10 << 6, "Fn -> type=code");
+        // No descriptor -> complete frame, no security bit.
+        let h2 = header_for_code(Kind::Fn, "g", &[], None, None);
+        assert!(h2.word_flags.has(cubecoords::WordFlags::START_RECORD));
+        assert!(!h2.word_flags.has(cubecoords::WordFlags::ENCRYPTED));
+    }
 }
@@ -276,6 +276,7 @@ fn write_bucket(store: &ConcurrentStore, grants: &[Grant]) {
 pub fn grant_header() -> CubeHeader {
    CubeHeader {
        flags: cubecoords::HeaderFlags(cubecoords::HeaderFlags::DOC_TYPE),
+        word_flags: cubecoords::WordFlags::default(),
        title: None,
        doc_type: Some("grant-table".to_string()),
        created_at: None,
@@ -134,6 +134,34 @@ pub fn load_code_cell<B: CubeBackend>(
 /// Build a [`CodeCell`] from parts and write it back as a record at `path`,
 /// so code written by the VM (or tooling) is visible to cubefs at that path.
 #[allow(clippy::too_many_arguments)]
+/// Per-word flags (tag 14) stamped on OS-created records: a complete single
+/// record frame + a type note (code cells = type=code) + a security note, so the
+/// store actually carries WordFlags metadata (not just stores it).
+pub(crate) fn default_word_flags(
+    kind: Kind,
+    descriptor: Option<cubecode::Behavior>,
+) -> cubecoords::WordFlags {
+    let mut wf = cubecoords::WordFlags::from_bits(
+        cubecoords::WordFlags::START_RECORD
+            | cubecoords::WordFlags::END_RECORD
+            | cubecoords::WordFlags::IS_HEADER,
+    );
+    if matches!(kind, Kind::Fn | Kind::Kernel) {
+        wf.0 |= 0b10 << 6; // type=code
+    }
+    if let Some(b) = descriptor {
+        if b.0 & cubecode::Behavior::SECURITY_SENSITIVE != 0 {
+            wf.0 |= cubecoords::WordFlags::ENCRYPTED;
+        }
+    }
+    wf
+}
+
+/// Store a CUBEVM code cell at the coordinate derived from `path`. Builds the
+/// record header (title/doc_type/owner) and stamps per-word flags (tag 14) via
+/// [`default_word_flags`], so OS-created code cells carry WordFlags metadata.
+/// Returns the resolved coordinate; used by the `prog`/`write`/`kernel`/`fn`
+/// OS commands and the `cubefs` socket backend.
 pub fn store_code_cell<B: CubeBackend>(
    store: &mut CubeStore<B>,
    path: &str,
@@ -153,6 +181,7 @@ pub fn store_code_cell<B: CubeBackend>(
    h.doc_type = Some(kind.as_str().to_string());
    h.linked_records = links.to_vec();
    h.owner_local_user = owner.map(|o| o.to_string());
+    h.word_flags = default_word_flags(kind, descriptor);
    if let Some(b) = descriptor {
        h.flags.0 |= b.to_flags();
    }
@@ -587,6 +587,12 @@ impl ConcurrentStore {
        self.inner.read().unwrap().scan_by_path(path)
    }

+    /// Query records by per-word flag (the 16-bit WordFlags metadata stripe in
+    /// the header, tag 14). Delegates to [`CubeStore::scan_by_word_flag`].
+    pub fn scan_by_word_flag(&self, flag: u16) -> Vec<Czyx> {
+        self.inner.read().unwrap().scan_by_word_flag(flag)
+    }
+
    /// The `owner_local_user` stamped on the record at `key`, if it has one.
    /// Used by owner enforcement (Task 6): a mutating command may only
    /// overwrite a record whose owner matches the session's identity owner.