CUBELinux-2
7ccb29aa6a
- Add persist.rs: std-only NDJSON snapshot of the HashBackend store (no serde) for the durable checkpoint + load_into_store replay. - Add store.rs: ConcurrentStore = Mutex<HashBackend> live store + WAL (newline-delimited JSON, group-commit fsync, idempotent seq-numbered replay) + durable JSON checkpoint + bg flusher + startup replay. - Recovery events (checkpoint failure, WAL fsync failure, WAL replay) are written to a recovery.ndjson you asked to keep as the written backup log, so any fall-back to JSON is recorded 'in writing'. - Refactor cube-server to thread-per-connection over ConcurrentStore. - query_doc_type / scan_prefix / linked_to / delete_raw added. Verified: ./check (fmt, 7 unit tests, clippy -D warnings) all green; ./check stress drove 22,080 prog+run pairs (~368/s) over 60s, daemon survived, latency prog~9us/run~13us mean.
367 lines
14 KiB
Rust
367 lines
14 KiB
Rust
//! Shared CUBELinux-2 system command interpreter.
|
|
//!
|
|
//! This module holds the *single* implementation of the cube command language
|
|
//! (`prog`, `write`, `run`, `ls`, `stat`, `seal`, `open`, `query`). It is used
|
|
//! by every front-end — the local `cube` REPL, the `cubec` socket client, and
|
|
//! the `cube-server` daemon — so the behaviour can never drift between them.
|
|
//!
|
|
//! A [`Session`] wraps one [`ConcurrentStore`] (a mutex-wrapped `CubeStore`
|
|
//! with a write-ahead log + scheduled checkpoint behind it). Because the store
|
|
//! is concurrent and durable, the daemon can serve many connections at once and
|
|
//! survives restarts. Each `exec` takes and returns an `Arc<ConcurrentStore>`
|
|
//! so the server can hand a cloned handle to each worker thread.
|
|
|
|
use crate::store::ConcurrentStore;
|
|
use cubecode::{CodeCell, Kind, Op, Vm};
|
|
use cubecoords::CubeHeader;
|
|
use cubecrypt::{CubeEnv, KeySlot, Selector, TransformId};
|
|
use std::collections::BTreeMap;
|
|
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
|
|
/// Per-command latency accumulator (cumulative; the daemon reports these via
|
|
/// the `stats` command). Counts and sums are exact; mean/max are derived.
|
|
#[derive(Default, Clone)]
|
|
struct CmdStat {
|
|
count: u64,
|
|
total_ns: u128,
|
|
max_ns: u128,
|
|
}
|
|
|
|
/// One cube command session: a concurrent, durable store plus the interpreter.
|
|
/// The REPL holds a transient one; the daemon shares a single `Arc<Session>`
|
|
/// across all connection threads.
|
|
pub struct Session {
|
|
store: Arc<ConcurrentStore>,
|
|
/// Total commands executed under this session (telemetry).
|
|
calls: u64,
|
|
/// Per top-level command latency histogram (command name -> stats).
|
|
per_cmd: BTreeMap<String, CmdStat>,
|
|
}
|
|
|
|
impl Default for Session {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Session {
|
|
/// A fresh, empty session over an in-memory (non-durable) store.
|
|
pub fn new() -> Self {
|
|
Session {
|
|
store: Arc::new(ConcurrentStore::memory()),
|
|
calls: 0,
|
|
per_cmd: BTreeMap::new(),
|
|
}
|
|
}
|
|
|
|
/// A session over a durable, concurrent store (used by the daemon).
|
|
pub fn with_store(store: Arc<ConcurrentStore>) -> Self {
|
|
Session {
|
|
store,
|
|
calls: 0,
|
|
per_cmd: BTreeMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Shared handle to the underlying concurrent store.
|
|
pub fn store(&self) -> Arc<ConcurrentStore> {
|
|
self.store.clone()
|
|
}
|
|
|
|
/// Snapshot of the session's telemetry: total commands serviced, per-command
|
|
/// latency distribution (mean/max in µs), and the occupancy of each `C`
|
|
/// namespace (number of records whose class axis equals `c`).
|
|
pub fn stats(&self) -> String {
|
|
let mut lines = Vec::new();
|
|
lines.push(format!("total commands serviced: {}", self.calls));
|
|
lines.push("per-command latency (µs, mean / max / count):".into());
|
|
if self.per_cmd.is_empty() {
|
|
lines.push(" (no commands timed yet)".into());
|
|
} else {
|
|
for (name, st) in &self.per_cmd {
|
|
let mean_us = if st.count > 0 {
|
|
(st.total_ns as f64) / (st.count as f64) / 1e3
|
|
} else {
|
|
0.0
|
|
};
|
|
let max_us = st.max_ns as f64 / 1e3;
|
|
lines.push(format!(
|
|
" {:<8} mean {:8.2} max {:8.2} n={}",
|
|
name, mean_us, max_us, st.count
|
|
));
|
|
}
|
|
}
|
|
let keys = self.store.keys();
|
|
let mut by_c: BTreeMap<u8, usize> = BTreeMap::new();
|
|
for k in &keys {
|
|
*by_c.entry(k.c).or_insert(0) += 1;
|
|
}
|
|
lines.push(format!("records by C namespace ({} total):", keys.len()));
|
|
if by_c.is_empty() {
|
|
lines.push(" (store empty)".into());
|
|
} else {
|
|
for (c, n) in &by_c {
|
|
let label = if *c == 0 { "Null(0)" } else { "" };
|
|
lines.push(format!(" C={c:<3} {n:>6} records {label}"));
|
|
}
|
|
}
|
|
lines.join("\n")
|
|
}
|
|
|
|
/// Execute one command line. `Ok(out)` is a (possibly multi-line) result to
|
|
/// print; `Err(e)` is a human-readable error. Also records per-command
|
|
/// latency into the session telemetry (see [`Session::stats`]).
|
|
pub fn exec(&mut self, line: &str) -> Result<String, String> {
|
|
let t0 = Instant::now();
|
|
let cmd_name = line.split_whitespace().next().unwrap_or("").to_string();
|
|
let result = self.exec_inner(line);
|
|
self.calls += 1;
|
|
let st = self.per_cmd.entry(cmd_name).or_default();
|
|
let elapsed = t0.elapsed().as_nanos();
|
|
st.count += 1;
|
|
st.total_ns += elapsed;
|
|
if elapsed > st.max_ns {
|
|
st.max_ns = elapsed;
|
|
}
|
|
result
|
|
}
|
|
|
|
/// The real interpreter (separated so [`exec`] can wrap it with timing).
|
|
fn exec_inner(&mut self, line: &str) -> Result<String, String> {
|
|
let mut it = line.split_whitespace();
|
|
let cmd = it.next().ok_or_else(|| "empty line".to_string())?;
|
|
let store = &self.store;
|
|
match cmd {
|
|
"stats" => Ok(self.stats()),
|
|
"query" => {
|
|
let dt = it
|
|
.next()
|
|
.ok_or_else(|| "query needs <doc_type>".to_string())?;
|
|
let coords = store.query_doc_type(dt);
|
|
if coords.is_empty() {
|
|
Ok(format!("query {dt} -> (no matches)"))
|
|
} else {
|
|
let names: Vec<String> =
|
|
coords.iter().map(|c| c.pack_u32().to_string()).collect();
|
|
Ok(format!(
|
|
"query {dt} -> {} matches: {}",
|
|
coords.len(),
|
|
names.join(" ")
|
|
))
|
|
}
|
|
}
|
|
"prog" => {
|
|
let path = it.next().ok_or_else(|| "prog needs <path>".to_string())?;
|
|
let mut ops: Vec<Op> = Vec::new();
|
|
while let Some(tok) = it.next() {
|
|
let arg = if takes_arg(tok) {
|
|
it.next()
|
|
.and_then(|a| a.parse::<u8>().ok())
|
|
.ok_or_else(|| format!("prog: {tok} needs a u8 argument"))?
|
|
} else {
|
|
0
|
|
};
|
|
ops.push(make_op(tok, arg)?);
|
|
}
|
|
if ops.is_empty() {
|
|
return Err("prog: no ops given".to_string());
|
|
}
|
|
let name = path.rsplit('/').next().unwrap_or(path);
|
|
let coord = store
|
|
.put_code_cell(path, Kind::Fn, name, &[], &ops)
|
|
.map_err(|e| e.to_string())?;
|
|
Ok(format!(
|
|
"wrote program {path} -> coord {} ({} ops)",
|
|
coord.pack_u32(),
|
|
ops.len()
|
|
))
|
|
}
|
|
"write" => {
|
|
let path = it.next().ok_or_else(|| "write needs <path>".to_string())?;
|
|
let bytes: Vec<u8> = it
|
|
.map(parse_byte)
|
|
.collect::<Option<_>>()
|
|
.ok_or_else(|| "write: every byte must be hex/dec 0..255".to_string())?;
|
|
let code = cubecode::decode(&bytes)
|
|
.map_err(|e| format!("bytecode decode error: {e:?}"))?;
|
|
let name = path.rsplit('/').next().unwrap_or(path);
|
|
let coord = store
|
|
.put_code_cell(path, Kind::Fn, name, &[], &code)
|
|
.map_err(|e| e.to_string())?;
|
|
Ok(format!("wrote {path} -> coord {}", coord.pack_u32()))
|
|
}
|
|
"run" => {
|
|
let path = it.next().ok_or_else(|| "run needs <path>".to_string())?;
|
|
let _coord = crate::path_to_czyx(path).map_err(|e| e.to_string())?;
|
|
let sn = store.read_snapshot();
|
|
let cell = crate::load_code_cell(&sn, path).map_err(|e| e.to_string())?;
|
|
let mut vm = Vm::new(sn);
|
|
let res = vm.run(cell.label);
|
|
let mut out = format!("run {path} => {res:?}");
|
|
if !vm.output().is_empty() {
|
|
out.push_str(&format!(
|
|
"\n trace: {}",
|
|
String::from_utf8_lossy(vm.output()).trim_end()
|
|
));
|
|
}
|
|
Ok(out)
|
|
}
|
|
"ls" => {
|
|
let dir = it.next().ok_or_else(|| "ls needs <dir>".to_string())?;
|
|
let fs = cubefs::CubeFs::new(store.read_snapshot());
|
|
let entries = fs.readdir(dir).map_err(|e| format!("ls {dir}: {e:?}"))?;
|
|
if entries.is_empty() {
|
|
Ok(format!("ls {dir} -> (empty)"))
|
|
} else {
|
|
let names: Vec<String> = entries.into_iter().map(|(n, _)| n).collect();
|
|
Ok(format!("ls {dir} -> {}", names.join(" ")))
|
|
}
|
|
}
|
|
"stat" => {
|
|
let path = it.next().ok_or_else(|| "stat needs <path>".to_string())?;
|
|
let fs = cubefs::CubeFs::new(store.read_snapshot());
|
|
let a = fs
|
|
.getattr(path)
|
|
.map_err(|e| format!("stat {path}: {e:?}"))?;
|
|
Ok(format!(
|
|
"stat {path} -> ino={} kind={:?} size={} mode={:o}",
|
|
a.ino, a.kind, a.size, a.mode
|
|
))
|
|
}
|
|
"seal" | "open" => {
|
|
let path = it.next().ok_or_else(|| format!("{cmd} needs <path>"))?;
|
|
let keyc = it
|
|
.next()
|
|
.ok_or_else(|| format!("{cmd} needs <K.Z.Y.X> key cell"))?;
|
|
let tf = it
|
|
.next()
|
|
.ok_or_else(|| format!("{cmd} needs <transform>"))?;
|
|
let coord = crate::path_to_czyx(path).map_err(|e| e.to_string())?;
|
|
let kc = parse_coord(keyc)
|
|
.ok_or_else(|| "bad key-cell coord (use C.Z.Y.X)".to_string())?;
|
|
let transform = parse_transform(tf)
|
|
.ok_or_else(|| "unknown transform (none|gcm|chacha|xts)".to_string())?;
|
|
|
|
if store.get_record(&kc).is_none() {
|
|
store.put_raw(kc, b"demo-key-material-32-bytes-long!!".to_vec());
|
|
}
|
|
let env = CubeEnv::new(
|
|
vec![KeySlot {
|
|
key_cell: kc,
|
|
transform,
|
|
salt: vec![],
|
|
}],
|
|
vec![],
|
|
);
|
|
|
|
if cmd == "seal" {
|
|
let (h, body) = store
|
|
.get_record(&coord)
|
|
.ok_or_else(|| format!("seal: no record at {path}"))?;
|
|
store
|
|
.with_mut(|s| env.put_encrypted(s, coord, Selector::Slot(0), &body, h))
|
|
.map_err(|e| format!("seal: {e:?}"))?;
|
|
// Log the re-written (encrypted) record to the WAL.
|
|
if let Some(v) = store.get_raw(&coord) {
|
|
store.log_put(coord, v);
|
|
}
|
|
Ok(format!("sealed {path} under key {} ({tf})", kc.pack_u32()))
|
|
} else {
|
|
let (_, envelope) = store
|
|
.get_record(&coord)
|
|
.ok_or_else(|| format!("open: no record at {path}"))?;
|
|
let pt = env
|
|
.open(&store.read_snapshot(), Selector::Slot(0), &envelope)
|
|
.map_err(|e| format!("open: {e:?}"))?;
|
|
let cell = CodeCell::from_record(coord, &CubeHeader::new(), &pt)
|
|
.ok_or_else(|| "open: decrypted body is not valid bytecode".to_string())?;
|
|
store.put_raw(coord, pt);
|
|
let sn = store.read_snapshot();
|
|
let mut vm = Vm::new(sn);
|
|
let res = vm.run(cell.label);
|
|
Ok(format!(
|
|
"open+run {path} (key {}) => {res:?}",
|
|
kc.pack_u32()
|
|
))
|
|
}
|
|
}
|
|
other => Err(format!("unknown command: {other}")),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Parse a byte token: decimal (`42`) or hex (`0x2a`).
|
|
fn parse_byte(t: &str) -> Option<u8> {
|
|
if let Ok(v) = t.parse::<u8>() {
|
|
return Some(v);
|
|
}
|
|
u8::from_str_radix(t.trim_start_matches("0x"), 16).ok()
|
|
}
|
|
|
|
/// Parse a coordinate `C.Z.Y.X` (decimal, allows 0 for Null space).
|
|
pub fn parse_coord(s: &str) -> Option<cubecoords::Czyx> {
|
|
let parts: Vec<&str> = s.split('.').collect();
|
|
if parts.len() != 4 {
|
|
return None;
|
|
}
|
|
let nums: Option<Vec<u8>> = parts.iter().map(|p| p.parse::<u8>().ok()).collect();
|
|
let nums = nums?;
|
|
Some(cubecoords::Czyx::new(nums[0], nums[1], nums[2], nums[3]))
|
|
}
|
|
|
|
/// Parse a cubevm op name (case-insensitive) into an [`Op`]. `arg` is the
|
|
/// operand byte for ops that take one (const/load/store/jmp/jz/jnz/call/ret/
|
|
/// syscall); it is ignored for argument-less ops.
|
|
fn make_op(t: &str, arg: u8) -> Result<Op, String> {
|
|
Ok(match t.to_ascii_lowercase().as_str() {
|
|
"nop" => Op::Nop,
|
|
"halt" => Op::Halt,
|
|
"const" => Op::Const(arg),
|
|
"load" => Op::Load(arg),
|
|
"store" => Op::Store(arg),
|
|
"add" => Op::Add,
|
|
"sub" => Op::Sub,
|
|
"mul" => Op::Mul,
|
|
"div" => Op::Div,
|
|
"mod" => Op::Mod,
|
|
"and" => Op::And,
|
|
"or" => Op::Or,
|
|
"xor" => Op::Xor,
|
|
"shl" => Op::Shl,
|
|
"shr" => Op::Shr,
|
|
"eq" => Op::Eq,
|
|
"ne" => Op::Ne,
|
|
"lt" => Op::Lt,
|
|
"gt" => Op::Gt,
|
|
"le" => Op::Le,
|
|
"ge" => Op::Ge,
|
|
"jmp" => Op::Jmp(arg),
|
|
"jz" => Op::Jz(arg),
|
|
"jnz" => Op::Jnz(arg),
|
|
"call" => Op::CallLink(arg),
|
|
"ret" => Op::Ret,
|
|
"syscall" => Op::Syscall(arg),
|
|
other => return Err(format!("prog: unknown op {other}")),
|
|
})
|
|
}
|
|
|
|
/// True for ops that consume the next token as a u8 operand.
|
|
fn takes_arg(t: &str) -> bool {
|
|
matches!(
|
|
t.to_ascii_lowercase().as_str(),
|
|
"const" | "load" | "store" | "jmp" | "jz" | "jnz" | "call" | "syscall"
|
|
)
|
|
}
|
|
|
|
/// Parse a transform token into a [`TransformId`].
|
|
pub fn parse_transform(s: &str) -> Option<TransformId> {
|
|
match s {
|
|
"none" => Some(TransformId::None),
|
|
"gcm" => Some(TransformId::Aes256Gcm),
|
|
"chacha" => Some(TransformId::ChaCha20Poly1305),
|
|
"xts" => Some(TransformId::Aes256Xts),
|
|
_ => None,
|
|
}
|
|
}
|