feat(metatag-dirs): nested directory hierarchy via path metatag + flags
Implements the source PDF's prescribed model for directory structure: 'treat the CZYX cube as the only persistent store, with system calls that operate on CZYX records and Null-space flags rather than paths and inodes.' Nested dirs are RECONSTRUCTED from a per-record `path` metatag, so the filesystem does not need recursive inode trees. - cubecoords: add CubeHeader.path (Option<String>) + HAS_PATH flag bit (1<<8); refresh_flags() sets it. Cubestore TLV codec now serializes/deserializes the path as tag 13 (persists across checkpoint/reopen). - cubestore: scan_by_path (exact), scan_by_path_prefix (dir listing by path prefix; bare '/etc' normalized to '/etc/'), plus query_by_path/_prefix returning decoded (coord,header,body). - cubesys: `put` verb gains path=<p>; new `query-path <dir>` verb reconstructs a directory listing from metatags. - ConcurrentStore: query_by_path(_prefix) delegate to inner CubeStore. Verified live in VM: migrated /etc/passwd, /etc/network/interfaces, /usr/bin/ls at unrelated coordinates; 'query-path /etc' returns the two /etc files, '/usr/bin/ls' correctly excluded. cubestore 8/8 tests pass. Refs: OS-in-CUBE migration, query-layer prototype.
This commit is contained in:
@@ -185,6 +185,13 @@ impl HeaderFlags {
|
|||||||
pub const PERM_REMOTE_USER: u16 = 1 << 6;
|
pub const PERM_REMOTE_USER: u16 = 1 << 6;
|
||||||
/// Flag 8: has outgoing association links.
|
/// Flag 8: has outgoing association links.
|
||||||
pub const HAS_ASSOCIATIONS: u16 = 1 << 7;
|
pub const HAS_ASSOCIATIONS: u16 = 1 << 7;
|
||||||
|
/// Flag 9: carries a POSIX `path` metatag (the original filesystem path
|
||||||
|
/// the record was migrated from). Lets the associative query layer
|
||||||
|
/// reconstruct a directory hierarchy WITHOUT the filesystem needing to
|
||||||
|
/// support arbitrary recursive nesting — the path is data, addressed by
|
||||||
|
/// flag, exactly as the source PDF prescribes ("operate on CZYX records
|
||||||
|
/// and Null-space flags rather than paths and inodes").
|
||||||
|
pub const HAS_PATH: u16 = 1 << 8;
|
||||||
// Flag 255 (conceptual end-of-header) is represented out-of-band by the
|
// Flag 255 (conceptual end-of-header) is represented out-of-band by the
|
||||||
// record serializer; there is no bit for it.
|
// record serializer; there is no bit for it.
|
||||||
|
|
||||||
@@ -245,6 +252,13 @@ pub struct CubeHeader {
|
|||||||
pub owner_remote_user: Option<String>,
|
pub owner_remote_user: Option<String>,
|
||||||
/// Association links to other records (flags 5–19).
|
/// Association links to other records (flags 5–19).
|
||||||
pub linked_records: Vec<Czyx>,
|
pub linked_records: Vec<Czyx>,
|
||||||
|
/// POSIX path metatag: the original filesystem path this record was
|
||||||
|
/// migrated from (e.g. "/etc/network/interfaces"). Empty/absent means the
|
||||||
|
/// record has no path identity. When present, the `HAS_PATH` flag bit is
|
||||||
|
/// set so `scan_by_path`/`query_by_path` can recover the hierarchy from
|
||||||
|
/// metatags alone (see cubestore). This is the PDF's "path is a view over
|
||||||
|
/// coordinates + flags" made concrete.
|
||||||
|
pub path: Option<String>,
|
||||||
/// Total local accesses (from the PDF's association/permission flags).
|
/// Total local accesses (from the PDF's association/permission flags).
|
||||||
pub total_accesses: u64,
|
pub total_accesses: u64,
|
||||||
/// Total remote accesses.
|
/// Total remote accesses.
|
||||||
@@ -292,6 +306,9 @@ impl CubeHeader {
|
|||||||
if !self.linked_records.is_empty() {
|
if !self.linked_records.is_empty() {
|
||||||
f |= HeaderFlags::HAS_ASSOCIATIONS;
|
f |= HeaderFlags::HAS_ASSOCIATIONS;
|
||||||
}
|
}
|
||||||
|
if self.path.is_some() {
|
||||||
|
f |= HeaderFlags::HAS_PATH;
|
||||||
|
}
|
||||||
// Preserve spare/out-of-band flag bits (bits 8..=15) that are not
|
// Preserve spare/out-of-band flag bits (bits 8..=15) that are not
|
||||||
// derived from structured fields. This includes
|
// derived from structured fields. This includes
|
||||||
// `cubecrypt::HEADER_FLAG_ENCRYPTED` (bit 12) and the behavior-descriptor
|
// `cubecrypt::HEADER_FLAG_ENCRYPTED` (bit 12) and the behavior-descriptor
|
||||||
|
|||||||
@@ -170,6 +170,10 @@ mod record_codec {
|
|||||||
// 9 total_remote_accesses (u64 le)
|
// 9 total_remote_accesses (u64 le)
|
||||||
// 10 last_access (u64 le)
|
// 10 last_access (u64 le)
|
||||||
// 11 last_remote_access (u64 le)
|
// 11 last_remote_access (u64 le)
|
||||||
|
// 13 path (utf8) — POSIX path metatag (the original filesystem path a
|
||||||
|
// record was migrated from). Carries the directory hierarchy as DATA
|
||||||
|
// so the associative query layer can reconstruct nesting from flags.
|
||||||
|
// (Tag 12, raw flag bits, is documented at its emit site below.)
|
||||||
|
|
||||||
pub fn encode_header(h: &CubeHeader) -> Vec<u8> {
|
pub fn encode_header(h: &CubeHeader) -> Vec<u8> {
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
@@ -213,6 +217,9 @@ mod record_codec {
|
|||||||
if let Some(a) = h.last_remote_access {
|
if let Some(a) = h.last_remote_access {
|
||||||
put_u64(&mut out, 11, a);
|
put_u64(&mut out, 11, a);
|
||||||
}
|
}
|
||||||
|
if let Some(p) = &h.path {
|
||||||
|
put_utf8(&mut out, 13, p);
|
||||||
|
}
|
||||||
// Tag 12: raw flag bits. Serializes out-of-band/spare bits (e.g.
|
// Tag 12: raw flag bits. Serializes out-of-band/spare bits (e.g.
|
||||||
// `cubecrypt::HEADER_FLAG_ENCRYPTED`) that are not derived from
|
// `cubecrypt::HEADER_FLAG_ENCRYPTED`) that are not derived from
|
||||||
// structured fields, so they survive an encode/decode round-trip.
|
// structured fields, so they survive an encode/decode round-trip.
|
||||||
@@ -297,6 +304,11 @@ mod record_codec {
|
|||||||
h.last_remote_access = Some(v);
|
h.last_remote_access = Some(v);
|
||||||
b = rest;
|
b = rest;
|
||||||
}
|
}
|
||||||
|
13 => {
|
||||||
|
let (v, rest) = take_utf8(b)?;
|
||||||
|
h.path = Some(v);
|
||||||
|
b = rest;
|
||||||
|
}
|
||||||
12 => {
|
12 => {
|
||||||
if b.len() < 2 {
|
if b.len() < 2 {
|
||||||
return None;
|
return None;
|
||||||
@@ -579,6 +591,78 @@ impl<B: CubeBackend> CubeStore<B> {
|
|||||||
.filter_map(|k| self.get_record(&k).map(|(h, b)| (k, h, b)))
|
.filter_map(|k| self.get_record(&k).map(|(h, b)| (k, h, b)))
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// PDF Package 2 / OS-in-CUBE: reconstruct a *directory hierarchy from
|
||||||
|
/// metatags*. Returns every coordinate whose header carries a `path`
|
||||||
|
/// metatag equal to `path` (exact) — i.e. "the file at this path".
|
||||||
|
///
|
||||||
|
/// Combined with [`CubeStore::scan_by_path_prefix`], this lets the cube
|
||||||
|
/// answer "everything under /etc" WITHOUT the filesystem supporting
|
||||||
|
/// recursive nesting: the path is stored as a flag-addressed field, not
|
||||||
|
/// as an inode tree. This is the source PDF's prescribed model ("operate
|
||||||
|
/// on CZYX records and Null-space flags rather than paths and inodes").
|
||||||
|
pub fn scan_by_path(&self, path: &str) -> Vec<Czyx> {
|
||||||
|
let mut out: Vec<Czyx> = self
|
||||||
|
.backend
|
||||||
|
.keys()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|k| {
|
||||||
|
self.get_record(k)
|
||||||
|
.map(|(h, _)| h.path.as_deref() == Some(path))
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// "List a directory": every record whose `path` metatag is *under* the
|
||||||
|
/// given directory prefix (e.g. `scan_by_path_prefix("/etc")` returns
|
||||||
|
/// `/etc/passwd`, `/etc/network/interfaces`, ...). The prefix must end
|
||||||
|
/// with `/` or be empty to mean "all paths". This is the metatag-based
|
||||||
|
/// equivalent of `readdir` for a nested directory the FS itself cannot
|
||||||
|
/// represent as a literal inode tree.
|
||||||
|
pub fn scan_by_path_prefix(&self, dir: &str) -> Vec<Czyx> {
|
||||||
|
let norm = if dir.is_empty() {
|
||||||
|
""
|
||||||
|
} else if dir.ends_with('/') {
|
||||||
|
dir
|
||||||
|
} else {
|
||||||
|
// Treat a bare "dir" as "dir/" so "/etc" matches "/etc/passwd".
|
||||||
|
return self.scan_by_path_prefix(&format!("{dir}/"));
|
||||||
|
};
|
||||||
|
let mut out: Vec<Czyx> = self
|
||||||
|
.backend
|
||||||
|
.keys()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|k| {
|
||||||
|
self.get_record(k)
|
||||||
|
.map(|(h, _)| match &h.path {
|
||||||
|
Some(p) => p != norm && p.starts_with(norm),
|
||||||
|
None => false,
|
||||||
|
})
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(label, header, body)` for [`CubeStore::scan_by_path`].
|
||||||
|
pub fn query_by_path(&self, path: &str) -> Vec<(Czyx, CubeHeader, Vec<u8>)> {
|
||||||
|
self.scan_by_path(path)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|k| self.get_record(&k).map(|(h, b)| (k, h, b)))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(label, header, body)` for [`CubeStore::scan_by_path_prefix`].
|
||||||
|
pub fn query_by_path_prefix(&self, dir: &str) -> Vec<(Czyx, CubeHeader, Vec<u8>)> {
|
||||||
|
self.scan_by_path_prefix(dir)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|k| self.get_record(&k).map(|(h, b)| (k, h, b)))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -733,4 +817,59 @@ mod tests {
|
|||||||
assert_eq!(store.scan_by_type("state"), vec![Czyx::new(200, 1, 1, 1)]);
|
assert_eq!(store.scan_by_type("state"), vec![Czyx::new(200, 1, 1, 1)]);
|
||||||
assert!(store.scan_by_type("nonexistent").is_empty());
|
assert!(store.scan_by_type("nonexistent").is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Path metatags reconstruct a nested directory hierarchy WITHOUT the FS
|
||||||
|
/// needing recursive inode trees (source PDF: "operate on CZYX records and
|
||||||
|
/// Null-space flags rather than paths and inodes"). `/etc/passwd` and
|
||||||
|
/// `/etc/network/interfaces` live at unrelated coordinates but share the
|
||||||
|
/// `/etc` prefix, so `scan_by_path_prefix("/etc")` finds both.
|
||||||
|
#[test]
|
||||||
|
fn scan_by_path_prefix_lists_directory() {
|
||||||
|
let mut store = CubeStore::new(HashBackend::new());
|
||||||
|
let mut mk = |c: u8, path: &str, dt: &str| {
|
||||||
|
let mut h = CubeHeader::new();
|
||||||
|
h.path = Some(path.to_string());
|
||||||
|
h.doc_type = Some(dt.to_string());
|
||||||
|
h.refresh_flags();
|
||||||
|
store.put_record(Czyx::new(200, 1, c, 1), &h, b"body");
|
||||||
|
};
|
||||||
|
mk(1, "/etc/passwd", "file");
|
||||||
|
mk(2, "/etc/network/interfaces", "file");
|
||||||
|
mk(3, "/etc/hosts", "file");
|
||||||
|
mk(4, "/usr/bin/ls", "file");
|
||||||
|
|
||||||
|
// Exact match.
|
||||||
|
assert_eq!(
|
||||||
|
store.scan_by_path("/etc/passwd"),
|
||||||
|
vec![Czyx::new(200, 1, 1, 1)]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Directory listing via metatag prefix.
|
||||||
|
let etc = store.scan_by_path_prefix("/etc");
|
||||||
|
assert_eq!(etc.len(), 3);
|
||||||
|
assert!(etc.contains(&Czyx::new(200, 1, 1, 1)));
|
||||||
|
assert!(etc.contains(&Czyx::new(200, 1, 2, 1)));
|
||||||
|
assert!(etc.contains(&Czyx::new(200, 1, 3, 1)));
|
||||||
|
assert!(
|
||||||
|
!etc.contains(&Czyx::new(200, 1, 4, 1)),
|
||||||
|
"/usr must not appear under /etc"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Bare prefix (no trailing slash) is normalized to "dir/".
|
||||||
|
assert_eq!(store.scan_by_path_prefix("/etc").len(), 3);
|
||||||
|
|
||||||
|
// Empty prefix = all paths.
|
||||||
|
assert_eq!(store.scan_by_path_prefix("").len(), 4);
|
||||||
|
|
||||||
|
// The path flag bit is set so the record is also flag-discoverable.
|
||||||
|
let by_path_flag = store.scan_by_flag(cubecoords::HeaderFlags::HAS_PATH);
|
||||||
|
assert_eq!(by_path_flag.len(), 4);
|
||||||
|
|
||||||
|
// query_by_path_prefix returns decoded payloads.
|
||||||
|
let full = store.query_by_path_prefix("/etc");
|
||||||
|
assert_eq!(full.len(), 3);
|
||||||
|
assert!(full
|
||||||
|
.iter()
|
||||||
|
.all(|(_, h, _)| h.path.as_deref().unwrap().starts_with("/etc")));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -448,6 +448,35 @@ impl Session {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
"query-path" => {
|
||||||
|
// Reconstruct a directory listing from path metatags (no FS
|
||||||
|
// inode tree needed). `query-path /etc` returns every record
|
||||||
|
// whose `path` metatag is under /etc — the PDF's "operate on
|
||||||
|
// CZYX records and Null-space flags rather than paths/inodes".
|
||||||
|
let dir = it
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "query-path needs <dir-prefix>".to_string())?;
|
||||||
|
let matches = store.query_by_path_prefix(dir);
|
||||||
|
if matches.is_empty() {
|
||||||
|
Ok(format!("query-path {dir} -> (no matches)"))
|
||||||
|
} else {
|
||||||
|
let mut names: Vec<String> = matches
|
||||||
|
.iter()
|
||||||
|
.filter_map(|c| {
|
||||||
|
store.get_record(c).map(|(h, _)| {
|
||||||
|
let p = h.path.as_deref().unwrap_or("");
|
||||||
|
format!("{}:{}", c.pack_u32(), p)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
names.sort();
|
||||||
|
Ok(format!(
|
||||||
|
"query-path {dir} -> {} matches:\n {}",
|
||||||
|
names.len(),
|
||||||
|
names.join("\n ")
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
"prog" => {
|
"prog" => {
|
||||||
let path = it.next().ok_or_else(|| "prog needs <path>".to_string())?;
|
let path = it.next().ok_or_else(|| "prog needs <path>".to_string())?;
|
||||||
let mut ops: Vec<Op> = Vec::new();
|
let mut ops: Vec<Op> = Vec::new();
|
||||||
@@ -964,6 +993,10 @@ impl Session {
|
|||||||
header.doc_type = Some(v.to_string());
|
header.doc_type = Some(v.to_string());
|
||||||
} else if let Some(v) = tok.strip_prefix("title=") {
|
} else if let Some(v) = tok.strip_prefix("title=") {
|
||||||
header.title = Some(v.to_string());
|
header.title = Some(v.to_string());
|
||||||
|
} else if let Some(v) = tok.strip_prefix("path=") {
|
||||||
|
// POSIX path metatag: the original filesystem path.
|
||||||
|
// Lets scan_by_path_prefix reconstruct nesting from flags.
|
||||||
|
header.path = Some(v.to_string());
|
||||||
} else {
|
} else {
|
||||||
return Err(format!("put: unknown option {tok}"));
|
return Err(format!("put: unknown option {tok}"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -283,6 +283,7 @@ pub fn grant_header() -> CubeHeader {
|
|||||||
owner_local_user: None,
|
owner_local_user: None,
|
||||||
owner_remote_user: None,
|
owner_remote_user: None,
|
||||||
linked_records: Vec::new(),
|
linked_records: Vec::new(),
|
||||||
|
path: None,
|
||||||
total_accesses: 0,
|
total_accesses: 0,
|
||||||
total_remote_accesses: 0,
|
total_remote_accesses: 0,
|
||||||
last_access: None,
|
last_access: None,
|
||||||
|
|||||||
@@ -574,6 +574,19 @@ impl ConcurrentStore {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a directory listing from `path` metatags: every record
|
||||||
|
/// whose `path` header field is under `dir` (the PDF's "operate on CZYX
|
||||||
|
/// records and Null-space flags rather than paths and inodes"). Delegates
|
||||||
|
/// to [`CubeStore::scan_by_path_prefix`].
|
||||||
|
pub fn query_by_path_prefix(&self, dir: &str) -> Vec<Czyx> {
|
||||||
|
self.inner.read().unwrap().scan_by_path_prefix(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exact-path metatag lookup. Delegates to [`CubeStore::scan_by_path`].
|
||||||
|
pub fn query_by_path(&self, path: &str) -> Vec<Czyx> {
|
||||||
|
self.inner.read().unwrap().scan_by_path(path)
|
||||||
|
}
|
||||||
|
|
||||||
/// The `owner_local_user` stamped on the record at `key`, if it has one.
|
/// The `owner_local_user` stamped on the record at `key`, if it has one.
|
||||||
/// Used by owner enforcement (Task 6): a mutating command may only
|
/// Used by owner enforcement (Task 6): a mutating command may only
|
||||||
/// overwrite a record whose owner matches the session's identity owner.
|
/// overwrite a record whose owner matches the session's identity owner.
|
||||||
|
|||||||
Reference in New Issue
Block a user