#!/usr/bin/env bash
# Canonical verification for the CUBELinux-2 workspace.
#
#   ./check          full gate (fmt, tests, clippy -D warnings)
#   ./check quick    tests only
#   ./check mount    full gate + live FUSE mount end-to-end (needs root)
#   ./check bench    full gate + cube-bench correctness-gated microbenchmarks
#   ./check stress   full gate + ~150s sustained stress against a real daemon
#                     (override duration with STRESS_SECONDS=NN)
#
# Exit 0 means the tree on disk is green. This is the single source of truth;
# do not claim verification from an ad-hoc run.
#
# Why `mount` is opt-in: it needs root, /dev/fuse, and the `attr` package, and
# it is the only stage that exercises the real kernel VFS path. Two Package 3
# defects (mkdir -p to depth 4, cross-user ACLs) passed every unit test and
# were caught only here — so run it before calling filesystem work done.
# Why `bench` is opt-in: it is a release build + timed run of cube-bench, which
# is slow and machine-noise-sensitive; it asserts correctness on every path but
# the numbers are informational, not a pass/fail gate.
# Why `stress` is opt-in: it spins a fresh daemon built from this tree and drives
# it hard for ~150s to exercise the per-command latency + per-C telemetry under
# load; the numbers are informational, and it obviously takes ~150s to run.
set -euo pipefail
cd "$(dirname "$0")"

command -v cargo >/dev/null || PATH="$HOME/.cargo/bin:$PATH"
command -v cargo >/dev/null || { echo "cargo not found; add ~/.cargo/bin to PATH" >&2; exit 127; }

FEAT="--features cubefs/mount"   # the FUSE adapter must compile in every gate
JOBS="${JOBS:-4}"                # never saturate all 8 cores by default

step() { printf '\n\033[1m### %s\033[0m\n' "$1"; }

if [ "${1:-}" = quick ]; then
  step "tests"
  exec cargo test --workspace $FEAT --jobs "$JOBS"
fi

step "1/3  format"
cargo fmt --all -- --check

step "2/3  tests"
cargo test --workspace $FEAT --jobs "$JOBS"

step "3/3  clippy"
cargo clippy --workspace --all-targets $FEAT --jobs "$JOBS" -- -D warnings

[ "${1:-}" = mount ]  && { run_mount=1; }
[ "${1:-}" = bench ]  && { run_bench=1; }
[ "${1:-}" = stress ] && { run_stress=1; }
[ "${1:-}" = daemon ] && { run_daemon=1; }

[ -z "${run_mount:-}${run_bench:-}${run_stress:-}${run_daemon:-}" ] && {
  printf '\n\033[32mALL CHECKS PASSED\033[0m  (run: ./check mount | bench | stress | daemon for those stages)\n'
  exit 0
}

if [ "${run_daemon:-}" ]; then
  step "4/4  daemon-backed integration tests (live cube-server)"
  # Spin up a real daemon on a temp socket/store, then run the `#[ignore]`d
  # tests that require CUBE_SOCK. Tears the daemon down afterwards.
  DSOCK=$(mktemp -u /tmp/cube-check-daemon.XXXXXX.sock)
  DDIR=$(mktemp -d /tmp/cube-check-daemon.XXXXXX)
  DSTORE=$DDIR/cube-store.json
  ./target/debug/cube-server --socket "$DSOCK" --store "$DSTORE" \
    --allow-anonymous --checkpoint-ms 400 --wal-fsync-ms 30 \
    >/tmp/cube-check-daemon.log 2>&1 &
  DPID=$!
  for i in $(seq 1 30); do [ -S "$DSOCK" ] && break; sleep 0.3; done
  if [ ! -S "$DSOCK" ]; then
    echo "DAEMON FAILED to start:"; cat /tmp/cube-check-daemon.log
    exit 1
  fi
  export CUBE_SOCK="$DSOCK"
  cargo test -p cubefs --features cubefs/mount \
    --test cubefs_daemon_smoke --test daemon_backend_smoke \
    -- --ignored --nocapture
  RC=$?
  kill -9 "$DPID" 2>/dev/null || true
  rm -rf "$DDIR" 2>/dev/null || true
  [ "$RC" -eq 0 ] || exit "$RC"
  printf '\n\033[32mALL CHECKS PASSED\033[0m (incl. daemon-backed tests)\n'
  exit 0
fi

if [ "${run_bench:-}" ]; then
  step "4/4  cube-bench"
  # Release build + timed run. Asserts correctness on every path; numbers are
  # informational. JOBS kept modest so we don't saturate the box.
  cargo run -p cube-bench --release --jobs "$JOBS"
  printf '\n\033[32mALL CHECKS PASSED\033[0m (incl. cube-bench)\n'
  exit 0
fi

if [ "${run_stress:-}" ]; then
  step "4/4  sustained stress ($((${STRESS_SECONDS:-150}))s) against a fresh daemon"
  STRESS_SECONDS="${STRESS_SECONDS:-150}" ./tools/stress.sh
  printf '\n\033[32mALL CHECKS PASSED\033[0m (incl. sustained stress)\n'
  exit 0
fi

step "4/4  live FUSE mount (in-memory --seed AND daemon-backed --socket)"
[ "$(id -u)" -eq 0 ] || { echo "SKIP: live mount needs root" >&2; exit 1; }
command -v getfattr >/dev/null || { echo "missing getfattr: apt-get install attr" >&2; exit 1; }

cargo build $FEAT --jobs "$JOBS"
BIN=$PWD/target/debug/cubefs-mount

# --- shared assertion helpers ---------------------------------------------
PASS=0; FAIL=0
ok()  { printf 'PASS %s\n' "$1"; PASS=$((PASS+1)); }
no()  { printf '\033[31mFAIL %s -- %s\033[0m\n' "$1" "${2-}"; FAIL=$((FAIL+1)); }
is()  { [ "$2" = "$3" ] && ok "$1" || no "$1" "want[$3] got[$2]"; }
# A command that must fail (permission, ENOTEMPTY, EINVAL...).
nay() { local m=$1; shift; "$@" >/dev/null 2>&1 && no "$m" "unexpectedly succeeded" || ok "$m"; }

# run_fs_tests MNT  — the regression suite, backend-agnostic.
# Establishes its own baseline record (does NOT assume any seeded data), so it
# works identically against an in-memory "--seed" mount or a daemon-backed
# "--socket" mount (the latter starts with an empty store).
run_fs_tests() {
  local MNT=$1
  local F=$MNT/c001/z001/y001/x001
  # Establish the baseline file ourselves (x001 AND x002, so the unlink
  # assertion holds on both backends — the in-memory seed mount also has both).
  echo "hello from the cube" > "$F"
  echo "second seeded record" > "$MNT/c001/z001/y001/x002"
  is "seed read"      "$(cat $F)"                                   "hello from the cube"
  # root listing: both backends guarantee at least c001 (the baseline file's
  # coordinate). The in-memory --seed mount also seeds c002, but the daemon
  # store starts empty, so we assert c001 presence, not an exact set.
  is "root listing" "$(ls $MNT | tr '\n' ' ' | grep -o c001 | head -1)" "c001"
  is "stat size"      "$(stat -c %s $F)"                            "20"
  is "partial read"   "$(dd if=$F bs=1 skip=6 count=4 2>/dev/null)" "from"

  echo posix-write > $MNT/c001/z001/y001/x003
  is "write+read"     "$(cat $MNT/c001/z001/y001/x003)" "posix-write"
  truncate -s 5 $MNT/c001/z001/y001/x003
  is "truncate"       "$(cat $MNT/c001/z001/y001/x003)" "posix"
  printf XY >> $MNT/c001/z001/y001/x003
  is "append"         "$(cat $MNT/c001/z001/y001/x003)" "posixXY"
  rm $MNT/c001/z001/y001/x003
  is "unlink"         "$(ls $MNT/c001/z001/y001 | tr '\n' ' ')" "x001 x002 "

  # Regression: empty directories must be representable, or mkdir -p can never
  # reach depth 4 (the kernel's revalidating lookup returned ENOENT).
  mkdir -p $MNT/c050/z001/y001 && ok "mkdir -p depth3" || no "mkdir -p depth3"
  is "empty dir listed" "$(test -d "$MNT/c050" && echo 1 || echo 0)" "1"
  is "empty dir empty"  "$(ls $MNT/c050/z001/y001 | wc -l)"    "0"
  echo deep > $MNT/c050/z001/y001/x001
  is "create under new" "$(cat $MNT/c050/z001/y001/x001)"      "deep"
  nay "rmdir nonempty refused" rmdir $MNT/c050
  mkdir $MNT/c060 && rmdir $MNT/c060
  is "mkdir/rmdir empty" "$(ls $MNT | grep -c c060)"           "0"

  mkdir -p $MNT/c255/z255/y255 && echo edge > $MNT/c255/z255/y255/x255
  is "axis-255 corner" "$(cat $MNT/c255/z255/y255/x255)" "edge"
  nay "reject non-canonical x1" tee $MNT/c001/z001/y001/x1 <<<x

  cp /etc/hostname $MNT/c050/z001/y001/x002
  diff -q /etc/hostname $MNT/c050/z001/y001/x002 >/dev/null && ok "cp byte-identical" || no "cp byte-identical"

  setfattr -n user.a -v 1 $F; setfattr -n user.b -v 2 $F; setfattr -n user.d -v D $MNT/c001
  is "xattr a present" "$(getfattr -n user.a --only-values $F 2>/dev/null)" "1"
  is "xattr b present" "$(getfattr -n user.b --only-values $F 2>/dev/null)" "2"
  is "xattr on dir"    "$(getfattr -n user.d --only-values $MNT/c001 2>/dev/null)" "D"
  setfattr -x user.a $F
  is "xattr removed"    "$(getfattr -d $F 2>/dev/null | grep -c '^user\.a')"      "0"

  # Regression: cross-user ACLs need --allow-other, else the kernel returns
  # EACCES at the mountpoint before any request reaches us.
  if id luulu >/dev/null 2>&1; then
    chown 0:0 $F; chmod 644 $F
    is "644 root, other reads" "$(sudo -u luulu cat $F)" "hello from the cube"
    chmod 600 $F
    nay "600 root denies other" sudo -u luulu cat $F
    chown 1000:1000 $F; chmod 600 $F
    is "600 owned, owner reads" "$(sudo -u luulu cat $F)" "hello from the cube"
    sudo -u luulu tee $F >/dev/null <<<by-other
    is "other writes"           "$(cat $F)"              "by-other"
  else
    echo "SKIP: user luulu absent, cross-user ACL stage not run"
  fi

  mkdir -p $MNT/c100/z001/y001
  for i in $(seq 1 200); do printf 'rec%03d\n' $i > $MNT/c100/z001/y001/x$(printf %03d $i); done
  is "200 records" "$(ls $MNT/c100/z001/y001 | wc -l)" "200"
  is "record 137"  "$(cat $MNT/c100/z001/y001/x137)"   "rec137"
}

# --- LEG 1: in-memory seed mount (baseline, no daemon) --------------------
MNT=$(mktemp -d /tmp/cubefs-check.XXXXXX)
cleanup_seed() { fusermount3 -u "$MNT" 2>/dev/null || true; sleep 0.5; rmdir "$MNT" 2>/dev/null || true; }
set +e
"$BIN" "$MNT" --seed --label check --allow-other >/dev/null 2>&1 &
sleep 2
mount | grep -q "on $MNT type fuse" && ok mounted || { no mounted; exit 1; }
run_fs_tests "$MNT"
set -e
fusermount3 -u "$MNT" 2>/dev/null || true; sleep 0.5; rmdir "$MNT" 2>/dev/null || true

# --- LEG 2: daemon-backed --socket mount (regression guard for the durable
#           FUSE view; this is what the deployed VM uses and what would have
#           caught a daemon lacking the raw* commands) -----------------------
MNT=$(mktemp -d /tmp/cubefs-check.XXXXXX)
DSOCK=$(mktemp -u /tmp/cubefs-check-daemon.XXXXXX.sock)
DDIR=$(mktemp -d /tmp/cubefs-check-daemon.XXXXXX)
DSTORE=$DDIR/cube-store.json
cleanup_sock() { fusermount3 -u "$MNT" 2>/dev/null || true; sleep 0.5; rmdir "$MNT" 2>/dev/null || true; kill -9 $DPID 2>/dev/null || true; rm -rf "$DDIR" 2>/dev/null || true; }
set +e
./target/debug/cube-server --socket "$DSOCK" --store "$DSTORE" \
  --allow-anonymous --checkpoint-ms 400 --wal-fsync-ms 30 >/tmp/cubefs-check-daemon.log 2>&1 &
DPID=$!
for i in $(seq 1 30); do [ -S "$DSOCK" ] && break; sleep 0.3; done
if [ ! -S "$DSOCK" ]; then echo "DAEMON FAILED to start:"; cat /tmp/cubefs-check-daemon.log; cleanup_sock; exit 1; fi
"$BIN" "$MNT" --socket "$DSOCK" --label check --allow-other >/dev/null 2>&1 &
sleep 2
mount | grep -q "on $MNT type fuse" && ok "socket-mounted" || { no "socket-mounted"; cleanup_sock; exit 1; }
run_fs_tests "$MNT"
# Durability: a record written through the socket mount must survive a daemon
# restart (WAL recovery), proving the FUSE view is a real durable store.
echo durable-restart-proof > $MNT/c200/z001/y001/x001
sync; sleep 1   # let the WAL fsync the write before we kill the daemon
kill -9 $DPID 2>/dev/null || true; sleep 1
./target/debug/cube-server --socket "$DSOCK" --store "$DSTORE" \
  --allow-anonymous --checkpoint-ms 400 --wal-fsync-ms 30 >/tmp/cubefs-check-daemon.log 2>&1 &
DPID=$!
for i in $(seq 1 30); do [ -S "$DSOCK" ] && break; sleep 0.3; done
sleep 1   # give the remounted FUSE view a beat to re-resolve against the new daemon
is "durable across restart" "$(cat $MNT/c200/z001/y001/x001 2>&1)" "durable-restart-proof"
set -e
cleanup_sock

printf '\n%d passed, %d failed\n' "$PASS" "$FAIL"
[ "$FAIL" -eq 0 ] || exit 1
printf '\n\033[32mALL CHECKS PASSED\033[0m\n'