integration - zygaena/zyginit

root / tests / integration
integration Plain Text 1792 lines 61.8 KB
[service]
name = "cron"
type = "daemon"

[exec]
start = "/usr/sbin/cron"
../cron.toml../filesystem.toml[service]
name = "filesystem"
type = "oneshot"

[exec]
start = "./start.sh"
#!/bin/sh
mount -a
#!/bin/sh
# Tests for scripts/migrate-layout.sh
set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
TOOL="$PROJECT_DIR/scripts/migrate-layout.sh"
FIXTURE_SRC="$SCRIPT_DIR/fixtures/old-layout"

PASS=0
FAIL=0

pass() { echo "  [pass]   $1";  PASS=$((PASS+1)); }
fail() { echo "  [FAIL]   $1"; echo "    $2"; FAIL=$((FAIL+1)); }

setup_tmp() {
    TMP=$(mktemp -d /tmp/migrate-test.XXXXXX)
    cp -a "$FIXTURE_SRC"/. "$TMP"/
    echo "$TMP"
}

# Test 1: --dry-run does not modify anything
TMP=$(setup_tmp)
BEFORE=$(find "$TMP" | sort)
"$TOOL" --dry-run --config-dir "$TMP" > /dev/null
AFTER=$(find "$TMP" | sort)
if [ "$BEFORE" = "$AFTER" ]; then
    pass "dry-run leaves tree unchanged"
else
    fail "dry-run leaves tree unchanged" "tree changed"
fi
rm -rf "$TMP"

# Test 2: --apply migrates flat layout to per-service directories
TMP=$(setup_tmp)
"$TOOL" --apply --config-dir "$TMP" > /dev/null
if [ -f "$TMP/services/cron/service.toml" ] && \
   [ -f "$TMP/services/filesystem/service.toml" ] && \
   [ -f "$TMP/services/filesystem/start.sh" ]; then
    pass "apply creates services/<name>/service.toml"
else
    fail "apply creates services/<name>/service.toml" "missing expected files"
fi
rm -rf "$TMP"

# Test 3: --apply retargets enabled.d symlinks to directories
TMP=$(setup_tmp)
"$TOOL" --apply --config-dir "$TMP" > /dev/null
target=$(readlink "$TMP/enabled.d/cron")
case "$target" in
    *services/cron) pass "enabled.d symlink targets directory" ;;
    *)              fail "enabled.d symlink targets directory" "got '$target'" ;;
esac
rm -rf "$TMP"

# Test 4: --apply is idempotent
TMP=$(setup_tmp)
"$TOOL" --apply --config-dir "$TMP" > /dev/null
SECOND=$("$TOOL" --apply --config-dir "$TMP" 2>&1)
if echo "$SECOND" | grep -q "already migrated\|nothing to migrate\|no changes"; then
    pass "apply is idempotent"
else
    fail "apply is idempotent" "second run did not report no-op"
fi
rm -rf "$TMP"

# Test 5: refuses when both old and new layout coexist
TMP=$(setup_tmp)
mkdir -p "$TMP/services/cron"
cp "$TMP/cron.toml" "$TMP/services/cron/service.toml"
set +e
"$TOOL" --apply --config-dir "$TMP" > /dev/null 2>&1
rc=$?
set -e
if [ "$rc" -ne 0 ]; then
    pass "refuses on ambiguous old+new coexistence"
else
    fail "refuses on ambiguous old+new coexistence" "exit was 0"
fi
rm -rf "$TMP"

echo
echo "passed: $PASS  failed: $FAIL"
[ "$FAIL" -eq 0 ]
#!/bin/sh
#
# zyginit integration test harness
#
# Black-box tests for zyginit daemon + zygctl CLI.
# Creates temporary service definitions, starts zyginit in non-PID-1 mode,
# exercises zygctl commands, and verifies expected behavior.
#
# Usage: ./tests/integration/run_tests.sh
#
# Both binaries (zyginit + zygctl) are rebuilt from source at startup so the
# tests always run against current code. Set ZYGINIT_TEST_NO_BUILD=1 to skip
# the rebuild (fast iteration when you know the binaries are already current).
#
# Requirements (Linux dev): clang + reefc on PATH.
#

set -e

# ============================================================================
# Configuration
# ============================================================================

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
ZYGINIT="$PROJECT_DIR/build/zyginit"
ZYGCTL="$PROJECT_DIR/tools/zygctl/build/zygctl"

TEST_DIR=$(mktemp -d /tmp/zyginit-test.XXXXXX)
CONFIG_DIR="$TEST_DIR/config"
ENABLED_DIR="$CONFIG_DIR/enabled.d"
BIN_DIR="$TEST_DIR/bin"
SOCKET="$TEST_DIR/zyginit.sock"
DAEMON_LOG="$TEST_DIR/zyginit.log"
DAEMON_PID=""

LOG_DIR="$TEST_DIR/log"
RUN_DIR="$TEST_DIR/run"
export ZYGINIT_CONFIG_DIR="$CONFIG_DIR"
export ZYGINIT_SOCKET="$SOCKET"
export ZYGINIT_LOG_DIR="$LOG_DIR"
export ZYGINIT_RUN_DIR="$RUN_DIR"
# Force the daemon into plain mode regardless of TTY state — keeps
# integration test diffs stable and prevents ANSI escapes from
# polluting captured assertions.
export ZYGINIT_NO_UI=1

PASS=0
FAIL=0
TOTAL=0

# ============================================================================
# Build binaries
# ============================================================================
#
# Always rebuild from source so the test binaries match current code. The
# harness used to only build when a binary was MISSING, which let a stale
# build/ artifact (e.g. an old version string) silently linger and fail tests.
# Set ZYGINIT_TEST_NO_BUILD=1 to skip (e.g. fast local iteration).

if [ -z "$ZYGINIT_TEST_NO_BUILD" ]; then
    printf "Building zyginit + zygctl from source...\n"
    mkdir -p "$PROJECT_DIR/build" "$PROJECT_DIR/tools/zygctl/build"
    (
        cd "$PROJECT_DIR" &&
        clang -c src/helpers.c -o build/helpers.o &&
        clang -c src/contract_linux_stubs.c -o build/contract_linux_stubs.o &&
        reefc build --obj build/helpers.o --obj build/contract_linux_stubs.o
    ) || { echo "ERROR: zyginit build failed"; exit 1; }
    (
        cd "$PROJECT_DIR/tools/zygctl" &&
        clang -c src/symlink_wrapper.c -o build/symlink_wrapper.o &&
        reefc build --obj build/symlink_wrapper.o
    ) || { echo "ERROR: zygctl build failed"; exit 1; }
fi

# ============================================================================
# Test framework
# ============================================================================

pass() {
    PASS=$((PASS + 1))
    TOTAL=$((TOTAL + 1))
    printf "  PASS: %s\n" "$1"
}

fail() {
    FAIL=$((FAIL + 1))
    TOTAL=$((TOTAL + 1))
    printf "  FAIL: %s\n" "$1"
    if [ -n "$2" ]; then
        printf "        expected: %s\n" "$2"
    fi
    if [ -n "$3" ]; then
        printf "        got:      %s\n" "$3"
    fi
}

section() {
    printf "\n=== %s ===\n" "$1"
}

# Assert output contains a substring
assert_contains() {
    local output="$1"
    local expected="$2"
    local label="$3"
    if echo "$output" | grep -q "$expected"; then
        pass "$label"
    else
        fail "$label" "contains '$expected'" "$output"
    fi
}

# Assert output does NOT contain a substring
assert_not_contains() {
    local output="$1"
    local expected="$2"
    local label="$3"
    if echo "$output" | grep -q "$expected"; then
        fail "$label" "not contains '$expected'" "$output"
    else
        pass "$label"
    fi
}

# Assert exact match (trimmed)
assert_equals() {
    local output="$(echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
    local expected="$(echo "$2" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
    local label="$3"
    if [ "$output" = "$expected" ]; then
        pass "$label"
    else
        fail "$label" "'$expected'" "'$output'"
    fi
}

# Run zygctl and capture output
zygctl() {
    "$ZYGCTL" "$@" 2>&1
}

# ============================================================================
# Service definition helpers
# ============================================================================

create_service() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local restart="${4:-never}"
    local delay="${5:-1}"
    local max_retries="${6:-3}"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"

[restart]
on = "$restart"
delay = $delay
max_retries = $max_retries
TOML
}

create_service_with_workdir() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local workdir="$4"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"
working_directory = "$workdir"

[restart]
on = "never"
TOML
}

create_service_with_env() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local workdir="$4"
    local env_array="$5"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"
working_directory = "$workdir"
environment = $env_array

[restart]
on = "never"
TOML
}

# Create a service with a [dependencies] conflicts array.
# Usage: create_service_with_conflicts <name> <type> <cmd> "\"other\""
create_service_with_conflicts() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local conflicts="$4"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"

[dependencies]
conflicts = [$conflicts]

[restart]
on = "never"
TOML
}

# Drop a helper script into a service's directory.
# Usage: create_service_script <svc> <script-name> "<content>"
create_service_script() {
    local svc="$1"
    local script="$2"
    local content="$3"
    mkdir -p "$CONFIG_DIR/services/$svc"
    printf '%s\n' "$content" > "$CONFIG_DIR/services/$svc/$script"
    chmod +x "$CONFIG_DIR/services/$svc/$script"
}

enable_service() {
    ln -sfn "../services/$1" "$ENABLED_DIR/$1"
}

disable_service() {
    rm -f "$ENABLED_DIR/$1"
}

# ============================================================================
# Daemon lifecycle
# ============================================================================

start_daemon() {
    # --supervisor-test: opt in to non-PID-1 supervisor mode (bug 004 gate).
    "$ZYGINIT" --supervisor-test > "$DAEMON_LOG" 2>&1 &
    DAEMON_PID=$!
    # Wait for socket to appear
    local retries=0
    while [ ! -S "$SOCKET" ] && [ $retries -lt 30 ]; do
        sleep 0.1
        retries=$((retries + 1))
    done
    if [ ! -S "$SOCKET" ]; then
        echo "ERROR: zyginit failed to start (no socket after 3s)"
        cat "$DAEMON_LOG"
        exit 1
    fi
    # Give services a moment to start
    sleep 1
}

stop_daemon() {
    if [ -n "$DAEMON_PID" ]; then
        kill "$DAEMON_PID" 2>/dev/null || true
        # Wait up to 5 seconds for graceful shutdown
        local retries=0
        while kill -0 "$DAEMON_PID" 2>/dev/null && [ $retries -lt 50 ]; do
            sleep 0.1
            retries=$((retries + 1))
        done
        # Force kill if still alive
        kill -9 "$DAEMON_PID" 2>/dev/null || true
        wait "$DAEMON_PID" 2>/dev/null || true
        DAEMON_PID=""
    fi
    # Kill any orphaned test service processes
    pkill -f "sleeper.sh" 2>/dev/null || true
    rm -f "$SOCKET"
}

# ============================================================================
# Cleanup
# ============================================================================

cleanup() {
    stop_daemon
    rm -rf "$TEST_DIR"
}
trap cleanup EXIT

# ============================================================================
# Create test helper scripts
# ============================================================================

mkdir -p "$BIN_DIR" "$RUN_DIR"

# Long-running daemon (sleeps via shell trap loop)
cat > "$BIN_DIR/sleeper.sh" << 'SCRIPT'
#!/bin/sh
trap 'exit 0' TERM INT
while true; do sleep 1; done
SCRIPT
chmod +x "$BIN_DIR/sleeper.sh"

# Successful oneshot
cat > "$BIN_DIR/succeed.sh" << 'SCRIPT'
#!/bin/sh
exit 0
SCRIPT
chmod +x "$BIN_DIR/succeed.sh"

# Failing oneshot/daemon
cat > "$BIN_DIR/fail.sh" << 'SCRIPT'
#!/bin/sh
exit 1
SCRIPT
chmod +x "$BIN_DIR/fail.sh"

# Write current directory to a marker file, then exit
cat > "$BIN_DIR/write_cwd.sh" << 'SCRIPT'
#!/bin/sh
pwd > cwd_marker.txt
exit 0
SCRIPT
chmod +x "$BIN_DIR/write_cwd.sh"

# Write an env var to a marker file, then exit
cat > "$BIN_DIR/write_env.sh" << 'SCRIPT'
#!/bin/sh
echo "$TEST_VAR" > env_marker.txt
exit 0
SCRIPT
chmod +x "$BIN_DIR/write_env.sh"

# Write to stderr and exit with failure (for log capture testing)
cat > "$BIN_DIR/log_fail.sh" << 'SCRIPT'
#!/bin/sh
echo "service starting up" >&1
echo "fatal: something went wrong" >&2
exit 1
SCRIPT
chmod +x "$BIN_DIR/log_fail.sh"

# ============================================================================
# Pre-flight checks
# ============================================================================

if [ ! -x "$ZYGINIT" ]; then
    echo "ERROR: zyginit binary not found at $ZYGINIT"
    echo "Run: reefc build --obj build/contract_stubs.o"
    exit 1
fi

if [ ! -x "$ZYGCTL" ]; then
    echo "ERROR: zygctl binary not found at $ZYGCTL"
    echo "Run: cd tools/zygctl && reefc build --obj build/symlink_wrapper.o"
    exit 1
fi

echo "zyginit integration tests"
echo "  zyginit: $ZYGINIT"
echo "  zygctl:  $ZYGCTL"
echo "  tmpdir:  $TEST_DIR"

# ============================================================================
# Test Suite 1: zygctl local commands (no daemon)
# ============================================================================

section "zygctl local commands"

# Setup config dir
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Test help
output=$(zygctl help)
assert_contains "$output" "zygctl" "help shows program name"
assert_contains "$output" "enable" "help lists enable command"
assert_contains "$output" "disable" "help lists disable command"
assert_contains "$output" "reload" "help lists reload command"

# Test enable — service does not exist
output=$(zygctl enable nonexistent)
assert_contains "$output" "error" "enable nonexistent shows error"

# Create a service definition and test enable
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
output=$(zygctl enable sleeper)
assert_contains "$output" "enabled" "enable sleeper succeeds"

# Verify symlink was created
if [ -L "$ENABLED_DIR/sleeper" ]; then
    pass "enable creates symlink"
else
    fail "enable creates symlink" "symlink at $ENABLED_DIR/sleeper" "not found"
fi

# Verify symlink target points at the new-layout directory (not flat .toml)
target=$(readlink "$ENABLED_DIR/sleeper")
case "$target" in
    *services/sleeper) pass "enable symlink targets services/sleeper" ;;
    *)                 fail "enable symlink targets services/sleeper" "*services/sleeper" "got: $target" ;;
esac

# Test enable again (already enabled)
output=$(zygctl enable sleeper)
assert_contains "$output" "already enabled" "enable again shows already enabled"

# Test disable
output=$(zygctl disable sleeper)
assert_contains "$output" "disabled" "disable sleeper succeeds"

# Verify symlink was removed
if [ ! -e "$ENABLED_DIR/sleeper" ]; then
    pass "disable removes symlink"
else
    fail "disable removes symlink" "no file at $ENABLED_DIR/sleeper" "file exists"
fi

# Test disable again (not enabled)
output=$(zygctl disable sleeper)
assert_contains "$output" "not enabled" "disable again shows not enabled"

# Test connect failure (no daemon running)
output=$(zygctl status)
assert_contains "$output" "cannot connect" "status without daemon shows connection error"

# ============================================================================
# Test Suite 2: Basic daemon operation
# ============================================================================

section "Basic daemon operation"

# Clean up and set up fresh
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create test services
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
create_service "greeter" "oneshot" "$BIN_DIR/succeed.sh"
enable_service "sleeper"
enable_service "greeter"

start_daemon

# Test list
output=$(zygctl list)
assert_contains "$output" "sleeper" "list shows sleeper"
assert_contains "$output" "greeter" "list shows greeter"
assert_contains "$output" "daemon" "list shows daemon type"
assert_contains "$output" "oneshot" "list shows oneshot type"

# Test status (all)
output=$(zygctl status)
assert_contains "$output" "sleeper" "status shows sleeper"
assert_contains "$output" "greeter" "status shows greeter"

# Test status of running daemon
output=$(zygctl status sleeper)
assert_contains "$output" "running" "sleeper is running"
assert_contains "$output" "sleeper" "sleeper status row present"

# Test status of completed oneshot
output=$(zygctl status greeter)
assert_contains "$output" "stopped" "greeter (oneshot) is stopped after completion"

# Test help via socket (this goes through socket since daemon is running... actually
# version and help are local in zygctl. Let me test an actual socket help)
# Actually, version/help in zygctl are intercepted locally. Let me test unknown command.
output=$(zygctl unknown_cmd 2>&1)
assert_contains "$output" "error" "unknown command returns error"

# Test stop
output=$(zygctl stop sleeper)
assert_contains "$output" "stopping" "stop sleeper succeeds"
sleep 1

output=$(zygctl status sleeper)
assert_contains "$output" "stopped" "sleeper is stopped after stop"

# Test start
output=$(zygctl start sleeper)
assert_contains "$output" "started" "start sleeper succeeds"
sleep 1

output=$(zygctl status sleeper)
assert_contains "$output" "running" "sleeper is running after start"

# Test restart
output=$(zygctl restart sleeper)
# restart of a running service sends stop first
sleep 2

output=$(zygctl status sleeper)
# It should be running or restarting
assert_contains "$output" "sleeper" "restart: service exists after restart"

# Test status of unknown service
output=$(zygctl status nonexistent)
assert_contains "$output" "error" "status of unknown service shows error"

# Ensure sleeper is running before testing double-start
sleep 1
output=$(zygctl status sleeper)
if echo "$output" | grep -q "running"; then
    output=$(zygctl start sleeper)
    assert_contains "$output" "already running" "start already running shows message"
else
    # Restart may have left it stopped; start it, wait, then try again
    zygctl start sleeper > /dev/null 2>&1
    sleep 1
    output=$(zygctl start sleeper)
    assert_contains "$output" "already running" "start already running shows message"
fi

stop_daemon

# ============================================================================
# Test Suite 3: Restart policies
# ============================================================================

section "Restart policies"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service that exits immediately with failure — restart on failure, max 2 retries
create_service "failer" "daemon" "$BIN_DIR/fail.sh" "failure" 1 2

enable_service "failer"

start_daemon

# Wait for restart attempts to play out (1s delay * 2 retries + startup time)
sleep 5

output=$(zygctl status failer)
assert_contains "$output" "maint." "failer enters maintenance after max retries"
assert_contains "$output" "restarts=" "failer shows restart count"
assert_contains "$output" "exit=" "failer shows exit code"

stop_daemon

# ============================================================================
# Test Suite 4: Enable/disable via socket + reload
# ============================================================================

section "Enable/disable via socket + reload"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Start with one service
create_service "alpha" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "alpha"

start_daemon

output=$(zygctl status)
assert_contains "$output" "alpha" "alpha is loaded"

# Create a new service and enable it
create_service "beta" "daemon" "$BIN_DIR/sleeper.sh"

# Enable via socket (server-side)
output=$(zygctl enable beta)
assert_contains "$output" "enabled" "enable beta via socket"

# Reload to pick up the new service
output=$(zygctl reload)
assert_contains "$output" "reload" "reload command accepted"
sleep 2

output=$(zygctl status beta)
assert_contains "$output" "beta" "beta appears after reload"
assert_contains "$output" "running" "beta is running after reload"

# Disable alpha and reload
output=$(zygctl disable alpha)
assert_contains "$output" "disabled" "disable alpha via socket"

output=$(zygctl reload)
sleep 2

output=$(zygctl status alpha)
assert_contains "$output" "disabled" "alpha is disabled after reload"

stop_daemon

# ============================================================================
# Test Suite 5: Oneshot service behavior
# ============================================================================

section "Oneshot services"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Successful oneshot
create_service "setup-ok" "oneshot" "$BIN_DIR/succeed.sh"
# Failing oneshot
create_service "setup-fail" "oneshot" "$BIN_DIR/fail.sh"

enable_service "setup-ok"
enable_service "setup-fail"

start_daemon

sleep 2

output=$(zygctl status setup-ok)
assert_contains "$output" "stopped" "successful oneshot shows stopped"

output=$(zygctl status setup-fail)
assert_contains "$output" "failed" "failing oneshot shows failed"
assert_contains "$output" "exit=" "failing oneshot shows exit code"

stop_daemon

# ============================================================================
# Test Suite 5b: Task type semantics
# ============================================================================

section "Task type semantics"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "task-success" "task" "$BIN_DIR/succeed.sh" "never"
enable_service "task-success"

create_service "task-failure" "task" "$BIN_DIR/fail.sh" "never"
enable_service "task-failure"

start_daemon
sleep 2

out=$(zygctl status)

echo "$out" | grep -q "task-success.*stopped" && \
    pass "task exit 0 = stopped" || \
    fail "task exit 0 = stopped" "status=stopped" "got: $(echo "$out" | grep task-success)"

echo "$out" | grep -q "task-failure.*failed" && \
    pass "task exit !=0 = failed" || \
    fail "task exit !=0 = failed" "status=failed" "got: $(echo "$out" | grep task-failure)"

# Verify no restart attempt — restart_count should be 0 (no restarts= field)
if echo "$out" | grep "task-failure" | grep -q "restarts="; then
    fail "task does not restart" "no restarts= field" "restarts field present"
else
    pass "task does not restart"
fi

stop_daemon

# ============================================================================
# Test Suite 6: Working directory
# ============================================================================

section "Working directory"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create a target directory for the service to run in
WORK_DIR="$TEST_DIR/workdir"
mkdir -p "$WORK_DIR"

create_service_with_workdir "cwd-test" "oneshot" "$BIN_DIR/write_cwd.sh" "$WORK_DIR"
enable_service "cwd-test"

start_daemon

sleep 2

# Check that the marker file was written in the working directory
if [ -f "$WORK_DIR/cwd_marker.txt" ]; then
    pass "working_directory: marker file created in workdir"
    marker_content=$(cat "$WORK_DIR/cwd_marker.txt" | tr -d '\n')
    assert_equals "$marker_content" "$WORK_DIR" "working_directory: cwd matches configured path"
else
    fail "working_directory: marker file created in workdir" "file at $WORK_DIR/cwd_marker.txt" "not found"
    fail "working_directory: cwd matches configured path" "$WORK_DIR" "marker file missing"
fi

stop_daemon

# ============================================================================
# Test Suite 7: Environment variables
# ============================================================================

section "Environment variables"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

ENV_DIR="$TEST_DIR/envdir"
mkdir -p "$ENV_DIR"

create_service_with_env "env-test" "oneshot" "$BIN_DIR/write_env.sh" "$ENV_DIR" '["TEST_VAR=hello_zyginit"]'
enable_service "env-test"

start_daemon

sleep 2

if [ -f "$ENV_DIR/env_marker.txt" ]; then
    pass "environment: marker file created"
    marker_content=$(cat "$ENV_DIR/env_marker.txt" | tr -d '\n')
    assert_equals "$marker_content" "hello_zyginit" "environment: env var passed to service"
else
    fail "environment: marker file created" "file at $ENV_DIR/env_marker.txt" "not found"
    fail "environment: env var passed to service" "hello_zyginit" "marker file missing"
fi

stop_daemon

# ============================================================================
# Test Suite 8: Privilege separation (error path)
# ============================================================================

section "Privilege separation"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service with nonexistent user — should fail with exit 126
mkdir -p "$CONFIG_DIR/services/baduser"
cat > "$CONFIG_DIR/services/baduser/service.toml" << TOML
[service]
description = "Test bad user"
type = "oneshot"

[exec]
start = "$BIN_DIR/succeed.sh"
user = "nonexistent_user_zyginit_test"

[restart]
on = "never"
TOML
enable_service "baduser"

start_daemon

sleep 2

output=$(zygctl status baduser)
assert_contains "$output" "failed" "privilege: bad user causes failure"
assert_contains "$output" "exit=" "privilege: exit code shown"

stop_daemon

# ============================================================================
# Test Suite 9: Service output capture + zygctl log
# ============================================================================

section "Service output capture"

rm -rf "$CONFIG_DIR" "$LOG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "log-test" "oneshot" "$BIN_DIR/log_fail.sh"
enable_service "log-test"

start_daemon

sleep 2

# Check that log file was created
if [ -f "$LOG_DIR/log-test.log" ]; then
    pass "log: per-service log file created"
else
    fail "log: per-service log file created" "file at $LOG_DIR/log-test.log" "not found"
fi

# Check log file contains expected output
log_content=$(cat "$LOG_DIR/log-test.log" 2>/dev/null)
assert_contains "$log_content" "something went wrong" "log: stderr captured in log file"
assert_contains "$log_content" "starting up" "log: stdout captured in log file"

# Check zygctl log command
output=$(zygctl log log-test)
assert_contains "$output" "something went wrong" "log: zygctl log shows stderr"
assert_contains "$output" "starting up" "log: zygctl log shows stdout"

# Check zygctl log for unknown service
output=$(zygctl log nonexistent)
assert_contains "$output" "error" "log: unknown service shows error"

# Check zygctl log for service with no log
output=$(zygctl log setup-fail 2>&1)
# setup-fail doesn't exist in this test run, so it'll be unknown
assert_contains "$output" "error" "log: missing service shows error"

stop_daemon

# ============================================================================
# Test Suite 9b: Spawn-failure capture (bug 003)
# ============================================================================
#
# When a service's start command cannot be exec'd (missing/unrunnable),
# the supervisor must NOT let the service vanish silently with an empty log.
# The child retries the exec a few times, then records the spawn failure to
# both the per-service log (fd 2) and the central zyginit.log before exiting.

section "Spawn-failure capture (bug 003)"

rm -rf "$CONFIG_DIR" "$LOG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Start command points at a path that does not exist — exec always fails.
create_service "spawn-fail" "oneshot" "/nonexistent/zyginit-bug003-missing --flag"
enable_service "spawn-fail"

start_daemon

# Child retries the exec (5 x 100ms) before giving up, so allow settle time.
sleep 3

# Central zyginit log must exist and record the spawn failure.
if [ -f "$LOG_DIR/zyginit.log" ]; then
    pass "spawn-fail: central zyginit.log created"
else
    fail "spawn-fail: central zyginit.log created" "file at $LOG_DIR/zyginit.log" "not found"
fi
central_content=$(cat "$LOG_DIR/zyginit.log" 2>/dev/null)
assert_contains "$central_content" "exec failed" "spawn-fail: central log records exec failure"
assert_contains "$central_content" "spawn-fail" "spawn-fail: central log names the service"

# The per-service log must NOT be empty (the bug-003 symptom was an empty log).
perservice_content=$(cat "$LOG_DIR/spawn-fail.log" 2>/dev/null)
assert_contains "$perservice_content" "exec failed" "spawn-fail: per-service log records exec failure"

# The service is reported failed (exit 127), never as exit=-1.
output=$(zygctl status spawn-fail)
assert_contains "$output" "failed" "spawn-fail: service reported failed"
assert_not_contains "$output" "exit=-1" "spawn-fail: not reported as exit=-1"

stop_daemon

# ============================================================================
# Test Suite 10: zygctl replace — command parsing and state-file write
# ============================================================================

section "zygctl replace — command parsing and state-file write"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create a minimal service
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sleeper"

start_daemon

# Test 1: zygctl replace with no args
out=$("$ZYGCTL" replace 2>&1 || true)
assert_contains "$out" "replace queued" "replace acks with no args"
assert_contains "$out" "wait=0" "replace acks with default wait=0"

# Wait briefly to let the daemon process the flag (it will attempt exec
# of /sbin/init, which fails on Linux because that's systemd, not us;
# zyginit logs FATAL: execve returned and continues running.)
sleep 1.5

# Verify state.toml was written before the exec attempt
if [ -f "$RUN_DIR/state.toml" ]; then
    pass "state.toml written by replace flow"
    out=$(cat "$RUN_DIR/state.toml")
    assert_contains "$out" "boot_time =" "state.toml has boot_time line"
else
    fail "state.toml written by replace flow" "file present at $RUN_DIR/state.toml" "missing"
fi

# Verify the daemon log shows the outgoing flow
log_out=$(cat "$DAEMON_LOG")
assert_contains "$log_out" "replace: requested" "daemon logged replace request"
assert_contains "$log_out" "replace: state written" "daemon logged state write"
assert_contains "$log_out" "replace: execve" "daemon logged execve attempt"

# Stop daemon (it survived the failed execve and is still running)
stop_daemon
# Clean up the leftover state.toml so the next test starts fresh
rm -f "$RUN_DIR/state.toml"

# Test 2: zygctl replace --wait=N
start_daemon
out=$("$ZYGCTL" replace --wait=10 2>&1 || true)
assert_contains "$out" "replace queued" "replace --wait=10 acks"
assert_contains "$out" "wait=10" "replace --wait=10 reflects wait value"
sleep 1.5
stop_daemon
rm -f "$RUN_DIR/state.toml"

# Test 3: zygctl replace --bogus
start_daemon
out=$("$ZYGCTL" replace --bogus 2>&1 || true)
assert_contains "$out" "error" "replace --bogus returns error"
assert_contains "$out" "unknown argument" "replace --bogus error mentions unknown argument"
stop_daemon

# ============================================================================
# Test Suite 11: [condition] block — SKIPPED state
# ============================================================================

section "[condition] block — SKIPPED state"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service whose exists_file condition references a path that cannot exist.
mkdir -p "$CONFIG_DIR/services/skip-missing-file"
cat > "$CONFIG_DIR/services/skip-missing-file/service.toml" << 'TOML'
[service]
type = "daemon"

[exec]
start = "/bin/sleep 1000"

[condition]
exists_file = "/this/path/definitely/does/not/exist"
TOML
enable_service "skip-missing-file"

# Service whose command condition exits non-zero (/bin/false).
mkdir -p "$CONFIG_DIR/services/skip-cmd-fail"
cat > "$CONFIG_DIR/services/skip-cmd-fail/service.toml" << 'TOML'
[service]
type = "daemon"

[exec]
start = "/bin/sleep 1000"

[condition]
command = "/bin/false"
TOML
enable_service "skip-cmd-fail"

# Service whose command condition exits zero — should start, not be skipped.
mkdir -p "$CONFIG_DIR/services/start-cmd-pass"
cat > "$CONFIG_DIR/services/start-cmd-pass/service.toml" << 'TOML'
[service]
type = "task"

[exec]
start = "/bin/true"

[condition]
command = "/bin/true"
TOML
enable_service "start-cmd-pass"

start_daemon
sleep 2

# Task 7 verifies the supervisor wiring (condition fails → no fork, state
# transitions to STATE_SKIPPED). The display label "skipped" comes in Task 8
# when ui_render.reef maps STATE_SKIPPED. For now, assert the wiring:
# a service with a failing condition must NOT appear as running.
out=$("$ZYGCTL" status)

echo "$out" | grep "skip-missing-file" | grep -q "running" && \
    fail "missing file → not running" "running" "service forked despite failing condition" || \
    pass "missing file → not running"

echo "$out" | grep "skip-cmd-fail" | grep -q "running" && \
    fail "command exit non-zero → not running" "running" "service forked despite failing command" || \
    pass "command exit non-zero → not running"

# A service whose condition passes (task type, /bin/true) ran to completion
# and is now stopped. Either "stopped" or "running" (briefly) is acceptable;
# what we want to confirm is the service was NOT skipped.
echo "$out" | grep "start-cmd-pass" | grep -qi "skipped" && \
    fail "command exit zero → NOT skipped" "not skipped" "incorrectly skipped: $(echo "$out" | grep -i start-cmd-pass || echo '(no line)')" || \
    pass "command exit zero → NOT skipped"

stop_daemon

# ============================================================================
# Test Suite 12: TTY-aware dispatch — plain output when piped
# ============================================================================

section "TTY-aware dispatch"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sleeper"

start_daemon

# zygctl status is piped here (stdout is not a TTY), so client_should_be_plain()
# returns true and the client sends "status plain". Output must contain no ANSI
# escape sequences.
out=$(zygctl status)
if printf '%s' "$out" | grep -qP '\x1b'; then
    fail "zygctl status returns no ANSI escapes when piped" "no ESC bytes" "ESC found"
else
    pass "zygctl status returns no ANSI escapes when piped"
fi

# Same check for list
out=$(zygctl list)
if printf '%s' "$out" | grep -qP '\x1b'; then
    fail "zygctl list returns no ANSI escapes when piped" "no ESC bytes" "ESC found"
else
    pass "zygctl list returns no ANSI escapes when piped"
fi

stop_daemon

# ============================================================================
# Shutdown orchestration tests (standalone script)
# ============================================================================

section "Shutdown orchestration"

if bash "$SCRIPT_DIR/test_shutdown.sh" >> "$TEST_DIR/shutdown.log" 2>&1; then
    pass "oneshot stop command runs at shutdown"
else
    fail "oneshot stop command runs at shutdown" "PASS" "FAIL — see $TEST_DIR/shutdown.log"
fi

# ============================================================================
# Test Suite: Service conflicts
# ============================================================================
section "Service conflicts"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Two daemons that declare each other as conflicting; both enabled.
create_service_with_conflicts "mailer-a" "daemon" "$BIN_DIR/sleeper.sh" "\"mailer-b\""
create_service "mailer-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "mailer-a"
enable_service "mailer-b"

start_daemon

# Boot resolution: both enabled + conflicting => start neither, fail both.
output=$(zygctl status mailer-a)
assert_contains "$output" "failed" "conflict: mailer-a failed at boot"
assert_contains "$output" "conflicts with" "conflict: mailer-a status shows reason"
output=$(zygctl status mailer-b)
assert_contains "$output" "failed" "conflict: mailer-b failed at boot"

stop_daemon

# --- Runtime refusal: starting a service whose conflicting peer runs is refused.
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# svc-b conflicts with svc-a. Enable only svc-a at boot; add svc-b via reload so
# it joins the table WAITING without the boot pre-flight failing both.
create_service "svc-a" "daemon" "$BIN_DIR/sleeper.sh"
create_service_with_conflicts "svc-b" "daemon" "$BIN_DIR/sleeper.sh" "\"svc-a\""
enable_service "svc-a"
enable_service "svc-b"
disable_service "svc-b"

start_daemon

output=$(zygctl status svc-a)
assert_contains "$output" "running" "runtime: svc-a is running"

enable_service "svc-b"
zygctl reload > /dev/null 2>&1
sleep 1
output=$(zygctl start svc-b)
assert_contains "$output" "conflicts with running service" "runtime: start svc-b refused"
output=$(zygctl status svc-a)
assert_contains "$output" "running" "runtime: svc-a still running after refusal"

stop_daemon

# --- Stale reason cleared on successful start after conflict-fail.
# conflict-fail both at boot, then start one after the other is gone (already
# failed so not running); the restarted service must show "running", NOT
# "conflicts with".
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service_with_conflicts "clr-a" "daemon" "$BIN_DIR/sleeper.sh" "\"clr-b\""
create_service "clr-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "clr-a"
enable_service "clr-b"

start_daemon

# Both should be failed at boot (mutual conflict)
output=$(zygctl status clr-a)
assert_contains "$output" "failed" "stale-clear: clr-a failed at boot"
output=$(zygctl status clr-b)
assert_contains "$output" "failed" "stale-clear: clr-b failed at boot"

# Now start clr-a — clr-b is failed (not running), so the start should succeed
output=$(zygctl start clr-a)
sleep 1
output=$(zygctl status clr-a)
assert_contains "$output" "running" "stale-clear: clr-a running after manual start"
assert_not_contains "$output" "conflicts with" "stale-clear: clr-a status has no stale conflict reason"

stop_daemon

# --- Symmetry: declare the conflict on ONE side only; both still fail at boot.
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# only one side declares the conflict
create_service_with_conflicts "sym-a" "daemon" "$BIN_DIR/sleeper.sh" "\"sym-b\""
create_service "sym-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sym-a"
enable_service "sym-b"

start_daemon

output=$(zygctl status sym-a)
assert_contains "$output" "failed" "symmetry: sym-a failed (declared side)"
output=$(zygctl status sym-b)
assert_contains "$output" "failed" "symmetry: sym-b failed (non-declared side)"

stop_daemon

# ============================================================================
# Version reporting
# ============================================================================

section "Version reporting"

# Both binaries should respond to --version (and `version` subcommand for zygctl)
# with a single line "<binary> <version>" matching reef.toml.

EXPECTED_VERSION=$(grep '^version' "$PROJECT_DIR/reef.toml" | head -1 | sed 's/.*"\([^"]*\)".*/\1/')

OUT="$("$ZYGINIT" --version 2>&1)"
assert_equals "$OUT" "zyginit $EXPECTED_VERSION" "zyginit --version output"

OUT="$(zygctl --version)"
assert_equals "$OUT" "zygctl $EXPECTED_VERSION" "zygctl --version output"

OUT="$(zygctl version)"
assert_equals "$OUT" "zygctl $EXPECTED_VERSION" "zygctl version subcommand output"

# ============================================================================
# Test Suite: BUG-004 — non-PID-1 invocation must not clobber the live socket
# ============================================================================

section "BUG-004: non-PID-1 socket safety"

rm -rf "$CONFIG_DIR" "$LOG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sleeper"

# --- fix 1: `zyginit -v` prints version and exits, touching no socket ---
B4_SOCK="$TEST_DIR/bug004.sock"
rm -f "$B4_SOCK"
OUT="$(ZYGINIT_SOCKET="$B4_SOCK" "$ZYGINIT" -v 2>&1)"
assert_contains "$OUT" "zyginit $EXPECTED_VERSION" "bug004: zyginit -v prints version"
if [ ! -e "$B4_SOCK" ]; then
    pass "bug004: zyginit -v created/clobbered no socket"
else
    fail "bug004: zyginit -v created/clobbered no socket" "no socket" "socket present"
fi

# --- fix 3: bare `zyginit` (no --supervisor-test) refuses and exits ---
rm -f "$B4_SOCK"
OUT="$(ZYGINIT_SOCKET="$B4_SOCK" "$ZYGINIT" 2>&1)"
assert_contains "$OUT" "refusing to run as a non-PID-1 supervisor" \
    "bug004: bare zyginit refuses without --supervisor-test"
if [ ! -e "$B4_SOCK" ]; then
    pass "bug004: refused invocation created/clobbered no socket"
else
    fail "bug004: refused invocation created/clobbered no socket" "no socket" "socket present"
fi

# --- fix 2: a second instance must not clobber a live server's socket ---
start_daemon    # daemon A on $SOCKET (launched with --supervisor-test)
OUT="$(zygctl status sleeper)"
assert_contains "$OUT" "sleeper" "bug004: live daemon A reachable before second instance"

# Second instance targets the SAME socket; it must refuse and self-exit,
# leaving A's socket intact.
SECOND="$(ZYGINIT_SOCKET="$SOCKET" "$ZYGINIT" --supervisor-test 2>&1)"
assert_contains "$SECOND" "REFUSING" "bug004: second instance refuses to clobber live socket"

OUT="$(zygctl status sleeper)"
assert_contains "$OUT" "sleeper" "bug004: daemon A still reachable after second instance refused"
stop_daemon

# ============================================================================
# Results
# ============================================================================

echo ""
echo "========================================"
echo "Results: $PASS passed, $FAIL failed (of $TOTAL)"
echo "========================================"

if [ $FAIL -gt 0 ]; then
    exit 1
fi

exit 0
  [36m[Z][0m zyginit 0.2.2 · [37mmulti-user[0m   [36melapsed 5.0s[0m
  [37m17 services · tier 5 · 9 done · 1 failed · 0 skipped[0m
  [31mfailed: network  exit=1[0m
  [36m──────────────────────────────────────────────────────────────[0m
  0.04  [32m#[0m  root-fs
  0.16  [32m#[0m  crypto
  0.31  [32m#[0m  devfs
  0.49  [32m#[0m  swap
  1.04  [32m#[0m  filesystem
  1.08  [32m#[0m  identity
  1.36  [32m#[0m  sysconfig
  1.55  [32m#[0m  dlmgmtd
  5.02  [31mX[0m  network  [2m(exit=1)[0m










  [36m──────────────────────────────────────────────────────────────[0m
  [#################                 ]  9/17  52%  0.04  info  started  root-fs
  0.16  info  started  crypto
  0.31  info  started  devfs
  0.49  info  started  swap
  1.04  info  started  filesystem
  1.08  info  started  identity
  1.36  info  started  sysconfig
  1.55  info  started  dlmgmtd
  5.02  err  failed  network  exit=1
  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111melapsed 5.0s[0m
  [38;2;110;123;139m17 services · tier 5 · 9 done · 1 failed · 0 skipped[0m
  [38;2;200;32;31mfailed: network  exit=1[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  0.04  [38;2;46;139;87m●[0m  root-fs
  0.16  [38;2;46;139;87m●[0m  crypto
  0.31  [38;2;46;139;87m●[0m  devfs
  0.49  [38;2;46;139;87m●[0m  swap
  1.04  [38;2;46;139;87m●[0m  filesystem
  1.08  [38;2;46;139;87m●[0m  identity
  1.36  [38;2;46;139;87m●[0m  sysconfig
  1.55  [38;2;46;139;87m●[0m  dlmgmtd
  5.02  [38;2;200;32;31m✕[0m  network  [2m(exit=1)[0m










  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m  9/17  52%[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111melapsed 0.4s[0m
  [38;2;110;123;139m5 services · tier 0 · 2 done · 0 failed · 1 skipped[0m
  [2mfailed: none[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  0.04  [38;2;46;139;87m●[0m  root-fs
  0.16  [2m·[0m  acpihpd  [2m(skipped: missing /dev/acpihp)[0m
  0.31  [38;2;46;139;87m●[0m  cron
















  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m  3/5  60%mode=0
mode=0
mode=2
mode=3
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111mboot 8.3s[0m
  [38;2;110;123;139m17 services — 16 online, 1 failed, 0 skipped[0m

  [38;2;200;32;31mfailed: network  exit=1[0m
      [38;2;0;106;111m→ zygctl log network[0m[2m    to see why[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;110;123;139mslowest:  network 5.02s  filesystem 0.55s  sshd 0.37s[0m
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111mboot 8.3s[0m
  [38;2;110;123;139m17 services — 17 online, 0 failed, 0 skipped[0m

  [38;2;46;139;87mall services online[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;110;123;139mslowest:  dlmgmtd 1.2s  filesystem 0.55s  sshd 0.37s[0m
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111mboot 8.3s[0m
  [38;2;110;123;139m23 services — 19 online, 0 failed, 4 skipped[0m

  [38;2;46;139;87mall services online[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
# o X .
● ◉ ✕ ·
[37mframe text[0m
[36maccent text[0m
[32mok text[0m
[33mwarn text[0m
[31mfail text[0m
[2mmute text[0m
[38;2;110;123;139mframe text[0m
[38;2;0;106;111maccent text[0m
[38;2;46;139;87mok text[0m
[38;2;255;191;0mwarn text[0m
[38;2;200;32;31mfail text[0m
[2mmute text[0m
  0.04  info  started  root-fs
  0.12  info  started  crypto  dur_ms=80
  0.31  info  started  devfs  dur_ms=190
  5.02  err  failed  network  exit=1 dur_ms=4023
  [#######################           ]  12/17  70%  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m  17/17  100%  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m  12/17  70%[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mreboot complete[0m   [38;2;0;106;111mreboot 2.1s[0m
  [38;2;110;123;139m17 services stopped cleanly[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [2minvoking uadmin(A_SHUTDOWN, AD_BOOT)[0m
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mstopping for reboot[0m   [38;2;0;106;111melapsed 0.4s[0m
  [38;2;110;123;139m17 services · tier 6 · 4 done · 0 failed · 0 skipped[0m
  [2mfailed: none[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  0.04  [2m·[0m  sshd
  0.16  [2m·[0m  console-login
  0.31  [2m·[0m  cron
  0.48  [38;2;255;191;0m/[0m  syslogd















  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m  4/17 down  23%[38;2;0;106;111m[Z][0m zyginit 0.2.2
ui.reef skeleton ok, mode=0
/-\|/-\|
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111melapsed 2.1s[0m
  [38;2;110;123;139m17 services · tier 0 · 3 done · 0 failed · 0 skipped[0m
  [2mfailed: none[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  0.04  [38;2;46;139;87m●[0m  root-fs
  0.16  [38;2;46;139;87m●[0m  crypto
  0.31  [38;2;46;139;87m●[0m  devfs
  2.10  [38;2;255;191;0m\[0m  network















  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
  [38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[38;2;0;106;111m█[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m[2m░[0m  3/17  17%|\-/|\-/
status_skipped: deferred — zygctl status snapshot via --ui-demo not yet supported
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111mboot 0.2s[0m
  [38;2;110;123;139m5 services — 4 online, 1 failed, 0 skipped[0m

  [38;2;200;32;31mfailed: task-svc  exit=1[0m
      [38;2;0;106;111m→ zygctl log task-svc[0m[2m    to see why[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
[2J[H  [38;2;0;106;111m[Z][0m zyginit 0.2.2 · [38;2;110;123;139mmulti-user[0m   [38;2;0;106;111mboot 0.2s[0m
  [38;2;110;123;139m5 services — 5 online, 0 failed, 0 skipped[0m

  [38;2;46;139;87mall services online[0m
  [38;2;0;106;111m──────────────────────────────────────────────────────────────[0m
# Live Replace — Hammerhead Test Checklist

Manual checklist for validating `zygctl replace` on `hh-prototest`
(192.168.122.50). Companion to the spec at
`docs/superpowers/specs/2026-05-02-contract-readoption-design.md`.

## Build + deploy

```bash
# From dev host
scp src/replace.reef src/main.reef src/contract.reef src/socket.reef \
    src/contract_linux_stubs.c src/helpers.c \
    root@192.168.122.50:/root/zyginit/src/
scp tools/zygctl/src/zygctl.reef \
    root@192.168.122.50:/root/zyginit/tools/zygctl/src/

# On hh-prototest
ssh root@192.168.122.50 'cd /root/zyginit && \
    gcc -c src/helpers.c -o build/helpers.o && \
    reefc build -l contract --obj build/helpers.o && \
    cp build/zyginit /sbin/init.new && mv /sbin/init.new /sbin/init && \
    cd tools/zygctl && \
    gcc -c src/symlink_wrapper.c -o build/symlink_wrapper.o && \
    reefc build --obj build/symlink_wrapper.o && \
    cp build/zygctl /sbin/zygctl'
```

After the binary swap, the running zyginit is still the OLD one. The
first `zygctl replace` will swap to the new binary.

## Test cases

### 1. Smoke: replace into the same binary

- [ ] `zygctl status` — note all PIDs and ctids; capture uptimes
- [ ] `zygctl replace` — should print `replace queued (wait=0)` then close
- [ ] Wait ~1 second; `zygctl status` again
- [ ] **All services have the same PIDs** (kernel-level continuity)
- [ ] **All services have the same contract IDs**
- [ ] **Uptimes are continuous** (delta from last_start_time, not reset)
- [ ] **The SSH session driving this test is still alive**
- [ ] `who -b` and `who -r` still report the same boot_time / runlevel

### 2. Pre-condition refusal

- [ ] `zygctl restart sshd` (puts sshd briefly in STOPPING/STARTING)
- [ ] Immediately: `zygctl replace`
- [ ] Expect server log line: `zyginit: replace: blocked: sshd:stopping`
- [ ] Verify the running zyginit did NOT exec (check uptime via /proc/1/start)

### 3. `--wait=N` success

- [ ] `zygctl restart syslogd && zygctl replace --wait=10`
- [ ] Replace should succeed once syslogd stabilizes
- [ ] Confirm syslogd's restart_count incremented and is preserved post-replace

### 4. Dirty disable

- [ ] `rm /etc/zyginit/enabled.d/cron.toml`  (no `zygctl reload`)
- [ ] `zygctl replace`
- [ ] After replace, `zygctl status` does NOT list cron
- [ ] But: `pgrep cron` still shows the cron pid running (process not killed)
- [ ] Server log shows `zyginit: replace: orphaned cron`

### 5. Crash during exec gap

- [ ] Pick a service with `restart.on = "always"`, e.g. sshd
- [ ] `pkill -9 -f "sshd -D"` (or whatever the daemon's argv looks like)
- [ ] Within ~0.5 seconds: `zygctl replace`
- [ ] After replace, server log includes `zyginit: replace: sshd contract empty post-recovery, applying restart policy`
- [ ] sshd is restarted with restart_count incremented

### 6. Stale state.toml across real reboot

- [ ] `cp /var/run/zyginit/state.toml /tmp/saved-state.toml` while a replace
      hasn't been issued (file may not exist; create one by issuing replace
      first then copy quickly before the new zyginit deletes it)
- [ ] `zygctl reboot` (real reboot)
- [ ] After boot: `cp /tmp/saved-state.toml /var/run/zyginit/state.toml`
- [ ] Stop+start zyginit somehow OR observe that the next replace handles it
- [ ] Server log includes `replace: stale state file (boot_time ...)`
- [ ] state.toml is deleted after that read

### 7. Repeated replace

- [ ] Note sshd's restart_count and uptime
- [ ] `zygctl replace` × 5 in a row, ~5 seconds apart
- [ ] After all 5: restart_count is unchanged (replace doesn't bump it)
- [ ] Uptime is continuous from the original start (no resets)
- [ ] No leftover state.toml.tmp files in /var/run/zyginit/

## Reverting

If a replace puts the system in a bad state and SSH is still alive:

```bash
mv /sbin/init.bak /sbin/init  # if you saved the old binary
zygctl replace                # swap back to known-good
```

If SSH is dead but the VM is still running, `virsh reset hh-prototest`
will reboot. The new binary at /sbin/init persists (filesystem-level
swap), so a reboot uses the new binary cleanly — re-test from cold boot.
#!/bin/sh
# zyginit — Init System for Zygaena/Hammerhead
# File: tests/integration/test_shutdown.sh
# Description: Integration test for tier-reversed shutdown orchestration
#              and oneshot stop command invocation.
#
# Copyright (c) 2024, Chris Tusa <chris.tusa@leafscale.com>
# SPDX-License-Identifier: CDDL-1.0

# Verifies tier-reversed shutdown walk and oneshot stop command invocation.

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
ZYGINIT="$PROJECT_DIR/build/zyginit"
ZYGCTL="$PROJECT_DIR/tools/zygctl/build/zygctl"

TEST_DIR=$(mktemp -d /tmp/zyginit-shutdown-test.XXXXXX)
trap "rm -rf $TEST_DIR; kill %1 2>/dev/null || true" EXIT

CONFIG_DIR="$TEST_DIR/config"
ENABLED_DIR="$CONFIG_DIR/enabled.d"
BIN_DIR="$TEST_DIR/bin"
LOG_DIR="$TEST_DIR/log"

mkdir -p "$CONFIG_DIR" "$ENABLED_DIR" "$BIN_DIR" "$LOG_DIR"

# Trivial oneshot: start prints "START alpha", stop prints "STOP alpha"
cat > "$BIN_DIR/alpha-start" <<EOF
#!/bin/sh
echo "START alpha" >> "$LOG_DIR/trace"
EOF
cat > "$BIN_DIR/alpha-stop" <<EOF
#!/bin/sh
echo "STOP alpha" >> "$LOG_DIR/trace"
EOF
chmod +x "$BIN_DIR/alpha-start" "$BIN_DIR/alpha-stop"

mkdir -p "$CONFIG_DIR/services/alpha"
cat > "$CONFIG_DIR/services/alpha/service.toml" <<EOF
[service]
type = "oneshot"
[exec]
start = "$BIN_DIR/alpha-start"
stop = "$BIN_DIR/alpha-stop"
[restart]
on = "never"
EOF

ln -sfn ../services/alpha "$ENABLED_DIR/alpha"

# Start zyginit in background
ZYGINIT_CONFIG_DIR="$CONFIG_DIR" \
  ZYGINIT_SOCKET="$TEST_DIR/zyginit.sock" \
  ZYGINIT_LOG_DIR="$LOG_DIR/zyg" \
  "$ZYGINIT" --supervisor-test > "$LOG_DIR/zyginit.out" 2>&1 &
ZPID=$!

# Wait for socket to appear
for i in 1 2 3 4 5; do
    [ -S "$TEST_DIR/zyginit.sock" ] && break
    sleep 1
done
[ -S "$TEST_DIR/zyginit.sock" ] || { echo "FAIL: socket never appeared"; exit 1; }

# Give alpha oneshot a moment to run
sleep 1

# Confirm START alpha ran
grep -q "START alpha" "$LOG_DIR/trace" || { echo "FAIL: alpha start didn't run"; exit 1; }

# Issue halt via zygctl
ZYGINIT_SOCKET="$TEST_DIR/zyginit.sock" "$ZYGCTL" halt

# Wait for zyginit to exit (non-PID-1 path returns from main)
for i in 1 2 3 4 5; do
    kill -0 $ZPID 2>/dev/null || break
    sleep 1
done

# Confirm STOP alpha ran during shutdown
grep -q "STOP alpha" "$LOG_DIR/trace" || {
    echo "FAIL: alpha stop didn't run during shutdown"
    echo "trace contents:"
    cat "$LOG_DIR/trace"
    exit 1
}

echo "PASS: oneshot stop command runs at shutdown"
#!/bin/sh
# Snapshot tests for the ui.reef renderer.
# Each scenario invokes `zyginit --ui-demo <name>` and diffs against a golden file.
set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
ZYGINIT="$PROJECT_DIR/build/zyginit"
SNAP_DIR="$SCRIPT_DIR/snapshots"

mkdir -p "$SNAP_DIR"
PASS=0
FAIL=0

check_scenario() {
    name=$1
    env_prefix=$2
    expected="$SNAP_DIR/$name.txt"
    got=$(mktemp)
    eval "$env_prefix $ZYGINIT --ui-demo $name" > "$got" 2>&1 || true
    if [ ! -f "$expected" ]; then
        cp "$got" "$expected"
        echo "  [created]  $name"
        PASS=$((PASS+1))
        rm -f "$got"
        return
    fi
    if diff -u "$expected" "$got" > /dev/null; then
        echo "  [pass]     $name"
        PASS=$((PASS+1))
    else
        echo "  [FAIL]     $name"
        diff -u "$expected" "$got" || true
        FAIL=$((FAIL+1))
    fi
    rm -f "$got"
}

echo "ui snapshot tests"
# ZYGINIT_FORCE_80x25=1 locks the TIOCGWINSZ path to 80x25 regardless
# of the host terminal size, keeping snapshot output deterministic.
F80="ZYGINIT_FORCE_80x25=1"
check_scenario skeleton "ZYGINIT_NO_UI=1 $F80"
check_scenario plain_boot "ZYGINIT_NO_UI=1 $F80"
check_scenario detect_no_color "NO_COLOR=1 TERM=xterm-256color $F80"
check_scenario detect_dumb "TERM=dumb $F80"
check_scenario detect_sun "TERM=sun $F80"
check_scenario detect_sun_color "TERM=sun-color $F80"

check_scenario palette_true "TERM=xterm-256color $F80"
check_scenario palette_16   "TERM=sun $F80"
check_scenario glyphs_unicode "TERM=xterm-256color $F80"
check_scenario glyphs_ascii   "TERM=xterm-256color ZYGINIT_ASCII=1 $F80"
check_scenario sigil_rich   "TERM=sun-color $F80"

check_scenario boot_tape_rich  "TERM=xterm-256color $F80"
check_scenario boot_tape_ascii "TERM=xterm ZYGINIT_ASCII=1 $F80"
check_scenario boot_tape_plain "ZYGINIT_NO_UI=1 $F80"

check_scenario progress_rich  "TERM=xterm-256color $F80"
check_scenario progress_ascii "TERM=xterm ZYGINIT_ASCII=1 $F80"
check_scenario progress_full  "TERM=xterm-256color $F80"

check_scenario spinner_forward  "TERM=xterm-256color $F80"
check_scenario spinner_reverse  "TERM=xterm-256color $F80"
check_scenario spinner_in_tape  "TERM=xterm-256color $F80"

check_scenario final_card_ok      "TERM=xterm-256color $F80"
check_scenario final_card_failed  "TERM=xterm-256color $F80"

check_scenario shutdown_mirror   "TERM=xterm-256color $F80"
check_scenario shutdown_final    "TERM=xterm-256color $F80"

check_scenario task_ok                "ZYGINIT_FORCE_80x25=1 TERM=xterm-256color"
check_scenario task_failed            "ZYGINIT_FORCE_80x25=1 TERM=xterm-256color"
check_scenario boot_with_skipped      "ZYGINIT_FORCE_80x25=1 TERM=xterm-256color"
check_scenario final_card_with_skipped "ZYGINIT_FORCE_80x25=1 TERM=xterm-256color"
check_scenario status_skipped         "ZYGINIT_FORCE_80x25=1 TERM=xterm-256color"

echo
echo "passed: $PASS  failed: $FAIL"
[ "$FAIL" -eq 0 ]