run_tests.sh - zygaena/zyginit

root / tests / integration / run_tests.sh
run_tests.sh Bash 1182 lines 35.8 KB
#!/bin/sh
#
# zyginit integration test harness
#
# Black-box tests for zyginit daemon + zygctl CLI.
# Creates temporary service definitions, starts zyginit in non-PID-1 mode,
# exercises zygctl commands, and verifies expected behavior.
#
# Usage: ./tests/integration/run_tests.sh
#
# Both binaries (zyginit + zygctl) are rebuilt from source at startup so the
# tests always run against current code. Set ZYGINIT_TEST_NO_BUILD=1 to skip
# the rebuild (fast iteration when you know the binaries are already current).
#
# Requirements (Linux dev): clang + reefc on PATH.
#

set -e

# ============================================================================
# Configuration
# ============================================================================

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
ZYGINIT="$PROJECT_DIR/build/zyginit"
ZYGCTL="$PROJECT_DIR/tools/zygctl/build/zygctl"

TEST_DIR=$(mktemp -d /tmp/zyginit-test.XXXXXX)
CONFIG_DIR="$TEST_DIR/config"
ENABLED_DIR="$CONFIG_DIR/enabled.d"
BIN_DIR="$TEST_DIR/bin"
SOCKET="$TEST_DIR/zyginit.sock"
DAEMON_LOG="$TEST_DIR/zyginit.log"
DAEMON_PID=""

LOG_DIR="$TEST_DIR/log"
RUN_DIR="$TEST_DIR/run"
export ZYGINIT_CONFIG_DIR="$CONFIG_DIR"
export ZYGINIT_SOCKET="$SOCKET"
export ZYGINIT_LOG_DIR="$LOG_DIR"
export ZYGINIT_RUN_DIR="$RUN_DIR"
# Force the daemon into plain mode regardless of TTY state — keeps
# integration test diffs stable and prevents ANSI escapes from
# polluting captured assertions.
export ZYGINIT_NO_UI=1

PASS=0
FAIL=0
TOTAL=0

# ============================================================================
# Build binaries
# ============================================================================
#
# Always rebuild from source so the test binaries match current code. The
# harness used to only build when a binary was MISSING, which let a stale
# build/ artifact (e.g. an old version string) silently linger and fail tests.
# Set ZYGINIT_TEST_NO_BUILD=1 to skip (e.g. fast local iteration).

if [ -z "$ZYGINIT_TEST_NO_BUILD" ]; then
    printf "Building zyginit + zygctl from source...\n"
    mkdir -p "$PROJECT_DIR/build" "$PROJECT_DIR/tools/zygctl/build"
    (
        cd "$PROJECT_DIR" &&
        clang -c src/helpers.c -o build/helpers.o &&
        clang -c src/contract_linux_stubs.c -o build/contract_linux_stubs.o &&
        reefc build --obj build/helpers.o --obj build/contract_linux_stubs.o
    ) || { echo "ERROR: zyginit build failed"; exit 1; }
    (
        cd "$PROJECT_DIR/tools/zygctl" &&
        clang -c src/symlink_wrapper.c -o build/symlink_wrapper.o &&
        reefc build --obj build/symlink_wrapper.o
    ) || { echo "ERROR: zygctl build failed"; exit 1; }
fi

# ============================================================================
# Test framework
# ============================================================================

pass() {
    PASS=$((PASS + 1))
    TOTAL=$((TOTAL + 1))
    printf "  PASS: %s\n" "$1"
}

fail() {
    FAIL=$((FAIL + 1))
    TOTAL=$((TOTAL + 1))
    printf "  FAIL: %s\n" "$1"
    if [ -n "$2" ]; then
        printf "        expected: %s\n" "$2"
    fi
    if [ -n "$3" ]; then
        printf "        got:      %s\n" "$3"
    fi
}

section() {
    printf "\n=== %s ===\n" "$1"
}

# Assert output contains a substring
assert_contains() {
    local output="$1"
    local expected="$2"
    local label="$3"
    if echo "$output" | grep -q "$expected"; then
        pass "$label"
    else
        fail "$label" "contains '$expected'" "$output"
    fi
}

# Assert output does NOT contain a substring
assert_not_contains() {
    local output="$1"
    local expected="$2"
    local label="$3"
    if echo "$output" | grep -q "$expected"; then
        fail "$label" "not contains '$expected'" "$output"
    else
        pass "$label"
    fi
}

# Assert exact match (trimmed)
assert_equals() {
    local output="$(echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
    local expected="$(echo "$2" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
    local label="$3"
    if [ "$output" = "$expected" ]; then
        pass "$label"
    else
        fail "$label" "'$expected'" "'$output'"
    fi
}

# Run zygctl and capture output
zygctl() {
    "$ZYGCTL" "$@" 2>&1
}

# ============================================================================
# Service definition helpers
# ============================================================================

create_service() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local restart="${4:-never}"
    local delay="${5:-1}"
    local max_retries="${6:-3}"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"

[restart]
on = "$restart"
delay = $delay
max_retries = $max_retries
TOML
}

create_service_with_workdir() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local workdir="$4"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"
working_directory = "$workdir"

[restart]
on = "never"
TOML
}

create_service_with_env() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local workdir="$4"
    local env_array="$5"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"
working_directory = "$workdir"
environment = $env_array

[restart]
on = "never"
TOML
}

# Create a service with a [dependencies] conflicts array.
# Usage: create_service_with_conflicts <name> <type> <cmd> "\"other\""
create_service_with_conflicts() {
    local name="$1"
    local type="$2"
    local cmd="$3"
    local conflicts="$4"

    mkdir -p "$CONFIG_DIR/services/$name"
    cat > "$CONFIG_DIR/services/$name/service.toml" << TOML
[service]
description = "Test service: $name"
type = "$type"

[exec]
start = "$cmd"

[dependencies]
conflicts = [$conflicts]

[restart]
on = "never"
TOML
}

# Drop a helper script into a service's directory.
# Usage: create_service_script <svc> <script-name> "<content>"
create_service_script() {
    local svc="$1"
    local script="$2"
    local content="$3"
    mkdir -p "$CONFIG_DIR/services/$svc"
    printf '%s\n' "$content" > "$CONFIG_DIR/services/$svc/$script"
    chmod +x "$CONFIG_DIR/services/$svc/$script"
}

enable_service() {
    ln -sfn "../services/$1" "$ENABLED_DIR/$1"
}

disable_service() {
    rm -f "$ENABLED_DIR/$1"
}

# ============================================================================
# Daemon lifecycle
# ============================================================================

start_daemon() {
    "$ZYGINIT" > "$DAEMON_LOG" 2>&1 &
    DAEMON_PID=$!
    # Wait for socket to appear
    local retries=0
    while [ ! -S "$SOCKET" ] && [ $retries -lt 30 ]; do
        sleep 0.1
        retries=$((retries + 1))
    done
    if [ ! -S "$SOCKET" ]; then
        echo "ERROR: zyginit failed to start (no socket after 3s)"
        cat "$DAEMON_LOG"
        exit 1
    fi
    # Give services a moment to start
    sleep 1
}

stop_daemon() {
    if [ -n "$DAEMON_PID" ]; then
        kill "$DAEMON_PID" 2>/dev/null || true
        # Wait up to 5 seconds for graceful shutdown
        local retries=0
        while kill -0 "$DAEMON_PID" 2>/dev/null && [ $retries -lt 50 ]; do
            sleep 0.1
            retries=$((retries + 1))
        done
        # Force kill if still alive
        kill -9 "$DAEMON_PID" 2>/dev/null || true
        wait "$DAEMON_PID" 2>/dev/null || true
        DAEMON_PID=""
    fi
    # Kill any orphaned test service processes
    pkill -f "sleeper.sh" 2>/dev/null || true
    rm -f "$SOCKET"
}

# ============================================================================
# Cleanup
# ============================================================================

cleanup() {
    stop_daemon
    rm -rf "$TEST_DIR"
}
trap cleanup EXIT

# ============================================================================
# Create test helper scripts
# ============================================================================

mkdir -p "$BIN_DIR" "$RUN_DIR"

# Long-running daemon (sleeps via shell trap loop)
cat > "$BIN_DIR/sleeper.sh" << 'SCRIPT'
#!/bin/sh
trap 'exit 0' TERM INT
while true; do sleep 1; done
SCRIPT
chmod +x "$BIN_DIR/sleeper.sh"

# Successful oneshot
cat > "$BIN_DIR/succeed.sh" << 'SCRIPT'
#!/bin/sh
exit 0
SCRIPT
chmod +x "$BIN_DIR/succeed.sh"

# Failing oneshot/daemon
cat > "$BIN_DIR/fail.sh" << 'SCRIPT'
#!/bin/sh
exit 1
SCRIPT
chmod +x "$BIN_DIR/fail.sh"

# Write current directory to a marker file, then exit
cat > "$BIN_DIR/write_cwd.sh" << 'SCRIPT'
#!/bin/sh
pwd > cwd_marker.txt
exit 0
SCRIPT
chmod +x "$BIN_DIR/write_cwd.sh"

# Write an env var to a marker file, then exit
cat > "$BIN_DIR/write_env.sh" << 'SCRIPT'
#!/bin/sh
echo "$TEST_VAR" > env_marker.txt
exit 0
SCRIPT
chmod +x "$BIN_DIR/write_env.sh"

# Write to stderr and exit with failure (for log capture testing)
cat > "$BIN_DIR/log_fail.sh" << 'SCRIPT'
#!/bin/sh
echo "service starting up" >&1
echo "fatal: something went wrong" >&2
exit 1
SCRIPT
chmod +x "$BIN_DIR/log_fail.sh"

# ============================================================================
# Pre-flight checks
# ============================================================================

if [ ! -x "$ZYGINIT" ]; then
    echo "ERROR: zyginit binary not found at $ZYGINIT"
    echo "Run: reefc build --obj build/contract_stubs.o"
    exit 1
fi

if [ ! -x "$ZYGCTL" ]; then
    echo "ERROR: zygctl binary not found at $ZYGCTL"
    echo "Run: cd tools/zygctl && reefc build --obj build/symlink_wrapper.o"
    exit 1
fi

echo "zyginit integration tests"
echo "  zyginit: $ZYGINIT"
echo "  zygctl:  $ZYGCTL"
echo "  tmpdir:  $TEST_DIR"

# ============================================================================
# Test Suite 1: zygctl local commands (no daemon)
# ============================================================================

section "zygctl local commands"

# Setup config dir
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Test help
output=$(zygctl help)
assert_contains "$output" "zygctl" "help shows program name"
assert_contains "$output" "enable" "help lists enable command"
assert_contains "$output" "disable" "help lists disable command"
assert_contains "$output" "reload" "help lists reload command"

# Test enable — service does not exist
output=$(zygctl enable nonexistent)
assert_contains "$output" "error" "enable nonexistent shows error"

# Create a service definition and test enable
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
output=$(zygctl enable sleeper)
assert_contains "$output" "enabled" "enable sleeper succeeds"

# Verify symlink was created
if [ -L "$ENABLED_DIR/sleeper" ]; then
    pass "enable creates symlink"
else
    fail "enable creates symlink" "symlink at $ENABLED_DIR/sleeper" "not found"
fi

# Verify symlink target points at the new-layout directory (not flat .toml)
target=$(readlink "$ENABLED_DIR/sleeper")
case "$target" in
    *services/sleeper) pass "enable symlink targets services/sleeper" ;;
    *)                 fail "enable symlink targets services/sleeper" "*services/sleeper" "got: $target" ;;
esac

# Test enable again (already enabled)
output=$(zygctl enable sleeper)
assert_contains "$output" "already enabled" "enable again shows already enabled"

# Test disable
output=$(zygctl disable sleeper)
assert_contains "$output" "disabled" "disable sleeper succeeds"

# Verify symlink was removed
if [ ! -e "$ENABLED_DIR/sleeper" ]; then
    pass "disable removes symlink"
else
    fail "disable removes symlink" "no file at $ENABLED_DIR/sleeper" "file exists"
fi

# Test disable again (not enabled)
output=$(zygctl disable sleeper)
assert_contains "$output" "not enabled" "disable again shows not enabled"

# Test connect failure (no daemon running)
output=$(zygctl status)
assert_contains "$output" "cannot connect" "status without daemon shows connection error"

# ============================================================================
# Test Suite 2: Basic daemon operation
# ============================================================================

section "Basic daemon operation"

# Clean up and set up fresh
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create test services
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
create_service "greeter" "oneshot" "$BIN_DIR/succeed.sh"
enable_service "sleeper"
enable_service "greeter"

start_daemon

# Test list
output=$(zygctl list)
assert_contains "$output" "sleeper" "list shows sleeper"
assert_contains "$output" "greeter" "list shows greeter"
assert_contains "$output" "daemon" "list shows daemon type"
assert_contains "$output" "oneshot" "list shows oneshot type"

# Test status (all)
output=$(zygctl status)
assert_contains "$output" "sleeper" "status shows sleeper"
assert_contains "$output" "greeter" "status shows greeter"

# Test status of running daemon
output=$(zygctl status sleeper)
assert_contains "$output" "running" "sleeper is running"
assert_contains "$output" "sleeper" "sleeper status row present"

# Test status of completed oneshot
output=$(zygctl status greeter)
assert_contains "$output" "stopped" "greeter (oneshot) is stopped after completion"

# Test help via socket (this goes through socket since daemon is running... actually
# version and help are local in zygctl. Let me test an actual socket help)
# Actually, version/help in zygctl are intercepted locally. Let me test unknown command.
output=$(zygctl unknown_cmd 2>&1)
assert_contains "$output" "error" "unknown command returns error"

# Test stop
output=$(zygctl stop sleeper)
assert_contains "$output" "stopping" "stop sleeper succeeds"
sleep 1

output=$(zygctl status sleeper)
assert_contains "$output" "stopped" "sleeper is stopped after stop"

# Test start
output=$(zygctl start sleeper)
assert_contains "$output" "started" "start sleeper succeeds"
sleep 1

output=$(zygctl status sleeper)
assert_contains "$output" "running" "sleeper is running after start"

# Test restart
output=$(zygctl restart sleeper)
# restart of a running service sends stop first
sleep 2

output=$(zygctl status sleeper)
# It should be running or restarting
assert_contains "$output" "sleeper" "restart: service exists after restart"

# Test status of unknown service
output=$(zygctl status nonexistent)
assert_contains "$output" "error" "status of unknown service shows error"

# Ensure sleeper is running before testing double-start
sleep 1
output=$(zygctl status sleeper)
if echo "$output" | grep -q "running"; then
    output=$(zygctl start sleeper)
    assert_contains "$output" "already running" "start already running shows message"
else
    # Restart may have left it stopped; start it, wait, then try again
    zygctl start sleeper > /dev/null 2>&1
    sleep 1
    output=$(zygctl start sleeper)
    assert_contains "$output" "already running" "start already running shows message"
fi

stop_daemon

# ============================================================================
# Test Suite 3: Restart policies
# ============================================================================

section "Restart policies"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service that exits immediately with failure — restart on failure, max 2 retries
create_service "failer" "daemon" "$BIN_DIR/fail.sh" "failure" 1 2

enable_service "failer"

start_daemon

# Wait for restart attempts to play out (1s delay * 2 retries + startup time)
sleep 5

output=$(zygctl status failer)
assert_contains "$output" "maint." "failer enters maintenance after max retries"
assert_contains "$output" "restarts=" "failer shows restart count"
assert_contains "$output" "exit=" "failer shows exit code"

stop_daemon

# ============================================================================
# Test Suite 4: Enable/disable via socket + reload
# ============================================================================

section "Enable/disable via socket + reload"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Start with one service
create_service "alpha" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "alpha"

start_daemon

output=$(zygctl status)
assert_contains "$output" "alpha" "alpha is loaded"

# Create a new service and enable it
create_service "beta" "daemon" "$BIN_DIR/sleeper.sh"

# Enable via socket (server-side)
output=$(zygctl enable beta)
assert_contains "$output" "enabled" "enable beta via socket"

# Reload to pick up the new service
output=$(zygctl reload)
assert_contains "$output" "reload" "reload command accepted"
sleep 2

output=$(zygctl status beta)
assert_contains "$output" "beta" "beta appears after reload"
assert_contains "$output" "running" "beta is running after reload"

# Disable alpha and reload
output=$(zygctl disable alpha)
assert_contains "$output" "disabled" "disable alpha via socket"

output=$(zygctl reload)
sleep 2

output=$(zygctl status alpha)
assert_contains "$output" "disabled" "alpha is disabled after reload"

stop_daemon

# ============================================================================
# Test Suite 5: Oneshot service behavior
# ============================================================================

section "Oneshot services"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Successful oneshot
create_service "setup-ok" "oneshot" "$BIN_DIR/succeed.sh"
# Failing oneshot
create_service "setup-fail" "oneshot" "$BIN_DIR/fail.sh"

enable_service "setup-ok"
enable_service "setup-fail"

start_daemon

sleep 2

output=$(zygctl status setup-ok)
assert_contains "$output" "stopped" "successful oneshot shows stopped"

output=$(zygctl status setup-fail)
assert_contains "$output" "failed" "failing oneshot shows failed"
assert_contains "$output" "exit=" "failing oneshot shows exit code"

stop_daemon

# ============================================================================
# Test Suite 5b: Task type semantics
# ============================================================================

section "Task type semantics"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "task-success" "task" "$BIN_DIR/succeed.sh" "never"
enable_service "task-success"

create_service "task-failure" "task" "$BIN_DIR/fail.sh" "never"
enable_service "task-failure"

start_daemon
sleep 2

out=$(zygctl status)

echo "$out" | grep -q "task-success.*stopped" && \
    pass "task exit 0 = stopped" || \
    fail "task exit 0 = stopped" "status=stopped" "got: $(echo "$out" | grep task-success)"

echo "$out" | grep -q "task-failure.*failed" && \
    pass "task exit !=0 = failed" || \
    fail "task exit !=0 = failed" "status=failed" "got: $(echo "$out" | grep task-failure)"

# Verify no restart attempt — restart_count should be 0 (no restarts= field)
if echo "$out" | grep "task-failure" | grep -q "restarts="; then
    fail "task does not restart" "no restarts= field" "restarts field present"
else
    pass "task does not restart"
fi

stop_daemon

# ============================================================================
# Test Suite 6: Working directory
# ============================================================================

section "Working directory"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create a target directory for the service to run in
WORK_DIR="$TEST_DIR/workdir"
mkdir -p "$WORK_DIR"

create_service_with_workdir "cwd-test" "oneshot" "$BIN_DIR/write_cwd.sh" "$WORK_DIR"
enable_service "cwd-test"

start_daemon

sleep 2

# Check that the marker file was written in the working directory
if [ -f "$WORK_DIR/cwd_marker.txt" ]; then
    pass "working_directory: marker file created in workdir"
    marker_content=$(cat "$WORK_DIR/cwd_marker.txt" | tr -d '\n')
    assert_equals "$marker_content" "$WORK_DIR" "working_directory: cwd matches configured path"
else
    fail "working_directory: marker file created in workdir" "file at $WORK_DIR/cwd_marker.txt" "not found"
    fail "working_directory: cwd matches configured path" "$WORK_DIR" "marker file missing"
fi

stop_daemon

# ============================================================================
# Test Suite 7: Environment variables
# ============================================================================

section "Environment variables"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

ENV_DIR="$TEST_DIR/envdir"
mkdir -p "$ENV_DIR"

create_service_with_env "env-test" "oneshot" "$BIN_DIR/write_env.sh" "$ENV_DIR" '["TEST_VAR=hello_zyginit"]'
enable_service "env-test"

start_daemon

sleep 2

if [ -f "$ENV_DIR/env_marker.txt" ]; then
    pass "environment: marker file created"
    marker_content=$(cat "$ENV_DIR/env_marker.txt" | tr -d '\n')
    assert_equals "$marker_content" "hello_zyginit" "environment: env var passed to service"
else
    fail "environment: marker file created" "file at $ENV_DIR/env_marker.txt" "not found"
    fail "environment: env var passed to service" "hello_zyginit" "marker file missing"
fi

stop_daemon

# ============================================================================
# Test Suite 8: Privilege separation (error path)
# ============================================================================

section "Privilege separation"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service with nonexistent user — should fail with exit 126
mkdir -p "$CONFIG_DIR/services/baduser"
cat > "$CONFIG_DIR/services/baduser/service.toml" << TOML
[service]
description = "Test bad user"
type = "oneshot"

[exec]
start = "$BIN_DIR/succeed.sh"
user = "nonexistent_user_zyginit_test"

[restart]
on = "never"
TOML
enable_service "baduser"

start_daemon

sleep 2

output=$(zygctl status baduser)
assert_contains "$output" "failed" "privilege: bad user causes failure"
assert_contains "$output" "exit=" "privilege: exit code shown"

stop_daemon

# ============================================================================
# Test Suite 9: Service output capture + zygctl log
# ============================================================================

section "Service output capture"

rm -rf "$CONFIG_DIR" "$LOG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "log-test" "oneshot" "$BIN_DIR/log_fail.sh"
enable_service "log-test"

start_daemon

sleep 2

# Check that log file was created
if [ -f "$LOG_DIR/log-test.log" ]; then
    pass "log: per-service log file created"
else
    fail "log: per-service log file created" "file at $LOG_DIR/log-test.log" "not found"
fi

# Check log file contains expected output
log_content=$(cat "$LOG_DIR/log-test.log" 2>/dev/null)
assert_contains "$log_content" "something went wrong" "log: stderr captured in log file"
assert_contains "$log_content" "starting up" "log: stdout captured in log file"

# Check zygctl log command
output=$(zygctl log log-test)
assert_contains "$output" "something went wrong" "log: zygctl log shows stderr"
assert_contains "$output" "starting up" "log: zygctl log shows stdout"

# Check zygctl log for unknown service
output=$(zygctl log nonexistent)
assert_contains "$output" "error" "log: unknown service shows error"

# Check zygctl log for service with no log
output=$(zygctl log setup-fail 2>&1)
# setup-fail doesn't exist in this test run, so it'll be unknown
assert_contains "$output" "error" "log: missing service shows error"

stop_daemon

# ============================================================================
# Test Suite 9b: Spawn-failure capture (bug 003)
# ============================================================================
#
# When a service's start command cannot be exec'd (missing/unrunnable),
# the supervisor must NOT let the service vanish silently with an empty log.
# The child retries the exec a few times, then records the spawn failure to
# both the per-service log (fd 2) and the central zyginit.log before exiting.

section "Spawn-failure capture (bug 003)"

rm -rf "$CONFIG_DIR" "$LOG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Start command points at a path that does not exist — exec always fails.
create_service "spawn-fail" "oneshot" "/nonexistent/zyginit-bug003-missing --flag"
enable_service "spawn-fail"

start_daemon

# Child retries the exec (5 x 100ms) before giving up, so allow settle time.
sleep 3

# Central zyginit log must exist and record the spawn failure.
if [ -f "$LOG_DIR/zyginit.log" ]; then
    pass "spawn-fail: central zyginit.log created"
else
    fail "spawn-fail: central zyginit.log created" "file at $LOG_DIR/zyginit.log" "not found"
fi
central_content=$(cat "$LOG_DIR/zyginit.log" 2>/dev/null)
assert_contains "$central_content" "exec failed" "spawn-fail: central log records exec failure"
assert_contains "$central_content" "spawn-fail" "spawn-fail: central log names the service"

# The per-service log must NOT be empty (the bug-003 symptom was an empty log).
perservice_content=$(cat "$LOG_DIR/spawn-fail.log" 2>/dev/null)
assert_contains "$perservice_content" "exec failed" "spawn-fail: per-service log records exec failure"

# The service is reported failed (exit 127), never as exit=-1.
output=$(zygctl status spawn-fail)
assert_contains "$output" "failed" "spawn-fail: service reported failed"
assert_not_contains "$output" "exit=-1" "spawn-fail: not reported as exit=-1"

stop_daemon

# ============================================================================
# Test Suite 10: zygctl replace — command parsing and state-file write
# ============================================================================

section "zygctl replace — command parsing and state-file write"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Create a minimal service
create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sleeper"

start_daemon

# Test 1: zygctl replace with no args
out=$("$ZYGCTL" replace 2>&1 || true)
assert_contains "$out" "replace queued" "replace acks with no args"
assert_contains "$out" "wait=0" "replace acks with default wait=0"

# Wait briefly to let the daemon process the flag (it will attempt exec
# of /sbin/init, which fails on Linux because that's systemd, not us;
# zyginit logs FATAL: execve returned and continues running.)
sleep 1.5

# Verify state.toml was written before the exec attempt
if [ -f "$RUN_DIR/state.toml" ]; then
    pass "state.toml written by replace flow"
    out=$(cat "$RUN_DIR/state.toml")
    assert_contains "$out" "boot_time =" "state.toml has boot_time line"
else
    fail "state.toml written by replace flow" "file present at $RUN_DIR/state.toml" "missing"
fi

# Verify the daemon log shows the outgoing flow
log_out=$(cat "$DAEMON_LOG")
assert_contains "$log_out" "replace: requested" "daemon logged replace request"
assert_contains "$log_out" "replace: state written" "daemon logged state write"
assert_contains "$log_out" "replace: execve" "daemon logged execve attempt"

# Stop daemon (it survived the failed execve and is still running)
stop_daemon
# Clean up the leftover state.toml so the next test starts fresh
rm -f "$RUN_DIR/state.toml"

# Test 2: zygctl replace --wait=N
start_daemon
out=$("$ZYGCTL" replace --wait=10 2>&1 || true)
assert_contains "$out" "replace queued" "replace --wait=10 acks"
assert_contains "$out" "wait=10" "replace --wait=10 reflects wait value"
sleep 1.5
stop_daemon
rm -f "$RUN_DIR/state.toml"

# Test 3: zygctl replace --bogus
start_daemon
out=$("$ZYGCTL" replace --bogus 2>&1 || true)
assert_contains "$out" "error" "replace --bogus returns error"
assert_contains "$out" "unknown argument" "replace --bogus error mentions unknown argument"
stop_daemon

# ============================================================================
# Test Suite 11: [condition] block — SKIPPED state
# ============================================================================

section "[condition] block — SKIPPED state"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Service whose exists_file condition references a path that cannot exist.
mkdir -p "$CONFIG_DIR/services/skip-missing-file"
cat > "$CONFIG_DIR/services/skip-missing-file/service.toml" << 'TOML'
[service]
type = "daemon"

[exec]
start = "/bin/sleep 1000"

[condition]
exists_file = "/this/path/definitely/does/not/exist"
TOML
enable_service "skip-missing-file"

# Service whose command condition exits non-zero (/bin/false).
mkdir -p "$CONFIG_DIR/services/skip-cmd-fail"
cat > "$CONFIG_DIR/services/skip-cmd-fail/service.toml" << 'TOML'
[service]
type = "daemon"

[exec]
start = "/bin/sleep 1000"

[condition]
command = "/bin/false"
TOML
enable_service "skip-cmd-fail"

# Service whose command condition exits zero — should start, not be skipped.
mkdir -p "$CONFIG_DIR/services/start-cmd-pass"
cat > "$CONFIG_DIR/services/start-cmd-pass/service.toml" << 'TOML'
[service]
type = "task"

[exec]
start = "/bin/true"

[condition]
command = "/bin/true"
TOML
enable_service "start-cmd-pass"

start_daemon
sleep 2

# Task 7 verifies the supervisor wiring (condition fails → no fork, state
# transitions to STATE_SKIPPED). The display label "skipped" comes in Task 8
# when ui_render.reef maps STATE_SKIPPED. For now, assert the wiring:
# a service with a failing condition must NOT appear as running.
out=$("$ZYGCTL" status)

echo "$out" | grep "skip-missing-file" | grep -q "running" && \
    fail "missing file → not running" "running" "service forked despite failing condition" || \
    pass "missing file → not running"

echo "$out" | grep "skip-cmd-fail" | grep -q "running" && \
    fail "command exit non-zero → not running" "running" "service forked despite failing command" || \
    pass "command exit non-zero → not running"

# A service whose condition passes (task type, /bin/true) ran to completion
# and is now stopped. Either "stopped" or "running" (briefly) is acceptable;
# what we want to confirm is the service was NOT skipped.
echo "$out" | grep "start-cmd-pass" | grep -qi "skipped" && \
    fail "command exit zero → NOT skipped" "not skipped" "incorrectly skipped: $(echo "$out" | grep -i start-cmd-pass || echo '(no line)')" || \
    pass "command exit zero → NOT skipped"

stop_daemon

# ============================================================================
# Test Suite 12: TTY-aware dispatch — plain output when piped
# ============================================================================

section "TTY-aware dispatch"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service "sleeper" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sleeper"

start_daemon

# zygctl status is piped here (stdout is not a TTY), so client_should_be_plain()
# returns true and the client sends "status plain". Output must contain no ANSI
# escape sequences.
out=$(zygctl status)
if printf '%s' "$out" | grep -qP '\x1b'; then
    fail "zygctl status returns no ANSI escapes when piped" "no ESC bytes" "ESC found"
else
    pass "zygctl status returns no ANSI escapes when piped"
fi

# Same check for list
out=$(zygctl list)
if printf '%s' "$out" | grep -qP '\x1b'; then
    fail "zygctl list returns no ANSI escapes when piped" "no ESC bytes" "ESC found"
else
    pass "zygctl list returns no ANSI escapes when piped"
fi

stop_daemon

# ============================================================================
# Shutdown orchestration tests (standalone script)
# ============================================================================

section "Shutdown orchestration"

if bash "$SCRIPT_DIR/test_shutdown.sh" >> "$TEST_DIR/shutdown.log" 2>&1; then
    pass "oneshot stop command runs at shutdown"
else
    fail "oneshot stop command runs at shutdown" "PASS" "FAIL — see $TEST_DIR/shutdown.log"
fi

# ============================================================================
# Test Suite: Service conflicts
# ============================================================================
section "Service conflicts"

rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# Two daemons that declare each other as conflicting; both enabled.
create_service_with_conflicts "mailer-a" "daemon" "$BIN_DIR/sleeper.sh" "\"mailer-b\""
create_service "mailer-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "mailer-a"
enable_service "mailer-b"

start_daemon

# Boot resolution: both enabled + conflicting => start neither, fail both.
output=$(zygctl status mailer-a)
assert_contains "$output" "failed" "conflict: mailer-a failed at boot"
assert_contains "$output" "conflicts with" "conflict: mailer-a status shows reason"
output=$(zygctl status mailer-b)
assert_contains "$output" "failed" "conflict: mailer-b failed at boot"

stop_daemon

# --- Runtime refusal: starting a service whose conflicting peer runs is refused.
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# svc-b conflicts with svc-a. Enable only svc-a at boot; add svc-b via reload so
# it joins the table WAITING without the boot pre-flight failing both.
create_service "svc-a" "daemon" "$BIN_DIR/sleeper.sh"
create_service_with_conflicts "svc-b" "daemon" "$BIN_DIR/sleeper.sh" "\"svc-a\""
enable_service "svc-a"
enable_service "svc-b"
disable_service "svc-b"

start_daemon

output=$(zygctl status svc-a)
assert_contains "$output" "running" "runtime: svc-a is running"

enable_service "svc-b"
zygctl reload > /dev/null 2>&1
sleep 1
output=$(zygctl start svc-b)
assert_contains "$output" "conflicts with running service" "runtime: start svc-b refused"
output=$(zygctl status svc-a)
assert_contains "$output" "running" "runtime: svc-a still running after refusal"

stop_daemon

# --- Stale reason cleared on successful start after conflict-fail.
# conflict-fail both at boot, then start one after the other is gone (already
# failed so not running); the restarted service must show "running", NOT
# "conflicts with".
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

create_service_with_conflicts "clr-a" "daemon" "$BIN_DIR/sleeper.sh" "\"clr-b\""
create_service "clr-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "clr-a"
enable_service "clr-b"

start_daemon

# Both should be failed at boot (mutual conflict)
output=$(zygctl status clr-a)
assert_contains "$output" "failed" "stale-clear: clr-a failed at boot"
output=$(zygctl status clr-b)
assert_contains "$output" "failed" "stale-clear: clr-b failed at boot"

# Now start clr-a — clr-b is failed (not running), so the start should succeed
output=$(zygctl start clr-a)
sleep 1
output=$(zygctl status clr-a)
assert_contains "$output" "running" "stale-clear: clr-a running after manual start"
assert_not_contains "$output" "conflicts with" "stale-clear: clr-a status has no stale conflict reason"

stop_daemon

# --- Symmetry: declare the conflict on ONE side only; both still fail at boot.
stop_daemon
rm -rf "$CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$ENABLED_DIR"

# only one side declares the conflict
create_service_with_conflicts "sym-a" "daemon" "$BIN_DIR/sleeper.sh" "\"sym-b\""
create_service "sym-b" "daemon" "$BIN_DIR/sleeper.sh"
enable_service "sym-a"
enable_service "sym-b"

start_daemon

output=$(zygctl status sym-a)
assert_contains "$output" "failed" "symmetry: sym-a failed (declared side)"
output=$(zygctl status sym-b)
assert_contains "$output" "failed" "symmetry: sym-b failed (non-declared side)"

stop_daemon

# ============================================================================
# Version reporting
# ============================================================================

section "Version reporting"

# Both binaries should respond to --version (and `version` subcommand for zygctl)
# with a single line "<binary> <version>" matching reef.toml.

EXPECTED_VERSION=$(grep '^version' "$PROJECT_DIR/reef.toml" | head -1 | sed 's/.*"\([^"]*\)".*/\1/')

OUT="$("$ZYGINIT" --version 2>&1)"
assert_equals "$OUT" "zyginit $EXPECTED_VERSION" "zyginit --version output"

OUT="$(zygctl --version)"
assert_equals "$OUT" "zygctl $EXPECTED_VERSION" "zygctl --version output"

OUT="$(zygctl version)"
assert_equals "$OUT" "zygctl $EXPECTED_VERSION" "zygctl version subcommand output"

# ============================================================================
# Results
# ============================================================================

echo ""
echo "========================================"
echo "Results: $PASS passed, $FAIL failed (of $TOTAL)"
echo "========================================"

if [ $FAIL -gt 0 ]; then
    exit 1
fi

exit 0