supervisor.reef - zygaena/zyginit

root / src / supervisor.reef
supervisor.reef Reef 1149 lines 38.0 KB
/******************************************************************************
                __               ____                __
               / /   ___  ____ _/ __/_____________ _/ /__
              / /   / _ \/ __ `/ /_/ ___/ ___/ __ `/ / _ \
             / /___/  __/ /_/ / __(__  ) /__/ /_/ / /  __/
            /_____/\___/\__,_/_/ /____/\___/\__,_/_/\___/

    (C)opyright 2025-2026, Leafscale, LLC -  https://www.leafscale.com

    Project: zyginit
   Filename: supervisor.reef
    Authors: Chris Tusa <chris.tusa@leafscale.com>
    License: <see LICENSE file included with this source code>
Description: Service lifecycle management — fork/exec, state, restart

******************************************************************************/

module supervisor

import condition
import config
import contract
import ui
import io.dir
import io.file
import sys.env as sysenv
import sys.fd as fd
import sys.process as process
import sys.signal as signal
import time.time as time
import time.clock as clock
import collections.hashmap as hashmap
import core.str

export
    type ServiceRuntime
    type ServiceTable

    // State constants
    fn STATE_DISABLED(): int
    fn STATE_WAITING(): int
    fn STATE_STARTING(): int
    fn STATE_RUNNING(): int
    fn STATE_STOPPING(): int
    fn STATE_STOPPED(): int
    fn STATE_FAILED(): int
    fn STATE_MAINTENANCE(): int
    fn STATE_SKIPPED(): int
    fn state_name(state: int): string

    // ServiceRuntime accessors
    fn rt_state(rt: ServiceRuntime): int
    fn rt_pid(rt: ServiceRuntime): int
    fn rt_contract_id(rt: ServiceRuntime): int
    fn rt_restart_count(rt: ServiceRuntime): int
    fn rt_last_exit_code(rt: ServiceRuntime): int
    fn rt_last_start_time(rt: ServiceRuntime): int
    fn rt_def(rt: ServiceRuntime): config.ServiceDef
    fn rt_skip_reason(rt: ServiceRuntime): string

    // ServiceTable operations
    fn new_service_table(max_services: int): ServiceTable
    fn add_service(table: ServiceTable, def: config.ServiceDef): int
    fn find_service(table: ServiceTable, name: string): int
    fn find_by_contract(table: ServiceTable, contract_id: int): int
    fn service_count(table: ServiceTable): int
    fn get_runtime(table: ServiceTable, idx: int): ServiceRuntime
    fn conflicts_with(table: ServiceTable, a_idx: int, b_idx: int): bool
    fn conflicting_running_peer(table: ServiceTable, idx: int): int
    fn service_name_at(table: ServiceTable, idx: int): string
    proc mark_conflict_failed(table: ServiceTable, idx: int, other_name: string)

    // Lifecycle
    fn start_service(table: ServiceTable, idx: int): bool
    fn stop_service(table: ServiceTable, idx: int): bool
    proc handle_contract_event(table: ServiceTable, contract_id: int, hint_exit_code: int)
    proc handle_child_exit(table: ServiceTable, pid: int, exit_code: int)
    proc check_stop_timeouts(table: ServiceTable)
    proc check_restart_delays(table: ServiceTable)

    // Oneshot teardown
    fn stop_oneshot(table: ServiceTable, idx: int): bool

    // Reload support
    proc mark_runlevel_filtered(table: ServiceTable, idx: int)
    proc disable_service(table: ServiceTable, idx: int)

    // Adopt runtime fields directly — used by the live-replace recovery path
    // (main.apply_recovered_state) and by utils/replace_probe. Production
    // startup and supervision go through start_service / handle_contract_event.
    proc adopt_runtime(table: ServiceTable, idx: int, state: int, pid: int,
                       contract_id: int, restart_count: int,
                       last_exit_code: int, last_start_time: int)
    fn make_def_with_name(name: string): config.ServiceDef

    // Status
    fn get_status_line(table: ServiceTable, idx: int): string
    fn format_duration(seconds: int): string

    // Log directory
    fn log_dir(): string
    fn get_service_log(name: string): string
end export

fn LOG_DIR(): string
    // Persistent on-disk default. /var/run is tmpfs and gets wiped on every
    // reboot, which makes diagnosing PID-1 boot failures nearly impossible —
    // the log of what zyginit did is gone before the next boot can read it.
    // /var/log lives on the BE's ZFS dataset so it survives across reboots
    // and is readable by mounting the BE from another BE if boot fails.
    return sysenv.get_env_or("ZYGINIT_LOG_DIR", "/var/log/zyginit")
end LOG_DIR

// Privilege separation FFI wrappers
extern "C" fn zyginit_getuid_by_name(name: string): int
extern "C" fn zyginit_getgid_by_username(name: string): int
extern "C" fn zyginit_getgid_by_name(name: string): int
extern "C" fn zyginit_drop_privileges(username: string, uid: int, gid: int): int

// ============================================================================
// State constants
// ============================================================================

fn STATE_DISABLED(): int
    return 0
end STATE_DISABLED

fn STATE_WAITING(): int
    return 1
end STATE_WAITING

fn STATE_STARTING(): int
    return 2
end STATE_STARTING

fn STATE_RUNNING(): int
    return 3
end STATE_RUNNING

fn STATE_STOPPING(): int
    return 4
end STATE_STOPPING

fn STATE_STOPPED(): int
    return 5
end STATE_STOPPED

fn STATE_FAILED(): int
    return 6
end STATE_FAILED

fn STATE_MAINTENANCE(): int
    return 7
end STATE_MAINTENANCE

fn STATE_SKIPPED(): int
    return 8
end STATE_SKIPPED

fn state_name(state: int): string
    if state == 0
        return "disabled"
    elif state == 1
        return "waiting"
    elif state == 2
        return "starting"
    elif state == 3
        return "running"
    elif state == 4
        return "stopping"
    elif state == 5
        return "stopped"
    elif state == 6
        return "failed"
    elif state == 7
        return "maintenance"
    elif state == 8
        return "skipped"
    end if
    return "unknown"
end state_name

// ============================================================================
// ServiceRuntime type
// ============================================================================

type ServiceRuntime = struct
    def: config.ServiceDef
    state: int
    contract_id: int
    pid: int
    restart_count: int
    last_exit_code: int
    last_start_time: int
    stop_requested_time: int
    restart_scheduled_time: int
    skip_reason: string
end ServiceRuntime

fn new_runtime(def: config.ServiceDef): ServiceRuntime
    return ServiceRuntime{
        def: def,
        state: STATE_WAITING(),
        contract_id: 0 - 1,
        pid: 0 - 1,
        restart_count: 0,
        last_exit_code: 0 - 1,
        last_start_time: 0,
        stop_requested_time: 0,
        restart_scheduled_time: 0,
        skip_reason: ""
    }
end new_runtime

fn rt_state(rt: ServiceRuntime): int
    return rt.state
end rt_state

fn rt_pid(rt: ServiceRuntime): int
    return rt.pid
end rt_pid

fn rt_contract_id(rt: ServiceRuntime): int
    return rt.contract_id
end rt_contract_id

fn rt_restart_count(rt: ServiceRuntime): int
    return rt.restart_count
end rt_restart_count

fn rt_last_exit_code(rt: ServiceRuntime): int
    return rt.last_exit_code
end rt_last_exit_code

fn rt_last_start_time(rt: ServiceRuntime): int
    return rt.last_start_time
end rt_last_start_time

fn rt_def(rt: ServiceRuntime): config.ServiceDef
    return rt.def
end rt_def

fn rt_skip_reason(rt: ServiceRuntime): string
    return rt.skip_reason
end rt_skip_reason

// ============================================================================
// ServiceTable type
// ============================================================================

type ServiceTable = struct
    runtimes: [ServiceRuntime]
    count: int
    max: int
    name_map: hashmap.HashMap[int]
    contract_map: hashmap.HashMap[int]
end ServiceTable

fn new_service_table(max_services: int): ServiceTable
    return ServiceTable{
        runtimes: new [ServiceRuntime](max_services),
        count: 0,
        max: max_services,
        name_map: hashmap.create[:int](0 - 1),
        contract_map: hashmap.create[:int](0 - 1)
    }
end new_service_table

fn add_service(table: ServiceTable, def: config.ServiceDef): int
    if table.count >= table.max
        return 0 - 1
    end if

    let idx = table.count
    table.runtimes[idx] = new_runtime(def)
    table.name_map.set(config.svc_name(def), idx)
    table.count = table.count + 1
    return idx
end add_service

fn find_service(table: ServiceTable, name: string): int
    if table.name_map.has(name)
        return table.name_map.get(name)
    end if
    return 0 - 1
end find_service

fn find_by_contract(table: ServiceTable, contract_id: int): int
    let key = int_to_str(contract_id)
    if table.contract_map.has(key)
        return table.contract_map.get(key)
    end if
    return 0 - 1
end find_by_contract

fn service_count(table: ServiceTable): int
    return table.count
end service_count

fn get_runtime(table: ServiceTable, idx: int): ServiceRuntime
    return table.runtimes[idx]
end get_runtime

// True if services a and b are declared mutually exclusive. Symmetric: a
// conflict declared on either side counts for both. Self-comparison and
// unknown names never match.
fn conflicts_with(table: ServiceTable, a_idx: int, b_idx: int): bool
    if a_idx == b_idx
        return false
    end if
    let a_def = table.runtimes[a_idx].def
    let b_def = table.runtimes[b_idx].def
    let a_name = config.svc_name(a_def)
    let b_name = config.svc_name(b_def)

    let a_conf = config.svc_conflicts(a_def)
    let a_n = config.svc_conflicts_count(a_def)
    mut i = 0
    while i < a_n
        if a_conf[i] == b_name
            return true
        end if
        i = i + 1
    end while

    let b_conf = config.svc_conflicts(b_def)
    let b_n = config.svc_conflicts_count(b_def)
    i = 0
    while i < b_n
        if b_conf[i] == a_name
            return true
        end if
        i = i + 1
    end while

    return false
end conflicts_with

// Index of a RUNNING/STARTING service that conflicts with idx, or -1 if none.
// Shared by the start guard and by zygctl start for messaging.
fn conflicting_running_peer(table: ServiceTable, idx: int): int
    mut i = 0
    while i < table.count
        if i != idx and conflicts_with(table, idx, i)
            let st = table.runtimes[i].state
            if st == STATE_RUNNING() or st == STATE_STARTING()
                return i
            end if
        end if
        i = i + 1
    end while
    return 0 - 1
end conflicting_running_peer

// Name of the service at idx (convenience for callers outside this module).
fn service_name_at(table: ServiceTable, idx: int): string
    return config.svc_name(table.runtimes[idx].def)
end service_name_at

// Mark a service FAILED because of a conflict, recording the reason for
// `zygctl status`. The skip_reason field doubles as a generic status reason.
proc mark_conflict_failed(table: ServiceTable, idx: int, other_name: string)
    let rt = table.runtimes[idx]
    rt.state = STATE_FAILED()
    rt.skip_reason = str.concat("conflicts with ", other_name)
    table.runtimes[idx] = rt
end mark_conflict_failed

// ============================================================================
// Signal name to number mapping
// ============================================================================

fn signal_from_name(name: string): int
    if name == "TERM" or name == "SIGTERM"
        return signal.SIGTERM()
    elif name == "HUP" or name == "SIGHUP"
        return signal.SIGHUP()
    elif name == "INT" or name == "SIGINT"
        return signal.SIGINT()
    elif name == "KILL" or name == "SIGKILL"
        return signal.SIGKILL()
    elif name == "USR1" or name == "SIGUSR1"
        return signal.SIGUSR1()
    elif name == "USR2" or name == "SIGUSR2"
        return signal.SIGUSR2()
    end if
    return signal.SIGTERM()
end signal_from_name

// ============================================================================
// Service lifecycle
// ============================================================================

// Start a service: set up contract template, fork, exec.
// Returns true on success.
fn start_service(table: ServiceTable, idx: int): bool
    let rt = table.runtimes[idx]
    let def = rt.def
    let name = config.svc_name(def)

    // Evaluate [condition] block before forking. If conditions fail, the
    // service is transitioned to STATE_SKIPPED — no process is ever forked.
    let cond_result = condition.evaluate_conditions(def)
    if condition.cond_passed(cond_result) == false
        rt.state = STATE_SKIPPED()
        rt.skip_reason = condition.cond_reason(cond_result)
        table.runtimes[idx] = rt
        ui.ui_event_skipped(name, rt.skip_reason)
        return true
    end if

    // Conflict guard: refuse to start when a mutually-exclusive peer is already
    // running or starting. Leaves this service's state unchanged. Covers
    // zygctl start, runlevel transitions, and restarts (all funnel here).
    let conflict_peer = conflicting_running_peer(table, idx)
    if conflict_peer >= 0
        let other = config.svc_name(table.runtimes[conflict_peer].def)
        if ui.ui_mode() == ui.MODE_PLAIN()
            println(str.concat(str.concat("supervisor: cannot start ", name),
                    str.concat(": conflicts with running service ", other)))
        end if
        return false
    end if

    // Emit "starting" event so the UI spinner activates before the fork.
    ui.ui_event_starting(name)

    // Set up contract template (will fail on Linux — that's expected)
    let tmpl_fd = contract.setup_template()

    // Fork
    let pid = process.process_fork()

    if pid < 0
        rt.state = STATE_FAILED()
        table.runtimes[idx] = rt
        ui.ui_event_failed(name, 0 - 1, 0)
        if ui.ui_mode() == ui.MODE_PLAIN()
            println("supervisor: fork failed for " + name)
        end if
        if tmpl_fd >= 0
            contract.clear_template(tmpl_fd)
        end if
        return false
    end if

    if pid == 0
        // CHILD PROCESS

        // Redirect fd 0 to /dev/null BEFORE setsid. PID 1 has /dev/console on
        // fd 0/1/2 (from setup_pid1_console), and if the new session is born
        // holding that inherited tty fd, illumos may auto-claim /dev/console
        // as the session's ctty even without an explicit O_NOCTTY-less open.
        // That blocks ttymon (in console-login) from claiming the console
        // later. See bugs/001-supervisor-leaves-console-as-fd0-in-children.md.
        let null_fd = fd.fd_open("/dev/null", fd.O_RDWR(), 0)
        if null_fd >= 0
            fd.fd_dup2(null_fd, 0)
            fd.fd_close(null_fd)
        end if

        process.process_setsid()

        // Redirect stdout/stderr to per-service log file
        let log_path = str.concat(str.concat(LOG_DIR(), "/"), str.concat(name, ".log"))
        let log_flags = fd.O_WRONLY() + fd.O_CREAT() + fd.O_TRUNC()
        let log_fd = fd.fd_open(log_path, log_flags, 420)
        if log_fd >= 0
            fd.fd_dup2(log_fd, 1)
            fd.fd_dup2(log_fd, 2)
            fd.fd_close(log_fd)
        end if

        // Change working directory if specified
        let wdir = config.svc_working_dir(def)
        if str.length(wdir) > 0
            dir.change_dir(wdir)
        end if

        // Default PATH for service children. zyginit as PID 1 inherits
        // a sparse env from the kernel, so without this, services like
        // shell-script wrappers can't find bare command names. The
        // service's own [exec] environment list (below) can override.
        sysenv.set_env("PATH", "/usr/sbin:/usr/bin:/sbin")

        // Set environment variables
        let env_vars = config.svc_environment(def)
        let env_count = config.svc_environment_count(def)
        mut ei = 0
        while ei < env_count
            let entry = env_vars[ei]
            let eq_pos = str.index_of(entry, "=")
            if eq_pos > 0
                let key = str.substring(entry, 0, eq_pos)
                let val = str.substring(entry, eq_pos + 1, str.length(entry) - eq_pos - 1)
                sysenv.set_env(key, val)
            end if
            ei = ei + 1
        end while

        // Drop privileges if user/group specified
        let svc_user = config.svc_user(def)
        let svc_group = config.svc_group(def)
        if str.length(svc_user) > 0
            let uid = zyginit_getuid_by_name(svc_user)
            mut gid = zyginit_getgid_by_username(svc_user)

            // Override gid if explicit group specified
            if str.length(svc_group) > 0
                gid = zyginit_getgid_by_name(svc_group)
            end if

            if uid < 0 or gid < 0
                // User/group lookup failed — abort child
                process.exit_now(126)
            end if

            if zyginit_drop_privileges(svc_user, uid, gid) < 0
                process.exit_now(126)
            end if
        elif str.length(svc_group) > 0
            // Group only (no user change) — not typical but supported
            let gid = zyginit_getgid_by_name(svc_group)
            if gid < 0
                process.exit_now(126)
            end if
        end if

        // Parse start_cmd into program + args
        let cmd = config.svc_start_cmd(def)
        let parts = new [string](32)
        let argc = str.split(cmd, ' ', parts, 32)

        if argc > 0
            let program = parts[0]
            // argv[0] must be the program name (POSIX convention).
            // argv[1..] are the actual arguments.
            let args = new [string](argc)
            mut a = 0
            while a < argc
                args[a] = parts[a]
                a = a + 1
            end while
            process.process_exec(program, args)
        end if

        // exec failed
        process.exit_now(127)
    end if

    // PARENT PROCESS

    // Get contract ID (will be -1 on Linux)
    mut ctid = 0 - 1
    if tmpl_fd >= 0
        ctid = contract.get_latest_contract()
        contract.clear_template(tmpl_fd)
    end if

    // Update runtime
    rt.state = STATE_RUNNING()
    rt.pid = pid
    rt.contract_id = ctid
    rt.last_start_time = time.time_now()
    rt.skip_reason = ""    // clear any stale conflict/skip reason from a prior failure
    table.runtimes[idx] = rt

    // Register contract mapping
    if ctid >= 0
        table.contract_map.set(int_to_str(ctid), idx)
    end if

    // Fire ui_event_started only for daemons entering RUNNING state.
    // Oneshots must NOT fire here — they only call ui_event_started when
    // the contract-exit event confirms successful completion (in
    // handle_contract_event). Firing here AND on exit causes g_done to
    // double-increment, producing a count like "32/23 done".
    if config.svc_type(def) != config.SERVICE_TYPE_ONESHOT()
        ui.ui_event_started(name, 0)
    end if
    return true
end start_service

// Stop a service: signal the contract or exec the stop command.
// Returns true if stop was initiated.
fn stop_service(table: ServiceTable, idx: int): bool
    let rt = table.runtimes[idx]
    let def = rt.def
    let name = config.svc_name(def)

    if rt.state != STATE_RUNNING()
        return false
    end if

    rt.state = STATE_STOPPING()
    rt.stop_requested_time = time.time_now()
    table.runtimes[idx] = rt

    let stop_method = config.svc_stop_method(def)

    if stop_method == config.STOP_METHOD_EXEC() and str.length(config.svc_stop_cmd(def)) > 0
        // Exec the stop command
        process.process_spawn_shell(config.svc_stop_cmd(def))
        ui.ui_event_stopping(name)
    else
        // Signal the contract (default)
        let sig = signal_from_name(config.svc_stop_signal(def))
        if rt.contract_id >= 0
            contract.kill_contract(rt.contract_id, sig)
        elif rt.pid > 0
            // Fallback: signal the process directly (Linux or no contract)
            process.process_kill(rt.pid, sig)
        end if
        ui.ui_event_stopping(name)
    end if

    return true
end stop_service

// Run a oneshot's declared exec.stop command synchronously.
// Used during shutdown to reverse setup that the matching exec.start did.
// Does nothing if no stop command is declared.
// Returns true if stop was attempted (whether or not it succeeded).
fn stop_oneshot(table: ServiceTable, idx: int): bool
    let rt = table.runtimes[idx]
    let def = rt.def
    let name = config.svc_name(def)

    // Guard (C-1): only oneshots get exec.stop dispatched at shutdown.
    // Daemons that land in STATE_STOPPED already had their stop_method run
    // (via stop_service). Re-running it would be incorrect.
    if config.svc_type(def) != config.SERVICE_TYPE_ONESHOT()
        return false
    end if

    let stop_cmd = config.svc_stop_cmd(def)

    if str.length(stop_cmd) == 0
        // Nothing to reverse — normal for oneshots with no teardown.
        return false
    end if

    if ui.ui_mode() == ui.MODE_PLAIN()
        println("supervisor: running stop for " + name + " (" + stop_cmd + ")")
    end if

    let pid = process.process_spawn_shell(stop_cmd)
    if pid < 0
        if ui.ui_mode() == ui.MODE_PLAIN()
            println("supervisor: " + name + ": stop spawn failed")
        end if
        return true
    end if

    // Bounded wait (C-2): stop scripts that hang shouldn't lock up shutdown.
    // Try-wait loop with a 10s deadline (20 x 500ms).
    mut waits = 0
    mut exited = false
    while not exited and waits < 20
        if process.process_try_wait(pid)
            exited = true
        else
            clock.sleep_millis(500)
            waits = waits + 1
        end if
    end while

    if not exited
        if ui.ui_mode() == ui.MODE_PLAIN()
            println("supervisor: " + name + ": stop timed out, killing")
        end if
        let _ = process.process_kill(pid, signal.SIGKILL())
        // Reap the now-dead child
        let _ = process.process_wait(pid)
    elif process.process_exit_code() != 0
        if ui.ui_mode() == ui.MODE_PLAIN()
            println("supervisor: " + name + ": stop exited non-zero (" +
                    int_to_str(process.process_exit_code()) + ")")
        end if
    end if
    return true
end stop_oneshot

// Handle a contract event (typically "empty" — all processes exited).
// Looks up the service by contract ID and applies restart policy.
proc handle_contract_event(table: ServiceTable, contract_id: int, hint_exit_code: int)
    let idx = find_by_contract(table, contract_id)
    if idx < 0
        return
    end if

    let rt = table.runtimes[idx]
    let def = rt.def
    let name = config.svc_name(def)

    // Resolve exit code: prefer the hint (caller already reaped via SIGCHLD
    // path); else attempt to reap ourselves (contract-fd path arrived first).
    // hint_exit_code is non-negative when the caller has a real exit code;
    // -1 is the "no hint" sentinel matching the post-reap default below.
    mut exit_code = 0 - 1
    if hint_exit_code >= 0
        exit_code = hint_exit_code
    elif rt.pid > 0
        if process.process_try_wait(rt.pid)
            exit_code = process.process_exit_code()
        end if
    end if

    let svc_type = config.svc_type(def)

    // Task: may run for any duration. Exit 0 = STOPPED (counts as online).
    // Exit != 0 = FAILED. Never restarts. Skips the daemonize handshake.
    if svc_type == config.SERVICE_TYPE_TASK()
        let dur_ms = (time.time_now() - rt.last_start_time) * 1000
        let ec: int = if exit_code >= 0 then exit_code else 0 end if
        rt.pid = 0 - 1
        rt.contract_id = 0 - 1
        rt.last_exit_code = ec
        table.contract_map.remove(int_to_str(contract_id))
        if ec == 0
            rt.state = STATE_STOPPED()
            table.runtimes[idx] = rt
            ui.ui_event_stopped(name, dur_ms)
        else
            rt.state = STATE_FAILED()
            table.runtimes[idx] = rt
            ui.ui_event_failed(name, ec, dur_ms)
        end if
        return
    end if

    // Daemonize handshake: a double-forking daemon (dlmgmtd, syslogd,
    // utmpd, etc.) signals readiness by having its parent exit 0 once
    // the daemonized child is set up. The child stays alive in the same
    // contract (CT_PR_INHERIT). If this call came from the PID-reap
    // path (hint_exit_code >= 0) and the parent exited cleanly while
    // we're still RUNNING, this is the daemonize handshake — NOT a
    // service stop. Clear the original PID so reap_children stops
    // chasing it, but keep contract_id and contract_map intact: the
    // eventual contract-empty event from the bundle fd will route back
    // here with hint_exit_code = -1 and we'll process the real stop
    // through the normal logic below.
    if svc_type == config.SERVICE_TYPE_DAEMON() and hint_exit_code >= 0 and exit_code == 0 and rt.state == STATE_RUNNING()
        rt.pid = 0 - 1
        rt.last_exit_code = exit_code
        table.runtimes[idx] = rt
        return
    end if

    rt.last_exit_code = exit_code
    rt.pid = 0 - 1
    rt.contract_id = 0 - 1

    // Remove contract mapping
    table.contract_map.remove(int_to_str(contract_id))

    // Apply restart policy
    let policy = config.svc_restart_policy(def)

    // Oneshot services are "done" after running — don't restart
    if svc_type == config.SERVICE_TYPE_ONESHOT()
        let dur_ms = (time.time_now() - rt.last_start_time) * 1000
        if exit_code == 0
            rt.state = STATE_STOPPED()
            table.runtimes[idx] = rt
            ui.ui_event_started(name, dur_ms)
        else
            rt.state = STATE_FAILED()
            table.runtimes[idx] = rt
            ui.ui_event_failed(name, exit_code, dur_ms)
        end if
        return
    end if

    // Check if we were in the process of stopping
    if rt.state == STATE_STOPPING()
        let dur_ms = (time.time_now() - rt.last_start_time) * 1000
        rt.state = STATE_STOPPED()
        table.runtimes[idx] = rt
        ui.ui_event_stopped(name, dur_ms)
        return
    end if

    // Daemon restart logic
    mut should_restart = false

    if policy == config.RESTART_ALWAYS()
        should_restart = true
    elif policy == config.RESTART_FAILURE() and exit_code != 0
        should_restart = true
    end if

    if should_restart
        // Check max_retries
        let max = config.svc_max_retries(def)
        if max >= 0 and rt.restart_count >= max
            rt.state = STATE_MAINTENANCE()
            table.runtimes[idx] = rt
            if ui.ui_mode() == ui.MODE_PLAIN()
                println("supervisor: " + name + " entered maintenance (max retries)")
            end if
            return
        end if

        // Schedule restart after delay
        rt.state = STATE_WAITING()
        rt.restart_count = rt.restart_count + 1
        rt.restart_scheduled_time = time.time_now() + config.svc_restart_delay(def)
        if ui.ui_mode() == ui.MODE_PLAIN()
            println("supervisor: " + name + " scheduled for restart")
        end if
    else
        let dur_ms = (time.time_now() - rt.last_start_time) * 1000
        rt.state = STATE_STOPPED()
        table.runtimes[idx] = rt
        if exit_code == 0
            ui.ui_event_stopped(name, dur_ms)
        else
            ui.ui_event_failed(name, exit_code, dur_ms)
        end if
    end if

    table.runtimes[idx] = rt
end handle_contract_event

// Handle a child exit by PID (for Linux/non-contract mode).
// Scans the table for a matching PID.
proc handle_child_exit(table: ServiceTable, pid: int, exit_code: int)
    mut i = 0
    while i < table.count
        let rt = table.runtimes[i]
        if rt.pid == pid
            // Simulate a contract event with a fake contract id
            // Reuse handle_contract_event logic by temporarily setting fields
            rt.last_exit_code = exit_code
            rt.pid = 0 - 1
            table.runtimes[i] = rt

            // Use contract_id if available, otherwise handle directly
            if rt.contract_id >= 0
                handle_contract_event(table, rt.contract_id, exit_code)
            else
                // Inline the restart logic for non-contract mode
                let def = rt.def
                let name = config.svc_name(def)
                let policy = config.svc_restart_policy(def)
                let svc_type = config.svc_type(def)

                if svc_type == config.SERVICE_TYPE_TASK()
                    let dur_ms = (time.time_now() - rt.last_start_time) * 1000
                    if exit_code == 0
                        rt.state = STATE_STOPPED()
                        table.runtimes[i] = rt
                        ui.ui_event_stopped(name, dur_ms)
                    else
                        rt.state = STATE_FAILED()
                        rt.last_exit_code = exit_code
                        table.runtimes[i] = rt
                        ui.ui_event_failed(name, exit_code, dur_ms)
                    end if
                elif svc_type == config.SERVICE_TYPE_ONESHOT()
                    if exit_code == 0
                        rt.state = STATE_STOPPED()
                    else
                        rt.state = STATE_FAILED()
                    end if
                elif rt.state == STATE_STOPPING()
                    rt.state = STATE_STOPPED()
                else
                    mut should_restart = false
                    if policy == config.RESTART_ALWAYS()
                        should_restart = true
                    elif policy == config.RESTART_FAILURE() and exit_code != 0
                        should_restart = true
                    end if

                    if should_restart
                        let max = config.svc_max_retries(def)
                        if max >= 0 and rt.restart_count >= max
                            rt.state = STATE_MAINTENANCE()
                        else
                            rt.state = STATE_WAITING()
                            rt.restart_count = rt.restart_count + 1
                            rt.restart_scheduled_time = time.time_now() + config.svc_restart_delay(def)
                        end if
                    else
                        rt.state = STATE_STOPPED()
                    end if
                end if

                table.runtimes[i] = rt
            end if

            return
        end if
        i = i + 1
    end while
end handle_child_exit

// Check for services whose stop timeout has expired and escalate to SIGKILL.
proc check_stop_timeouts(table: ServiceTable)
    let now = time.time_now()
    mut i = 0
    while i < table.count
        let rt = table.runtimes[i]
        if rt.state == STATE_STOPPING() and rt.stop_requested_time > 0
            let timeout = config.svc_stop_timeout(rt.def)
            if now - rt.stop_requested_time >= timeout
                let name = config.svc_name(rt.def)
                if ui.ui_mode() == ui.MODE_PLAIN()
                    println("supervisor: " + name + " stop timeout, sending SIGKILL")
                end if
                if rt.contract_id >= 0
                    contract.kill_contract(rt.contract_id, signal.SIGKILL())
                elif rt.pid > 0
                    process.process_kill(rt.pid, signal.SIGKILL())
                end if
            end if
        end if
        i = i + 1
    end while
end check_stop_timeouts

// Check for services whose restart delay has elapsed and start them.
proc check_restart_delays(table: ServiceTable)
    let now = time.time_now()
    mut i = 0
    while i < table.count
        let rt = table.runtimes[i]
        if rt.state == STATE_WAITING() and rt.restart_scheduled_time > 0
            if now >= rt.restart_scheduled_time
                rt.restart_scheduled_time = 0
                table.runtimes[i] = rt
                let name = config.svc_name(rt.def)
                if ui.ui_mode() == ui.MODE_PLAIN()
                    println("supervisor: restarting " + name)
                end if
                start_service(table, i)
            end if
        end if
        i = i + 1
    end while
end check_restart_delays

// ============================================================================
// Reload support
// ============================================================================

// Mark a service as disabled and stop it if running.
// Does not remove from the table (to avoid index invalidation).
proc disable_service(table: ServiceTable, idx: int)
    let rt = table.runtimes[idx]

    if rt.state == STATE_RUNNING()
        stop_service(table, idx)
    end if

    rt.state = STATE_DISABLED()
    rt.restart_scheduled_time = 0
    table.runtimes[idx] = rt
end disable_service

// Mark a service as filtered-out-by-runlevel at boot time. Sets state
// to STOPPED, the same terminal state that runtime runlevel transitions
// produce for services that exit because they're no longer in the
// active set. Without this, runlevel-filtered services sit in
// STATE_WAITING forever — misleading in status output and tripping
// pre-condition checks (e.g., replace's check_preconditions). The
// state machine is more consistent if all "service didn't run / no
// longer running" cases collapse to STOPPED regardless of how they
// got there.
proc mark_runlevel_filtered(table: ServiceTable, idx: int)
    let rt = table.runtimes[idx]
    rt.state = STATE_STOPPED()
    table.runtimes[idx] = rt
end mark_runlevel_filtered

// Adopt runtime fields directly on a ServiceRuntime, bypassing the
// normal start/stop lifecycle. Used by main.apply_recovered_state for
// live-replace recovery (the new zyginit doesn't fork the service —
// it inherits a contract held by the same PID-1 proc_t and rebuilds
// in-memory bookkeeping) and by utils/replace_probe for state-file
// round-trip tests. Production startup and supervision still go
// through start_service / handle_contract_event.
proc adopt_runtime(table: ServiceTable, idx: int, state: int, pid: int,
                   contract_id: int, restart_count: int,
                   last_exit_code: int, last_start_time: int)
    let rt = table.runtimes[idx]
    rt.state = state
    rt.pid = pid
    rt.contract_id = contract_id
    rt.restart_count = restart_count
    rt.last_exit_code = last_exit_code
    rt.last_start_time = last_start_time
    table.runtimes[idx] = rt

    // Register contract mapping so handle_contract_event can route
    // events for this contract back to the right service.
    if contract_id >= 0
        table.contract_map.set(int_to_str(contract_id), idx)
    end if
end adopt_runtime

// Test/probe helper — build a ServiceDef with a given name and all other
// fields set to defaults. Wraps config.new_service_def() so external
// callers (utils/replace_probe) don't need to reconstruct the struct
// literal (cross-module struct literal syntax is not supported in Reef).
fn make_def_with_name(name: string): config.ServiceDef
    mut def = config.new_service_def()
    def.name = name
    return def
end make_def_with_name

// ============================================================================
// Status
// ============================================================================

fn get_status_line(table: ServiceTable, idx: int): string
    let rt = table.runtimes[idx]
    let name = config.svc_name(rt.def)
    let st = state_name(rt.state)
    mut line = name + " " + st

    if rt.pid > 0
        line = str.concat(line, " pid=")
        line = str.concat(line, int_to_str(rt.pid))
    end if

    if rt.contract_id >= 0
        line = str.concat(line, " ctid=")
        line = str.concat(line, int_to_str(rt.contract_id))
    end if

    // Uptime (only for running services with a valid start time)
    if rt.state == STATE_RUNNING() and rt.last_start_time > 0
        let uptime = time.time_now() - rt.last_start_time
        line = str.concat(line, " uptime=")
        line = str.concat(line, format_duration(uptime))
    end if

    // Exit code (show for stopped/failed/maintenance states)
    if rt.last_exit_code >= 0 and rt.state != STATE_RUNNING() and rt.state != STATE_WAITING()
        line = str.concat(line, " exit=")
        line = str.concat(line, int_to_str(rt.last_exit_code))
    end if

    // Restart count (show if non-zero)
    if rt.restart_count > 0
        line = str.concat(line, " restarts=")
        line = str.concat(line, int_to_str(rt.restart_count))
    end if

    return line
end get_status_line

// Format a duration in seconds to a human-readable string.
// Examples: "45s", "5m30s", "2h15m", "1d3h"
fn format_duration(seconds: int): string
    if seconds < 0
        return "0s"
    end if

    let days = seconds / 86400
    let remaining_after_days = seconds % 86400
    let hours = remaining_after_days / 3600
    let remaining_after_hours = remaining_after_days % 3600
    let minutes = remaining_after_hours / 60
    let secs = remaining_after_hours % 60

    if days > 0
        mut result = int_to_str(days) + "d"
        if hours > 0
            result = str.concat(result, int_to_str(hours))
            result = str.concat(result, "h")
        end if
        return result
    end if

    if hours > 0
        mut result = int_to_str(hours) + "h"
        if minutes > 0
            result = str.concat(result, int_to_str(minutes))
            result = str.concat(result, "m")
        end if
        return result
    end if

    if minutes > 0
        mut result = int_to_str(minutes) + "m"
        if secs > 0
            result = str.concat(result, int_to_str(secs))
            result = str.concat(result, "s")
        end if
        return result
    end if

    return int_to_str(secs) + "s"
end format_duration

// ============================================================================
// Log access
// ============================================================================

fn log_dir(): string
    return LOG_DIR()
end log_dir

fn get_service_log(name: string): string
    let log_path = str.concat(str.concat(LOG_DIR(), "/"), str.concat(name, ".log"))
    if not file.fileExists(log_path)
        return "no log available for " + name + "\n"
    end if
    let content = file.readFile(log_path)
    if str.length(content) == 0
        return "(empty log)\n"
    end if
    return content
end get_service_log

// ============================================================================
// Helpers
// ============================================================================

fn int_to_str(n: int): string
    if n == 0
        return "0"
    end if

    mut value = n
    if n < 0
        value = 0 - n
    end if

    mut result = ""
    while value > 0
        let digit = value % 10
        result = str.concat(str.substring("0123456789", digit, 1), result)
        value = value / 10
    end while

    if n < 0
        result = str.concat("-", result)
    end if

    return result
end int_to_str

end module