libc - zygaena/hammerhead

root / base / usr / src / cmd / mdb / common / modules / libc
libc Plain Text 2175 lines 57.1 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <mdb/mdb_modapi.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/avl.h>
#include <sys/lwp.h>
#include <thr_uberdata.h>
#include <stddef.h>
#include "findstack.h"

#if defined(__i386) || defined(__amd64)
struct rwindow {
	uintptr_t rw_fp;
	uintptr_t rw_rtn;
};
#endif

#ifndef STACK_BIAS
#define	STACK_BIAS	0
#endif

#ifdef __amd64
#define	STACKS_REGS_FP	"rbp"
#define	STACKS_REGS_RC	"rip"
#else
#ifdef __i386
#define	STACKS_REGS_FP	"ebp"
#define	STACKS_REGS_RC	"eip"
#else
#define	STACKS_REGS_FP	"fp"
#define	STACKS_REGS_RC	"pc"
#endif
#endif

#define	STACKS_SOBJ_MX	(uintptr_t)"MX"
#define	STACKS_SOBJ_CV	(uintptr_t)"CV"

int
thread_text_to_state(const char *state, uint_t *out)
{
	if (strcmp(state, "PARKED") == 0) {
		*out = B_TRUE;
	} else if (strcmp(state, "UNPARKED") == 0) {
		*out = B_FALSE;
	} else if (strcmp(state, "FREE") == 0) {
		/*
		 * When run with "-i", ::stacks filters out "FREE" threads.
		 * We therefore need to recognize "FREE", and set it to a
		 * value that will never match fsi_tstate.
		 */
		*out = UINT_MAX;
	} else {
		return (-1);
	}

	return (0);
}

void
thread_state_to_text(uint_t state, char *out, size_t out_sz)
{
	(void) snprintf(out, out_sz, state ? "PARKED" : "UNPARKED");
}

int
sobj_text_to_ops(const char *name, uintptr_t *sobj_ops_out)
{
	if (strcmp(name, "MX") == 0) {
		*sobj_ops_out = STACKS_SOBJ_MX;
	} else if (strcmp(name, "CV") == 0) {
		*sobj_ops_out = STACKS_SOBJ_CV;
	} else {
		mdb_warn("sobj \"%s\" not recognized\n", name);
		return (-1);
	}

	return (0);
}

void
sobj_ops_to_text(uintptr_t addr, char *out, size_t sz)
{
	(void) snprintf(out, sz, "%s", addr == 0 ? "<none>" : (char *)addr);
}

static int
stacks_module_callback(mdb_object_t *obj, void *arg)
{
	stacks_module_t *smp = arg;
	boolean_t match = (strcmp(obj->obj_name, smp->sm_name) == 0);
	char *suffix = ".so";
	const char *s, *next;
	size_t len;

	if (smp->sm_size != 0)
		return (0);

	/*
	 * It doesn't match the name, but -- for convenience -- we want to
	 * allow matches before ".so.[suffix]".  An aside:  why doesn't
	 * strrstr() exist?  (Don't google that.  I'm serious, don't do it.
	 * If you do, and you read the thread of "why doesn't strrstr() exist?"
	 * circa 2005 you will see things that you will NEVER be able to unsee!)
	 */
	if (!match && (s = strstr(obj->obj_name, suffix)) != NULL) {
		while ((next = strstr(s + 1, suffix)) != NULL) {
			s = next;
			continue;
		}

		len = s - obj->obj_name;

		match = (strncmp(smp->sm_name, obj->obj_name, len) == 0 &&
		    smp->sm_name[len] == '\0');
	}

	/*
	 * If we have a library that has the libc directory in the path, we
	 * want to match against anything that would match libc.so.1.  (This
	 * is necessary to be able to easily deal with libc implementations
	 * that have alternate hardware capabilities.)
	 */
	if (!match && strstr(obj->obj_fullname, "/libc/") != NULL) {
		mdb_object_t libc = *obj;

		libc.obj_name = "libc.so.1";
		libc.obj_fullname = "";

		return (stacks_module_callback(&libc, arg));
	}

	if (match) {
		smp->sm_text = obj->obj_base;
		smp->sm_size = obj->obj_size;
	}

	return (0);
}

int
stacks_module(stacks_module_t *smp)
{
	if (mdb_object_iter(stacks_module_callback, smp) != 0)
		return (-1);

	return (0);
}

typedef struct stacks_ulwp {
	avl_node_t sulwp_node;
	lwpid_t sulwp_id;
	uintptr_t sulwp_addr;
} stacks_ulwp_t;

boolean_t stacks_ulwp_initialized;
avl_tree_t stacks_ulwp_byid;

/*ARGSUSED*/
int
stacks_ulwp_walk(uintptr_t addr, ulwp_t *ulwp, void *ignored)
{
	stacks_ulwp_t *sulwp = mdb_alloc(sizeof (stacks_ulwp_t), UM_SLEEP);

	sulwp->sulwp_id = ulwp->ul_lwpid;
	sulwp->sulwp_addr = addr;

	if (avl_find(&stacks_ulwp_byid, sulwp, NULL) != NULL) {
		mdb_warn("found multiple LWPs with ID %d!", ulwp->ul_lwpid);
		return (WALK_ERR);
	}

	avl_add(&stacks_ulwp_byid, sulwp);

	return (WALK_NEXT);
}

static int
stacks_ulwp_compare(const void *l, const void *r)
{
	const stacks_ulwp_t *lhs = l;
	const stacks_ulwp_t *rhs = r;

	if (lhs->sulwp_id > rhs->sulwp_id)
		return (1);

	if (lhs->sulwp_id < rhs->sulwp_id)
		return (-1);

	return (0);
}

/*ARGSUSED*/
int
stacks_findstack(uintptr_t addr, findstack_info_t *fsip, uint_t print_warnings)
{
	mdb_reg_t reg;
	uintptr_t fp;
	struct rwindow frame;
	avl_tree_t *tree = &stacks_ulwp_byid;
	stacks_ulwp_t *sulwp, cmp;
	ulwp_t ulwp;

	fsip->fsi_failed = 0;
	fsip->fsi_pc = 0;
	fsip->fsi_sp = 0;
	fsip->fsi_depth = 0;
	fsip->fsi_overflow = 0;

	if (!stacks_ulwp_initialized) {
		avl_create(tree, stacks_ulwp_compare, sizeof (stacks_ulwp_t),
		    offsetof(stacks_ulwp_t, sulwp_node));

		if (mdb_walk("ulwp",
		    (mdb_walk_cb_t)stacks_ulwp_walk, NULL) != 0) {
			mdb_warn("couldn't walk 'ulwp'");
			return (-1);
		}

		stacks_ulwp_initialized = B_TRUE;
	}

	bzero(&cmp, sizeof (cmp));
	cmp.sulwp_id = (lwpid_t)addr;

	if ((sulwp = avl_find(tree, &cmp, NULL)) == NULL) {
		mdb_warn("couldn't find ulwp_t for tid %d\n", cmp.sulwp_id);
		return (-1);
	}

	if (mdb_vread(&ulwp, sizeof (ulwp), sulwp->sulwp_addr) == -1) {
		mdb_warn("couldn't read ulwp_t for tid %d at %p",
		    cmp.sulwp_id, sulwp->sulwp_addr);
		return (-1);
	}

	fsip->fsi_tstate = ulwp.ul_sleepq != NULL;
	fsip->fsi_sobj_ops = (uintptr_t)(ulwp.ul_sleepq == NULL ? 0 :
	    (ulwp.ul_qtype == MX ? STACKS_SOBJ_MX : STACKS_SOBJ_CV));

	if (mdb_getareg(addr, STACKS_REGS_FP, &reg) != 0) {
		mdb_warn("couldn't read frame pointer for thread 0x%p", addr);
		return (-1);
	}

	fsip->fsi_sp = fp = (uintptr_t)reg;

#if !defined(__i386)
	if (mdb_getareg(addr, STACKS_REGS_RC, &reg) != 0) {
		mdb_warn("couldn't read program counter for thread 0x%p", addr);
		return (-1);
	}

	fsip->fsi_pc = (uintptr_t)reg;
#endif

	while (fp != 0) {
		if (mdb_vread(&frame, sizeof (frame), fp) == -1) {
			mdb_warn("couldn't read frame for thread 0x%p at %p",
			    addr, fp);
			return (-1);
		}

		if (frame.rw_rtn == 0)
			break;

		if (fsip->fsi_depth < fsip->fsi_max_depth) {
			fsip->fsi_stack[fsip->fsi_depth++] = frame.rw_rtn;
		} else {
			fsip->fsi_overflow = 1;
			break;
		}

		fp = frame.rw_fp + STACK_BIAS;
	}

	return (0);
}

void
stacks_findstack_cleanup()
{
	avl_tree_t *tree = &stacks_ulwp_byid;
	void *cookie = NULL;
	stacks_ulwp_t *sulwp;

	if (!stacks_ulwp_initialized)
		return;

	while ((sulwp = avl_destroy_nodes(tree, &cookie)) != NULL)
		mdb_free(sulwp, sizeof (stacks_ulwp_t));

	bzero(tree, sizeof (*tree));
	stacks_ulwp_initialized = B_FALSE;
}

void
stacks_help(void)
{
	mdb_printf(
"::stacks processes all of the thread stacks in the process, grouping\n"
"together threads which have the same:\n"
"\n"
"  * Thread state,\n"
"  * Sync object type, and\n"
"  * PCs in their stack trace.\n"
"\n"
"The default output (no address or options) is just a dump of the thread\n"
"groups in the process.  For a view of active threads, use \"::stacks -i\",\n"
"which filters out threads sleeping on a CV.  More general filtering options\n"
"are described below, in the \"FILTERS\" section.\n"
"\n"
"::stacks can be used in a pipeline.  The input to ::stacks is one or more\n"
"thread IDs.  When output into a pipe, ::stacks prints all of the threads \n"
"input, filtered by the given filtering options.  This means that multiple\n"
"::stacks invocations can be piped together to achieve more complicated\n"
"filters.  For example, to get threads which have both '__door_return' and\n"
"'mutex_lock' in their stack trace, you could do:\n"
"\n"
"  ::stacks -c __door_return | ::stacks -c mutex_lock\n"
"\n"
"To get the full list of threads in each group, use the '-a' flag:\n"
"\n"
"  ::stacks -a\n"
"\n");
	mdb_dec_indent(2);
	mdb_printf("%<b>OPTIONS%</b>\n");
	mdb_inc_indent(2);
	mdb_printf("%s",
"  -a    Print all of the grouped threads, instead of just a count.\n"
"  -f    Force a re-run of the thread stack gathering.\n"
"  -v    Be verbose about thread stack gathering.\n"
"\n");
	mdb_dec_indent(2);
	mdb_printf("%<b>FILTERS%</b>\n");
	mdb_inc_indent(2);
	mdb_printf("%s",
"  -i    Show active threads; equivalent to '-S CV'.\n"
"  -c func[+offset]\n"
"        Only print threads whose stacks contain func/func+offset.\n"
"  -C func[+offset]\n"
"        Only print threads whose stacks do not contain func/func+offset.\n"
"  -m module\n"
"        Only print threads whose stacks contain functions from module.\n"
"  -M module\n"
"        Only print threads whose stacks do not contain functions from\n"
"        module.\n"
"  -s {type | ALL}\n"
"        Only print threads which are on a 'type' synchronization object\n"
"        (SOBJ).\n"
"  -S {type | ALL}\n"
"        Only print threads which are not on a 'type' SOBJ.\n"
"  -t tstate\n"
"        Only print threads which are in thread state 'tstate'.\n"
"  -T tstate\n"
"        Only print threads which are not in thread state 'tstate'.\n"
"\n");
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2012 by Delphix. All rights reserved.
 * Copyright (c) 2019 Carlos Neira <cneirabustos@gmail.com>
 * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2024 Oxide Computer Company
 */

#include <sys/mdb_modapi.h>
#include <mdb/mdb_whatis.h>
#include <mdb/mdb_ctf.h>
#include <procfs.h>
#include <ucontext.h>
#include <siginfo.h>
#include <signal.h>
#include <setjmp.h>
#include <string.h>
#include <thr_uberdata.h>
#include "findstack.h"
#include <libproc.h>

static const char *
stack_flags(const stack_t *sp)
{
	static char buf[32];

	if (sp->ss_flags == 0)
		(void) strcpy(buf, " 0");
	else if (sp->ss_flags & ~(SS_ONSTACK | SS_DISABLE))
		(void) mdb_snprintf(buf, sizeof (buf), " 0x%x", sp->ss_flags);
	else {
		buf[0] = '\0';
		if (sp->ss_flags & SS_ONSTACK)
			(void) strcat(buf, "|ONSTACK");
		if (sp->ss_flags & SS_DISABLE)
			(void) strcat(buf, "|DISABLE");
	}

	return (buf + 1);
}

/*ARGSUSED*/
static int
d_jmp_buf(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	jmp_buf jb;
	const ulong_t *b = (const ulong_t *)jb;

	if (argc != 0)
		return (DCMD_USAGE);

	if (mdb_vread(&jb, sizeof (jb), addr) != sizeof (jb)) {
		mdb_warn("failed to read jmp_buf at %p", addr);
		return (DCMD_ERR);
	}

#if defined(__sparc)
	mdb_printf("  %%sp = 0x%lx\n", b[1]);
	mdb_printf("  %%pc = 0x%lx %lA\n", b[2], b[2]);
	mdb_printf("  %%fp = 0x%lx\n", b[3]);
	mdb_printf("  %%i7 = 0x%lx %lA\n", b[4], b[4]);
#elif defined(__amd64)
	mdb_printf("  %%rbx = 0x%lx\n", b[0]);
	mdb_printf("  %%r12 = 0x%lx\n", b[1]);
	mdb_printf("  %%r13 = 0x%lx\n", b[2]);
	mdb_printf("  %%r14 = 0x%lx\n", b[3]);
	mdb_printf("  %%r15 = 0x%lx\n", b[4]);
	mdb_printf("  %%rbp = 0x%lx\n", b[5]);
	mdb_printf("  %%rsp = 0x%lx\n", b[6]);
	mdb_printf("  %%rip = 0x%lx %lA\n", b[7], b[7]);
#elif defined(__i386)
	mdb_printf("  %%ebx = 0x%lx\n", b[0]);
	mdb_printf("  %%esi = 0x%lx\n", b[1]);
	mdb_printf("  %%edi = 0x%lx\n", b[2]);
	mdb_printf("  %%ebp = 0x%lx\n", b[3]);
	mdb_printf("  %%esp = 0x%lx\n", b[4]);
	mdb_printf("  %%eip = 0x%lx %lA\n", b[5], b[5]);
#endif
	return (DCMD_OK);
}

const mdb_bitmask_t uc_flags_bits[] = {
	{ "UC_SIGMASK", UC_SIGMASK, UC_SIGMASK },
	{ "UC_STACK", UC_STACK, UC_STACK },
	{ "UC_CPU", UC_CPU, UC_CPU },
	{ "UC_FPU", UC_FPU, UC_FPU },
#if defined(UC_INTR)
	{ "UC_INTR", UC_INTR, UC_INTR },
#endif
#if defined(UC_ASR)
	{ "UC_ASR", UC_ASR, UC_ASR },
#endif
	{ NULL, 0, 0 }
};

/*ARGSUSED*/
static int
d_ucontext(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	ucontext_t uc;

	if (argc != 0)
		return (DCMD_USAGE);

	if (mdb_vread(&uc, sizeof (uc), addr) != sizeof (uc)) {
		mdb_warn("failed to read ucontext at %p", addr);
		return (DCMD_ERR);
	}

	mdb_printf("  flags    = 0x%lx <%b>\n", uc.uc_flags,
	    (uint_t)uc.uc_flags, uc_flags_bits);
	mdb_printf("  link     = 0x%p\n", uc.uc_link);
	mdb_printf("  sigmask  = 0x%08x 0x%08x 0x%08x 0x%08x\n",
	    uc.uc_sigmask.__sigbits[0], uc.uc_sigmask.__sigbits[1],
	    uc.uc_sigmask.__sigbits[2], uc.uc_sigmask.__sigbits[3]);
	mdb_printf("  stack    = sp 0x%p size 0x%lx flags %s\n",
	    uc.uc_stack.ss_sp, uc.uc_stack.ss_size, stack_flags(&uc.uc_stack));
	mdb_printf("  mcontext = 0x%p\n",
	    addr + OFFSETOF(ucontext_t, uc_mcontext));

	return (DCMD_OK);
}

/*ARGSUSED*/
static int
d_sigjmp_buf(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
#if defined(__sparc)
	struct {
		int sjs_flags;
		greg_t sjs_sp;
		greg_t sjs_pc;
		greg_t sjs_fp;
		greg_t sjs_i7;
		ucontext_t *sjs_uclink;
		ulong_t sjs_pad[_JBLEN - 6];
		sigset_t sjs_sigmask;
#if defined(_LP64)
		greg_t sjs_asi;
		greg_t sjs_fprs;
#endif
		stack_t sjs_stack;
	} s;

	if (argc != 0)
		return (DCMD_USAGE);

	if (mdb_vread(&s, sizeof (s), addr) != sizeof (s)) {
		mdb_warn("failed to read sigjmp_buf at %p", addr);
		return (DCMD_ERR);
	}

	mdb_printf("  flags  = 0x%x\n", s.sjs_flags);
	mdb_printf("  %%sp    = 0x%lx %lA\n", s.sjs_sp, s.sjs_sp);
	mdb_printf("  %%pc    = 0x%lx %lA\n", s.sjs_pc, s.sjs_pc);
	mdb_printf("  %%fp    = 0x%lx %lA\n", s.sjs_fp, s.sjs_fp);
	mdb_printf("  %%i7    = 0x%lx %lA\n", s.sjs_i7, s.sjs_i7);
	mdb_printf("  uclink = %p\n", s.sjs_uclink);
	mdb_printf("  sigset = 0x%08x 0x%08x 0x%08x 0x%08x\n",
	    s.sjs_sigmask.__sigbits[0], s.sjs_sigmask.__sigbits[1],
	    s.sjs_sigmask.__sigbits[2], s.sjs_sigmask.__sigbits[3]);
#if defined(_LP64)
	mdb_printf("  %%asi   = 0x%lx\n", s.sjs_asi);
	mdb_printf("  %%fprs  = 0x%lx\n", s.sjs_fprs);
#endif
	mdb_printf("  stack  = sp 0x%p size 0x%lx flags %s\n",
	    s.sjs_stack.ss_sp, s.sjs_stack.ss_size, stack_flags(&s.sjs_stack));

	return (DCMD_OK);

#elif defined(__i386) || defined(__amd64)
	return (d_ucontext(addr, flags, argc, argv));
#endif
}

/*ARGSUSED*/
static int
d_siginfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	static const char *const msname[] = {
		"USER", "SYSTEM", "TRAP", "TFAULT", "DFAULT", "KFAULT",
		"USER_LOCK", "SLEEP", "WAIT_CPU", "STOPPED"
	};

	char signame[SIG2STR_MAX];
	siginfo_t si;
	int i;

	if (argc != 0)
		return (DCMD_USAGE);

	if (mdb_vread(&si, sizeof (si), addr) != sizeof (si)) {
		mdb_warn("failed to read siginfo at %p", addr);
		return (DCMD_ERR);
	}

	if (sig2str(si.si_signo, signame) == -1)
		(void) strcpy(signame, "unknown");

	mdb_printf("  signal %5d (%s)\n", si.si_signo, signame);
	mdb_printf("  code   %5d (", si.si_code);

	switch (si.si_code) {
	case SI_NOINFO:
		mdb_printf("no info");
		break;
	case SI_DTRACE:
		mdb_printf("from DTrace raise() action");
		break;
	case SI_RCTL:
		mdb_printf("from rctl action");
		break;
	case SI_USER:
		mdb_printf("user generated via kill");
		break;
	case SI_LWP:
		mdb_printf("user generated via lwp_kill");
		break;
	case SI_QUEUE:
		mdb_printf("user generated via sigqueue");
		break;
	case SI_TIMER:
		mdb_printf("from timer expiration");
		break;
	case SI_ASYNCIO:
		mdb_printf("from async i/o completion");
		break;
	case SI_MESGQ:
		mdb_printf("from message arrival");
		break;
	default:
		if (SI_FROMUSER(&si))
			mdb_printf("from user process");
		else
			mdb_printf("from kernel");
	}

	mdb_printf(")\n  errno  %5d (%s)\n",
	    si.si_errno, strerror(si.si_errno));

	if (si.si_code == SI_USER || si.si_code == SI_QUEUE) {
		mdb_printf("  signal sent from PID %d (uid %d)\n",
		    si.si_pid, si.si_uid);
	}

	if (si.si_code == SI_QUEUE) {
		mdb_printf("  signal value = 0t%d / %p\n",
		    si.si_value.sival_int, si.si_value.sival_ptr);
	}

	switch (si.si_signo) {
	case SIGCLD:
		mdb_printf("  signal sent from child PID %d (uid %d)\n",
		    si.si_pid, si.si_uid);
		mdb_printf("  usr time = 0t%ld ticks, sys time = 0t%ld ticks\n",
		    si.si_utime, si.si_stime);
		mdb_printf("  wait status = 0x%x\n", si.si_status);
		break;

	case SIGSEGV:
	case SIGBUS:
	case SIGILL:
	case SIGTRAP:
	case SIGFPE:
		mdb_printf("  fault address = 0x%p\n  trapno = %d\n",
		    si.si_addr, si.si_trapno);
		mdb_printf("  instruction address = 0x%p %lA\n",
		    si.si_pc, si.si_pc);
		break;

	case SIGPOLL:
	case SIGXFSZ:
		mdb_printf("  fd = %d  band = 0x%lx\n",
		    si.si_fd, si.si_band);
		break;

	case SIGPROF:
		mdb_printf("  last fault address = 0x%p fault type = %d\n",
		    si.si_faddr, si.si_fault);
		mdb_printf("  timestamp = 0t%ld sec 0t%ld nsec\n",
		    si.si_tstamp.tv_sec, si.si_tstamp.tv_nsec);

		if (si.__data.__prof.__syscall != 0) {
			mdb_printf("  system call %d (", si.si_syscall);
			if (si.si_nsysarg > 0) {
				mdb_printf("%lx", si.si_sysarg[0]);
				for (i = 1; i < si.si_nsysarg; i++)
					mdb_printf(", %lx", si.si_sysarg[i]);
			}
			mdb_printf("  )\n");
		}

		for (i = 0; i < sizeof (msname) / sizeof (msname[0]); i++) {
			mdb_printf("  mstate[\"%s\"] = %d\n",
			    msname[i], si.si_mstate[i]);
		}
		break;
	}

	return (DCMD_OK);
}

static int
uc_walk_step(mdb_walk_state_t *wsp)
{
	uintptr_t addr = wsp->walk_addr;
	ucontext_t uc;

	if (addr == 0)
		return (WALK_DONE);

	if (mdb_vread(&uc, sizeof (uc), addr) != sizeof (uc)) {
		mdb_warn("failed to read ucontext at %p", addr);
		return (WALK_ERR);
	}

	wsp->walk_addr = (uintptr_t)uc.uc_link;
	return (wsp->walk_callback(addr, &uc, wsp->walk_cbdata));
}

static int
oldc_walk_init(mdb_walk_state_t *wsp)
{
	ssize_t nbytes = mdb_get_xdata("lwpstatus", NULL, 0);

	if (nbytes <= 0) {
		mdb_warn("lwpstatus information not available");
		return (WALK_ERR);
	}

	if (wsp->walk_addr != 0) {
		mdb_warn("walker only supports global walk\n");
		return (WALK_ERR);
	}

	wsp->walk_addr = nbytes; /* Use walk_addr to track size */
	wsp->walk_data = mdb_alloc(nbytes, UM_SLEEP);

	if (mdb_get_xdata("lwpstatus", wsp->walk_data, nbytes) != nbytes) {
		mdb_warn("failed to read lwpstatus information");
		mdb_free(wsp->walk_data, nbytes);
		return (WALK_ERR);
	}

	wsp->walk_arg = wsp->walk_data; /* Use walk_arg to track pointer */
	return (WALK_NEXT);
}

static int
oldc_walk_step(mdb_walk_state_t *wsp)
{
	const lwpstatus_t *lsp, *end;

	end = (const lwpstatus_t *)((uintptr_t)wsp->walk_data + wsp->walk_addr);
	lsp = wsp->walk_arg;

	wsp->walk_arg = (void *)(lsp + 1);

	if (lsp < end) {
		uintptr_t addr = lsp->pr_oldcontext;
		ucontext_t uc;

		if (addr == 0)
			return (WALK_NEXT);

		if (mdb_vread(&uc, sizeof (uc), addr) != sizeof (uc)) {
			mdb_warn("failed to read ucontext at %p", addr);
			return (WALK_NEXT);
		}

		return (wsp->walk_callback(addr, &uc, wsp->walk_cbdata));
	}

	return (WALK_DONE);
}

static void
oldc_walk_fini(mdb_walk_state_t *wsp)
{
	mdb_free(wsp->walk_data, wsp->walk_addr); /* walk_addr has size */
}

/*
 * ==================== threads ==========================
 * These are the interfaces that used to require libthread.
 * Now, libthread has been folded into libc.
 * =======================================================
 */

/*
 * prt_addr() is called up to three times to generate arguments for
 * one call to mdb_printf().  We must return at least three different
 * pointers to static storage for consecutive calls to prt_addr().
 */
static const char *
prt_addr(void *addr, int pad)
{
	static char buffer[4][24];
	static int ix = 0;
	char *buf;

	if (ix == 4)	/* use buffers in sequence: 0, 1, 2, 3 */
		ix = 0;
	buf = buffer[ix++];
	if (addr == NULL)
		return (pad? "<NULL>               " : "<NULL>");
	else {
#ifdef _LP64
		(void) mdb_snprintf(buf, sizeof (buffer[0]), "0x%016lx", addr);
		if (pad)
			(void) strcpy(buf + 18, "   ");
#else
		(void) mdb_snprintf(buf, sizeof (buffer[0]), "0x%08lx", addr);
		if (pad)
			(void) strcpy(buf + 10, "           ");
#endif	/* _LP64 */
		return (buf);
	}
}

#define	HD(str)		mdb_printf("           " str "\n")
#define	OFFSTR		"+0x%-7lx "
#define	OFFSET(member)	((size_t)OFFSETOF(ulwp_t, member))

/*ARGSUSED*/
static int
d_ulwp(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	ulwp_t ulwp;

	if (argc != 0 || !(flags & DCMD_ADDRSPEC))
		return (DCMD_USAGE);

	if (mdb_vread(&ulwp, sizeof (ulwp), addr) != sizeof (ulwp) &&
	    (bzero(&ulwp, sizeof (ulwp)),
	    mdb_vread(&ulwp, REPLACEMENT_SIZE, addr)) != REPLACEMENT_SIZE) {
		mdb_warn("failed to read ulwp at 0x%p", addr);
		return (DCMD_ERR);
	}

	mdb_printf("%#a\n", addr);

	HD("self                  uberdata");
	mdb_printf(OFFSTR "%s %s\n",
	    OFFSET(ul_self),
	    prt_addr(ulwp.ul_self, 1),
	    prt_addr(ulwp.ul_uberdata, 0));

	HD("tlsent                ntlsent");
	mdb_printf(OFFSTR "%s %ld\n",
	    OFFSET(ul_tlsent),
	    prt_addr(ulwp.ul_tlsent, 1),
	    ulwp.ul_ntlsent);

	HD("forw                  back                  next");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_forw),
	    prt_addr(ulwp.ul_forw, 1),
	    prt_addr(ulwp.ul_back, 1),
	    prt_addr(ulwp.ul_next, 0));

	HD("hash                  rval                  stk");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_hash),
	    prt_addr(ulwp.ul_hash, 1),
	    prt_addr(ulwp.ul_rval, 1),
	    prt_addr(ulwp.ul_stk, 0));

	HD("mapsiz     guardsize  stktop                stksiz");
	mdb_printf(OFFSTR "%-10ld %-10ld %s %ld\n",
	    OFFSET(ul_mapsiz),
	    ulwp.ul_mapsiz,
	    ulwp.ul_guardsize,
	    prt_addr((void *)ulwp.ul_stktop, 1),
	    ulwp.ul_stksiz);

	HD("ustack.ss_sp          ustack.ss_size        ustack.ss_flags");
	mdb_printf(OFFSTR "%s %-21ld %s\n",
	    OFFSET(ul_ustack.ss_sp),
	    prt_addr(ulwp.ul_ustack.ss_sp, 1),
	    ulwp.ul_ustack.ss_size,
	    stack_flags(&ulwp.ul_ustack));

	HD("ix         lwpid      pri        epri       policy     cid");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_ix),
	    ulwp.ul_ix,
	    ulwp.ul_lwpid,
	    ulwp.ul_pri,
	    ulwp.ul_epri,
	    ulwp.ul_policy,
	    ulwp.ul_cid);

	HD("cursig     pleasestop stop       signalled  dead       unwind");
	mdb_printf(OFFSTR "%-10d ",
	    OFFSET(ul_cursig),
	    ulwp.ul_cursig);
	mdb_printf(ulwp.ul_pleasestop? "0x%-8x " : "%-10d ",
	    ulwp.ul_pleasestop);
	mdb_printf(ulwp.ul_stop? "0x%-8x " : "%-10d ",
	    ulwp.ul_stop);
	mdb_printf("%-10d %-10d %d\n",
	    ulwp.ul_signalled,
	    ulwp.ul_dead,
	    ulwp.ul_unwind);

	HD("detached   writer     stopping   can'prolog preempt    savpreempt");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_detached),
	    ulwp.ul_detached,
	    ulwp.ul_writer,
	    ulwp.ul_stopping,
	    ulwp.ul_cancel_prologue,
	    ulwp.ul_preempt,
	    ulwp.ul_savpreempt);

	HD("sigsuspend main       fork       primarymap m'spinners d'noreserv");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_sigsuspend),
	    ulwp.ul_sigsuspend,
	    ulwp.ul_main,
	    ulwp.ul_fork,
	    ulwp.ul_primarymap,
	    ulwp.ul_max_spinners,
	    ulwp.ul_door_noreserve);

	HD("queue_fifo c'w'defer  e'detect'  async_safe rt         rtqueued");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_queue_fifo),
	    ulwp.ul_queue_fifo,
	    ulwp.ul_cond_wait_defer,
	    ulwp.ul_error_detection,
	    ulwp.ul_async_safe,
	    ulwp.ul_rt,
	    ulwp.ul_rtqueued);

	HD("misaligned adapt'spin queue_spin critical   sigdefer   vfork");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_misaligned),
	    ulwp.ul_misaligned,
	    ulwp.ul_adaptive_spin,
	    ulwp.ul_queue_spin,
	    ulwp.ul_critical,
	    ulwp.ul_sigdefer,
	    ulwp.ul_vfork);

	HD("cancelable c'pending  c'disabled c'async    save_async mutator");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(ul_cancelable),
	    ulwp.ul_cancelable,
	    ulwp.ul_cancel_pending,
	    ulwp.ul_cancel_disabled,
	    ulwp.ul_cancel_async,
	    ulwp.ul_save_async,
	    ulwp.ul_mutator);

	HD("created    replace    nocancel   errno      errnop");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %s\n",
	    OFFSET(ul_created),
	    ulwp.ul_created,
	    ulwp.ul_replace,
	    ulwp.ul_nocancel,
	    ulwp.ul_errno,
	    prt_addr(ulwp.ul_errnop, 0));

	HD("clnup_hdr             schedctl_called       schedctl");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_clnup_hdr),
	    prt_addr(ulwp.ul_clnup_hdr, 1),
	    prt_addr(ulwp.ul_schedctl_called, 1),
	    prt_addr((void *)ulwp.ul_schedctl, 0));

	HD("bindflags  libc_locks stsd                  &ftsd");
	mdb_printf(OFFSTR,
	    OFFSET(ul_bindflags));
	mdb_printf(ulwp.ul_bindflags? "0x%-8x " : "%-10d ",
	    ulwp.ul_bindflags);
	mdb_printf("%-10d ", ulwp.ul_libc_locks);
	mdb_printf("%s %s\n",
	    prt_addr(ulwp.ul_stsd, 1),
	    prt_addr((void *)(addr + OFFSET(ul_ftsd[0])), 0));

	HD("eventmask[0..1]       eventnum              eventdata");
	mdb_printf(OFFSTR "0x%08x 0x%08x %-21d %s\n",
	    OFFSET(ul_td_evbuf.eventmask.event_bits[0]),
	    ulwp.ul_td_evbuf.eventmask.event_bits[0],
	    ulwp.ul_td_evbuf.eventmask.event_bits[1],
	    ulwp.ul_td_evbuf.eventnum,
	    prt_addr(ulwp.ul_td_evbuf.eventdata, 0));

	HD("td'enable  sync'reg   qtype      cv_wake    rtld       usropts");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d ",
	    OFFSET(ul_td_events_enable),
	    ulwp.ul_td_events_enable,
	    ulwp.ul_sync_obj_reg,
	    ulwp.ul_qtype,
	    ulwp.ul_cv_wake,
	    ulwp.ul_rtld);
	mdb_printf(ulwp.ul_usropts? "0x%x\n" : "%d\n",
	    ulwp.ul_usropts);

	HD("startpc               startarg              wchan");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_startpc),
	    prt_addr((void *)ulwp.ul_startpc, 1),
	    prt_addr(ulwp.ul_startarg, 1),
	    prt_addr(ulwp.ul_wchan, 0));

	HD("link                  sleepq                cvmutex");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_link),
	    prt_addr(ulwp.ul_link, 1),
	    prt_addr(ulwp.ul_sleepq, 1),
	    prt_addr(ulwp.ul_cvmutex, 0));

	HD("mxchain               save_state");
	mdb_printf(OFFSTR "%s %d\n",
	    OFFSET(ul_mxchain),
	    prt_addr(ulwp.ul_mxchain, 1),
	    ulwp.ul_save_state);

	HD("rdlockcnt             rd_rwlock             rd_count");
	mdb_printf(OFFSTR "%-21d %s %d\n",
	    OFFSET(ul_rdlockcnt),
	    ulwp.ul_rdlockcnt,
	    prt_addr(ulwp.ul_readlock.single.rd_rwlock, 1),
	    ulwp.ul_readlock.single.rd_count);

	HD("heldlockcnt           heldlocks             tpdp");
	mdb_printf(OFFSTR "%-21d %s %s\n",
	    OFFSET(ul_heldlockcnt),
	    ulwp.ul_heldlockcnt,
	    prt_addr(ulwp.ul_heldlocks.single, 1),
	    prt_addr(ulwp.ul_tpdp, 0));

	HD("siglink               s'l'spin   s'l'spin2  s'l'sleep  s'l'wakeup");
	mdb_printf(OFFSTR "%s %-10d %-10d %-10d %d\n",
	    OFFSET(ul_siglink),
	    prt_addr(ulwp.ul_siglink, 1),
	    ulwp.ul_spin_lock_spin,
	    ulwp.ul_spin_lock_spin2,
	    ulwp.ul_spin_lock_sleep,
	    ulwp.ul_spin_lock_wakeup);

	HD("&queue_root           rtclassid  pilocks");
	mdb_printf(OFFSTR "%s %-10d %d\n",
	    OFFSET(ul_queue_root),
	    prt_addr((void *)(addr + OFFSET(ul_queue_root)), 1),
	    ulwp.ul_rtclassid,
	    ulwp.ul_pilocks);

	/*
	 * The remainder of the ulwp_t structure
	 * is invalid if this is a replacement.
	 */
	if (ulwp.ul_replace)
		return (DCMD_OK);

	HD("sigmask[0..3]");
	mdb_printf(OFFSTR "0x%08x 0x%08x 0x%08x 0x%08x\n",
	    OFFSET(ul_sigmask.__sigbits[0]),
	    ulwp.ul_sigmask.__sigbits[0],
	    ulwp.ul_sigmask.__sigbits[1],
	    ulwp.ul_sigmask.__sigbits[2],
	    ulwp.ul_sigmask.__sigbits[3]);

	HD("tmpmask[0..3]");
	mdb_printf(OFFSTR "0x%08x 0x%08x 0x%08x 0x%08x\n",
	    OFFSET(ul_tmpmask.__sigbits[0]),
	    ulwp.ul_tmpmask.__sigbits[0],
	    ulwp.ul_tmpmask.__sigbits[1],
	    ulwp.ul_tmpmask.__sigbits[2],
	    ulwp.ul_tmpmask.__sigbits[3]);

	HD("&siginfo              &spinlock             &fpuenv");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ul_siginfo),
	    prt_addr((void *)(addr + OFFSET(ul_siginfo)), 1),
	    prt_addr((void *)(addr + OFFSET(ul_spinlock)), 1),
	    prt_addr((void *)(addr + OFFSET(ul_fpuenv)), 0));

	HD("tmem.size             &tmem.roots");
	mdb_printf(OFFSTR "%-21H %s\n",
	    OFFSET(ul_tmem),
	    ulwp.ul_tmem.tm_size,
	    prt_addr((void *)(addr + OFFSET(ul_tmem) + sizeof (size_t)), 0));

	return (DCMD_OK);
}

/*
 * Get the address of the unique uberdata_t structure.
 */
static uintptr_t
uberdata_addr(void)
{
	uintptr_t uaddr;
	uintptr_t addr;
	GElf_Sym sym;

	if (mdb_lookup_by_obj("libc.so.1", "_tdb_bootstrap", &sym) != 0) {
		mdb_warn("cannot find libc.so.1`_tdb_bootstrap");
		return (0);
	}
	if (mdb_vread(&addr, sizeof (addr), sym.st_value) == sizeof (addr) &&
	    addr != 0 &&
	    mdb_vread(&uaddr, sizeof (uaddr), addr) == sizeof (uaddr) &&
	    uaddr != 0) {
		return (uaddr);
	}
	if (mdb_lookup_by_obj("libc.so.1", "_uberdata", &sym) != 0) {
		mdb_warn("cannot find libc.so.1`_uberdata");
		return (0);
	}
	return ((uintptr_t)sym.st_value);
}

#undef OFFSET
#define	OFFSET(member)	((size_t)OFFSETOF(uberdata_t, member))

/*ARGSUSED*/
static int
d_uberdata(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	uberdata_t uberdata;
	int i;

	if (argc != 0)
		return (DCMD_USAGE);
	if (!(flags & DCMD_ADDRSPEC) && (addr = uberdata_addr()) == 0)
		return (DCMD_ERR);

	if (mdb_vread(&uberdata, sizeof (uberdata), addr) !=
	    sizeof (uberdata)) {
		mdb_warn("failed to read uberdata at 0x%p", addr);
		return (DCMD_ERR);
	}

	mdb_printf("%#a\n", addr);

	HD("&link_lock            &ld_lock              &fork_lock");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(link_lock),
	    prt_addr((void *)(addr + OFFSET(link_lock)), 1),
	    prt_addr((void *)(addr + OFFSET(ld_lock)), 1),
	    prt_addr((void *)(addr + OFFSET(fork_lock)), 0));

	HD("&atfork_lock          &callout_lock         &tdb_hash_lock");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(atfork_lock),
	    prt_addr((void *)(addr + OFFSET(atfork_lock)), 1),
	    prt_addr((void *)(addr + OFFSET(callout_lock)), 1),
	    prt_addr((void *)(addr + OFFSET(tdb_hash_lock)), 0));

	HD("&tdb_hash_lock_stats  &siguaction[0]");
	mdb_printf(OFFSTR "%s %s\n",
	    OFFSET(tdb_hash_lock_stats),
	    prt_addr((void *)(addr + OFFSET(tdb_hash_lock_stats)), 1),
	    prt_addr((void *)(addr + OFFSET(siguaction)), 0));

	HD("&bucket               free_list             chunks");
	for (i = 0; i < NBUCKETS; i++) {
		mdb_printf(OFFSTR "%s %s %ld\n",
		    OFFSET(bucket[i]),
		    prt_addr((void *)(addr + OFFSET(bucket[i])), 1),
		    prt_addr(uberdata.bucket[i].free_list, 1),
		    uberdata.bucket[i].chunks);
	}

	HD("&atexit_root          head                  exit_frame_monitor");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(atexit_root),
	    prt_addr((void *)(addr + OFFSET(atexit_root.exitfns_lock)), 1),
	    prt_addr(uberdata.atexit_root.head, 1),
	    prt_addr(uberdata.atexit_root.exit_frame_monitor, 0));

	HD("&quickexit_root       head");
	mdb_printf(OFFSTR "%s %s\n",
	    OFFSET(quickexit_root),
	    prt_addr((void *)(addr + OFFSET(quickexit_root.exitfns_lock)), 1),
	    prt_addr(uberdata.quickexit_root.head, 0));


	HD("&tsd_metadata         tsdm_nkeys tsdm_nused tsdm_destro");
	mdb_printf(OFFSTR "%s %-10d %-10d %s\n",
	    OFFSET(tsd_metadata),
	    prt_addr((void *)(addr + OFFSET(tsd_metadata.tsdm_lock)), 1),
	    uberdata.tsd_metadata.tsdm_nkeys,
	    uberdata.tsd_metadata.tsdm_nused,
	    prt_addr((void *)uberdata.tsd_metadata.tsdm_destro, 0));

	HD("&tls_metadata         tls_modinfo.data      tls_modinfo.size");
	mdb_printf(OFFSTR "%s %s %ld\n",
	    OFFSET(tls_metadata),
	    prt_addr((void *)(addr + OFFSET(tls_metadata.tls_lock)), 1),
	    prt_addr(uberdata.tls_metadata.tls_modinfo.tls_data, 1),
	    uberdata.tls_metadata.tls_modinfo.tls_size);

	HD("                      static_tls.data       static_tls.size");
	mdb_printf(OFFSTR "%s %s %ld\n",
	    OFFSET(tls_metadata.static_tls),
	    "                     ",
	    prt_addr(uberdata.tls_metadata.static_tls.tls_data, 1),
	    uberdata.tls_metadata.static_tls.tls_size);

	HD("primary_ma bucket_ini uflags.mt  uflags.pad uflags.trs uflags.ted");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %-10d %d\n",
	    OFFSET(primary_map),
	    uberdata.primary_map,
	    uberdata.bucket_init,
	    uberdata.uberflags.uf_x.x_mt,
	    uberdata.uberflags.uf_x.x_pad,
	    uberdata.uberflags.uf_x.x_tdb_register_sync,
	    uberdata.uberflags.uf_x.x_thread_error_detection);

	HD("queue_head            thr_hash_table        hash_size  hash_mask");
	mdb_printf(OFFSTR "%s %s %-10d 0x%x\n",
	    OFFSET(queue_head),
	    prt_addr(uberdata.queue_head, 1),
	    prt_addr(uberdata.thr_hash_table, 1),
	    uberdata.hash_size,
	    uberdata.hash_mask);

	HD("ulwp_one              all_lwps              all_zombies");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ulwp_one),
	    prt_addr(uberdata.ulwp_one, 1),
	    prt_addr(uberdata.all_lwps, 1),
	    prt_addr(uberdata.all_zombies, 0));

	HD("nthreads   nzombies   ndaemons   pid        sigacthandler");
	mdb_printf(OFFSTR "%-10d %-10d %-10d %-10d %s\n",
	    OFFSET(nthreads),
	    uberdata.nthreads,
	    uberdata.nzombies,
	    uberdata.ndaemons,
	    (int)uberdata.pid,
	    prt_addr((void *)uberdata.sigacthandler, 0));

	HD("lwp_stacks            lwp_laststack         nfreestack stk_cache");
	mdb_printf(OFFSTR "%s %s %-10d %d\n",
	    OFFSET(lwp_stacks),
	    prt_addr(uberdata.lwp_stacks, 1),
	    prt_addr(uberdata.lwp_laststack, 1),
	    uberdata.nfreestack,
	    uberdata.thread_stack_cache);

	HD("ulwp_freelist         ulwp_lastfree         ulwp_replace_free");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(ulwp_freelist),
	    prt_addr(uberdata.ulwp_freelist, 1),
	    prt_addr(uberdata.ulwp_lastfree, 1),
	    prt_addr(uberdata.ulwp_replace_free, 0));

	HD("ulwp_replace_last     atforklist");
	mdb_printf(OFFSTR "%s %s\n",
	    OFFSET(ulwp_replace_last),
	    prt_addr(uberdata.ulwp_replace_last, 1),
	    prt_addr(uberdata.atforklist, 0));

	HD("robustlocks           robustlist            progname");
	mdb_printf(OFFSTR "%s %s %s\n",
	    OFFSET(robustlocks),
	    prt_addr(uberdata.robustlocks, 1),
	    prt_addr(uberdata.robustlist, 1),
	    prt_addr(uberdata.progname, 0));

	HD("tdb_bootstrap         tdb_sync_addr_hash    tdb_'count tdb_'fail");
	mdb_printf(OFFSTR "%s %s %-10d %d\n",
	    OFFSET(tdb_bootstrap),
	    prt_addr(uberdata.tdb_bootstrap, 1),
	    prt_addr(uberdata.tdb.tdb_sync_addr_hash, 1),
	    uberdata.tdb.tdb_register_count,
	    uberdata.tdb.tdb_hash_alloc_failed);

	HD("tdb_sync_addr_free    tdb_sync_addr_last    tdb_sync_alloc");
	mdb_printf(OFFSTR "%s %s %ld\n",
	    OFFSET(tdb.tdb_sync_addr_free),
	    prt_addr(uberdata.tdb.tdb_sync_addr_free, 1),
	    prt_addr(uberdata.tdb.tdb_sync_addr_last, 1),
	    uberdata.tdb.tdb_sync_alloc);

	HD("tdb_ev_global_mask    tdb_events");
	mdb_printf(OFFSTR "0x%08x 0x%08x %s\n",
	    OFFSET(tdb.tdb_ev_global_mask),
	    uberdata.tdb.tdb_ev_global_mask.event_bits[0],
	    uberdata.tdb.tdb_ev_global_mask.event_bits[1],
	    prt_addr((void *)uberdata.tdb.tdb_events, 0));

	return (DCMD_OK);
}

static int
ulwp_walk_init(mdb_walk_state_t *wsp)
{
	uintptr_t addr = wsp->walk_addr;
	uintptr_t uber_addr;
	int offset;

	offset = mdb_ctf_offsetof_by_name("uberdata_t", "all_lwps");
	if (offset == -1) {
		offset = OFFSETOF(uberdata_t, all_lwps);
		mdb_warn("CTF data is missing for uberdata_t; using current "
		    "platform's offset for uberdata.all_lwps");
	}

	if (addr == 0 &&
	    ((uber_addr = uberdata_addr()) == 0 ||
	    mdb_vread(&addr, sizeof (addr), uber_addr + offset)
	    != sizeof (addr))) {
		mdb_warn("cannot find 'uberdata.all_lwps'");
		return (WALK_ERR);
	}
	if (addr == 0)
		return (WALK_DONE);
	wsp->walk_addr = addr;
	wsp->walk_data = (void *)addr;
	return (WALK_NEXT);
}

static int
ulwp_walk_step(mdb_walk_state_t *wsp)
{
	uintptr_t addr = wsp->walk_addr;
	ulwp_t ulwp;

	if (addr == 0)
		return (WALK_DONE);
	if (mdb_vread(&ulwp, sizeof (ulwp), addr) != sizeof (ulwp) &&
	    (bzero(&ulwp, sizeof (ulwp)),
	    mdb_vread(&ulwp, REPLACEMENT_SIZE, addr)) != REPLACEMENT_SIZE) {
		mdb_warn("failed to read ulwp at 0x%p", addr);
		return (WALK_ERR);
	}
	/*
	 * If we have looped around to the beginning
	 * of the circular linked list, we are done.
	 */
	if ((wsp->walk_addr = (uintptr_t)ulwp.ul_forw)
	    == (uintptr_t)wsp->walk_data)
		wsp->walk_addr = 0;
	return (wsp->walk_callback(addr, &ulwp, wsp->walk_cbdata));
}

typedef struct lwp_wchan {
	uint8_t		wchan_flag[4];
	uint16_t	wchan_type;
	uint16_t	wchan_magic;
} lwp_wchan_t;

static int
wchan_walk_init(mdb_walk_state_t *wsp)
{
	if (wsp->walk_addr != 0) {
		mdb_warn("wchan walk only supports global walks");
		return (WALK_ERR);
	}

	if (mdb_layered_walk("ulwp", wsp) == -1) {
		mdb_warn("couldn't walk ulwp");
		return (WALK_ERR);
	}

	return (WALK_NEXT);
}

static int
wchan_walk_step(mdb_walk_state_t *wsp)
{
	uintptr_t addr = (uintptr_t)(((ulwp_t *)wsp->walk_layer)->ul_wchan);
	lwp_wchan_t wchan;

	if (addr == (uintptr_t)NULL) {
		return (WALK_NEXT);
	}

	if (mdb_vread(&wchan, sizeof (wchan), addr) != sizeof (wchan)) {
		mdb_warn("failed to read wchan at 0x%p", addr);
		return (WALK_ERR);
	}

	return (wsp->walk_callback(addr, &wchan, wsp->walk_cbdata));
}

/* Avoid classifying NULL pointers as part of the main stack on x86 */
#define	MIN_STACK_ADDR		(0x10000ul)

static int
whatis_walk_ulwp(uintptr_t addr, const ulwp_t *ulwp, mdb_whatis_t *w)
{
	uintptr_t cur;
	lwpid_t id = ulwp->ul_lwpid;
	uintptr_t top, base, size;

	while (mdb_whatis_match(w, addr, sizeof (ulwp_t), &cur))
		mdb_whatis_report_object(w, cur, addr,
		    "allocated as thread %#r's ulwp_t\n", id);

	top = (uintptr_t)ulwp->ul_stktop;
	size = ulwp->ul_stksiz;

	/*
	 * The main stack ends up being a little weird, especially if
	 * the stack ulimit is unlimited.  This tries to take that into
	 * account.
	 */
	if (size > top)
		size = top;
	if (top > MIN_STACK_ADDR && top - size < MIN_STACK_ADDR)
		size = top - MIN_STACK_ADDR;

	base = top - size;

	while (mdb_whatis_match(w, base, size, &cur))
		mdb_whatis_report_address(w, cur, "in [ stack tid=%#r ]\n", id);

	if (ulwp->ul_ustack.ss_flags & SS_ONSTACK) {
		base = (uintptr_t)ulwp->ul_ustack.ss_sp;
		size = ulwp->ul_ustack.ss_size;

		while (mdb_whatis_match(w, base, size, &cur))
			mdb_whatis_report_address(w, cur,
			    "in [ altstack tid=%#r ]\n", id);
	}

	return (WHATIS_WALKRET(w));
}

/*ARGSUSED*/
static int
whatis_run_ulwps(mdb_whatis_t *w, void *arg)
{
	if (mdb_walk("ulwps", (mdb_walk_cb_t)whatis_walk_ulwp, w) == -1) {
		mdb_warn("couldn't find ulwps walker");
		return (1);
	}
	return (0);
}

/*
 * =======================================================
 * End of thread (previously libthread) interfaces.
 * ==================== threads ==========================
 */

int
stacks_dcmd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	int rval = stacks(addr, flags, argc, argv);

	/*
	 * For the user-level variant of ::stacks, we don't bother caching
	 * state, as even a very large program is unlikely to compare to the
	 * kernel in terms of number of threads.  (And if you find yourself
	 * here in anger, frustrated about how long ::stacks is running on
	 * your galactically complicated zillion-thread program, hopefully
	 * you will find some solace in the irony.  Okay, probably not...)
	 */
	stacks_cleanup(B_TRUE);
	return (rval);
}

typedef struct tid2ulwp_walk {
	lwpid_t t2u_tid;
	uintptr_t t2u_lwp;
	boolean_t t2u_found;
} tid2ulwp_walk_t;

/*ARGSUSED*/
static int
tid2ulwp_walk(uintptr_t addr, ulwp_t *ulwp, tid2ulwp_walk_t *t2u)
{
	if (ulwp->ul_lwpid == t2u->t2u_tid) {
		t2u->t2u_lwp = addr;
		t2u->t2u_found = B_TRUE;
		return (WALK_DONE);
	}

	return (WALK_NEXT);
}

static int
tid2ulwp_impl(uintptr_t tid_addr, uintptr_t *ulwp_addrp)
{
	tid2ulwp_walk_t t2u;

	bzero(&t2u, sizeof (t2u));
	t2u.t2u_tid = (lwpid_t)tid_addr;

	if (mdb_walk("ulwp", (mdb_walk_cb_t)tid2ulwp_walk, &t2u) != 0) {
		mdb_warn("can't walk 'ulwp'");
		return (DCMD_ERR);
	}

	if (!t2u.t2u_found) {
		mdb_warn("thread ID %d not found", t2u.t2u_tid);
		return (DCMD_ERR);
	}
	*ulwp_addrp = t2u.t2u_lwp;
	return (DCMD_OK);
}

/*ARGSUSED*/
static int
tid2ulwp(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	uintptr_t ulwp_addr;
	int error;

	if (argc != 0)
		return (DCMD_USAGE);

	error = tid2ulwp_impl(addr, &ulwp_addr);
	if (error == DCMD_OK)
		mdb_printf("%p\n", ulwp_addr);
	return (error);
}

/*
 * This is used by both d_tsd and d_errno, and contains the sum of all
 * members used by both commands.
 */
typedef struct mdb_libc_ulwp {
	void *ul_ftsd[TSD_NFAST];
	tsd_t *ul_stsd;
	int *ul_errnop;
} mdb_libc_ulwp_t;

/*
 * Map from thread pointer to tsd for given key
 */
static int
d_tsd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	mdb_libc_ulwp_t u;
	uintptr_t ulwp_addr;
	uintptr_t key = 0;
	void *element = NULL;

	if (mdb_getopts(argc, argv, 'k', MDB_OPT_UINTPTR, &key, NULL) != argc)
		return (DCMD_USAGE);

	if (!(flags & DCMD_ADDRSPEC) || key == 0)
		return (DCMD_USAGE);

	if (tid2ulwp_impl(addr, &ulwp_addr) != DCMD_OK)
		return (DCMD_ERR);

	if (mdb_ctf_vread(&u, "ulwp_t", "mdb_libc_ulwp_t", ulwp_addr, 0) == -1)
		return (DCMD_ERR);

	if (key < TSD_NFAST) {
		element = u.ul_ftsd[key];
	} else if (u.ul_stsd != NULL) {
		uint_t nalloc;
		/* tsd_t is a union, so we can't use ctf_vread() on it. */
		if (mdb_vread(&nalloc, sizeof (nalloc),
		    (uintptr_t)&u.ul_stsd->tsd_nalloc) == -1) {
			mdb_warn("failed to read tsd_t at %p", u.ul_stsd);
			return (DCMD_ERR);
		}
		if (key < nalloc) {
			if (mdb_vread(&element, sizeof (element),
			    (uintptr_t)&u.ul_stsd->tsd_data[key]) == -1) {
				mdb_warn("failed to read tsd_t at %p",
				    u.ul_stsd);
				return (DCMD_ERR);
			}
		}
	}

	if (element == NULL && (flags & DCMD_PIPE))
		return (DCMD_OK);

	mdb_printf("%p\n", element);
	return (DCMD_OK);
}

/*
 * Print percent from 16-bit binary fraction [0 .. 1]
 * Round up .01 to .1 to indicate some small percentage (the 0x7000 below).
 *
 * Note: This routine was copied from elfdump/common/corenote.c	and modified.
 *
 */
static uint_t
pct_value(ushort_t pct)
{
	uint_t value = pct;

	value = ((value * 1000) + 0x7000) >> 15;	/* [0 .. 1000] */
	if (value >= 1000)
		value = 999;

	return (value);
}

static void
psinfo_raw(psinfo_t psinfo)
{
	const int minspaces = 2;
	const int spbcols = 20;
	char sysname[SYS2STR_MAX];
	uint_t cpu, mem;
	char buff[32];
	int bufflen;

	mdb_printf("[ NT_PRPSINFO ]\n");

	mdb_printf("\tpr_state:   %d\t\t\tpr_sname:   %c\n",
	    psinfo.pr_lwp.pr_state, psinfo.pr_lwp.pr_sname);

	mdb_printf("\tpr_zomb:    %d\t\t\tpr_nice:    %d\n",
	    psinfo.pr_nzomb, psinfo.pr_lwp.pr_nice);

	mdb_printf("\tpr_uid:     %u\t\t\tpr_gid:     %u\n",
	    psinfo.pr_uid, psinfo.pr_gid);

	mdb_snprintf(buff, sizeof (buff),
	    "%d", psinfo.pr_pid);

	bufflen = strlen(buff);
	mdb_printf("\tpr_pid:     %s%*spr_ppid:    %d\n",
	    buff, strlen(buff) > spbcols ? minspaces : (spbcols - bufflen), " ",
	    psinfo.pr_ppid);

	mdb_printf("\tpr_pgid:    %u\t\t\tpr_sid:     %d\n",
	    psinfo.pr_gid, psinfo.pr_sid);

	mdb_snprintf(buff, sizeof (buff),
	    "0x%lx", (ulong_t)psinfo.pr_addr);

	bufflen = strlen(buff);

	mdb_printf("\tpr_addr:    %s%*spr_size:    %#x\n",
	    buff, strlen(buff) > spbcols ? minspaces : (spbcols - bufflen), " ",
	    (ulong_t)psinfo.pr_size);

	mdb_printf("\tpr_rssize:  %#lx\t\tpr_wchan:   %#lx\n",
	    (ulong_t)psinfo.pr_rssize, (ulong_t)psinfo.pr_lwp.pr_wchan);

	mdb_printf("\tpr_start:\n\t    tv_sec: %ld\t\ttv_nsec:    %ld\n",
	    psinfo.pr_start.tv_sec, psinfo.pr_start.tv_nsec);

	mdb_printf("\tpr_time:\n\t    tv_sec: %ld\t\t\ttv_nsec:    %ld\n",
	    psinfo.pr_time.tv_sec, psinfo.pr_time.tv_nsec);

	mdb_printf("\tpr_pri:     %d\t\t\tpr_oldpri:  %d\n",
	    psinfo.pr_lwp.pr_pri, psinfo.pr_lwp.pr_oldpri);

	mdb_printf("\tpr_cpu:     %d\n", psinfo.pr_lwp.pr_cpu);

	mdb_printf("\tpr_clname:  %s\n", psinfo.pr_lwp.pr_clname);

	mdb_printf("\tpr_fname:   %s\n", psinfo.pr_fname);

	mdb_printf("\tpr_psargs:  %s\n", psinfo.pr_psargs);


	mdb_printf("\tpr_syscall: [ %s ]\n",
	    proc_sysname(psinfo.pr_lwp.pr_syscall, sysname,
	    sizeof (sysname)));

	mdb_printf("\tpr_ctime:\n\t    tv_sec: %ld\t\t\ttv_nsec:    %ld\n",
	    psinfo.pr_ctime.tv_sec, psinfo.pr_ctime.tv_nsec);

	mdb_printf("\tpr_argc:    %d\t\t\tpr_argv:    0x%lx\n",
	    psinfo.pr_argc, (ulong_t)psinfo.pr_argv);

	mdb_snprintf(buff, sizeof (buff), "0x%lx", (ulong_t)psinfo.pr_envp);

	bufflen = strlen(buff);

	mdb_printf("\tpr_envp:    %s%*spr_wstat:   %d\n",
	    buff, strlen(buff) > spbcols ? minspaces : (spbcols - bufflen), " ",
	    psinfo.pr_wstat);

	cpu = pct_value(psinfo.pr_pctcpu);
	mem = pct_value(psinfo.pr_pctmem);

	mdb_printf("\tpr_pctcpu:  %u.%u%%\t\tpr_pctmem:  %u.%u%%\n",
	    cpu / 10, cpu % 10, mem / 10, mem % 10);

	mdb_printf("\tpr_euid:    %u\t\t\tpr_egid:    %u\n",
	    psinfo.pr_euid, psinfo.pr_egid);

	mdb_printf("\tpr_dmodel:  [%s]\n",
	    proc_dmodelname(psinfo.pr_dmodel, buff, sizeof (buff)));
}

static void
psinfo_sum(psinfo_t psinfo)
{
	const int minspaces = 2;
	const int spbcols = 23;
	char buff[64];
	int bufflen;
	int ms;

	mdb_printf("PID:    %6d  (process id)\t\t"
	    "UID:     %4u  (real user id)\n",
	    psinfo.pr_pid, psinfo.pr_uid);

	mdb_printf("PPID:   %6d  (parent process id)\tEUID:    %4d"
	    "  (effective user id)\n", psinfo.pr_ppid, psinfo.pr_euid);

	mdb_printf("PGID:   %6d  (process group id)\tGID:     %4u"
	    "  (real group id)\n", psinfo.pr_pgid, psinfo.pr_gid);

	mdb_printf("SID:    %6d  (session id)\t\tEGID:    %4u"
	    "  (effective group id)\n",
	    psinfo.pr_sid, psinfo.pr_egid);

	mdb_printf("ZONEID: %6d\t\t\t\tCONTRACT:%4d\n",
	    psinfo.pr_zoneid, psinfo.pr_contract);

	mdb_printf("PROJECT:%6d \t\t\t\tTASK:    %4d\n\n",
	    psinfo.pr_projid, psinfo.pr_taskid);

	mdb_printf("START: %Y   (wall timestamp when the process started)\n",
	    psinfo.pr_start);

	ms = NSEC2MSEC(psinfo.pr_time.tv_nsec);

	mdb_snprintf(buff, sizeof (buff), "%ld.%d seconds",
	    psinfo.pr_time.tv_sec, ms);

	bufflen = strlen(buff);

	mdb_printf("TIME:  %s%*s"
	    "(CPU time used by this process)\n",
	    buff, bufflen > spbcols ? minspaces : (spbcols - bufflen), " ");

	ms = NSEC2MSEC(psinfo.pr_ctime.tv_nsec);

	mdb_snprintf(buff, sizeof (buff), "%ld.%d seconds",
	    psinfo.pr_ctime.tv_sec, ms);

	mdb_printf("CTIME: %s%*s"
	    "(CPU time used by child processes)\n",
	    buff, bufflen > spbcols ? minspaces : (spbcols - bufflen), " ");

	mdb_snprintf(buff, sizeof (buff), "%s", psinfo.pr_fname);
	bufflen = strlen(buff);

	mdb_printf("FNAME: %s%*s(name of the program executed)\n",
	    buff, bufflen > spbcols ? minspaces : (spbcols - bufflen), " ");

	mdb_printf("PSARGS: \"%s\"\n", psinfo.pr_psargs);
}

void
d_psinfo_dcmd_help(void)
{
	mdb_printf(
	    "Prints relevant fields from psinfo_t data and\n"
	    "most fields from NT_PRPSINFO note section\n\n"
	    "Usage:  ::psinfo [-v]\n"
	    "Options:\n"
	    "   -v   verbose output\n");
}

static int
d_psinfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	psinfo_t psinfo;
	uint_t opt_v = FALSE;
	ssize_t nbytes;

	if (mdb_getopts(argc, argv, 'v',
	    MDB_OPT_SETBITS, TRUE, &opt_v, NULL) != argc)
		return (DCMD_USAGE);

	nbytes = mdb_get_xdata("psinfo", NULL, 0);

	if (nbytes <= 0) {
		mdb_warn("information not available for analysis");
		return (DCMD_ERR);
	}

	if (mdb_get_xdata("psinfo", &psinfo, nbytes) != nbytes) {
		mdb_warn("failed to read psinfo information");
		return (DCMD_ERR);
	}

	if (opt_v) {
		psinfo_raw(psinfo);
	} else {
		psinfo_sum(psinfo);
	}

	return (DCMD_OK);
}

typedef struct d_mutex_output {
	char *mo_output;
	struct d_mutex_output *mo_next;
} d_mutex_output_t;

static void
d_mutex_output_push(d_mutex_output_t **head, const char *out)
{
	d_mutex_output_t *new;
	size_t len = strlen(out) + 1;

	new = mdb_alloc(sizeof (d_mutex_output_t), UM_SLEEP | UM_GC);
	new->mo_next = *head;
	new->mo_output = mdb_alloc(len, UM_SLEEP | UM_GC);
	bcopy(out, new->mo_output, len);

	*head = new;
}

void
d_mutex_output_reverse(d_mutex_output_t **head)
{
	d_mutex_output_t *current, *next, *last = NULL;

	for (current = *head; current != NULL; current = next) {
		next = current->mo_next;
		current->mo_next = last;
		last = current;

		if (next == NULL) {
			break;
		}
	}

	*head = current;
}

typedef struct d_mutex_walkdata {
	uintptr_t mow_target;
	d_mutex_output_t *mow_output;
} d_mutex_walkdata_t;

int
d_mutex_walk(uintptr_t addr, const ulwp_t *ulwp, d_mutex_walkdata_t *wd)
{
	char buf[256];

	if ((uintptr_t)ulwp->ul_wchan != wd->mow_target)
		return (WALK_NEXT);

	if (mdb_thread_name(ulwp->ul_lwpid, buf, sizeof (buf)) != 0) {
		mdb_snprintf(buf, sizeof (buf), "0x%p", addr);
	}

	d_mutex_output_push(&wd->mow_output, buf);
	return (WALK_NEXT);
}

static void
d_mutex_help(void)
{
	mdb_printf("%s\n",
"Dump a mutex, optionally decoding flags and displaying waiters.\n");
	mdb_dec_indent(2);
	mdb_printf("%<b>OPTIONS%</b>\n");
	mdb_inc_indent(2);
	mdb_printf("%s",
"  -v    Dump verbosely, decoding type and flags and showing waiters\n"
"  -f    force printing as a mutex, even if it doesn't appear to be one\n");
}

static int
d_mutex(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	lwp_mutex_t mutex;
	uintptr_t owner;
	char buf[256];
	uint_t opt_v = FALSE, opt_f = FALSE, warn;
	d_mutex_walkdata_t wd;
	d_mutex_output_t *toutput = NULL, *foutput = NULL;
	size_t i;


	if (!(flags & DCMD_ADDRSPEC))
		return (DCMD_USAGE);

	if (mdb_getopts(argc, argv,
	    'v', MDB_OPT_SETBITS, TRUE, &opt_v,
	    'f', MDB_OPT_SETBITS, TRUE, &opt_f,
	    NULL) != argc) {
		return (DCMD_USAGE);
	}

	if (DCMD_HDRSPEC(flags)) {
		mdb_printf("%-16s %4s %4s %4s %s\n",
		    "ADDR", "TYPE", "FLAG", "WTRS", "OWNER");
	}

	/*
	 * If we aren't in a loop or a pipe, we will warn explicitly when
	 * we can't make sense of a mutex.
	 */
	warn = (flags & (DCMD_LOOP | DCMD_PIPE)) ? FALSE : TRUE;

	if (mdb_vread(&mutex, sizeof (mutex), addr) != sizeof (mutex)) {
		if (warn)
			mdb_warn("failed to read mutex at 0x%p", addr);

		return (DCMD_ERR);
	}

	/*
	 * It's legal to have a zero'd mutex_t in BSS -- so we can only
	 * rely on the magic to disambiguate a mutex if it is non-zero.
	 */
	if (!opt_f && mutex.flags.magic != 0 &&
	    mutex.flags.magic != MUTEX_MAGIC) {
		if (!warn)
			return (DCMD_ERR);

		if (mutex.flags.magic == COND_MAGIC) {
			mdb_warn("0x%p is not a mutex (appears to be a "
			    "condition variable)\n", addr);
		} else if (mutex.flags.magic == SEMA_MAGIC) {
			mdb_warn("0x%p is not a mutex (appears to be a "
			    "semaphore)\n", addr);
		} else if (mutex.flags.magic == RWL_MAGIC) {
			mdb_warn("0x%p is not a mutex (appears to be a "
			    "readers/writer lock)\n", addr);
		} else {
			mdb_warn("0x%p does not appear to be a mutex (expected "
			    "0x%x, found 0x%x)\n", addr, MUTEX_MAGIC,
			    mutex.flags.magic);
		}

		return (DCMD_ERR);
	}

	if (!opt_f) {
		/*
		 * Sanity check that if we have an owner, it at least isn't
		 * obviously not a ulwp_t.
		 */
		uintptr_t owner = mutex.mutex_owner;
		ulwp_t u;

		if (owner == (uintptr_t)NULL) {
			if (mutex.mutex_waiters) {
				if (!warn)
					return (DCMD_ERR);

				mdb_warn("0x%p does not appear to be a mutex "
				    "(waiters, but no owner?)\n", addr);

				return (DCMD_ERR);
			}
		} else if (mdb_vread(&u, sizeof (u), owner) != sizeof (u) ||
		    (uintptr_t)u.ul_self != owner) {
			if (!warn)
				return (DCMD_ERR);

			mdb_warn("0x%p does not appear to be mutex "
			    "(owner 0x%p does not appear to be a ulwp_t)\n",
			    addr, owner);

			return (DCMD_ERR);
		}
	}

	mdb_printf("%-16p %4x %4x %4s ", addr, mutex.mutex_type,
	    mutex.mutex_flag,
	    mutex.mutex_waiters ? "yes" : "no");

	if ((owner = mutex.mutex_owner) == (uintptr_t)NULL) {
		mdb_printf("-\n");
	} else {
		ulwp_t u;

		if (mdb_vread(&u, sizeof (u), owner) != sizeof (u) ||
		    mdb_thread_name(u.ul_lwpid, buf, sizeof (buf)) != 0) {
			mdb_snprintf(buf, sizeof (buf), "%d", u.ul_lwpid);
		}

		mdb_printf("%p %s\n", owner, buf);
	}

	if (!opt_v)
		return (DCMD_OK);

	static struct {
		int val;
		char *str;
	} tvals[] = {
		{ 0x01, "LOCK_SHARED" },
		{ 0x02, "LOCK_ERRORCHECK" },
		{ 0x04, "LOCK_RECURSIVE" },
		{ 0x10, "LOCK_PRIO_INHERIT" },
		{ 0x20, "LOCK_PRIO_PROTECT" },
		{ 0x40, "LOCK_ROBUST" },
		/*
		 * This is a defunct type, but an ancient (or corrupt) mutex
		 * might have it set; indicate it if we see it.
		 */
		{ 0x08, "PROCESS_ROBUST" },
		{ 0, "" }
	};

	if (!(mutex.mutex_type & LOCK_SHARED)) {
		d_mutex_output_push(&toutput, "LOCK_NORMAL");
	}

	for (i = 0; tvals[i].val != 0; i++) {
		if ((mutex.mutex_type & tvals[i].val) != 0) {
			d_mutex_output_push(&toutput, tvals[i].str);
		}
	}

	static struct {
		int val;
		char *str;
	} fvals[] = {
		{ 0x1, "LOCK_OWNERDEAD" },
		{ 0x2, "LOCK_NOTRECOVERABLE" },
		{ 0x4, "LOCK_INITED" },
		{ 0x8, "LOCK_UNMAPPED" },
		{ 0x10, "LOCK_DEADLOCK" },
		{ 0, "" },
	};

	for (i = 0; fvals[i].val != 0; i++) {
		if ((mutex.mutex_flag & fvals[i].val) != 0) {
			d_mutex_output_push(&foutput, fvals[i].str);
		}
	}

	wd.mow_target = addr;
	wd.mow_output = NULL;

	if (mdb_walk("ulwp", (mdb_walk_cb_t)d_mutex_walk, &wd) != 0) {
		mdb_warn("can't walk \"ulwp\"");
		return (DCMD_ERR);
	}

	d_mutex_output_t *ooutput = wd.mow_output;
	d_mutex_output_reverse(&toutput);
	d_mutex_output_reverse(&foutput);
	d_mutex_output_reverse(&ooutput);

	d_mutex_output_t *thead = toutput;
	d_mutex_output_t *fhead = foutput;
	d_mutex_output_t *ohead = ooutput;

	mdb_printf("%21s", toutput != NULL ? "|" : "");
	mdb_printf("%5s", foutput != NULL ? "|" : "");
	mdb_printf("%3s\n", ooutput != NULL ? "|" : "");

	boolean_t needblank = ooutput != NULL;

	while (toutput != NULL || foutput != NULL || ooutput != NULL) {
		if (toutput != NULL) {
			mdb_printf("%17s %s", toutput->mo_output,
			    toutput == thead ? "<-+" : "   ");
			toutput = toutput->mo_next;
		} else {
			mdb_printf("%21s", "");
		}

		if (foutput != NULL) {
			if (ooutput != NULL) {
				mdb_printf("%5s", "|");
			} else {
				if (needblank) {
					mdb_printf("%5s", "|");
					needblank = B_FALSE;
				} else {
					mdb_printf("%7s %s",
					    foutput == fhead ? "+->" : "",
					    foutput->mo_output);
					foutput = foutput->mo_next;
				}
			}
		} else {
			mdb_printf("%5s", "");
		}

		if (ooutput != NULL) {
			mdb_printf("%5s %s",
			    ooutput == ohead ? "+->" : "", ooutput->mo_output);
			ooutput = ooutput->mo_next;
		}

		mdb_printf("\n");
	}

	return (DCMD_OK);
}

static int
d_errno(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	mdb_libc_ulwp_t u;
	uintptr_t ulwp_addr;
	int error, errval;

	if (argc != 0 || (flags & DCMD_ADDRSPEC) == 0)
		return (DCMD_USAGE);

	error = tid2ulwp_impl(addr, &ulwp_addr);
	if (error != DCMD_OK)
		return (error);

	/*
	 * For historical compatibility, thread 1's errno value is stored in
	 * a libc global variable 'errno', while each additional thread's
	 * errno value is stored in ulwp_t->ul_errno.  In addition,
	 * ulwp_t->ul_errnop is set to the address of the thread's errno value,
	 * (i.e. for tid 1, curthead->ul_errnop = &errno, for tid > 1,
	 * curthread->ul_errnop = &curthread->ul_errno).
	 *
	 * Since errno itself uses *curthread->ul_errnop (see ___errno()) to
	 * return the thread's current errno value, we do the same.
	 */
	if (mdb_ctf_vread(&u, "ulwp_t", "mdb_libc_ulwp_t", ulwp_addr, 0) == -1)
		return (DCMD_ERR);

	if (mdb_vread(&errval, sizeof (errval), (uintptr_t)u.ul_errnop) == -1) {
		mdb_warn("cannot read error value at 0x%p", u.ul_errnop);
		return (DCMD_ERR);
	}

	mdb_printf("%d\n", errval);
	return (DCMD_OK);
}

static const mdb_dcmd_t dcmds[] = {
	{ "errno", "?", "print errno of a given TID", d_errno, NULL },
	{ "jmp_buf", ":", "print jmp_buf contents", d_jmp_buf, NULL },
	{ "mutex", ":[-f|v]", "dump out a mutex", d_mutex, d_mutex_help },
	{ "psinfo", "[-v]", "prints relevant psinfo_t data", d_psinfo,
	    d_psinfo_dcmd_help },
	{ "siginfo", ":", "print siginfo_t structure", d_siginfo, NULL },
	{ "sigjmp_buf", ":", "print sigjmp_buf contents", d_sigjmp_buf, NULL },
	{ "stacks", "?[-afiv] [-c func] [-C func] [-m module] [-M module] ",
		"print unique thread stacks", stacks_dcmd, stacks_help },
	{ "tid2ulwp", "?", "convert TID to ulwp_t address", tid2ulwp },
	{ "tsd", ":-k key", "print tsd for this thread", d_tsd, NULL },
	{ "ucontext", ":", "print ucontext_t structure", d_ucontext, NULL },
	{ "ulwp", ":", "print ulwp_t structure", d_ulwp, NULL },
	{ "uberdata", ":", "print uberdata_t structure", d_uberdata, NULL },
	{ NULL }
};

static const mdb_walker_t walkers[] = {
	{ "ucontext", "walk ucontext_t uc_link list",
		NULL, uc_walk_step, NULL, NULL },
	{ "oldcontext", "walk per-lwp oldcontext pointers",
		oldc_walk_init, oldc_walk_step, oldc_walk_fini, NULL },
	{ "ulwps", "walk list of ulwp_t pointers",
		ulwp_walk_init, ulwp_walk_step, NULL, NULL },
	{ "ulwp", "walk list of ulwp_t pointers",
		ulwp_walk_init, ulwp_walk_step, NULL, NULL },
	{ "wchan", "walk wait channels",
		wchan_walk_init, wchan_walk_step, NULL, NULL },
	{ NULL }
};

static const mdb_modinfo_t modinfo = { MDB_API_VERSION, dcmds, walkers };

const mdb_modinfo_t *
_mdb_init(void)
{
	mdb_whatis_register("threads", whatis_run_ulwps, NULL,
	    WHATIS_PRIO_EARLY, WHATIS_REG_NO_ID);

	return (&modinfo);
}