libumem - zygaena/hammerhead

root / base / usr / src / lib / libumem
libumem Plain Text 10991 lines 288.8 KB
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

include ../Makefile.lib

HDRS=		umem.h umem_impl.h sys/vmem_impl_user.h
HDRDIR=		common
# Hammerhead: amd64-only
SUBDIRS = $(MACH64)

all	: TARGET= all
clean	: TARGET= clean
clobber	: TARGET= clobber
install	: TARGET= install

.KEEP_STATE:

all clean clobber install: $(SUBDIRS)

install_h: $(ROOTHDRS)

check: $(CHECKHDRS)

$(SUBDIRS): FRC
	@cd $@; pwd; $(MAKE) $(TARGET)

FRC:

include ../Makefile.targ
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# Copyright 2019 Joyent, Inc.
#

#
# The build process for libumem is sightly different from that used by other
# libraries, because libumem must be built in two flavors - as a standalone
# for use by kmdb and as a normal library.  We use $(CURTYPE) to indicate the
# current flavor being built.
#

LIBRARY = libumem.a
STANDLIBRARY = libstandumem.so
VERS = .1

# By default, we build the shared library.  Construction of the standalone
# is specifically requested by architecture-specific Makefiles.
TYPES = library
CURTYPE = library

# This would be much prettier if a) Makefile.lib didn't reqire both $(SRCS) and
# $(OBJECTS) to be set or b) make gave us a nice way to do basename in pattern
# replacement definitions.

# Files specific to the library version of libumem
OBJECTS_library = \
	init_lib.o \
	umem_agent_support.o \
	umem_fail.o \
	umem_fork.o \
	umem_genasm.o \
	umem_update_thread.o \
	vmem_mmap.o \
	vmem_sbrk.o

SRCS_common_library = \
	$(ISASRCDIR)/umem_genasm.c

SRCS_library = $(OBJECTS_library:%.o=../common/%.c) $(SRC_common_library)

# Files specific to the standalone version of libumem
OBJECTS_standalone = \
	init_stand.o \
	stub_stand.o \
	vmem_stand.o

SRCS_standalone = $(OBJECTS_standalone:%.o=../common/%.c)

# Architecture-dependent files common to both versions of libumem
OBJECTS_common_isadep = \
	asm_subr.o

SRCS_common_isadep = \
	$(ISASRCDIR)/asm_subr.S

# Architecture-independent files common to both versions  of libumem
OBJECTS_common_common = \
	envvar.o \
	getpcstack.o \
	malloc.o \
	misc.o \
	vmem_base.o \
	umem.o \
	vmem.o

SRCS_common_common = $(OBJECTS_common_common:%.o=../common/%.c)

OBJECTS = \
	$(OBJECTS_$(CURTYPE)) \
	$(OBJECTS_common_isadep) \
	$(OBJECTS_common_common)

include ../../Makefile.lib
include ../../Makefile.rootfs

SRCS = \
	$(SRCS_$(CURTYPE)) \
	$(SRCS_common_common)

SRCDIR = ../common

#
# Used to verify that the standalone doesn't have any unexpected external
# dependencies.
#
LINKTEST_OBJ = objs/linktest_stand.o

CLOBBERFILES_standalone = $(LINKTEST_OBJ)
CLOBBERFILES += $(CLOBBERFILES_$(CURTYPE))

LIBS_standalone = $(STANDLIBRARY)
LIBS_library = $(DYNLIB)
LIBS =	$(LIBS_$(CURTYPE))

MAPFILE_SUPPLEMENTAL_standalone = ../common/stand_mapfile
MAPFILE_SUPPLEMENTAL = $(MAPFILE_SUPPLEMENTAL_$(CURTYPE))

# Hammerhead: Add -ldl for dlopen/dlsym/dlclose/dladdr1 used by the library build
# (standalone doesn't use these functions)
LDLIBS_library = -ldl
LDLIBS +=	$(LDLIBS_$(CURTYPE)) -lc

# Hammerhead: Skip mapfiles for standalone build with GNU ld.
# GNU ld has issues combining multiple converted illumos mapfiles with -r.
# The standalone is only used by kmdb, and extra symbol visibility is acceptable.
LDFLAGS_standalone = $(ZNOVERSION) $(BREDUCE) -dy -r
LDFLAGS = $(LDFLAGS_$(CURTYPE))

ASFLAGS_standalone = -DUMEM_STANDALONE
ASFLAGS_library =
ASFLAGS += $(ASFLAGS_$(CURTYPE)) -D_ASM
ASFLAGS64 += $(ASFLAGS_$(CURTYPE)) -D_ASM

# We want the thread-specific errno in the library, but we don't want it in
# the standalone.  $(DTS_ERRNO) is designed to add -D_TS_ERRNO to $(CPPFLAGS),
# in order to enable this feature.  Conveniently, -D_REENTRANT does the same
# thing.  As such, we null out $(DTS_ERRNO) to ensure that the standalone
# doesn't get it.
DTS_ERRNO=

# We need to rename some standard functions so we can easily implement them
# in consumers.
STAND_RENAMED_FUNCS= \
	atomic_add_64 \
	atomic_add_32_nv \
	atomic_swap_64 \
	snprintf \
	vsnprintf

# Hammerhead: Use $(foreach) for GNU Make - the :%=-D%=umem_% pattern doesn't
# expand both % symbols correctly in GNU Make
CPPFLAGS_standalone = -DUMEM_STANDALONE $(foreach f,$(STAND_RENAMED_FUNCS),-D$(f)=umem_$(f))
CPPFLAGS_library = -D_REENTRANT
CPPFLAGS +=	-I../common -I../../common/inc $(CPPFLAGS_$(CURTYPE))

CFLAGS_standalone = $(STAND_FLAGS_32)
CFLAGS_common =
CFLAGS += $(CFLAGS_$(CURTYPE)) $(CFLAGS_common)

CFLAGS64_standalone = $(STAND_FLAGS_64)
CFLAGS64 += $(CCVERBOSE) $(CFLAGS64_$(CURTYPE)) $(CFLAGS64_common)

#
# For the standalone environment, disable the stack protector for the
# time being.
#
$(STANDLIBRARY) : STACKPROTECT = none

# false positive for umem_alloc_sizes_add()
pics/umem.o : SMOFF += index_overflow
objs/umem.o : SMOFF += index_overflow

INSTALL_DEPS_library =	$(ROOTLINKS) $(ROOTLIBS) $(ROOTCOMPATLINKS)

DYNFLAGS +=     $(ZINTERPOSE)

.KEEP_STATE:
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
# Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
# Copyright (c) 2019, Joyent, Inc.
#

#
# We build each flavor in a separate make invocation to improve clarity(!) in
# Makefile.com.  The subordinate makes have $(CURTYPE) set to indicate the
# flavor they're supposed to build.  This causes the correct set of source
# files and compiler and linker flags to be selected.
#
# Hammerhead: Disable parallel builds to prevent race conditions between
# library and standalone flavors which share object directories.
.NOTPARALLEL:

install: $(TYPES:%=install.%)

install.library: all.library $(INSTALL_DEPS_library)
install.standalone: all.standalone $(INSTALL_DEPS_standalone)

all: $(TYPES:%=all.%)

$(TYPES:%=all.%):
	@$(MAKE) $@.targ CURTYPE=$(@:all.%=%)

all.library.targ: $(LIBS)
all.standalone.targ: $(STANDLIBRARY)

# Hammerhead: Split standalone linking into linktest (no mapfiles) and final link.
# GNU ld has issues combining multiple converted illumos mapfiles with -r.
# The linktest only needs -zdefs to catch undefined symbols.
LDFLAGS_LINKTEST = $(ZNOVERSION) $(BREDUCE) -dy -r

$(STANDLIBRARY): $(OBJS) $(LINKTEST_OBJ)
	$(LD) $(LDFLAGS_LINKTEST) $(ZDEFS) -o $@.linktest $(OBJS) $(LINKTEST_OBJ)
	rm $@.linktest
	$(LD) $(LDFLAGS) -o $@ $(OBJS)

clobber: $(TYPES:%=clobber.%)

$(TYPES:%=clobber.%):
	@$(MAKE) clobber.targ CURTYPE=$(@:clobber.%=%)

clobber.targ: clean
	-$(RM) $(CLOBBERTARGFILES)

# include library targets
include ../../Makefile.targ

$(PICS): pics
$(OBJS): objs

objs/%.o: $(ISASRCDIR)/%.c
	$(COMPILE.c) -o $@ $<
	$(POST_PROCESS_O)

pics/%.o: $(ISASRCDIR)/%.c
	$(COMPILE.c) -o $@ $<
	$(POST_PROCESS_O)

objs/%.o: $(ISASRCDIR)/%.S
	$(COMPILE.s) -o $@ $<
	$(POST_PROCESS_S_O)

pics/%.o: $(ISASRCDIR)/%.S
	$(COMPILE.s) -o $@ $<
	$(POST_PROCESS_S_O)

objs/%.o: ../common/%.c
	$(COMPILE.c) -o $@ $<
	$(POST_PROCESS_O)

pics/%.o: ../common/%.c
	$(COMPILE.c) -o $@ $<
	$(POST_PROCESS_O)
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License, Version 1.0 only
# (the "License").  You may not use this file except in compliance
# with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

# Hammerhead: Changed from ../$(MACH)/ to . since we're amd64-only
# and the ISA-specific source files are in this directory
ISASRCDIR=.

include ../Makefile.com
include ../../Makefile.lib.64

TYPES=library standalone

INSTALL_DEPS_library = $(ROOTLINKS64) $(ROOTLIBS64) \
	$(ROOTCOMPATLINKS64)

include ../Makefile.targ
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/asm_linkage.h>

#define	NOP4	\
	nop;	\
	nop;	\
	nop;	\
	nop;

#define NOP16	\
	NOP4	\
	NOP4	\
	NOP4	\
	NOP4

#define	NOP64	\
	NOP16	\
	NOP16	\
	NOP16	\
	NOP16

#define	NOP256	\
	NOP64	\
	NOP64	\
	NOP64	\
	NOP64

#if defined(lint)

void *
getfp(void)
{
	return (NULL);
}

#ifndef UMEM_STANDALONE
void
_breakpoint(void)
{
	return;
}
#endif

#else	/* lint */

#if defined(__amd64)

	ENTRY(getfp)
	movq	%rbp, %rax
	ret
	SET_SIZE(getfp)

#else	/* __i386 */

	ENTRY(getfp)
	movl	%ebp, %eax
	ret
	SET_SIZE(getfp)

#endif

#ifndef UMEM_STANDALONE
	ENTRY(_breakpoint)
	int	$3
	ret
	SET_SIZE(_breakpoint)
#endif
	.section ".ptctext", "ax"
	.globl _malloc
	.type _malloc, @function
	.align ASM_ENTRY_ALIGN
_malloc:
	jmp umem_malloc@PLT;
	NOP256
	NOP256
#if defined(__amd64)
	NOP64
#endif
	SET_SIZE(_malloc)

	.section ".ptctext","ax"
	.globl _free
	.type _free, @function
	.align ASM_ENTRY_ALIGN
_free:
	jmp umem_malloc_free@PLT;
	NOP256
	NOP256
#if defined(__amd64)
	NOP64
#endif
	SET_SIZE(_free)

	ANSI_PRAGMA_WEAK2(malloc,_malloc,function)
	ANSI_PRAGMA_WEAK2(free,_free,function)

#endif	/* lint */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2019 Joyent, Inc.  All rights reserved.
 */

/*
 * Don't Panic! If you find the blocks of assembly that follow confusing and
 * you're questioning why they exist, please go read section 8 of the umem.c big
 * theory statement. Next familiarize yourself with the malloc and free
 * implementations in libumem's malloc.c.
 *
 * What follows is the amd64 implementation of the thread caching automatic
 * assembly generation. The amd64 calling conventions are documented in the
 * 64-bit System V ABI. For our purposes what matters is that our first argument
 * will come in rdi. Our functions have to preserve rbp, rbx, and r12->r15. We
 * are free to do whatever we want with rax, rcx, rdx, rsi, rdi, and r8->r11.
 *
 * For both our implementation of malloc and free we only use the registers we
 * don't have to preserve.
 *
 * Malloc register usage:
 * 	o. rdi: Original size to malloc. This never changes and is preserved.
 * 	o. rsi: Adjusted malloc size for malloc_data_tag(s).
 * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
 * 	o. rdx: Pointer to the tmem_t array of roots
 * 	o. r8:  Size of the cache
 * 	o. r9:  Scratch register
 *
 * Free register usage:
 *	o. rdi: Original buffer to free. This never changes and is preserved.
 *	o. rax: The actual buffer, adjusted for the hidden malloc_data_t(s).
 * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
 * 	o. rdx: Pointer to the tmem_t array of roots
 * 	o. r8:  Size of the cache
 * 	o. r9:  Scratch register
 *
 * Once we determine what cache we are using, we increment %rdx to the
 * appropriate offset and set %r8 with the size of the cache. This means that
 * when we break out to the normal buffer allocation point %rdx contains the
 * head of the linked list and %r8 is the amount that we have to adjust the
 * thread's cached amount by.
 *
 * Each block of assembly has psuedocode that describes its purpose.
 */

/*
 * umem_base must be first.
 */
#include "umem_base.h"

#include <inttypes.h>
#include <strings.h>
#include <umem_impl.h>
#include <atomic.h>
#include <sys/mman.h>
#include <errno.h>


#include <stdio.h>

const int umem_genasm_supported = 1;
static uintptr_t umem_genasm_mptr = (uintptr_t)&_malloc;
static size_t umem_genasm_msize = 576;
static uintptr_t umem_genasm_fptr = (uintptr_t)&_free;
static size_t umem_genasm_fsize = 576;
static uintptr_t umem_genasm_omptr = (uintptr_t)umem_malloc;
static uintptr_t umem_genasm_ofptr = (uintptr_t)umem_malloc_free;

#define	UMEM_GENASM_MAX64	(UINT32_MAX / sizeof (uintptr_t))
#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
#define	MULTINOP	0x0000441f0f

/*
 * void *ptcmalloc(size_t orig_size);
 *
 * size_t size = orig_size + 8;
 * if (size > UMEM_SECOND_ALIGN)
 * 	size += 8;
 *
 * if (size < orig_size)
 * 	goto tomalloc;		! This is overflow
 *
 * if (size > cache_max)
 * 	goto tomalloc
 *
 * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
 * void **roots = t->tm_roots;
 */
#define	PTC_MALINIT_JOUT	0x13
#define	PTC_MALINIT_MCS	0x1a
#define	PTC_MALINIT_JOV	0x20
#define	PTC_MALINIT_SOFF	0x30
static const uint8_t malinit[] =  {
	0x48, 0x8d, 0x77, 0x08,		/* leaq 0x8(%rdi),%rsi */
	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10, %rsi */
	0x76, 0x04,			/* jbe +0x4 */
	0x48, 0x8d, 0x77, 0x10,		/* leaq 0x10(%rdi),%rsi */
	0x48, 0x39, 0xfe,		/* cmpq %rdi,%rsi */
	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00,	/* jb +errout */
	0x48, 0x81, 0xfe,
	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
	0x64, 0x48, 0x8b, 0x0c, 0x25,
	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
	0x48, 0x81, 0xc1,
	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
};

/*
 * void ptcfree(void *buf);
 *
 * if (buf == NULL)
 * 	return;
 *
 * malloc_data_t *tag = buf;
 * tag--;
 * int size = tag->malloc_size;
 * int tagval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
 * if (tagval == MALLOC_SECOND_MAGIC) {
 * 	tag--;
 * } else if (tagval != MALLOC_MAGIC) {
 * 	goto tofree;
 * }
 *
 * if (size > cache_max)
 * 	goto tofree;
 *
 * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
 * void **roots = t->tm_roots;
 */
#define	PTC_FRINI_JDONE	0x05
#define	PTC_FRINI_JFREE	0x25
#define	PTC_FRINI_MCS	0x30
#define	PTC_FRINI_JOV	0x36
#define	PTC_FRINI_SOFF	0x46
static const uint8_t freeinit[] = {
	0x48, 0x85, 0xff,		/* testq %rdi,%rdi */
	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* jmp $JDONE (done) */
	0x8b, 0x77, 0xf8,		/* movl -0x8(%rdi),%esi */
	0x8b, 0x47, 0xfc,		/* movl -0x4(%rdi),%eax */
	0x01, 0xf0,			/* addl %esi,%eax */
	0x3d, 0x00, 0x70, 0xba, 0x16,	/* cmpl $MALLOC_2_MAGIC, %eax */
	0x75, 0x06,			/* jne +0x6 (checkover) */
	0x48, 0x8d, 0x47, 0xf0,		/* leaq -0x10(%rdi),%eax */
	0xeb, 0x0f,			/* jmp +0xf (freebuf) */
	0x3d, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl $MALLOC_MAGIC, %eax */
	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jmp +JFREE (goto torfree) */
	0x48, 0x8d, 0x47, 0xf8,		/* leaq -0x8(%rdi),%rax */
	0x48, 0x81, 0xfe,
	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
	0x64, 0x48, 0x8b, 0x0c, 0x25,
	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
	0x48, 0x81, 0xc1,
	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
};

/*
 * if (size <= $CACHE_SIZE) {
 *	csize = $CACHE_SIZE;
 * } else ...				! goto next cache
 */
#define	PTC_INICACHE_CMP	0x03
#define	PTC_INICACHE_SIZE	0x0c
#define	PTC_INICACHE_JMP	0x11
static const uint8_t inicache[] = {
	0x48, 0x81, 0xfe,
	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
	0x77, 0x0c,			/* ja +0xc (next cache) */
	0x49, 0xc7, 0xc0,
	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
	0xe9, 0x00, 0x00, 0x00, 0x00,	/* jmp $JMP (allocbuf) */
};

/*
 * if (size <= $CACHE_SIZE) {
 *	csize = $CACHE_SIZE;
 *	roots += $CACHE_NUM;
 * } else ...				! goto next cache
 */
#define	PTC_GENCACHE_CMP	0x03
#define	PTC_GENCACHE_SIZE	0x0c
#define	PTC_GENCACHE_NUM	0x13
#define	PTC_GENCACHE_JMP	0x18
static const uint8_t gencache[] = {
	0x48, 0x81, 0xfe,
	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
	0x77, 0x14,			/* ja +0xc (next cache) */
	0x49, 0xc7, 0xc0,
	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
	0x48, 0x81, 0xc2,
	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp +$JMP (allocbuf ) */
};

/*
 * else if (size <= $CACHE_SIZE) {
 *	csize = $CACHE_SIZE;
 *	roots += $CACHE_NUM;
 * } else {
 *	goto tofunc; 			! goto tomalloc if ptcmalloc.
 * }					! goto tofree if ptcfree.
 */
#define	PTC_FINCACHE_CMP	0x03
#define	PTC_FINCACHE_JMP	0x08
#define	PTC_FINCACHE_SIZE	0x0c
#define	PTC_FINCACHE_NUM	0x13
static const uint8_t fincache[] = {
	0x48, 0x81, 0xfe,
	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
	0x77, 0x00,			/* ja +JMP (to real malloc) */
	0x49, 0xc7, 0xc0,
	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
	0x48, 0x81, 0xc2,
	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */

};

/*
 * if (*root == NULL)
 * 	goto tomalloc;
 *
 * malloc_data_t *ret = *root;
 * *root = *(void **)ret;
 * t->tm_size += csize;
 * ret->malloc_size = size;
 *
 * if (size > UMEM_SECOND_ALIGN) {
 *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
 *	ret += 2;
 * } else {
 *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
 *	ret += 1;
 * }
 *
 * return ((void *)ret);
 * tomalloc:
 * 	return (malloc(orig_size));
 */
#define	PTC_MALFINI_ALLABEL	0x00
#define	PTC_MALFINI_JMLABEL	0x40
#define	PTC_MALFINI_JMADDR	0x41
static const uint8_t malfini[] = {
	0x48, 0x8b, 0x02,		/* movl (%rdx),%rax */
	0x48, 0x85, 0xc0,		/* testq %rax,%rax */
	0x74, 0x38,			/* je +0x38 (errout) */
	0x4c, 0x8b, 0x08,		/* movq (%rax),%r9 */
	0x4c, 0x89, 0x0a,		/* movq %r9,(%rdx) */
	0x4c, 0x29, 0x01,		/* subq %rsi,(%rcx) */
	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10,%rsi */
	0x76, 0x15,			/* jbe +0x15 */
	0x41, 0xb9, 0x00, 0x70, 0xba, 0x16, /* movl $MALLOC_MAGIC_2, %r9d */
	0x89, 0x70, 0x08,		/* movl %r9d,0x8(%rax) */
	0x41, 0x29, 0xf1,		/* subl %esi, %r9d */
	0x44, 0x89, 0x48, 0x0c,		/* movl %r9d, 0xc(%rax) */
	0x48, 0x83, 0xc0, 0x10,		/* addq $0x10, %rax */
	0xc3,				/* ret */
	0x41, 0xb9, 0x00, 0xc0, 0x10, 0x3a,	/* movl %MALLOC_MAGIC, %r9d */
	0x89, 0x30,			/* movl %esi,(%rax) */
	0x41, 0x29, 0xf1,		/* subl %esi,%r9d */
	0x44, 0x89, 0x48, 0x04,		/* movl %r9d,0x4(%rax) */
	0x48, 0x83, 0xc0, 0x08,		/* addq $0x8,%rax */
	0xc3,				/* ret */
	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $MALLOC */
};

/*
 * if (t->tm_size + csize > umem_ptc_size)
 * 	goto tofree;
 *
 * t->tm_size += csize
 * *(void **)tag = *root;
 * *root = tag;
 * return;
 * tofree:
 * 	free(buf);
 * 	return;
 */
#define	PTC_FRFINI_RBUFLABEL	0x00
#define	PTC_FRFINI_CACHEMAX	0x09
#define	PTC_FRFINI_DONELABEL	0x1b
#define	PTC_FRFINI_JFLABEL	0x1c
#define	PTC_FRFINI_JFADDR	0x1d
static const uint8_t freefini[] = {
	0x4c, 0x8b, 0x09,		/* movq (%rcx),%r9 */
	0x4d, 0x01, 0xc1,		/* addq %r8, %r9 */
	0x49, 0x81, 0xf9,
	0x00, 0x00, 0x00, 0x00,		/* cmpl $THR_CACHE_MAX, %r9 */
	0x77, 0x0d,			/* jae +0xd (torfree) */
	0x4c, 0x01, 0x01,		/* addq %r8,(%rcx) */
	0x4c, 0x8b, 0x0a,		/* movq (%rdx),%r9 */
	0x4c, 0x89, 0x08,		/* movq %r9,(%rax) */
	0x48, 0x89, 0x02,		/* movq %rax,(%rdx) */
	0xc3,				/* ret */
	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp free */
};

/*
 * Construct the initial part of malloc. off contains the offset from curthread
 * to the root of the tmem structure. ep is the address of the label to error
 * and jump to free. csize is the size of the largest umem_cache in ptcumem.
 */
static int
genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
{
	uint32_t addr;

	bcopy(malinit, bp, sizeof (malinit));
	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));

	return (sizeof (malinit));
}

static int
genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mcs)
{
	uint32_t addr;

	bcopy(freeinit, bp, sizeof (freeinit));
	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
	bcopy(&mcs, bp + PTC_FRINI_MCS, sizeof (mcs));
	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
	return (sizeof (freeinit));
}


/*
 * Create the initial cache entry of the specified size. The value of ap tells
 * us what the address of the label to try and allocate a buffer. This value is
 * an offset from the current base to that value.
 */
static int
genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
{
	uint32_t addr;

	bcopy(inicache, bp, sizeof (inicache));
	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
	ASSERT(addr != 0);
	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));

	return (sizeof (inicache));
}

static int
genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
{
	uint32_t addr;
	uint32_t coff;

	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
	ASSERT(num != 0);
	bcopy(gencache, bp, sizeof (gencache));
	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
	coff = num * PTC_ROOT_SIZE;
	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));

	return (sizeof (gencache));
}

static int
genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
{
	uint8_t eap;
	uint32_t coff;

	ASSERT(ep <= 0xff && ep > 7);
	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
	bcopy(fincache, bp, sizeof (fincache));
	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
	coff = num * PTC_ROOT_SIZE;
	bcopy(&coff, bp + PTC_FINCACHE_NUM, sizeof (coff));
	eap = ep - PTC_FINCACHE_JMP - 1;
	bcopy(&eap, bp + PTC_FINCACHE_JMP, sizeof (eap));

	return (sizeof (fincache));
}

static int
genasm_malfini(uint8_t *bp, uintptr_t mptr)
{
	uint32_t addr;

	bcopy(malfini, bp, sizeof (malfini));
	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));

	return (sizeof (malfini));
}

static int
genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
{
	uint32_t addr;

	bcopy(freefini, bp, sizeof (freefini));
	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));

	return (sizeof (freefini));
}

/*
 * The malloc inline assembly is constructed as follows:
 *
 * o Malloc prologue assembly
 * o Generic first-cache check
 * o n Generic cache checks (where n = _tmem_get_entries() - 2)
 * o Generic last-cache check
 * o Malloc epilogue assembly
 *
 * Generally there are at least three caches. When there is only one cache we
 * only use the generic last-cache. In the case where there are two caches, we
 * just leave out the middle ones.
 */
static int
genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
{
	int ii, off;
	uint8_t *bp;
	size_t total;
	uint32_t allocoff, erroff;

	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);

	if (nents >= 2)
		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);

	if (total > len)
		return (1);

	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;

	bp = base;

	off = genasm_malinit(bp, umem_tmem_off, erroff,
	    umem_alloc_sizes[nents-1]);
	bp += off;
	allocoff -= off;
	erroff -= off;

	if (nents > 1) {
		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
		bp += off;
		allocoff -= off;
		erroff -= off;
	}

	for (ii = 1; ii < nents - 1; ii++) {
		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
		bp += off;
		allocoff -= off;
		erroff -= off;
	}

	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
	    erroff);
	bp += genasm_malfini(bp, umem_genasm_omptr);
	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);

	return (0);
}

static int
genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
{
	uint8_t *bp;
	int ii, off;
	size_t total;
	uint32_t rbufoff, retoff, erroff;

	/* Assume that nents has already been audited for us */
	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
	if (nents >= 2)
		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);

	if (total > len)
		return (1);

	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);

	bp = base;

	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
	    umem_alloc_sizes[nents - 1]);
	bp += off;
	erroff -= off;
	rbufoff -= off;

	if (nents > 1) {
		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
		bp += off;
		erroff -= off;
		rbufoff -= off;
	}

	for (ii = 1; ii < nents - 1; ii++) {
		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
		bp += off;
		rbufoff -= off;
		erroff -= off;
	}

	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
	    erroff);
	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);

	return (0);
}

boolean_t
umem_genasm(int *cp, umem_cache_t **caches, int nc)
{
	int nents, i;
	uint8_t *mptr;
	uint8_t *fptr;
	uint64_t v, *vptr;
	size_t mplen, fplen;
	uintptr_t mpbase, fpbase;
	boolean_t ret = B_FALSE;

	mptr = (void *)((uintptr_t)umem_genasm_mptr + 5);
	fptr = (void *)((uintptr_t)umem_genasm_fptr + 5);
	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0) {
		return (B_FALSE);
	}

	mplen = P2ROUNDUP(umem_genasm_msize, pagesize);
	mpbase = P2ALIGN((uintptr_t)umem_genasm_mptr, pagesize);
	fplen = P2ROUNDUP(umem_genasm_fsize, pagesize);
	fpbase = P2ALIGN((uintptr_t)umem_genasm_mptr, pagesize);

	/*
	 * If the values straddle a page boundary, then we might need to
	 * actually remap two pages.
	 */
	if (P2ALIGN(umem_genasm_msize + (uintptr_t)umem_genasm_mptr,
	    pagesize) != mpbase) {
		mplen += pagesize;
	}

	if (P2ALIGN(umem_genasm_fsize + (uintptr_t)umem_genasm_fptr,
	    pagesize) != fpbase) {
		fplen += pagesize;
	}

	if (mprotect((void *)mpbase, mplen, PROT_READ | PROT_WRITE |
	    PROT_EXEC) != 0) {
		return (B_FALSE);
	}

	if (mprotect((void *)fpbase, fplen, PROT_READ | PROT_WRITE |
	    PROT_EXEC) != 0) {
		if (mprotect((void *)mpbase, mplen, PROT_READ | PROT_EXEC) !=
		    0) {
			umem_panic("genasm failed to restore memory "
			    "protection: %d", errno);
		}
		return (B_FALSE);
	}

	/*
	 * The total number of caches that we can service is the minimum of:
	 *  o the amount supported by libc
	 *  o the total number of umem caches
	 *  o we use a single byte addl, so it's MAX_UINT32 / sizeof (uintptr_t)
	 *    For 64-bit, this is MAX_UINT32 >> 3, a lot.
	 */
	nents = _tmem_get_nentries();

	if (UMEM_GENASM_MAX64 < nents)
		nents = UMEM_GENASM_MAX64;

	if (nc < nents)
		nents = nc;

	/*
	 * If the number of per-thread caches has been set to zero or the
	 * per-thread cache size has been set to zero, don't bother trying to
	 * write any assembly and just use the default malloc and free. When we
	 * return, indicate that there is no PTC support.
	 */
	if (nents == 0 || umem_ptc_size == 0) {
		goto out;
	}

	/* Take into account the jump */
	if (genasm_malloc(mptr, umem_genasm_msize, nents, cp) != 0) {
		goto out;
	}

	if (genasm_free(fptr, umem_genasm_fsize, nents, cp) != 0) {
		goto out;
	}

	/* nop out the jump with a multibyte jump */
	vptr = (void *)umem_genasm_mptr;
	v = MULTINOP;
	v |= *vptr & (0xffffffULL << 40);
	(void) atomic_swap_64(vptr, v);
	vptr = (void *)umem_genasm_fptr;
	v = MULTINOP;
	v |= *vptr & (0xffffffULL << 40);
	(void) atomic_swap_64(vptr, v);

	for (i = 0; i < nents; i++)
		caches[i]->cache_flags |= UMF_PTC;

	ret = B_TRUE;
out:
	if (mprotect((void *)mpbase, mplen, PROT_READ | PROT_EXEC) != 0) {
		umem_panic("genasm failed to restore memory protection: %d",
		    errno);
	}

	if (mprotect((void *)fpbase, fplen, PROT_READ | PROT_EXEC) != 0) {
		umem_panic("genasm failed to restore memory protection: %d",
		    errno);
	}

	return (ret);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2019 Joyent, Inc.
 * Copyright (c) 2015 by Delphix. All rights reserved.
 */

#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <dlfcn.h>
#include "umem_base.h"
#include "vmem_base.h"

/*
 * A umem environment variable, like UMEM_DEBUG, is set to a series
 * of items, seperated by ',':
 *
 *   UMEM_DEBUG="audit=10,guards,firewall=512"
 *
 * This structure describes items.  Each item has a name, type, and
 * description.  During processing, an item read from the user may
 * be either "valid" or "invalid".
 *
 * A valid item has an argument, if required, and it is of the right
 * form (doesn't overflow, doesn't contain any unexpected characters).
 *
 * If the item is valid, item_flag_target != NULL, and:
 *	type is not CLEARFLAG, then (*item_flag_target) |= item_flag_value
 *	type is CLEARFLAG, then (*item_flag_target) &= ~item_flag_value
 */

#define	UMEM_ENV_ITEM_MAX	512

struct umem_env_item;

typedef int arg_process_t(const struct umem_env_item *item, const char *value);
#define	ARG_SUCCESS	0	/* processing successful */
#define	ARG_BAD		1	/* argument had a bad value */

typedef struct umem_env_item {
	const char *item_name;	/* tag in environment variable */
	const char *item_interface_stability;
	enum {
	    ITEM_INVALID,
	    ITEM_FLAG,		/* only a flag.  No argument allowed */
	    ITEM_CLEARFLAG,	/* only a flag, but clear instead of set */
	    ITEM_OPTUINT,	/* optional integer argument */
	    ITEM_UINT,		/* required integer argument */
	    ITEM_OPTSIZE,	/* optional size_t argument */
	    ITEM_SIZE,		/* required size_t argument */
	    ITEM_SPECIAL	/* special argument processing */
	} item_type;
	const char *item_description;
	uint_t *item_flag_target; /* the variable containing the flag */
	uint_t item_flag_value;	/* the value to OR in */
	uint_t *item_uint_target; /* the variable to hold the integer */
	size_t *item_size_target;
	arg_process_t *item_special; /* callback for special handling */
} umem_env_item_t;

#ifndef UMEM_STANDALONE
static arg_process_t umem_backend_process;
static arg_process_t umem_allocator_process;
#endif

static arg_process_t umem_log_process;

static size_t umem_size_tempval;
static arg_process_t umem_size_process;

const char *____umem_environ_msg_options = "-- UMEM_OPTIONS --";

static umem_env_item_t umem_options_items[] = {
#ifndef UMEM_STANDALONE
	{ "backend",		"Evolving",	ITEM_SPECIAL,
		"=sbrk for sbrk(2), =mmap for mmap(2)",
		NULL, 0, NULL, NULL,
		&umem_backend_process
	},
	{ "allocator",		"Evolving",	ITEM_SPECIAL,
		"=best, =first, =next, or =instant",
		NULL, 0, NULL, NULL,
		&umem_allocator_process
	},
#endif

	{ "concurrency",	"Private",	ITEM_UINT,
		"Max concurrency",
		NULL, 0,	&umem_max_ncpus
	},
	{ "max_contention",	"Private",	ITEM_UINT,
		"Maximum contention in a reap interval before the depot is "
		    "resized.",
		NULL, 0,	&umem_depot_contention
	},
	{ "nomagazines",	"Private",	ITEM_FLAG,
		"no caches will be multithreaded, and no caching will occur.",
		&umem_flags,	UMF_NOMAGAZINE
	},
	{ "reap_interval",	"Private",	ITEM_UINT,
		"Minimum time between reaps and updates, in seconds.",
		NULL, 0,	&umem_reap_interval
	},

	{ "size_add",		"Private",	ITEM_SPECIAL,
		"add a size to the cache size table",
		NULL, 0, NULL,
		&umem_size_tempval,		&umem_size_process
	},
	{ "size_clear",		"Private",	ITEM_SPECIAL,
		"clear all but the largest size from the cache size table",
		NULL, 0, NULL,
		&umem_size_tempval,		&umem_size_process
	},
	{ "size_remove",	"Private",	ITEM_SPECIAL,
	    "remove a size from the cache size table",
		NULL, 0, NULL,
		&umem_size_tempval,		&umem_size_process
	},

#ifndef UMEM_STANDALONE
	{ "sbrk_minalloc",	"Private",	ITEM_SIZE,
		"The minimum allocation chunk for the sbrk(2) heap.",
		NULL, 0, NULL,	&vmem_sbrk_minalloc
	},
	{ "sbrk_pagesize",	"Private",	ITEM_SIZE,
		"The preferred page size for the sbrk(2) heap.",
		NULL, 0, NULL,	&vmem_sbrk_pagesize
	},
#endif
	{ "perthread_cache",	"Evolving",	ITEM_SIZE,
		"Size (in bytes) of per-thread allocation cache",
		NULL, 0, NULL, &umem_ptc_size
	},
	{ NULL, "-- end of UMEM_OPTIONS --",	ITEM_INVALID }
};

const char *____umem_environ_msg_debug = "-- UMEM_DEBUG --";

static umem_env_item_t umem_debug_items[] = {
	{ "default",		"Unstable",	ITEM_FLAG,
		"audit,contents,guards",
		&umem_flags,
		UMF_AUDIT | UMF_CONTENTS | UMF_DEADBEEF | UMF_REDZONE
	},
	{ "audit",		"Unstable",	ITEM_OPTUINT,
		"Enable auditing.  optionally =frames to set the number of "
		    "stored stack frames",
		&umem_flags,	UMF_AUDIT,	&umem_stack_depth
	},
	{ "contents",		"Unstable",	ITEM_OPTSIZE,
		"Enable contents storing.  UMEM_LOGGING=contents also "
		    "required.  optionally =bytes to set the number of stored "
		    "bytes",
		&umem_flags,	UMF_CONTENTS, NULL,	&umem_content_maxsave
	},
	{ "guards",		"Unstable",	ITEM_FLAG,
		"Enables guards and special patterns",
		&umem_flags,	UMF_DEADBEEF | UMF_REDZONE
	},
	{ "verbose",		"Unstable",	ITEM_FLAG,
		"Enables writing error messages to stderr",
		&umem_output,	1
	},

	{ "nosignal",	"Private",	ITEM_FLAG,
		"Abort if called from a signal handler.  Turns on 'audit'.  "
		    "Note that this is not always a bug.",
		&umem_flags,	UMF_AUDIT | UMF_CHECKSIGNAL
	},
	{ "firewall",		"Private",	ITEM_SIZE,
		"=minbytes.  Every object >= minbytes in size will have its "
		    "end against an unmapped page",
		&umem_flags,	UMF_FIREWALL,	NULL,	&umem_minfirewall
	},
	{ "lite",		"Private",	ITEM_FLAG,
		"debugging-lite",
		&umem_flags,	UMF_LITE
	},
	{ "maxverify",		"Private",	ITEM_SIZE,
		"=maxbytes, Maximum bytes to check when 'guards' is active. "
		    "Normally all bytes are checked.",
		NULL, 0, NULL,	&umem_maxverify
	},
	{ "noabort",		"Private",	ITEM_CLEARFLAG,
		"umem will not abort when a recoverable error occurs "
		    "(i.e. double frees, certain kinds of corruption)",
		&umem_abort,	1
	},
	{ "mtbf",		"Private",	ITEM_UINT,
		"=mtbf, the mean time between injected failures.  Works best "
		    "if prime.\n",
		NULL, 0,	&umem_mtbf
	},
	{ "random",		"Private",	ITEM_FLAG,
		"randomize flags on a per-cache basis",
		&umem_flags,	UMF_RANDOMIZE
	},
	{ "allverbose",		"Private",	ITEM_FLAG,
		"Enables writing all logged messages to stderr",
		&umem_output,	2
	},
	{ "checknull",		"Private",	ITEM_FLAG,
		"Abort if an allocation would return null",
		&umem_flags,	UMF_CHECKNULL
	},

	{ NULL, "-- end of UMEM_DEBUG --",	ITEM_INVALID }
};

const char *____umem_environ_msg_logging = "-- UMEM_LOGGING --";

static umem_env_item_t umem_logging_items[] = {
	{ "transaction",	"Unstable",	ITEM_SPECIAL,
		"If 'audit' is set in UMEM_DEBUG, the audit structures "
		    "from previous transactions are entered into this log.",
		NULL, 0, NULL,
		&umem_transaction_log_size,	&umem_log_process
	},
	{ "contents",		"Unstable",	ITEM_SPECIAL,
		"If 'audit' is set in UMEM_DEBUG, the contents of objects "
		    "are recorded in this log as they are freed.  If the "
		    "'contents' option is not set in UMEM_DEBUG, the first "
		    "256 bytes of each freed buffer will be saved.",
		&umem_flags,	UMF_CONTENTS,	NULL,
		&umem_content_log_size,		&umem_log_process
	},
	{ "fail",		"Unstable",	ITEM_SPECIAL,
		"Records are entered into this log for every failed "
		    "allocation.",
		NULL, 0, NULL,
		&umem_failure_log_size,		&umem_log_process
	},

	{ "slab",		"Private",	ITEM_SPECIAL,
		"Every slab created will be entered into this log.",
		NULL, 0, NULL,
		&umem_slab_log_size,		&umem_log_process
	},

	{ NULL, "-- end of UMEM_LOGGING --",	ITEM_INVALID }
};

typedef struct umem_envvar {
	const char *env_name;
	const char *env_func;
	umem_env_item_t	*env_item_list;
	const char *env_getenv_result;
	const char *env_func_result;
} umem_envvar_t;

static umem_envvar_t umem_envvars[] = {
	{ "UMEM_DEBUG",		"_umem_debug_init",	umem_debug_items },
	{ "UMEM_OPTIONS",	"_umem_options_init",	umem_options_items },
	{ "UMEM_LOGGING",	"_umem_logging_init",	umem_logging_items },
	{ NULL, NULL, NULL }
};

static umem_envvar_t *env_current;
#define	CURRENT		(env_current->env_name)

static int
empty(const char *str)
{
	char c;

	while ((c = *str) != '\0' && isspace(c))
		str++;

	return (*str == '\0');
}

static int
item_uint_process(const umem_env_item_t *item, const char *item_arg)
{
	ulong_t result;
	char *endptr = "";
	int olderrno;

	olderrno = errno;
	errno = 0;

	if (empty(item_arg)) {
		goto badnumber;
	}

	result = strtoul(item_arg, &endptr, 10);

	if (result == ULONG_MAX && errno == ERANGE) {
		errno = olderrno;
		goto overflow;
	}
	errno = olderrno;

	if (*endptr != '\0')
		goto badnumber;
	if ((uint_t)result != result)
		goto overflow;

	(*item->item_uint_target) = (uint_t)result;
	return (ARG_SUCCESS);

badnumber:
	log_message("%s: %s: not a number\n", CURRENT, item->item_name);
	return (ARG_BAD);

overflow:
	log_message("%s: %s: overflowed\n", CURRENT, item->item_name);
	return (ARG_BAD);
}

static int
item_size_process(const umem_env_item_t *item, const char *item_arg)
{
	ulong_t result;
	ulong_t result_arg;
	char *endptr = "";
	int olderrno;

	if (empty(item_arg))
		goto badnumber;

	olderrno = errno;
	errno = 0;

	result_arg = strtoul(item_arg, &endptr, 10);

	if (result_arg == ULONG_MAX && errno == ERANGE) {
		errno = olderrno;
		goto overflow;
	}
	errno = olderrno;

	result = result_arg;

	switch (*endptr) {
	case 't':
	case 'T':
		result *= 1024;
		if (result < result_arg)
			goto overflow;
		/*FALLTHRU*/
	case 'g':
	case 'G':
		result *= 1024;
		if (result < result_arg)
			goto overflow;
		/*FALLTHRU*/
	case 'm':
	case 'M':
		result *= 1024;
		if (result < result_arg)
			goto overflow;
		/*FALLTHRU*/
	case 'k':
	case 'K':
		result *= 1024;
		if (result < result_arg)
			goto overflow;
		endptr++;		/* skip over the size character */
		break;
	default:
		break;			/* handled later */
	}

	if (*endptr != '\0')
		goto badnumber;

	(*item->item_size_target) = result;
	return (ARG_SUCCESS);

badnumber:
	log_message("%s: %s: not a number\n", CURRENT, item->item_name);
	return (ARG_BAD);

overflow:
	log_message("%s: %s: overflowed\n", CURRENT, item->item_name);
	return (ARG_BAD);
}

static int
umem_log_process(const umem_env_item_t *item, const char *item_arg)
{
	if (item_arg != NULL) {
		int ret;
		ret = item_size_process(item, item_arg);
		if (ret != ARG_SUCCESS)
			return (ret);

		if (*item->item_size_target == 0)
			return (ARG_SUCCESS);
	} else
		*item->item_size_target = 64*1024;

	umem_logging = 1;
	return (ARG_SUCCESS);
}

static int
umem_size_process(const umem_env_item_t *item, const char *item_arg)
{
	const char *name = item->item_name;
	void (*action_func)(size_t);

	size_t result;

	int ret;

	if (strcmp(name, "size_clear") == 0) {
		if (item_arg != NULL) {
			log_message("%s: %s: does not take a value. ignored\n",
			    CURRENT, name);
			return (ARG_BAD);
		}
		umem_alloc_sizes_clear();
		return (ARG_SUCCESS);
	} else if (strcmp(name, "size_add") == 0) {
		action_func = umem_alloc_sizes_add;
	} else if (strcmp(name, "size_remove") == 0) {
		action_func = umem_alloc_sizes_remove;
	} else {
		log_message("%s: %s: internally unrecognized\n",
		    CURRENT, name, name, name);
		return (ARG_BAD);
	}

	if (item_arg == NULL) {
		log_message("%s: %s: requires a value. ignored\n",
		    CURRENT, name);
		return (ARG_BAD);
	}

	ret = item_size_process(item, item_arg);
	if (ret != ARG_SUCCESS)
		return (ret);

	result = *item->item_size_target;
	action_func(result);
	return (ARG_SUCCESS);
}

#ifndef UMEM_STANDALONE
static int
umem_backend_process(const umem_env_item_t *item, const char *item_arg)
{
	const char *name = item->item_name;

	if (item_arg == NULL)
		goto fail;

	if (strcmp(item_arg, "sbrk") == 0)
		vmem_backend |= VMEM_BACKEND_SBRK;
	else if (strcmp(item_arg, "mmap") == 0)
		vmem_backend |= VMEM_BACKEND_MMAP;
	else
		goto fail;

	return (ARG_SUCCESS);

fail:
	log_message("%s: %s: must be %s=sbrk or %s=mmap\n",
	    CURRENT, name, name, name);
	return (ARG_BAD);
}


static int
umem_allocator_process(const umem_env_item_t *item, const char *item_arg)
{
	const char *name = item->item_name;

	if (item_arg == NULL)
		goto fail;

	if (strcmp(item_arg, "best") == 0)
		vmem_allocator = VM_BESTFIT;
	else if (strcmp(item_arg, "next") == 0)
		vmem_allocator = VM_NEXTFIT;
	else if (strcmp(item_arg, "first") == 0)
		vmem_allocator = VM_FIRSTFIT;
	else if (strcmp(item_arg, "instant") == 0)
		vmem_allocator = 0;
	else
		goto fail;

	return (ARG_SUCCESS);

fail:
	log_message("%s: %s: must be %s=best, %s=next or %s=first\n",
	    CURRENT, name, name, name, name);
	return (ARG_BAD);

}
#endif

static int
process_item(const umem_env_item_t *item, const char *item_arg)
{
	int arg_required = 0;
	arg_process_t *processor;

	switch (item->item_type) {
	case ITEM_FLAG:
	case ITEM_CLEARFLAG:
	case ITEM_OPTUINT:
	case ITEM_OPTSIZE:
	case ITEM_SPECIAL:
		arg_required = 0;
		break;

	case ITEM_UINT:
	case ITEM_SIZE:
		arg_required = 1;
		break;

	default:
		/*
		 * These are flags that aren't supported, so they'll error out
		 * below.
		 */
		break;
	}

	switch (item->item_type) {
	case ITEM_FLAG:
	case ITEM_CLEARFLAG:
		if (item_arg != NULL) {
			log_message("%s: %s: does not take a value. ignored\n",
			    CURRENT, item->item_name);
			return (1);
		}
		processor = NULL;
		break;

	case ITEM_UINT:
	case ITEM_OPTUINT:
		processor = item_uint_process;
		break;

	case ITEM_SIZE:
	case ITEM_OPTSIZE:
		processor = item_size_process;
		break;

	case ITEM_SPECIAL:
		processor = item->item_special;
		break;

	default:
		log_message("%s: %s: Invalid type.  Ignored\n",
		    CURRENT, item->item_name);
		return (1);
	}

	if (arg_required && item_arg == NULL) {
		log_message("%s: %s: Required value missing\n",
		    CURRENT, item->item_name);
		goto invalid;
	}

	if (item_arg != NULL || item->item_type == ITEM_SPECIAL) {
		if (processor(item, item_arg) != ARG_SUCCESS)
			goto invalid;
	}

	if (item->item_flag_target) {
		if (item->item_type == ITEM_CLEARFLAG)
			(*item->item_flag_target) &= ~item->item_flag_value;
		else
			(*item->item_flag_target) |= item->item_flag_value;
	}
	return (0);

invalid:
	return (1);
}

#define	ENV_SHORT_BYTES	10	/* bytes to print on error */
void
umem_process_value(umem_env_item_t *item_list, const char *beg, const char *end)
{
	char buf[UMEM_ENV_ITEM_MAX];
	char *argptr;

	size_t count;

	while (beg < end && isspace(*beg))
		beg++;

	while (beg < end && isspace(*(end - 1)))
		end--;

	if (beg >= end) {
		log_message("%s: empty option\n", CURRENT);
		return;
	}

	count = end - beg;

	if (count + 1 > sizeof (buf)) {
		char outbuf[ENV_SHORT_BYTES + 1];
		/*
		 * Have to do this, since sprintf("%10s",...) calls malloc()
		 */
		(void) strncpy(outbuf, beg, ENV_SHORT_BYTES);
		outbuf[ENV_SHORT_BYTES] = 0;

		log_message("%s: argument \"%s...\" too long\n", CURRENT,
		    outbuf);
		return;
	}

	(void) strncpy(buf, beg, count);
	buf[count] = 0;

	argptr = strchr(buf, '=');

	if (argptr != NULL)
		*argptr++ = 0;

	for (; item_list->item_name != NULL; item_list++) {
		if (strcmp(buf, item_list->item_name) == 0) {
			(void) process_item(item_list, argptr);
			return;
		}
	}
	log_message("%s: '%s' not recognized\n", CURRENT, buf);
}

/*ARGSUSED*/
void
umem_setup_envvars(int invalid)
{
	umem_envvar_t *cur_env;
	static volatile enum {
		STATE_START,
		STATE_GETENV,
		STATE_DLOPEN,
		STATE_DLSYM,
		STATE_FUNC,
		STATE_DONE
	} state = STATE_START;
#ifndef UMEM_STANDALONE
	void *h;
#endif

	if (invalid) {
		const char *where;
		/*
		 * One of the calls below invoked malloc() recursively.  We
		 * remove any partial results and return.
		 */

		switch (state) {
		case STATE_START:
			where = "before getenv(3C) calls -- "
			    "getenv(3C) results ignored.";
			break;
		case STATE_GETENV:
			where = "during getenv(3C) calls -- "
			    "getenv(3C) results ignored.";
			break;
		case STATE_DLOPEN:
			where = "during dlopen(3C) call -- "
			    "_umem_*() results ignored.";
			break;
		case STATE_DLSYM:
			where = "during dlsym(3C) call -- "
			    "_umem_*() results ignored.";
			break;
		case STATE_FUNC:
			where = "during _umem_*() call -- "
			    "_umem_*() results ignored.";
			break;
		case STATE_DONE:
			where = "after dlsym() or _umem_*() calls.";
			break;
		default:
			where = "at unknown point -- "
			    "_umem_*() results ignored.";
			break;
		}

		log_message("recursive allocation %s\n", where);

		for (cur_env = umem_envvars; cur_env->env_name != NULL;
		    cur_env++) {
			if (state == STATE_GETENV)
				cur_env->env_getenv_result = NULL;
			if (state != STATE_DONE)
				cur_env->env_func_result = NULL;
		}

		state = STATE_DONE;
		return;
	}

	state = STATE_GETENV;

	for (cur_env = umem_envvars; cur_env->env_name != NULL; cur_env++) {
		cur_env->env_getenv_result = getenv(cur_env->env_name);
		if (state == STATE_DONE)
			return;		/* recursed */
	}

#ifndef UMEM_STANDALONE
	state = STATE_DLOPEN;

	/* get a handle to the "a.out" object */
	if ((h = dlopen(0, RTLD_FIRST | RTLD_LAZY)) != NULL) {
		for (cur_env = umem_envvars; cur_env->env_name != NULL;
		    cur_env++) {
			const char *(*func)(void);
			const char *value;

			state = STATE_DLSYM;
			func = (const char *(*)(void))dlsym(h,
			    cur_env->env_func);

			if (state == STATE_DONE)
				break;		/* recursed */

			state = STATE_FUNC;
			if (func != NULL) {
				value = func();
				if (state == STATE_DONE)
					break;		/* recursed */
				cur_env->env_func_result = value;
			}
		}
		(void) dlclose(h);
	} else {
		(void) dlerror();		/* snarf dlerror() */
	}
#endif /* UMEM_STANDALONE */

	state = STATE_DONE;
}

/*
 * Process the environment variables.
 */
void
umem_process_envvars(void)
{
	const char *value;
	const char *end, *next;
	umem_envvar_t *cur_env;

	for (cur_env = umem_envvars; cur_env->env_name != NULL; cur_env++) {
		env_current = cur_env;

		value = cur_env->env_getenv_result;
		if (value == NULL)
			value = cur_env->env_func_result;

		/* ignore if missing or empty */
		if (value == NULL)
			continue;

		for (end = value; *end != '\0'; value = next) {
			end = strchr(value, ',');
			if (end != NULL)
				next = end + 1;		/* skip the comma */
			else
				next = end = value + strlen(value);

			umem_process_value(cur_env->env_item_list, value, end);
		}
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 */

#include "misc.h"
#include <ucontext.h>
#include <sys/frame.h>
#include <sys/stack.h>
#include <stdio.h>

#if defined(__sparc) || defined(__sparcv9)
extern void flush_windows(void);
#define	UMEM_FRAMESIZE	MINFRAME

#elif defined(__i386) || defined(__amd64)
/*
 * On x86, MINFRAME is defined to be 0, but we want to be sure we can
 * dereference the entire frame structure.
 */
#define	UMEM_FRAMESIZE	(sizeof (struct frame))

#else
#error needs update for new architecture
#endif

/*
 * Get a pc-only stacktrace.  Used for kmem_alloc() buffer ownership tracking.
 * Returns MIN(current stack depth, pcstack_limit).
 */
/*ARGSUSED*/
int
getpcstack(uintptr_t *pcstack, int pcstack_limit, int check_signal)
{
	struct frame *fp;
	struct frame *nextfp, *minfp;
	int depth = 0;
	uintptr_t base = 0;
	size_t size = 0;
#ifndef UMEM_STANDALONE
	int on_altstack = 0;
	uintptr_t sigbase = 0;
	size_t sigsize = 0;

	stack_t st;

	if (stack_getbounds(&st) != 0) {
		if (thr_stksegment(&st) != 0 ||
		    (uintptr_t)st.ss_sp < st.ss_size) {
			return (0);		/* unable to get stack bounds */
		}
		/*
		 * thr_stksegment(3C) has a slightly different interface than
		 * stack_getbounds(3C) -- correct it
		 */
		st.ss_sp = (void *)(((uintptr_t)st.ss_sp) - st.ss_size);
		st.ss_flags = 0;		/* can't be on-stack */
	}
	on_altstack = (st.ss_flags & SS_ONSTACK);

	if (st.ss_size != 0) {
		base = (uintptr_t)st.ss_sp;
		size = st.ss_size;
	} else {
		/*
		 * If size == 0, then ss_sp is the *top* of the stack.
		 *
		 * Since we only allow increasing frame pointers, and we
		 * know our caller set its up correctly, we can treat ss_sp
		 * as an upper bound safely.
		 */
		base = 0;
		size = (uintptr_t)st.ss_sp;
	}

	if (check_signal != 0) {
		void (*sigfunc)() = NULL;
		int sigfuncsize = 0;
		extern void thr_sighndlrinfo(void (**)(), int *);

		thr_sighndlrinfo(&sigfunc, &sigfuncsize);
		sigbase = (uintptr_t)sigfunc;
		sigsize = sigfuncsize;
	}
#else /* UMEM_STANDALONE */
	base = (uintptr_t)umem_min_stack;
	size = umem_max_stack - umem_min_stack;
#endif

	/*
	 * shorten size so that fr_savfp and fr_savpc will be within the stack
	 * bounds.
	 */
	if (size >= UMEM_FRAMESIZE - 1)
		size -= (UMEM_FRAMESIZE - 1);
	else
		size = 0;

#if defined(__sparc) || defined(__sparcv9)
	flush_windows();
#endif

	/* LINTED alignment */
	fp = (struct frame *)((caddr_t)getfp() + STACK_BIAS);

	minfp = fp;

	if (((uintptr_t)fp - base) >= size)
		return (0);	/* the frame pointer isn't in our stack */

	while (depth < pcstack_limit) {
		uintptr_t tmp;

		/* LINTED alignment */
		nextfp = (struct frame *)((caddr_t)fp->fr_savfp + STACK_BIAS);
		tmp = (uintptr_t)nextfp;

		/*
		 * Check nextfp for validity.  It must be properly aligned,
		 * increasing compared to the last %fp (or the top of the
		 * stack we just switched to), and it must be inside
		 * [base, base + size).
		 */
		if (tmp != SA(tmp))
			break;
		else if (nextfp <= minfp || (tmp - base) >= size) {
#ifndef UMEM_STANDALONE
			if (tmp == (uintptr_t)NULL || !on_altstack)
				break;
			/*
			 * If we're on an alternate signal stack, try jumping
			 * to the main thread stack.
			 *
			 * If the main thread stack has an unlimited size, we
			 * punt, since we don't know where the frame pointer's
			 * been.
			 *
			 * (thr_stksegment() returns the *top of stack*
			 * in ss_sp, not the bottom)
			 */
			if (thr_stksegment(&st) == 0) {
				if (st.ss_size >= (uintptr_t)st.ss_sp ||
				    st.ss_size < UMEM_FRAMESIZE - 1)
					break;

				on_altstack = 0;
				base = (uintptr_t)st.ss_sp - st.ss_size;
				size = st.ss_size - (UMEM_FRAMESIZE - 1);
				minfp = (struct frame *)base;
				continue;		/* try again */
			}
#endif
			break;
		}

#ifndef UMEM_STANDALONE
		if (check_signal && (fp->fr_savpc - sigbase) <= sigsize)
			umem_panic("called from signal handler");
#endif
		pcstack[depth++] = fp->fr_savpc;
		fp = nextfp;
		minfp = fp;
	}
	return (depth);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Initialization routines for the library version of libumem.
 */

#include "umem_base.h"
#include "vmem_base.h"
#include <unistd.h>
#include <dlfcn.h>

void
vmem_heap_init(void)
{
	void *handle = dlopen("libmapmalloc.so.1", RTLD_NOLOAD);

	if (handle != NULL) {
		log_message("sbrk backend disabled\n");
		vmem_backend = VMEM_BACKEND_MMAP;
	}

	if ((vmem_backend & VMEM_BACKEND_MMAP) != 0) {
		vmem_backend = VMEM_BACKEND_MMAP;
		(void) vmem_mmap_arena(NULL, NULL);
	} else {
		vmem_backend = VMEM_BACKEND_SBRK;
		(void) vmem_sbrk_arena(NULL, NULL);
	}
}

/*ARGSUSED*/
void
umem_type_init(caddr_t start, size_t len, size_t pgsize)
{
	pagesize = _sysconf(_SC_PAGESIZE);
}

int
umem_get_max_ncpus(void)
{
	if (thr_main() != -1)
		return (2 * sysconf(_SC_NPROCESSORS_ONLN));
	else
		return (1);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Initialization routines for the standalone version of libumem.
 */

#include "umem_base.h"
#include "vmem_base.h"

#include "vmem_stand.h"

void
vmem_heap_init(void)
{
	vmem_backend = VMEM_BACKEND_STAND;
	(void) vmem_stand_arena(NULL, NULL);
}

void
umem_type_init(caddr_t base, size_t len, size_t pgsize)
{
	pagesize = pgsize;

	vmem_stand_init();
	(void) vmem_stand_add(base, len);
}

int
umem_get_max_ncpus(void)
{
	return (1);
}

int
umem_add(caddr_t base, size_t len)
{
	return (vmem_stand_add(base, len));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file is used to verify that the standalone's external dependencies
 * haven't changed in a way that'll break things that use it.
 */

void __umem_assert_failed(void) {}
void atomic_add_64(void) {}
void atomic_add_32_nv(void) {}
void atomic_swap_64(void) {}
void dladdr1(void) {}
void bcopy(void) {}
void bzero(void) {}
void exit(void) {}
void getenv(void) {}
void gethrtime(void) {}
void membar_producer(void) {}
void memcpy(void) {}
void _memcpy(void) {}
void memset(void) {}
void snprintf(void) {}
void strchr(void) {}
void strcmp(void) {}
void strlen(void) {}
void strncpy(void) {}
void strrchr(void) {}
void strtoul(void) {}
void umem_err_recoverable(void) {}
void umem_panic(void) {}
void vsnprintf(void) {}

#ifdef	__i386
void __mul64(void) {}
void __rem64(void) {}
void __div64(void) {}

#ifdef	__GNUC__
void __divdi3(void) {}
void __moddi3(void) {}
#endif	/* __GNUC__ */

#endif	/* __i386 */

int __ctype;
int errno;
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <sys/sysmacros.h>
#include "umem_base.h"
#include "misc.h"

/*
 * malloc_data_t is an 8-byte structure which is located "before" the pointer
 * returned from {m,c,re}alloc and memalign.  The first four bytes give
 * information about the buffer, and the second four bytes are a status byte.
 *
 * See umem_impl.h for the various magic numbers used, and the size
 * encode/decode macros.
 *
 * The 'size' of the buffer includes the tags.  That is, we encode the
 * argument to umem_alloc(), not the argument to malloc().
 */

typedef struct malloc_data {
	uint32_t malloc_size;
	uint32_t malloc_stat; /* = UMEM_MALLOC_ENCODE(state, malloc_size) */
} malloc_data_t;

/*
 * Because we do not support ptcumem on non-x86 today, we have to create these
 * weak aliases.
 */
#ifndef _x86
#pragma weak malloc = umem_malloc
#pragma weak free = umem_malloc_free
#endif /* !_x86 */

void *
umem_malloc(size_t size_arg)
{
#ifdef _LP64
	uint32_t high_size = 0;
#endif
	size_t size;

	malloc_data_t *ret;
	size = size_arg + sizeof (malloc_data_t);

#ifdef _LP64
	if (size > UMEM_SECOND_ALIGN) {
		size += sizeof (malloc_data_t);
		high_size = (size >> 32);
	}
#endif
	if (size < size_arg) {
		errno = ENOMEM;			/* overflow */
		return (NULL);
	}
	ret = (malloc_data_t *)_umem_alloc(size, UMEM_DEFAULT);
	if (ret == NULL) {
		if (size <= UMEM_MAXBUF)
			errno = EAGAIN;
		else
			errno = ENOMEM;
		return (NULL);
#ifdef _LP64
	} else if (high_size > 0) {
		uint32_t low_size = (uint32_t)size;

		/*
		 * uses different magic numbers to make it harder to
		 * undetectably corrupt
		 */
		ret->malloc_size = high_size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_MAGIC, high_size);
		ret++;

		ret->malloc_size = low_size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_OVERSIZE_MAGIC,
		    low_size);
		ret++;
	} else if (size > UMEM_SECOND_ALIGN) {
		uint32_t low_size = (uint32_t)size;

		ret++; /* leave the first 8 bytes alone */

		ret->malloc_size = low_size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC,
		    low_size);
		ret++;
#endif
	} else {
		ret->malloc_size = size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_MAGIC, size);
		ret++;
	}
	return ((void *)ret);
}

void *
calloc(size_t nelem, size_t elsize)
{
	size_t size = nelem * elsize;
	void *retval;

	if (nelem > 0 && elsize > 0 && size/nelem != elsize) {
		errno = ENOMEM;				/* overflow */
		return (NULL);
	}

	retval = malloc(size);
	if (retval == NULL)
		return (NULL);

	(void) memset(retval, 0, size);
	return (retval);
}

/*
 * memalign uses vmem_xalloc to do its work.
 *
 * in 64-bit, the memaligned buffer always has two tags.  This simplifies the
 * code.
 */

void *
memalign(size_t align, size_t size_arg)
{
	size_t size;
	uintptr_t phase;

	void *buf;
	malloc_data_t *ret;

	size_t overhead;

	if (size_arg == 0 || align == 0 || (align & (align - 1)) != 0) {
		errno = EINVAL;
		return (NULL);
	}

	/*
	 * if malloc provides the required alignment, use it.
	 */
	if (align <= UMEM_ALIGN ||
	    (align <= UMEM_SECOND_ALIGN && size_arg >= UMEM_SECOND_ALIGN))
		return (malloc(size_arg));

#ifdef _LP64
	overhead = 2 * sizeof (malloc_data_t);
#else
	overhead = sizeof (malloc_data_t);
#endif

	ASSERT(overhead <= align);

	size = size_arg + overhead;
	phase = align - overhead;

	if (umem_memalign_arena == NULL && umem_init() == 0) {
		errno = ENOMEM;
		return (NULL);
	}

	if (size < size_arg) {
		errno = ENOMEM;			/* overflow */
		return (NULL);
	}

	buf = vmem_xalloc(umem_memalign_arena, size, align, phase,
	    0, NULL, NULL, VM_NOSLEEP);

	if (buf == NULL) {
		if ((size_arg + align) <= UMEM_MAXBUF)
			errno = EAGAIN;
		else
			errno = ENOMEM;

		return (NULL);
	}

	ret = (malloc_data_t *)buf;
	{
		uint32_t low_size = (uint32_t)size;

#ifdef _LP64
		uint32_t high_size = (uint32_t)(size >> 32);

		ret->malloc_size = high_size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MEMALIGN_MAGIC,
		    high_size);
		ret++;
#endif

		ret->malloc_size = low_size;
		ret->malloc_stat = UMEM_MALLOC_ENCODE(MEMALIGN_MAGIC, low_size);
		ret++;
	}

	ASSERT(P2PHASE((uintptr_t)ret, align) == 0);
	ASSERT((void *)((uintptr_t)ret - overhead) == buf);

	return ((void *)ret);
}

void *
valloc(size_t size)
{
	return (memalign(pagesize, size));
}

/*
 * process_free:
 *
 * Pulls information out of a buffer pointer, and optionally free it.
 * This is used by free() and realloc() to process buffers.
 *
 * On failure, calls umem_err_recoverable() with an appropriate message
 * On success, returns the data size through *data_size_arg, if (!is_free).
 *
 * Preserves errno, since free()'s semantics require it.
 */

static int
process_free(void *buf_arg,
    int do_free,		/* free the buffer, or just get its size? */
    size_t *data_size_arg)	/* output: bytes of data in buf_arg */
{
	malloc_data_t *buf;

	void *base;
	size_t size;
	size_t data_size;

	const char *message;
	int old_errno = errno;

	buf = (malloc_data_t *)buf_arg;

	buf--;
	size = buf->malloc_size;

	switch (UMEM_MALLOC_DECODE(buf->malloc_stat, size)) {

	case MALLOC_MAGIC:
		base = (void *)buf;
		data_size = size - sizeof (malloc_data_t);

		if (do_free)
			buf->malloc_stat = UMEM_FREE_PATTERN_32;

		goto process_malloc;

#ifdef _LP64
	case MALLOC_SECOND_MAGIC:
		base = (void *)(buf - 1);
		data_size = size - 2 * sizeof (malloc_data_t);

		if (do_free)
			buf->malloc_stat = UMEM_FREE_PATTERN_32;

		goto process_malloc;

	case MALLOC_OVERSIZE_MAGIC: {
		size_t high_size;

		buf--;
		high_size = buf->malloc_size;

		if (UMEM_MALLOC_DECODE(buf->malloc_stat, high_size) !=
		    MALLOC_MAGIC) {
			message = "invalid or corrupted buffer";
			break;
		}

		size += high_size << 32;

		base = (void *)buf;
		data_size = size - 2 * sizeof (malloc_data_t);

		if (do_free) {
			buf->malloc_stat = UMEM_FREE_PATTERN_32;
			(buf + 1)->malloc_stat = UMEM_FREE_PATTERN_32;
		}

		goto process_malloc;
	}
#endif

	case MEMALIGN_MAGIC: {
		size_t overhead = sizeof (malloc_data_t);

#ifdef _LP64
		size_t high_size;

		overhead += sizeof (malloc_data_t);

		buf--;
		high_size = buf->malloc_size;

		if (UMEM_MALLOC_DECODE(buf->malloc_stat, high_size) !=
		    MEMALIGN_MAGIC) {
			message = "invalid or corrupted buffer";
			break;
		}
		size += high_size << 32;

		/*
		 * destroy the main tag's malloc_stat
		 */
		if (do_free)
			(buf + 1)->malloc_stat = UMEM_FREE_PATTERN_32;
#endif

		base = (void *)buf;
		data_size = size - overhead;

		if (do_free)
			buf->malloc_stat = UMEM_FREE_PATTERN_32;

		goto process_memalign;
	}
	default:
		if (buf->malloc_stat == UMEM_FREE_PATTERN_32)
			message = "double-free or invalid buffer";
		else
			message = "invalid or corrupted buffer";
		break;
	}

	umem_err_recoverable("%s(%p): %s\n",
	    do_free? "free" : "realloc", buf_arg, message);

	errno = old_errno;
	return (0);

process_malloc:
	if (do_free)
		_umem_free(base, size);
	else
		*data_size_arg = data_size;

	errno = old_errno;
	return (1);

process_memalign:
	if (do_free)
		vmem_xfree(umem_memalign_arena, base, size);
	else
		*data_size_arg = data_size;

	errno = old_errno;
	return (1);
}

void
umem_malloc_free(void *buf)
{
	if (buf == NULL)
		return;

	/*
	 * Process buf, freeing it if it is not corrupt.
	 */
	(void) process_free(buf, 1, NULL);
}

void *
realloc(void *buf_arg, size_t newsize)
{
	size_t oldsize;
	void *buf;

	if (buf_arg == NULL)
		return (malloc(newsize));

	if (newsize == 0) {
		free(buf_arg);
		return (NULL);
	}

	/*
	 * get the old data size without freeing the buffer
	 */
	if (process_free(buf_arg, 0, &oldsize) == 0) {
		errno = EINVAL;
		return (NULL);
	}

	if (newsize == oldsize)		/* size didn't change */
		return (buf_arg);

	buf = malloc(newsize);
	if (buf == NULL)
		return (NULL);

	(void) memcpy(buf, buf_arg, MIN(newsize, oldsize));
	free(buf_arg);
	return (buf);
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2019, Joyent, Inc.
#

#
# MAPFILE HEADER START
#
# WARNING:  STOP NOW.  DO NOT MODIFY THIS FILE.
# Object versioning must comply with the rules detailed in
#
#	usr/src/lib/README.mapfiles
#
# You should not be making modifications here until you've read the most current
# copy of that file. If you need help, contact a gatekeeper for guidance.
#
# MAPFILE HEADER END
#

$mapfile_version 2

$if _x86
LOAD_SEGMENT ptctext {
	FLAGS = READ EXECUTE;
	ALIGN = 0x1000;
	ROUND = 0x1000;
	ASSIGN_SECTION {
		IS_NAME = .ptctext;
	};
};
$endif

SYMBOL_VERSION ILLUMOS_0.1 {
    global:
	umem_setmtbf;
} SUNW_1.1;

SYMBOL_VERSION SUNW_1.1 {
    global:
	calloc			{ FLAGS = NODIRECT };
	free			{ FLAGS = NODIRECT };
	malloc			{ FLAGS = NODIRECT };
	memalign		{ FLAGS = NODIRECT };
	realloc			{ FLAGS = NODIRECT };
	umem_alloc;
	umem_cache_alloc;
	umem_cache_create;
	umem_cache_destroy;
	umem_cache_free;
	umem_free;
	umem_nofail_callback;
	umem_zalloc;
	valloc			{ FLAGS = NODIRECT };
};

SYMBOL_VERSION SUNWprivate_1.1 {
    global:
	umem_alloc_align;
	umem_free_align;
	umem_reap;
	vmem_add;
	vmem_alloc;
	vmem_contains;
	vmem_create;
	vmem_destroy;
	vmem_free;
	vmem_heap_arena;
	vmem_size;
	vmem_walk;
	vmem_xalloc;
	vmem_xfree;
    local:
	*;
};
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <unistd.h>
#include <dlfcn.h>
#include <signal.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#include <sys/machelf.h>

#include <umem_impl.h>
#include "misc.h"

#define	UMEM_ERRFD	2	/* goes to standard error */
#define	UMEM_MAX_ERROR_SIZE 4096 /* error messages are truncated to this */

/*
 * This is a circular buffer for holding error messages.
 * umem_error_enter appends to the buffer, adding "..." to the beginning
 * if data has been lost.
 */

#define	ERR_SIZE 8192		/* must be a power of 2 */

static mutex_t umem_error_lock = DEFAULTMUTEX;

char umem_error_buffer[ERR_SIZE] = "";
uint_t umem_error_begin = 0;
uint_t umem_error_end = 0;

#define	WRITE_AND_INC(var, value) { \
	umem_error_buffer[(var)++] = (value); \
	var = P2PHASE((var), ERR_SIZE); \
}

static void
log_enter(const char *error_str)
{
	int looped;
	char c;

	looped = 0;

	(void) mutex_lock(&umem_error_lock);

	while ((c = *error_str++) != '\0') {
		WRITE_AND_INC(umem_error_end, c);
		if (umem_error_end == umem_error_begin)
			looped = 1;
	}

	umem_error_buffer[umem_error_end] = 0;

	if (looped) {
		uint_t idx;
		umem_error_begin = P2PHASE(umem_error_end + 1, ERR_SIZE);

		idx = umem_error_begin;
		WRITE_AND_INC(idx, '.');
		WRITE_AND_INC(idx, '.');
		WRITE_AND_INC(idx, '.');
	}

	(void) mutex_unlock(&umem_error_lock);
}

void
umem_error_enter(const char *error_str)
{
#ifndef UMEM_STANDALONE
	if (umem_output && !issetugid())
		(void) write(UMEM_ERRFD, error_str, strlen(error_str));
#endif

	log_enter(error_str);
}

int
highbit(ulong_t i)
{
	register int h = 1;

	if (i == 0)
		return (0);
#ifdef _LP64
	if (i & 0xffffffff00000000ul) {
		h += 32; i >>= 32;
	}
#endif
	if (i & 0xffff0000) {
		h += 16; i >>= 16;
	}
	if (i & 0xff00) {
		h += 8; i >>= 8;
	}
	if (i & 0xf0) {
		h += 4; i >>= 4;
	}
	if (i & 0xc) {
		h += 2; i >>= 2;
	}
	if (i & 0x2) {
		h += 1;
	}
	return (h);
}

int
lowbit(ulong_t i)
{
	register int h = 1;

	if (i == 0)
		return (0);
#ifdef _LP64
	if (!(i & 0xffffffff)) {
		h += 32; i >>= 32;
	}
#endif
	if (!(i & 0xffff)) {
		h += 16; i >>= 16;
	}
	if (!(i & 0xff)) {
		h += 8; i >>= 8;
	}
	if (!(i & 0xf)) {
		h += 4; i >>= 4;
	}
	if (!(i & 0x3)) {
		h += 2; i >>= 2;
	}
	if (!(i & 0x1)) {
		h += 1;
	}
	return (h);
}

void
hrt2ts(hrtime_t hrt, timestruc_t *tsp)
{
	tsp->tv_sec = hrt / NANOSEC;
	tsp->tv_nsec = hrt % NANOSEC;
}

void
log_message(const char *format, ...)
{
	char buf[UMEM_MAX_ERROR_SIZE] = "";

	va_list va;

	va_start(va, format);
	(void) vsnprintf(buf, UMEM_MAX_ERROR_SIZE-1, format, va);
	va_end(va);

#ifndef UMEM_STANDALONE
	if (umem_output > 1)
		(void) write(UMEM_ERRFD, buf, strlen(buf));
#endif

	log_enter(buf);
}

#ifndef UMEM_STANDALONE
void
debug_printf(const char *format, ...)
{
	char buf[UMEM_MAX_ERROR_SIZE] = "";

	va_list va;

	va_start(va, format);
	(void) vsnprintf(buf, UMEM_MAX_ERROR_SIZE-1, format, va);
	va_end(va);

	(void) write(UMEM_ERRFD, buf, strlen(buf));
}
#endif

void
umem_vprintf(const char *format, va_list va)
{
	char buf[UMEM_MAX_ERROR_SIZE] = "";

	(void) vsnprintf(buf, UMEM_MAX_ERROR_SIZE-1, format, va);

	umem_error_enter(buf);
}

void
umem_printf(const char *format, ...)
{
	va_list va;

	va_start(va, format);
	umem_vprintf(format, va);
	va_end(va);
}

/*ARGSUSED*/
void
umem_printf_warn(void *ignored, const char *format, ...)
{
	va_list va;

	va_start(va, format);
	umem_vprintf(format, va);
	va_end(va);
}

/*
 * print_sym tries to print out the symbol and offset of a pointer
 */
int
print_sym(void *pointer)
{
	int result;
	Dl_info sym_info;

	uintptr_t end = (uintptr_t)NULL;

	Sym *ext_info = NULL;

	result = dladdr1(pointer, &sym_info, (void **)&ext_info,
	    RTLD_DL_SYMENT);

	if (result != 0) {
		const char *endpath;

		end = (uintptr_t)sym_info.dli_saddr + ext_info->st_size;

		endpath = strrchr(sym_info.dli_fname, '/');
		if (endpath)
			endpath++;
		else
			endpath = sym_info.dli_fname;
		umem_printf("%s'", endpath);
	}

	if (result == 0 || (uintptr_t)pointer > end) {
		umem_printf("?? (0x%p)", pointer);
		return (0);
	} else {
		umem_printf("%s+0x%p", sym_info.dli_sname,
		    (char *)pointer - (char *)sym_info.dli_saddr);
		return (1);
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_MISC_H
#define	_MISC_H

#include <sys/types.h>
#include <sys/time.h>
#include <thread.h>
#include <pthread.h>
#include <stdarg.h>
#include <stddef.h>

#ifdef	__cplusplus
extern "C" {
#endif

extern uint_t umem_abort;		/* abort when errors occur */
extern uint_t umem_output;		/* output error messages to stderr */
extern caddr_t umem_min_stack;		/* max stack address for audit log */
extern caddr_t umem_max_stack;		/* min stack address for audit log */

/*
 * a safe printf  -- do not use for error messages.
 */
void debug_printf(const char *format, ...);

/*
 * adds a message to the log without writing it out.
 */
void log_message(const char *format, ...);

/*
 * returns the index of the (high/low) bit + 1
 */
int highbit(ulong_t);
int lowbit(ulong_t);

/*
 * Converts a hrtime_t to a timestruc_t
 */
void hrt2ts(hrtime_t hrt, timestruc_t *tsp);

/*
 * tries to print out the symbol and offset of a pointer using umem_error_info
 */
int print_sym(void *pointer);

/*
 * Information about the current error.  Can be called multiple times, should
 * be followed eventually with a call to umem_err or umem_err_recoverable.
 */
void umem_printf(const char *format, ...);
void umem_vprintf(const char *format, va_list);

void umem_printf_warn(void *ignored, const char *format, ...);

void umem_error_enter(const char *);

/*
 * prints error message and stack trace, then aborts.  Cannot return.
 */
void umem_panic(const char *format, ...) __NORETURN;

/*
 * like umem_err, but only aborts if umem_abort > 0
 */
void umem_err_recoverable(const char *format, ...);

/*
 * We define our own assertion handling since libc's assert() calls malloc()
 */
#ifdef NDEBUG
#define	ASSERT(assertion) (void)0
#else
#define	ASSERT(assertion) (void)((assertion) || \
    __umem_assert_failed(#assertion, __FILE__, __LINE__))
#endif

int __umem_assert_failed(const char *assertion, const char *file, int line);
/*
 * These have architecture-specific implementations.
 */

/*
 * Returns the current function's frame pointer.
 */
extern void *getfp(void);

/*
 * puts a pc-only stack trace of up to pcstack_limit frames into pcstack.
 * Returns the number of stacks written.
 *
 * if check_sighandler != 0, and we are in a signal context, calls
 * umem_err_recoverable.
 */
extern int getpcstack(uintptr_t *pcstack, int pcstack_limit,
    int check_sighandler);

#ifdef	__cplusplus
}
#endif

#endif /* _MISC_H */
#
# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# MAPFILE HEADER START
#
# WARNING:  STOP NOW.  DO NOT MODIFY THIS FILE.
# Object versioning must comply with the rules detailed in
#
#	usr/src/lib/README.mapfiles
#
# You should not be making modifications here until you've read the most current
# copy of that file. If you need help, contact a gatekeeper for guidance.
#
# MAPFILE HEADER END
#

$mapfile_version 2

#
# This is a supplemental mapfile, used in addition to the standard one
# produced by the spec tools.  This mapfile exposes an additional
# symbol (umem_startup) that is only present in the standalone version
# of libumem.
#

SYMBOL_VERSION SUNWprivate1.1 {
	global:
		umem_startup;
		umem_add;
};
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
 */

/*
 * Stubs for the standalone to reduce the dependence on external libraries
 */

#include <string.h>
#include "misc.h"

/*ARGSUSED*/
int
cond_init(cond_t *cvp, int type, void *arg)
{
	return (0);
}

/*ARGSUSED*/
int
cond_destroy(cond_t *cvp)
{
	return (0);
}

/*ARGSUSED*/
int
cond_wait(cond_t *cv, mutex_t *mutex)
{
	umem_panic("attempt to wait on standumem cv %p", cv);

	/*NOTREACHED*/
	return (0);
}

/*ARGSUSED*/
int
cond_broadcast(cond_t *cvp)
{
	return (0);
}

/*ARGSUSED*/
int
pthread_setcancelstate(int state, int *oldstate)
{
	return (0);
}

thread_t
thr_self(void)
{
	return ((thread_t)1);
}

static mutex_t _mp = DEFAULTMUTEX;

/*ARGSUSED*/
int
mutex_init(mutex_t *mp, int type, void *arg)
{
	(void) memcpy(mp, &_mp, sizeof (mutex_t));
	return (0);
}

/*ARGSUSED*/
int
mutex_destroy(mutex_t *mp)
{
	return (0);
}

/*ARGSUSED*/
int
_mutex_held(void *mp)
{
	return (1);
}

/*ARGSUSED*/
int
mutex_lock(mutex_t *mp)
{
	return (0);
}

/*ARGSUSED*/
int
mutex_trylock(mutex_t *mp)
{
	return (0);
}

/*ARGSUSED*/
int
mutex_unlock(mutex_t *mp)
{
	return (0);
}

int
issetugid(void)
{
	return (1);
}

int
_tmem_get_nentries(void)
{
	return (0);
}

uintptr_t
_tmem_get_base(void)
{
	return (0);
}

/*ARGSUSED*/
void
_tmem_set_cleanup(void (*f)(int, void *))
{
}

/*
 * Hammerhead: Make isspace weak so libstand.a's strong definition wins
 * when kmdbmod.core links both libstandumem.so and libstand.a together.
 * In the original illumos build, mapfiles made this symbol LOCAL in the
 * .so, but we skip mapfiles for the standalone (GNU ld + -r issues).
 */
#pragma weak isspace
int
isspace(int c)
{
	switch (c) {
	case ' ':
	case '\t':
	case '\n':
	case '\r':
	case '\f':
	case '\v':
		return (1);
	}
	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 1999-2002 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _SYS_VMEM_IMPL_USER_H
#define	_SYS_VMEM_IMPL_USER_H

#include <sys/kstat.h>
#include <sys/time.h>
#include <sys/vmem.h>
#include <thread.h>
#include <synch.h>

#ifdef	__cplusplus
extern "C" {
#endif

typedef struct vmem_seg vmem_seg_t;

#define	VMEM_STACK_DEPTH	20

struct vmem_seg {
	/*
	 * The first four fields must match vmem_freelist_t exactly.
	 */
	uintptr_t	vs_start;	/* start of segment (inclusive) */
	uintptr_t	vs_end;		/* end of segment (exclusive) */
	vmem_seg_t	*vs_knext;	/* next of kin (alloc, free, span) */
	vmem_seg_t	*vs_kprev;	/* prev of kin */

	vmem_seg_t	*vs_anext;	/* next in arena */
	vmem_seg_t	*vs_aprev;	/* prev in arena */
	uint8_t		vs_type;	/* alloc, free, span */
	uint8_t		vs_import;	/* non-zero if segment was imported */
	uint8_t		vs_depth;	/* stack depth if UMF_AUDIT active */
	/*
	 * The following fields are present only when UMF_AUDIT is set.
	 */
	thread_t	vs_thread;
	hrtime_t	vs_timestamp;
	uintptr_t	vs_stack[VMEM_STACK_DEPTH];
};

typedef struct vmem_freelist {
	uintptr_t	vs_start;	/* always zero */
	uintptr_t	vs_end;		/* segment size */
	vmem_seg_t	*vs_knext;	/* next of kin */
	vmem_seg_t	*vs_kprev;	/* prev of kin */
} vmem_freelist_t;

#define	VS_SIZE(vsp)	((vsp)->vs_end - (vsp)->vs_start)

/*
 * Segment hashing
 */
#define	VMEM_HASH_INDEX(a, s, q, m)					\
	((((a) + ((a) >> (s)) + ((a) >> ((s) << 1))) >> (q)) & (m))

#define	VMEM_HASH(vmp, addr)						\
	(&(vmp)->vm_hash_table[VMEM_HASH_INDEX(addr,			\
	(vmp)->vm_hash_shift, (vmp)->vm_qshift, (vmp)->vm_hash_mask)])

#define	VMEM_NAMELEN		30
#define	VMEM_HASH_INITIAL	16
#define	VMEM_NQCACHE_MAX	16
#define	VMEM_FREELISTS		(sizeof (void *) * 8)

typedef struct vmem_kstat {
	uint64_t	vk_mem_inuse;	/* memory in use */
	uint64_t	vk_mem_import;	/* memory imported */
	uint64_t	vk_mem_total;	/* total memory in arena */
	uint32_t	vk_source_id;	/* vmem id of vmem source */
	uint64_t	vk_alloc;	/* number of allocations */
	uint64_t	vk_free;	/* number of frees */
	uint64_t	vk_wait;	/* number of allocations that waited */
	uint64_t	vk_fail;	/* number of allocations that failed */
	uint64_t	vk_lookup;	/* hash lookup count */
	uint64_t	vk_search;	/* freelist search count */
	uint64_t	vk_populate_wait;	/* populates that waited */
	uint64_t	vk_populate_fail;	/* populates that failed */
	uint64_t	vk_contains;		/* vmem_contains() calls */
	uint64_t	vk_contains_search;	/* vmem_contains() search cnt */
} vmem_kstat_t;

struct vmem {
	char		vm_name[VMEM_NAMELEN];	/* arena name */
	cond_t		vm_cv;		/* cv for blocking allocations */
	mutex_t		vm_lock;	/* arena lock */
	uint32_t	vm_id;		/* vmem id */
	uint32_t	vm_mtbf;	/* induced alloc failure rate */
	int		vm_cflags;	/* arena creation flags */
	int		vm_qshift;	/* log2(vm_quantum) */
	size_t		vm_quantum;	/* vmem quantum */
	size_t		vm_qcache_max;	/* maximum size to front by umem */
	vmem_alloc_t	*vm_source_alloc;
	vmem_free_t	*vm_source_free;
	vmem_t		*vm_source;	/* vmem source for imported memory */
	vmem_t		*vm_next;	/* next in vmem_list */
	ssize_t		vm_nsegfree;	/* number of free vmem_seg_t's */
	vmem_seg_t	*vm_segfree;	/* free vmem_seg_t list */
	vmem_seg_t	**vm_hash_table; /* allocated-segment hash table */
	size_t		vm_hash_mask;	/* hash_size - 1 */
	size_t		vm_hash_shift;	/* log2(vm_hash_mask + 1) */
	ulong_t		vm_freemap;	/* bitmap of non-empty freelists */
	vmem_seg_t	vm_seg0;	/* anchor segment */
	vmem_seg_t	vm_rotor;	/* rotor for VM_NEXTFIT allocations */
	vmem_seg_t	*vm_hash0[VMEM_HASH_INITIAL];	/* initial hash table */
	void		*vm_qcache[VMEM_NQCACHE_MAX];	/* quantum caches */
	vmem_freelist_t	vm_freelist[VMEM_FREELISTS + 1]; /* power-of-2 flists */
	vmem_kstat_t	vm_kstat;	/* kstat data */
};

/*
 * We cannot use a mutex_t and MUTEX_HELD, since that will not work
 * when libthread is not linked.
 */
typedef struct vmem_populate_lock {
	mutex_t		vmpl_mutex;
	thread_t	vmpl_thr;
} vmem_populate_lock_t;

#define	VM_UMFLAGS	VM_KMFLAGS

#ifdef	__cplusplus
}
#endif

#endif	/* _SYS_VMEM_IMPL_USER_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2019 Joyent, Inc.
 * Copyright (c) 2015 by Delphix. All rights reserved.
 */

/*
 * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18
 *
 * The slab allocator, as described in the following two papers:
 *
 *	Jeff Bonwick,
 *	The Slab Allocator: An Object-Caching Kernel Memory Allocator.
 *	Proceedings of the Summer 1994 Usenix Conference.
 *	Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
 *
 *	Jeff Bonwick and Jonathan Adams,
 *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
 *	Arbitrary Resources.
 *	Proceedings of the 2001 Usenix Conference.
 *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
 *
 * 1. Overview
 * -----------
 * umem is very close to kmem in implementation.  There are seven major
 * areas of divergence:
 *
 *	* Initialization
 *
 *	* CPU handling
 *
 *	* umem_update()
 *
 *	* KM_SLEEP v.s. UMEM_NOFAIL
 *
 *	* lock ordering
 *
 *	* changing UMEM_MAXBUF
 *
 *	* Per-thread caching for malloc/free
 *
 * 2. Initialization
 * -----------------
 * kmem is initialized early on in boot, and knows that no one will call
 * into it before it is ready.  umem does not have these luxuries. Instead,
 * initialization is divided into two phases:
 *
 *	* library initialization, and
 *
 *	* first use
 *
 * umem's full initialization happens at the time of the first allocation
 * request (via malloc() and friends, umem_alloc(), or umem_zalloc()),
 * or the first call to umem_cache_create().
 *
 * umem_free(), and umem_cache_alloc() do not require special handling,
 * since the only way to get valid arguments for them is to successfully
 * call a function from the first group.
 *
 * 2.1. Library Initialization: umem_startup()
 * -------------------------------------------
 * umem_startup() is libumem.so's .init section.  It calls pthread_atfork()
 * to install the handlers necessary for umem's Fork1-Safety.  Because of
 * race condition issues, all other pre-umem_init() initialization is done
 * statically (i.e. by the dynamic linker).
 *
 * For standalone use, umem_startup() returns everything to its initial
 * state.
 *
 * 2.2. First use: umem_init()
 * ------------------------------
 * The first time any memory allocation function is used, we have to
 * create the backing caches and vmem arenas which are needed for it.
 * umem_init() is the central point for that task.  When it completes,
 * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable
 * to initialize, probably due to lack of memory).
 *
 * There are four different paths from which umem_init() is called:
 *
 *	* from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
 *
 *	* from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
 *
 *	* from umem_cache_create(), and
 *
 *	* from memalign(), with align > UMEM_ALIGN.
 *
 * The last three just check if umem is initialized, and call umem_init()
 * if it is not.  For performance reasons, the first case is more complicated.
 *
 * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF
 * -----------------------------------------------------------------
 * In this case, umem_cache_alloc(&umem_null_cache, ...) is called.
 * There is special case code in which causes any allocation on
 * &umem_null_cache to fail by returning (NULL), regardless of the
 * flags argument.
 *
 * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call
 * umem_alloc_retry().  umem_alloc_retry() sees that the allocation
 * was agains &umem_null_cache, and calls umem_init().
 *
 * If initialization is successful, umem_alloc_retry() returns 1, which
 * causes umem_alloc()/umem_zalloc() to start over, which causes it to load
 * the (now valid) cache pointer from umem_alloc_table.
 *
 * 2.2.2. Dealing with race conditions
 * -----------------------------------
 * There are a couple race conditions resulting from the initialization
 * code that we have to guard against:
 *
 *	* In umem_cache_create(), there is a special UMC_INTERNAL cflag
 *	that is passed for caches created during initialization.  It
 *	is illegal for a user to try to create a UMC_INTERNAL cache.
 *	This allows initialization to proceed, but any other
 *	umem_cache_create()s will block by calling umem_init().
 *
 *	* Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
 *	is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
 *	mask the cpu number.  This prevents a race between grabbing a
 *	cache pointer out of umem_alloc_table and growing the cpu array.
 *
 *
 * 3. CPU handling
 * ---------------
 * kmem uses the CPU's sequence number to determine which "cpu cache" to
 * use for an allocation.  Currently, there is no way to get the sequence
 * number in userspace.
 *
 * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus
 * umem_cpu_t structures.  CURCPU() is a a "hint" function, which we then mask
 * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id.
 * The mechanics of this is all in the CPU(mask) macro.
 *
 * Currently, umem uses _lwp_self() as its hint.
 *
 *
 * 4. The update thread
 * --------------------
 * kmem uses a task queue, kmem_taskq, to do periodic maintenance on
 * every kmem cache.  vmem has a periodic timeout for hash table resizing.
 * The kmem_taskq also provides a separate context for kmem_cache_reap()'s
 * to be done in, avoiding issues of the context of kmem_reap() callers.
 *
 * Instead, umem has the concept of "updates", which are asynchronous requests
 * for work attached to single caches.  All caches with pending work are
 * on a doubly linked list rooted at the umem_null_cache.  All update state
 * is protected by the umem_update_lock mutex, and the umem_update_cv is used
 * for notification between threads.
 *
 * 4.1. Cache states with regards to updates
 * -----------------------------------------
 * A given cache is in one of three states:
 *
 * Inactive		cache_uflags is zero, cache_u{next,prev} are NULL
 *
 * Work Requested	cache_uflags is non-zero (but UMU_ACTIVE is not set),
 *			cache_u{next,prev} link the cache onto the global
 *			update list
 *
 * Active		cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
 *			are NULL, and either umem_update_thr or
 *			umem_st_update_thr are actively doing work on the
 *			cache.
 *
 * An update can be added to any cache in any state -- if the cache is
 * Inactive, it transitions to being Work Requested.  If the cache is
 * Active, the worker will notice the new update and act on it before
 * transitioning the cache to the Inactive state.
 *
 * If a cache is in the Active state, UMU_NOTIFY can be set, which asks
 * the worker to broadcast the umem_update_cv when it has finished.
 *
 * 4.2. Update interface
 * ---------------------
 * umem_add_update() adds an update to a particular cache.
 * umem_updateall() adds an update to all caches.
 * umem_remove_updates() returns a cache to the Inactive state.
 *
 * umem_process_updates() process all caches in the Work Requested state.
 *
 * 4.3. Reaping
 * ------------
 * When umem_reap() is called (at the time of heap growth), it schedule
 * UMU_REAP updates on every cache.  It then checks to see if the update
 * thread exists (umem_update_thr != 0).  If it is, it broadcasts
 * the umem_update_cv to wake the update thread up, and returns.
 *
 * If the update thread does not exist (umem_update_thr == 0), and the
 * program currently has multiple threads, umem_reap() attempts to create
 * a new update thread.
 *
 * If the process is not multithreaded, or the creation fails, umem_reap()
 * calls umem_st_update() to do an inline update.
 *
 * 4.4. The update thread
 * ----------------------
 * The update thread spends most of its time in cond_timedwait() on the
 * umem_update_cv.  It wakes up under two conditions:
 *
 *	* The timedwait times out, in which case it needs to run a global
 *	update, or
 *
 *	* someone cond_broadcast(3C)s the umem_update_cv, in which case
 *	it needs to check if there are any caches in the Work Requested
 *	state.
 *
 * When it is time for another global update, umem calls umem_cache_update()
 * on every cache, then calls vmem_update(), which tunes the vmem structures.
 * umem_cache_update() can request further work using umem_add_update().
 *
 * After any work from the global update completes, the update timer is
 * reset to umem_reap_interval seconds in the future.  This makes the
 * updates self-throttling.
 *
 * Reaps are similarly self-throttling.  After a UMU_REAP update has
 * been scheduled on all caches, umem_reap() sets a flag and wakes up the
 * update thread.  The update thread notices the flag, and resets the
 * reap state.
 *
 * 4.5. Inline updates
 * -------------------
 * If the update thread is not running, umem_st_update() is used instead.  It
 * immediately does a global update (as above), then calls
 * umem_process_updates() to process both the reaps that umem_reap() added and
 * any work generated by the global update.  Afterwards, it resets the reap
 * state.
 *
 * While the umem_st_update() is running, umem_st_update_thr holds the thread
 * id of the thread performing the update.
 *
 * 4.6. Updates and fork1()
 * ------------------------
 * umem has fork1() pre- and post-handlers which lock up (and release) every
 * mutex in every cache.  They also lock up the umem_update_lock.  Since
 * fork1() only copies over a single lwp, other threads (including the update
 * thread) could have been actively using a cache in the parent.  This
 * can lead to inconsistencies in the child process.
 *
 * Because we locked all of the mutexes, the only possible inconsistancies are:
 *
 *	* a umem_cache_alloc() could leak its buffer.
 *
 *	* a caller of umem_depot_alloc() could leak a magazine, and all the
 *	buffers contained in it.
 *
 *	* a cache could be in the Active update state.  In the child, there
 *	would be no thread actually working on it.
 *
 *	* a umem_hash_rescale() could leak the new hash table.
 *
 *	* a umem_magazine_resize() could be in progress.
 *
 *	* a umem_reap() could be in progress.
 *
 * The memory leaks we can't do anything about.  umem_release_child() resets
 * the update state, moves any caches in the Active state to the Work Requested
 * state.  This might cause some updates to be re-run, but UMU_REAP and
 * UMU_HASH_RESCALE are effectively idempotent, and the worst that can
 * happen from umem_magazine_resize() is resizing the magazine twice in close
 * succession.
 *
 * Much of the cleanup in umem_release_child() is skipped if
 * umem_st_update_thr == thr_self().  This is so that applications which call
 * fork1() from a cache callback does not break.  Needless to say, any such
 * application is tremendously broken.
 *
 *
 * 5. KM_SLEEP v.s. UMEM_NOFAIL
 * ----------------------------
 * Allocations against kmem and vmem have two basic modes:  SLEEP and
 * NOSLEEP.  A sleeping allocation is will go to sleep (waiting for
 * more memory) instead of failing (returning NULL).
 *
 * SLEEP allocations presume an extremely multithreaded model, with
 * a lot of allocation and deallocation activity.  umem cannot presume
 * that its clients have any particular type of behavior.  Instead,
 * it provides two types of allocations:
 *
 *	* UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
 *	failure)
 *
 *	* UMEM_NOFAIL, which, on failure, calls an optional callback
 *	(registered with umem_nofail_callback()).
 *
 * The callback is invoked with no locks held, and can do an arbitrary
 * amount of work.  It then has a choice between:
 *
 *	* Returning UMEM_CALLBACK_RETRY, which will cause the allocation
 *	to be restarted.
 *
 *	* Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
 *	to be invoked with status.  If multiple threads attempt to do
 *	this simultaneously, only one will call exit(2).
 *
 *	* Doing some kind of non-local exit (thr_exit(3C), longjmp(3C),
 *	etc.)
 *
 * The default callback returns UMEM_CALLBACK_EXIT(255).
 *
 * To have these callbacks without risk of state corruption (in the case of
 * a non-local exit), we have to ensure that the callbacks get invoked
 * close to the original allocation, with no inconsistent state or held
 * locks.  The following steps are taken:
 *
 *	* All invocations of vmem are VM_NOSLEEP.
 *
 *	* All constructor callbacks (which can themselves to allocations)
 *	are passed UMEM_DEFAULT as their required allocation argument.  This
 *	way, the constructor will fail, allowing the highest-level allocation
 *	invoke the nofail callback.
 *
 *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
 *	the nofail callback does a non-local exit, we will leak the
 *	partially-constructed buffer.
 *
 *
 * 6. Lock Ordering
 * ----------------
 * umem has a few more locks than kmem does, mostly in the update path.  The
 * overall lock ordering (earlier locks must be acquired first) is:
 *
 *	umem_init_lock
 *
 *	vmem_list_lock
 *	vmem_nosleep_lock.vmpl_mutex
 *	vmem_t's:
 *		vm_lock
 *	sbrk_lock
 *
 *	umem_cache_lock
 *	umem_update_lock
 *	umem_flags_lock
 *	umem_cache_t's:
 *		cache_cpu[*].cc_lock
 *		cache_depot_lock
 *		cache_lock
 *	umem_log_header_t's:
 *		lh_cpu[*].clh_lock
 *		lh_lock
 *
 * 7. Changing UMEM_MAXBUF
 * -----------------------
 *
 * When changing UMEM_MAXBUF extra care has to be taken. It is not sufficient to
 * simply increase this number. First, one must update the umem_alloc_table to
 * have the appropriate number of entires based upon the new size. If this is
 * not done, this will lead to libumem blowing an assertion.
 *
 * The second place to update, which is not required, is the umem_alloc_sizes.
 * These determine the default cache sizes that we're going to support.
 *
 * 8. Per-thread caching for malloc/free
 * -------------------------------------
 *
 * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams
 *
 * Time may be an illusion, but CPU cycles aren't.  While libumem is designed
 * to be a highly scalable allocator, that scalability comes with a fixed cycle
 * penalty even in the absence of contention: libumem must acquire (and release
 * a per-CPU lock for each allocation.  When contention is low and malloc(3C)
 * frequency is high, this overhead can dominate execution time.  To alleviate
 * this, we allow for per-thread caching, a lock-free means of caching recent
 * deallocations on a per-thread basis for use in satisfying subsequent calls
 *
 * In addition to improving performance, we also want to:
 *	* Minimize fragmentation
 *	* Not add additional memory overhead (no larger malloc tags)
 *
 * In the ulwp_t of each thread there is a private data structure called a
 * umem_t that looks like:
 *
 * typedef struct {
 *	size_t	tm_size;
 *	void	*tm_roots[NTMEMBASE];  (Currently 16)
 * } tmem_t;
 *
 * Each of the roots is treated as the head of a linked list. Each entry in the
 * list can be thought of as a void ** which points to the next entry, until one
 * of them points to NULL. If the head points to NULL, the list is empty.
 *
 * Each head corresponds to a umem_cache. Currently there is a linear mapping
 * where the first root corresponds to the first cache, second root to the
 * second cache, etc. This works because every allocation that malloc makes to
 * umem_alloc that can be satisified by a umem_cache will actually return a
 * number of bytes equal to the size of that cache. Because of this property and
 * a one to one mapping between caches and roots we can guarantee that every
 * entry in a given root's list will be able to satisfy the same requests as the
 * corresponding cache.
 *
 * The choice of sixteen roots is based on where we believe we get the biggest
 * bang for our buck. The per-thread caches will cache up to 256 byte and 448
 * byte allocations on ILP32 and LP64 respectively. Generally applications plan
 * more carefully how they do larger allocations than smaller ones. Therefore
 * sixteen roots is a reasonable compromise between the amount of additional
 * overhead per thread, and the likelihood of a program to benefit from it.
 *
 * The maximum amount of memory that can be cached in each thread is determined
 * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size
 * value. The default value for this is currently 1 MB. Once umem_init() has
 * finished this cannot be directly tuned without directly modifying the
 * instruction text. If, upon calling free(3C), the amount cached would exceed
 * this maximum, we instead actually return the buffer to the umem_cache instead
 * of holding onto it in the thread.
 *
 * When a thread calls malloc(3C) it first determines which umem_cache it
 * would be serviced by. If the allocation is not covered by ptcumem it goes to
 * the normal malloc instead.  Next, it checks if the tmem_root's list is empty
 * or not. If it is empty, we instead go and allocate the memory from
 * umem_alloc. If it is not empty, we remove the head of the list, set the
 * appropriate malloc tags, and return that buffer.
 *
 * When a thread calls free(3C) it first looks at the malloc tag and if it is
 * invalid or the allocation exceeds the largest cache in ptcumem and sends it
 * off to the original free() to handle and clean up appropriately. Next, it
 * checks if the allocation size is covered by one of the per-thread roots and
 * if it isn't, it passes it off to the original free() to be released. Finally,
 * before it inserts this buffer as the head, it checks if adding this buffer
 * would put the thread over its maximum cache size. If it would, it frees the
 * buffer back to the umem_cache. Otherwise it increments the threads total
 * cached amount and makes the buffer the new head of the appropriate tm_root.
 *
 * When a thread exits, all of the buffers that it has in its per-thread cache
 * will be passed to umem_free() and returned to the appropriate umem_cache.
 *
 * 8.1 Handling addition and removal of umem_caches
 * ------------------------------------------------
 *
 * The set of umem_caches that are used to back calls to umem_alloc() and
 * ultimately malloc() are determined at program execution time. The default set
 * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist
 * that modify the set of caches: size_add, size_clear, and size_remove. Because
 * the set of caches can only be determined once umem_init() has been called and
 * we have the additional goals of minimizing additional fragmentation and
 * metadata space overhead in the malloc tags, this forces our hand to go down a
 * slightly different path: the one tread by fasttrap and trapstat.
 *
 * During umem_init we're going to dynamically construct a new version of
 * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure
 * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If
 * ptcmalloc and ptcfree cannot handle a request, they simply jump to the
 * original libumem implementations.
 *
 * After creating all of the umem_caches, but before making them visible,
 * umem_cache_init checks that umem_genasm_supported is non-zero. This value is
 * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not
 * they support this. If the value is zero, then this process is skipped.
 * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then
 * this is also skipped.
 *
 * In umem_genasm.c, each architecture's implementation implements a single
 * function called umem_genasm() that is responsible for generating the
 * appropriate versions of ptcmalloc() and ptcfree(), placing them in the
 * appropriate memory location, and finally doing the switch from malloc() and
 * free() to ptcmalloc() and ptcfree().  Once the change has been made, there is
 * no way to switch back, short of restarting the program or modifying program
 * text with mdb.
 *
 * 8.2 Modifying the Procedure Linkage Table (PLT)
 * -----------------------------------------------
 *
 * The last piece of this puzzle is how we actually jam ptcmalloc() into the
 * PLT.  To handle this, we have defined two functions, _malloc and _free, we
 * use a standard #pragma weak for malloc and free and direct them to those
 * symbols. By default, those symbols have text defined as nops for our
 * generated functions and when they're invoked, they jump to the default
 * malloc and free functions.
 *
 * When umem_genasm() is called, it makes _malloc and _free writeable and goes
 * through and updates the text provided for by _malloc and _free just after
 * the jump. Once both have been successfully generated, umem_genasm() nops
 * over the original jump so that we now call into the genasm versions of
 * these functions, and makes the functions read-only once again.
 *
 * 8.3 umem_genasm()
 * -----------------
 *
 * umem_genasm() is currently implemented for i386 and amd64. This section
 * describes the theory behind the construction. For specific byte code to
 * assembly instructions and niceish C and asm versions of ptcmalloc and
 * ptcfree, see the individual umem_genasm.c files. The layout consists of the
 * following sections:
 *
 *	o. function-specfic prologue
 *	o. function-generic cache-selecting elements
 *	o. function-specific epilogue
 *
 * There are three different generic cache elements that exist:
 *
 *	o. the last or only cache
 *	o. the intermediary caches if more than two
 *	o. the first one if more than one cache
 *
 * The malloc and free prologues and epilogues mimic the necessary portions of
 * libumem's malloc and free. This includes things like checking for size
 * overflow, setting and verifying the malloc tags.
 *
 * It is an important constraint that these functions do not make use of the
 * call instruction. The only jmp outside of the individual functions is to the
 * original libumem malloc and free respectively. Because doing things like
 * setting errno or raising an internal umem error on improper malloc tags would
 * require using calls into the PLT, whenever we encounter one of those cases we
 * just jump to the original malloc and free functions reusing the same stack
 * frame.
 *
 * Each of the above sections, the three caches, and the malloc and free
 * prologue and epilogue are implemented as blocks of machine code with the
 * corresponding assembly in comments. There are known offsets into each block
 * that corresponds to locations of data and addresses that we only know at run
 * time. These blocks are copied as necessary and the blanks filled in
 * appropriately.
 *
 * As mentioned in section 8.2, the trampoline library uses specifically named
 * variables to communicate the buffers and size to use. These variables are:
 *
 *	o. umem_genasm_mptr: The buffer for ptcmalloc
 *	o. umem_genasm_msize: The size in bytes of the above buffer
 *	o. umem_genasm_fptr: The buffer for ptcfree
 *	o. umem_genasm_fsize: The size in bytes of the above buffer
 *
 * Finally, to enable the generated assembly we need to remove the previous jump
 * to the actual malloc that exists at the start of these buffers. On x86, this
 * is a five byte region. We could zero out the jump offset to be a jmp +0, but
 * using nops can be faster. We specifically use a single five byte nop on x86
 * as it is faster. When porting ptcumem to other architectures, the various
 * opcode changes and options should be analyzed.
 *
 * 8.4 Interface with libc.so
 * --------------------------
 *
 * The tmem_t structure as described in the beginning of section 8, is part of a
 * private interface with libc. There are three functions that exist to cover
 * this. They are not documented in man pages or header files. They are in the
 * SUNWprivate part of libc's mapfile.
 *
 *	o. _tmem_get_base(void)
 *
 *	Returns the offset from the ulwp_t (curthread) to the tmem_t structure.
 *	This is a constant for all threads and is effectively a way to to do
 *	::offsetof ulwp_t ul_tmem without having to know the specifics of the
 *	structure outside of libc.
 *
 *	o. _tmem_get_nentries(void)
 *
 *	Returns the number of roots that exist in the tmem_t. This is one part
 *	of the cap on the number of umem_caches that we can back with tmem.
 *
 *	o. _tmem_set_cleanup(void (*)(void *, int))
 *
 *	This sets a clean up handler that gets called back when a thread exits.
 *	There is one call per buffer, the void * is a pointer to the buffer on
 *	the list, the int is the index into the roots array for this buffer.
 *
 * 8.5 Tuning and disabling per-thread caching
 * -------------------------------------------
 *
 * There is only one tunable for per-thread caching:  the amount of memory each
 * thread should be able to cache.  This is specified via the perthread_cache
 * UMEM_OPTION option.  No attempt is made to to sanity check the specified
 * value; the limit is simply the maximum value of a size_t.
 *
 * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested,
 * or UMEM_DEBUG has been turned on then we will never call into umem_genasm;
 * however, the trampoline audit library and jump will still be in place.
 *
 * 8.6 Observing efficacy of per-thread caching
 * --------------------------------------------
 *
 * To understand the efficacy of per-thread caching, use the ::umastat dcmd
 * to see the percentage of capacity consumed on a per-thread basis, the
 * degree to which each umem cache contributes to per-thread cache consumption,
 * and the number of buffers in per-thread caches on a per-umem cache basis.
 * If more detail is required, the specific buffers in a per-thread cache can
 * be iterated over with the umem_ptc_* walkers. (These walkers allow an
 * optional ulwp_t to be specified to iterate only over a particular thread's
 * cache.)
 */

#include <umem_impl.h>
#include <sys/vmem_impl_user.h>
#include "umem_base.h"
#include "vmem_base.h"

#include <sys/processor.h>
#include <sys/sysmacros.h>

#include <alloca.h>
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <signal.h>
#include <unistd.h>
#include <atomic.h>

#include "misc.h"

#define	UMEM_VMFLAGS(umflag)	(VM_NOSLEEP)

size_t pagesize;

/*
 * The default set of caches to back umem_alloc().
 * These sizes should be reevaluated periodically.
 *
 * We want allocations that are multiples of the coherency granularity
 * (64 bytes) to be satisfied from a cache which is a multiple of 64
 * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
 * the next kmem_cache_size greater than or equal to it must be a
 * multiple of 64.
 *
 * This table must be in sorted order, from smallest to highest.  The
 * highest slot must be UMEM_MAXBUF, and every slot afterwards must be
 * zero.
 */
static int umem_alloc_sizes[] = {
#ifdef _LP64
	1 * 8,
	1 * 16,
	2 * 16,
	3 * 16,
#else
	1 * 8,
	2 * 8,
	3 * 8,
	4 * 8,		5 * 8,		6 * 8,		7 * 8,
#endif
	4 * 16,		5 * 16,		6 * 16,		7 * 16,
	4 * 32,		5 * 32,		6 * 32,		7 * 32,
	4 * 64,		5 * 64,		6 * 64,		7 * 64,
	4 * 128,	5 * 128,	6 * 128,	7 * 128,
	P2ALIGN(8192 / 7, 64),
	P2ALIGN(8192 / 6, 64),
	P2ALIGN(8192 / 5, 64),
	P2ALIGN(8192 / 4, 64), 2304,
	P2ALIGN(8192 / 3, 64),
	P2ALIGN(8192 / 2, 64), 4544,
	P2ALIGN(8192 / 1, 64), 9216,
	4096 * 3,
	8192 * 2,				/* = 8192 * 2 */
	24576, 32768, 40960, 49152, 57344, 65536, 73728, 81920,
	90112, 98304, 106496, 114688, 122880, UMEM_MAXBUF, /* 128k */
	/* 24 slots for user expansion */
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
};
#define	NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes))

static umem_magtype_t umem_magtype[] = {
	{ 1,	8,	3200,	65536	},
	{ 3,	16,	256,	32768	},
	{ 7,	32,	64,	16384	},
	{ 15,	64,	0,	8192	},
	{ 31,	64,	0,	4096	},
	{ 47,	64,	0,	2048	},
	{ 63,	64,	0,	1024	},
	{ 95,	64,	0,	512	},
	{ 143,	64,	0,	0	},
};

/*
 * umem tunables
 */
uint32_t umem_max_ncpus;	/* # of CPU caches. */

uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */
uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */
uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */
uint_t umem_abort = 1;		/* whether to abort on error */
uint_t umem_output = 0;		/* whether to write to standard error */
uint_t umem_logging = 0;	/* umem_log_enter() override */
uint32_t umem_mtbf = 0;		/* mean time between failures [default: off] */
size_t umem_transaction_log_size; /* size of transaction log */
size_t umem_content_log_size;	/* size of content log */
size_t umem_failure_log_size;	/* failure log [4 pages per CPU] */
size_t umem_slab_log_size;	/* slab create log [4 pages per CPU] */
size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */
size_t umem_lite_minsize = 0;	/* minimum buffer size for UMF_LITE */
size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
size_t umem_maxverify;		/* maximum bytes to inspect in debug routines */
size_t umem_minfirewall;	/* hardware-enforced redzone threshold */
size_t umem_ptc_size = 1048576;	/* size of per-thread cache (in bytes) */

uint_t umem_flags = 0;
uintptr_t umem_tmem_off;

mutex_t			umem_init_lock;		/* locks initialization */
cond_t			umem_init_cv;		/* initialization CV */
thread_t		umem_init_thr;		/* thread initializing */
int			umem_init_env_ready;	/* environ pre-initted */
int			umem_ready = UMEM_READY_STARTUP;

int			umem_ptc_enabled;	/* per-thread caching enabled */

static umem_nofail_callback_t *nofail_callback;
static mutex_t		umem_nofail_exit_lock;
static thread_t		umem_nofail_exit_thr;

static umem_cache_t	*umem_slab_cache;
static umem_cache_t	*umem_bufctl_cache;
static umem_cache_t	*umem_bufctl_audit_cache;

mutex_t			umem_flags_lock;

static vmem_t		*heap_arena;
static vmem_alloc_t	*heap_alloc;
static vmem_free_t	*heap_free;

static vmem_t		*umem_internal_arena;
static vmem_t		*umem_cache_arena;
static vmem_t		*umem_hash_arena;
static vmem_t		*umem_log_arena;
static vmem_t		*umem_oversize_arena;
static vmem_t		*umem_va_arena;
static vmem_t		*umem_default_arena;
static vmem_t		*umem_firewall_va_arena;
static vmem_t		*umem_firewall_arena;

vmem_t			*umem_memalign_arena;

umem_log_header_t *umem_transaction_log;
umem_log_header_t *umem_content_log;
umem_log_header_t *umem_failure_log;
umem_log_header_t *umem_slab_log;

#define	CPUHINT()		(thr_self())
#define	CPUHINT_MAX()		INT_MAX

#define	CPU(mask)		(umem_cpus + (CPUHINT() & (mask)))
static umem_cpu_t umem_startup_cpu = {	/* initial, single, cpu */
	UMEM_CACHE_SIZE(0),
	0
};

static uint32_t umem_cpu_mask = 0;			/* global cpu mask */
static umem_cpu_t *umem_cpus = &umem_startup_cpu;	/* cpu list */

volatile uint32_t umem_reaping;

thread_t		umem_update_thr;
struct timeval		umem_update_next;	/* timeofday of next update */
volatile thread_t	umem_st_update_thr;	/* only used when single-thd */

#define	IN_UPDATE()	(thr_self() == umem_update_thr || \
			    thr_self() == umem_st_update_thr)
#define	IN_REAP()	IN_UPDATE()

mutex_t			umem_update_lock;	/* cache_u{next,prev,flags} */
cond_t			umem_update_cv;

volatile hrtime_t umem_reap_next;	/* min hrtime of next reap */

mutex_t			umem_cache_lock;	/* inter-cache linkage only */

#ifdef UMEM_STANDALONE
umem_cache_t		umem_null_cache;
static const umem_cache_t umem_null_cache_template = {
#else
umem_cache_t		umem_null_cache = {
#endif
	0, 0, 0, 0, 0,
	0, 0,
	0, 0,
	0, 0,
	"invalid_cache",
	0, 0,
	NULL, NULL, NULL, NULL,
	NULL,
	0, 0, 0, 0,
	&umem_null_cache, &umem_null_cache,
	&umem_null_cache, &umem_null_cache,
	0,
	DEFAULTMUTEX,				/* start of slab layer */
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	&umem_null_cache.cache_nullslab,
	{
		&umem_null_cache,
		NULL,
		&umem_null_cache.cache_nullslab,
		&umem_null_cache.cache_nullslab,
		NULL,
		-1,
		0
	},
	NULL,
	NULL,
	DEFAULTMUTEX,				/* start of depot layer */
	NULL, {
		NULL, 0, 0, 0, 0
	}, {
		NULL, 0, 0, 0, 0
	}, {
		{
			DEFAULTMUTEX,		/* start of CPU cache */
			0, 0, NULL, NULL, -1, -1, 0
		}
	}
};

#define	ALLOC_TABLE_4 \
	&umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache

#define	ALLOC_TABLE_64 \
	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4

#define	ALLOC_TABLE_1024 \
	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64

static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = {
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024,
	ALLOC_TABLE_1024
};


/* Used to constrain audit-log stack traces */
caddr_t			umem_min_stack;
caddr_t			umem_max_stack;


#define	UMERR_MODIFIED	0	/* buffer modified while on freelist */
#define	UMERR_REDZONE	1	/* redzone violation (write past end of buf) */
#define	UMERR_DUPFREE	2	/* freed a buffer twice */
#define	UMERR_BADADDR	3	/* freed a bad (unallocated) address */
#define	UMERR_BADBUFTAG	4	/* buftag corrupted */
#define	UMERR_BADBUFCTL	5	/* bufctl corrupted */
#define	UMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
#define	UMERR_BADSIZE	7	/* alloc size != free size */
#define	UMERR_BADBASE	8	/* buffer base address wrong */

struct {
	hrtime_t	ump_timestamp;	/* timestamp of error */
	int		ump_error;	/* type of umem error (UMERR_*) */
	void		*ump_buffer;	/* buffer that induced abort */
	void		*ump_realbuf;	/* real start address for buffer */
	umem_cache_t	*ump_cache;	/* buffer's cache according to client */
	umem_cache_t	*ump_realcache;	/* actual cache containing buffer */
	umem_slab_t	*ump_slab;	/* slab accoring to umem_findslab() */
	umem_bufctl_t	*ump_bufctl;	/* bufctl */
} umem_abort_info;

static void
copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
{
	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
	uint64_t *buf = buf_arg;

	while (buf < bufend)
		*buf++ = pattern;
}

static void *
verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
{
	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
	uint64_t *buf;

	for (buf = buf_arg; buf < bufend; buf++)
		if (*buf != pattern)
			return (buf);
	return (NULL);
}

static void *
verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
{
	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
	uint64_t *buf;

	for (buf = buf_arg; buf < bufend; buf++) {
		if (*buf != old) {
			copy_pattern(old, buf_arg,
			    (char *)buf - (char *)buf_arg);
			return (buf);
		}
		*buf = new;
	}

	return (NULL);
}

void
umem_cache_applyall(void (*func)(umem_cache_t *))
{
	umem_cache_t *cp;

	(void) mutex_lock(&umem_cache_lock);
	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
	    cp = cp->cache_next)
		func(cp);
	(void) mutex_unlock(&umem_cache_lock);
}

static void
umem_add_update_unlocked(umem_cache_t *cp, int flags)
{
	umem_cache_t *cnext, *cprev;

	flags &= ~UMU_ACTIVE;

	if (!flags)
		return;

	if (cp->cache_uflags & UMU_ACTIVE) {
		cp->cache_uflags |= flags;
	} else {
		if (cp->cache_unext != NULL) {
			ASSERT(cp->cache_uflags != 0);
			cp->cache_uflags |= flags;
		} else {
			ASSERT(cp->cache_uflags == 0);
			cp->cache_uflags = flags;
			cp->cache_unext = cnext = &umem_null_cache;
			cp->cache_uprev = cprev = umem_null_cache.cache_uprev;
			cnext->cache_uprev = cp;
			cprev->cache_unext = cp;
		}
	}
}

static void
umem_add_update(umem_cache_t *cp, int flags)
{
	(void) mutex_lock(&umem_update_lock);

	umem_add_update_unlocked(cp, flags);

	if (!IN_UPDATE())
		(void) cond_broadcast(&umem_update_cv);

	(void) mutex_unlock(&umem_update_lock);
}

/*
 * Remove a cache from the update list, waiting for any in-progress work to
 * complete first.
 */
static void
umem_remove_updates(umem_cache_t *cp)
{
	(void) mutex_lock(&umem_update_lock);

	/*
	 * Get it out of the active state
	 */
	while (cp->cache_uflags & UMU_ACTIVE) {
		int cancel_state;

		ASSERT(cp->cache_unext == NULL);

		cp->cache_uflags |= UMU_NOTIFY;

		/*
		 * Make sure the update state is sane, before we wait
		 */
		ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0);
		ASSERT(umem_update_thr != thr_self() &&
		    umem_st_update_thr != thr_self());

		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
		    &cancel_state);
		(void) cond_wait(&umem_update_cv, &umem_update_lock);
		(void) pthread_setcancelstate(cancel_state, NULL);
	}
	/*
	 * Get it out of the Work Requested state
	 */
	if (cp->cache_unext != NULL) {
		cp->cache_uprev->cache_unext = cp->cache_unext;
		cp->cache_unext->cache_uprev = cp->cache_uprev;
		cp->cache_uprev = cp->cache_unext = NULL;
		cp->cache_uflags = 0;
	}
	/*
	 * Make sure it is in the Inactive state
	 */
	ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0);
	(void) mutex_unlock(&umem_update_lock);
}

static void
umem_updateall(int flags)
{
	umem_cache_t *cp;

	/*
	 * NOTE:  To prevent deadlock, umem_cache_lock is always acquired first.
	 *
	 * (umem_add_update is called from things run via umem_cache_applyall)
	 */
	(void) mutex_lock(&umem_cache_lock);
	(void) mutex_lock(&umem_update_lock);

	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
	    cp = cp->cache_next)
		umem_add_update_unlocked(cp, flags);

	if (!IN_UPDATE())
		(void) cond_broadcast(&umem_update_cv);

	(void) mutex_unlock(&umem_update_lock);
	(void) mutex_unlock(&umem_cache_lock);
}

/*
 * Debugging support.  Given a buffer address, find its slab.
 */
static umem_slab_t *
umem_findslab(umem_cache_t *cp, void *buf)
{
	umem_slab_t *sp;

	(void) mutex_lock(&cp->cache_lock);
	for (sp = cp->cache_nullslab.slab_next;
	    sp != &cp->cache_nullslab; sp = sp->slab_next) {
		if (UMEM_SLAB_MEMBER(sp, buf)) {
			(void) mutex_unlock(&cp->cache_lock);
			return (sp);
		}
	}
	(void) mutex_unlock(&cp->cache_lock);

	return (NULL);
}

static void
umem_error(int error, umem_cache_t *cparg, void *bufarg)
{
	umem_buftag_t *btp = NULL;
	umem_bufctl_t *bcp = NULL;
	umem_cache_t *cp = cparg;
	umem_slab_t *sp;
	uint64_t *off;
	void *buf = bufarg;

	int old_logging = umem_logging;

	umem_logging = 0;	/* stop logging when a bad thing happens */

	umem_abort_info.ump_timestamp = gethrtime();

	sp = umem_findslab(cp, buf);
	if (sp == NULL) {
		for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache;
		    cp = cp->cache_prev) {
			if ((sp = umem_findslab(cp, buf)) != NULL)
				break;
		}
	}

	if (sp == NULL) {
		cp = NULL;
		error = UMERR_BADADDR;
	} else {
		if (cp != cparg)
			error = UMERR_BADCACHE;
		else
			buf = (char *)bufarg - ((uintptr_t)bufarg -
			    (uintptr_t)sp->slab_base) % cp->cache_chunksize;
		if (buf != bufarg)
			error = UMERR_BADBASE;
		if (cp->cache_flags & UMF_BUFTAG)
			btp = UMEM_BUFTAG(cp, buf);
		if (cp->cache_flags & UMF_HASH) {
			(void) mutex_lock(&cp->cache_lock);
			for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
				if (bcp->bc_addr == buf)
					break;
			(void) mutex_unlock(&cp->cache_lock);
			if (bcp == NULL && btp != NULL)
				bcp = btp->bt_bufctl;
			if (umem_findslab(cp->cache_bufctl_cache, bcp) ==
			    NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) ||
			    bcp->bc_addr != buf) {
				error = UMERR_BADBUFCTL;
				bcp = NULL;
			}
		}
	}

	umem_abort_info.ump_error = error;
	umem_abort_info.ump_buffer = bufarg;
	umem_abort_info.ump_realbuf = buf;
	umem_abort_info.ump_cache = cparg;
	umem_abort_info.ump_realcache = cp;
	umem_abort_info.ump_slab = sp;
	umem_abort_info.ump_bufctl = bcp;

	umem_printf("umem allocator: ");

	switch (error) {

	case UMERR_MODIFIED:
		umem_printf("buffer modified after being freed\n");
		off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
		if (off == NULL)	/* shouldn't happen */
			off = buf;
		umem_printf("modification occurred at offset 0x%lx "
		    "(0x%llx replaced by 0x%llx)\n",
		    (uintptr_t)off - (uintptr_t)buf,
		    (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off);
		break;

	case UMERR_REDZONE:
		umem_printf("redzone violation: write past end of buffer\n");
		break;

	case UMERR_BADADDR:
		umem_printf("invalid free: buffer not in cache\n");
		break;

	case UMERR_DUPFREE:
		umem_printf("duplicate free: buffer freed twice\n");
		break;

	case UMERR_BADBUFTAG:
		umem_printf("boundary tag corrupted\n");
		umem_printf("bcp ^ bxstat = %lx, should be %lx\n",
		    (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
		    UMEM_BUFTAG_FREE);
		break;

	case UMERR_BADBUFCTL:
		umem_printf("bufctl corrupted\n");
		break;

	case UMERR_BADCACHE:
		umem_printf("buffer freed to wrong cache\n");
		umem_printf("buffer was allocated from %s,\n", cp->cache_name);
		umem_printf("caller attempting free to %s.\n",
		    cparg->cache_name);
		break;

	case UMERR_BADSIZE:
		umem_printf("bad free: free size (%u) != alloc size (%u)\n",
		    UMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
		    UMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
		break;

	case UMERR_BADBASE:
		umem_printf("bad free: free address (%p) != alloc address "
		    "(%p)\n", bufarg, buf);
		break;
	}

	umem_printf("buffer=%p  bufctl=%p  cache: %s\n",
	    bufarg, (void *)bcp, cparg->cache_name);

	if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) &&
	    error != UMERR_BADBUFCTL) {
		int d;
		timespec_t ts;
		hrtime_t diff;
		umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp;

		diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp;
		ts.tv_sec = diff / NANOSEC;
		ts.tv_nsec = diff % NANOSEC;

		umem_printf("previous transaction on buffer %p:\n", buf);
		umem_printf("thread=%p  time=T-%ld.%09ld  slab=%p  cache: %s\n",
		    (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
		    (void *)sp, cp->cache_name);
		for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) {
			(void) print_sym((void *)bcap->bc_stack[d]);
			umem_printf("\n");
		}
	}

	umem_err_recoverable("umem: heap corruption detected");

	umem_logging = old_logging;	/* resume logging */
}

void
umem_nofail_callback(umem_nofail_callback_t *cb)
{
	nofail_callback = cb;
}

static int
umem_alloc_retry(umem_cache_t *cp, int umflag)
{
	if (cp == &umem_null_cache) {
		if (umem_init())
			return (1);				/* retry */
		/*
		 * Initialization failed.  Do normal failure processing.
		 */
	}
	if (umem_flags & UMF_CHECKNULL) {
		umem_err_recoverable("umem: out of heap space");
	}
	if (umflag & UMEM_NOFAIL) {
		int def_result = UMEM_CALLBACK_EXIT(255);
		int result = def_result;
		umem_nofail_callback_t *callback = nofail_callback;

		if (callback != NULL)
			result = callback();

		if (result == UMEM_CALLBACK_RETRY)
			return (1);

		if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) {
			log_message("nofail callback returned %x\n", result);
			result = def_result;
		}

		/*
		 * only one thread will call exit
		 */
		if (umem_nofail_exit_thr == thr_self())
			umem_panic("recursive UMEM_CALLBACK_EXIT()\n");

		(void) mutex_lock(&umem_nofail_exit_lock);
		umem_nofail_exit_thr = thr_self();
		exit(result & 0xFF);
		/*NOTREACHED*/
	}
	return (0);
}

static umem_log_header_t *
umem_log_init(size_t logsize)
{
	umem_log_header_t *lhp;
	int nchunks = 4 * umem_max_ncpus;
	size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]);
	int i;

	if (logsize == 0)
		return (NULL);

	/*
	 * Make sure that lhp->lh_cpu[] is nicely aligned
	 * to prevent false sharing of cache lines.
	 */
	lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN);
	lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
	    NULL, NULL, VM_NOSLEEP);
	if (lhp == NULL)
		goto fail;

	bzero(lhp, lhsize);

	(void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL);
	lhp->lh_nchunks = nchunks;
	lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE);
	if (lhp->lh_chunksize == 0)
		lhp->lh_chunksize = PAGESIZE;

	lhp->lh_base = vmem_alloc(umem_log_arena,
	    lhp->lh_chunksize * nchunks, VM_NOSLEEP);
	if (lhp->lh_base == NULL)
		goto fail;

	lhp->lh_free = vmem_alloc(umem_log_arena,
	    nchunks * sizeof (int), VM_NOSLEEP);
	if (lhp->lh_free == NULL)
		goto fail;

	bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);

	for (i = 0; i < umem_max_ncpus; i++) {
		umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
		(void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL);
		clhp->clh_chunk = i;
	}

	for (i = umem_max_ncpus; i < nchunks; i++)
		lhp->lh_free[i] = i;

	lhp->lh_head = umem_max_ncpus;
	lhp->lh_tail = 0;

	return (lhp);

fail:
	if (lhp != NULL) {
		if (lhp->lh_base != NULL)
			vmem_free(umem_log_arena, lhp->lh_base,
			    lhp->lh_chunksize * nchunks);

		vmem_xfree(umem_log_arena, lhp, lhsize);
	}
	return (NULL);
}

static void *
umem_log_enter(umem_log_header_t *lhp, void *data, size_t size)
{
	void *logspace;
	umem_cpu_log_header_t *clhp;

	if (lhp == NULL || umem_logging == 0)
		return (NULL);

	clhp = &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number];

	(void) mutex_lock(&clhp->clh_lock);
	clhp->clh_hits++;
	if (size > clhp->clh_avail) {
		(void) mutex_lock(&lhp->lh_lock);
		lhp->lh_hits++;
		lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
		lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
		clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
		lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
		clhp->clh_current = lhp->lh_base +
		    clhp->clh_chunk * lhp->lh_chunksize;
		clhp->clh_avail = lhp->lh_chunksize;
		if (size > lhp->lh_chunksize)
			size = lhp->lh_chunksize;
		(void) mutex_unlock(&lhp->lh_lock);
	}
	logspace = clhp->clh_current;
	clhp->clh_current += size;
	clhp->clh_avail -= size;
	bcopy(data, logspace, size);
	(void) mutex_unlock(&clhp->clh_lock);
	return (logspace);
}

#define	UMEM_AUDIT(lp, cp, bcp)						\
{									\
	umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp);	\
	_bcp->bc_timestamp = gethrtime();				\
	_bcp->bc_thread = thr_self();					\
	_bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth,	\
	    (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL));	\
	_bcp->bc_lastlog = umem_log_enter((lp), _bcp,			\
	    UMEM_BUFCTL_AUDIT_SIZE);					\
}

static void
umem_log_event(umem_log_header_t *lp, umem_cache_t *cp,
    umem_slab_t *sp, void *addr)
{
	umem_bufctl_audit_t *bcp;
	UMEM_LOCAL_BUFCTL_AUDIT(&bcp);

	bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE);
	bcp->bc_addr = addr;
	bcp->bc_slab = sp;
	bcp->bc_cache = cp;
	UMEM_AUDIT(lp, cp, bcp);
}

/*
 * Create a new slab for cache cp.
 */
static umem_slab_t *
umem_slab_create(umem_cache_t *cp, int umflag)
{
	size_t slabsize = cp->cache_slabsize;
	size_t chunksize = cp->cache_chunksize;
	int cache_flags = cp->cache_flags;
	size_t color, chunks;
	char *buf, *slab;
	umem_slab_t *sp;
	umem_bufctl_t *bcp;
	vmem_t *vmp = cp->cache_arena;

	color = cp->cache_color + cp->cache_align;
	if (color > cp->cache_maxcolor)
		color = cp->cache_mincolor;
	cp->cache_color = color;

	slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag));

	if (slab == NULL)
		goto vmem_alloc_failure;

	ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);

	if (!(cp->cache_cflags & UMC_NOTOUCH) &&
	    (cp->cache_flags & UMF_DEADBEEF))
		copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize);

	if (cache_flags & UMF_HASH) {
		if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL)
			goto slab_alloc_failure;
		chunks = (slabsize - color) / chunksize;
	} else {
		sp = UMEM_SLAB(cp, slab);
		chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize;
	}

	sp->slab_cache	= cp;
	sp->slab_head	= NULL;
	sp->slab_refcnt	= 0;
	sp->slab_base	= buf = slab + color;
	sp->slab_chunks	= chunks;

	ASSERT(chunks > 0);
	while (chunks-- != 0) {
		if (cache_flags & UMF_HASH) {
			bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag);
			if (bcp == NULL)
				goto bufctl_alloc_failure;
			if (cache_flags & UMF_AUDIT) {
				umem_bufctl_audit_t *bcap =
				    (umem_bufctl_audit_t *)bcp;
				bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE);
				bcap->bc_cache = cp;
			}
			bcp->bc_addr = buf;
			bcp->bc_slab = sp;
		} else {
			bcp = UMEM_BUFCTL(cp, buf);
		}
		if (cache_flags & UMF_BUFTAG) {
			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
			btp->bt_redzone = UMEM_REDZONE_PATTERN;
			btp->bt_bufctl = bcp;
			btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
			if (cache_flags & UMF_DEADBEEF) {
				copy_pattern(UMEM_FREE_PATTERN, buf,
				    cp->cache_verify);
			}
		}
		bcp->bc_next = sp->slab_head;
		sp->slab_head = bcp;
		buf += chunksize;
	}

	umem_log_event(umem_slab_log, cp, sp, slab);

	return (sp);

bufctl_alloc_failure:

	while ((bcp = sp->slab_head) != NULL) {
		sp->slab_head = bcp->bc_next;
		_umem_cache_free(cp->cache_bufctl_cache, bcp);
	}
	_umem_cache_free(umem_slab_cache, sp);

slab_alloc_failure:

	vmem_free(vmp, slab, slabsize);

vmem_alloc_failure:

	umem_log_event(umem_failure_log, cp, NULL, NULL);
	atomic_add_64(&cp->cache_alloc_fail, 1);

	return (NULL);
}

/*
 * Destroy a slab.
 */
static void
umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp)
{
	vmem_t *vmp = cp->cache_arena;
	void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);

	if (cp->cache_flags & UMF_HASH) {
		umem_bufctl_t *bcp;
		while ((bcp = sp->slab_head) != NULL) {
			sp->slab_head = bcp->bc_next;
			_umem_cache_free(cp->cache_bufctl_cache, bcp);
		}
		_umem_cache_free(umem_slab_cache, sp);
	}
	vmem_free(vmp, slab, cp->cache_slabsize);
}

/*
 * Allocate a raw (unconstructed) buffer from cp's slab layer.
 */
static void *
umem_slab_alloc(umem_cache_t *cp, int umflag)
{
	umem_bufctl_t *bcp, **hash_bucket;
	umem_slab_t *sp;
	void *buf;

	(void) mutex_lock(&cp->cache_lock);
	cp->cache_slab_alloc++;
	sp = cp->cache_freelist;
	ASSERT(sp->slab_cache == cp);
	if (sp->slab_head == NULL) {
		/*
		 * The freelist is empty.  Create a new slab.
		 */
		(void) mutex_unlock(&cp->cache_lock);
		if (cp == &umem_null_cache)
			return (NULL);
		if ((sp = umem_slab_create(cp, umflag)) == NULL)
			return (NULL);
		(void) mutex_lock(&cp->cache_lock);
		cp->cache_slab_create++;
		if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
			cp->cache_bufmax = cp->cache_buftotal;
		sp->slab_next = cp->cache_freelist;
		sp->slab_prev = cp->cache_freelist->slab_prev;
		sp->slab_next->slab_prev = sp;
		sp->slab_prev->slab_next = sp;
		cp->cache_freelist = sp;
	}

	sp->slab_refcnt++;
	ASSERT(sp->slab_refcnt <= sp->slab_chunks);

	/*
	 * If we're taking the last buffer in the slab,
	 * remove the slab from the cache's freelist.
	 */
	bcp = sp->slab_head;
	if ((sp->slab_head = bcp->bc_next) == NULL) {
		cp->cache_freelist = sp->slab_next;
		ASSERT(sp->slab_refcnt == sp->slab_chunks);
	}

	if (cp->cache_flags & UMF_HASH) {
		/*
		 * Add buffer to allocated-address hash table.
		 */
		buf = bcp->bc_addr;
		hash_bucket = UMEM_HASH(cp, buf);
		bcp->bc_next = *hash_bucket;
		*hash_bucket = bcp;
		if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
			UMEM_AUDIT(umem_transaction_log, cp, bcp);
		}
	} else {
		buf = UMEM_BUF(cp, bcp);
	}

	ASSERT(UMEM_SLAB_MEMBER(sp, buf));

	(void) mutex_unlock(&cp->cache_lock);

	return (buf);
}

/*
 * Free a raw (unconstructed) buffer to cp's slab layer.
 */
static void
umem_slab_free(umem_cache_t *cp, void *buf)
{
	umem_slab_t *sp;
	umem_bufctl_t *bcp, **prev_bcpp;

	ASSERT(buf != NULL);

	(void) mutex_lock(&cp->cache_lock);
	cp->cache_slab_free++;

	if (cp->cache_flags & UMF_HASH) {
		/*
		 * Look up buffer in allocated-address hash table.
		 */
		prev_bcpp = UMEM_HASH(cp, buf);
		while ((bcp = *prev_bcpp) != NULL) {
			if (bcp->bc_addr == buf) {
				*prev_bcpp = bcp->bc_next;
				sp = bcp->bc_slab;
				break;
			}
			cp->cache_lookup_depth++;
			prev_bcpp = &bcp->bc_next;
		}
	} else {
		bcp = UMEM_BUFCTL(cp, buf);
		sp = UMEM_SLAB(cp, buf);
	}

	if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) {
		(void) mutex_unlock(&cp->cache_lock);
		umem_error(UMERR_BADADDR, cp, buf);
		return;
	}

	if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
		if (cp->cache_flags & UMF_CONTENTS)
			((umem_bufctl_audit_t *)bcp)->bc_contents =
			    umem_log_enter(umem_content_log, buf,
			    cp->cache_contents);
		UMEM_AUDIT(umem_transaction_log, cp, bcp);
	}

	/*
	 * If this slab isn't currently on the freelist, put it there.
	 */
	if (sp->slab_head == NULL) {
		ASSERT(sp->slab_refcnt == sp->slab_chunks);
		ASSERT(cp->cache_freelist != sp);
		sp->slab_next->slab_prev = sp->slab_prev;
		sp->slab_prev->slab_next = sp->slab_next;
		sp->slab_next = cp->cache_freelist;
		sp->slab_prev = cp->cache_freelist->slab_prev;
		sp->slab_next->slab_prev = sp;
		sp->slab_prev->slab_next = sp;
		cp->cache_freelist = sp;
	}

	bcp->bc_next = sp->slab_head;
	sp->slab_head = bcp;

	ASSERT(sp->slab_refcnt >= 1);
	if (--sp->slab_refcnt == 0) {
		/*
		 * There are no outstanding allocations from this slab,
		 * so we can reclaim the memory.
		 */
		sp->slab_next->slab_prev = sp->slab_prev;
		sp->slab_prev->slab_next = sp->slab_next;
		if (sp == cp->cache_freelist)
			cp->cache_freelist = sp->slab_next;
		cp->cache_slab_destroy++;
		cp->cache_buftotal -= sp->slab_chunks;
		(void) mutex_unlock(&cp->cache_lock);
		umem_slab_destroy(cp, sp);
		return;
	}
	(void) mutex_unlock(&cp->cache_lock);
}

static int
umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag)
{
	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
	uint32_t mtbf;
	int flags_nfatal;

	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
		umem_error(UMERR_BADBUFTAG, cp, buf);
		return (-1);
	}

	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC;

	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
		umem_error(UMERR_BADBUFCTL, cp, buf);
		return (-1);
	}

	btp->bt_redzone = UMEM_REDZONE_PATTERN;

	if (cp->cache_flags & UMF_DEADBEEF) {
		if (verify_and_copy_pattern(UMEM_FREE_PATTERN,
		    UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) {
			umem_error(UMERR_MODIFIED, cp, buf);
			return (-1);
		}
	}

	if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 &&
	    gethrtime() % mtbf == 0 &&
	    (umflag & (UMEM_FATAL_FLAGS)) == 0) {
		umem_log_event(umem_failure_log, cp, NULL, NULL);
	} else {
		mtbf = 0;
	}

	/*
	 * We do not pass fatal flags on to the constructor.  This prevents
	 * leaking buffers in the event of a subordinate constructor failing.
	 */
	flags_nfatal = UMEM_DEFAULT;
	if (mtbf || (cp->cache_constructor != NULL &&
	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) {
		atomic_add_64(&cp->cache_alloc_fail, 1);
		btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
		umem_slab_free(cp, buf);
		return (-1);
	}

	if (cp->cache_flags & UMF_AUDIT) {
		UMEM_AUDIT(umem_transaction_log, cp, bcp);
	}

	return (0);
}

static int
umem_cache_free_debug(umem_cache_t *cp, void *buf)
{
	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
	umem_slab_t *sp;

	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) {
		if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
			umem_error(UMERR_DUPFREE, cp, buf);
			return (-1);
		}
		sp = umem_findslab(cp, buf);
		if (sp == NULL || sp->slab_cache != cp)
			umem_error(UMERR_BADADDR, cp, buf);
		else
			umem_error(UMERR_REDZONE, cp, buf);
		return (-1);
	}

	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;

	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
		umem_error(UMERR_BADBUFCTL, cp, buf);
		return (-1);
	}

	if (btp->bt_redzone != UMEM_REDZONE_PATTERN) {
		umem_error(UMERR_REDZONE, cp, buf);
		return (-1);
	}

	if (cp->cache_flags & UMF_AUDIT) {
		if (cp->cache_flags & UMF_CONTENTS)
			bcp->bc_contents = umem_log_enter(umem_content_log,
			    buf, cp->cache_contents);
		UMEM_AUDIT(umem_transaction_log, cp, bcp);
	}

	if (cp->cache_destructor != NULL)
		cp->cache_destructor(buf, cp->cache_private);

	if (cp->cache_flags & UMF_DEADBEEF)
		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);

	return (0);
}

/*
 * Free each object in magazine mp to cp's slab layer, and free mp itself.
 */
static void
umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds)
{
	int round;

	ASSERT(cp->cache_next == NULL || IN_UPDATE());

	for (round = 0; round < nrounds; round++) {
		void *buf = mp->mag_round[round];

		if ((cp->cache_flags & UMF_DEADBEEF) &&
		    verify_pattern(UMEM_FREE_PATTERN, buf,
		    cp->cache_verify) != NULL) {
			umem_error(UMERR_MODIFIED, cp, buf);
			continue;
		}

		if (!(cp->cache_flags & UMF_BUFTAG) &&
		    cp->cache_destructor != NULL)
			cp->cache_destructor(buf, cp->cache_private);

		umem_slab_free(cp, buf);
	}
	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
	_umem_cache_free(cp->cache_magtype->mt_cache, mp);
}

/*
 * Allocate a magazine from the depot.
 */
static umem_magazine_t *
umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp)
{
	umem_magazine_t *mp;

	/*
	 * If we can't get the depot lock without contention,
	 * update our contention count.  We use the depot
	 * contention rate to determine whether we need to
	 * increase the magazine size for better scalability.
	 */
	if (mutex_trylock(&cp->cache_depot_lock) != 0) {
		(void) mutex_lock(&cp->cache_depot_lock);
		cp->cache_depot_contention++;
	}

	if ((mp = mlp->ml_list) != NULL) {
		ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
		mlp->ml_list = mp->mag_next;
		if (--mlp->ml_total < mlp->ml_min)
			mlp->ml_min = mlp->ml_total;
		mlp->ml_alloc++;
	}

	(void) mutex_unlock(&cp->cache_depot_lock);

	return (mp);
}

/*
 * Free a magazine to the depot.
 */
static void
umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp)
{
	(void) mutex_lock(&cp->cache_depot_lock);
	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
	mp->mag_next = mlp->ml_list;
	mlp->ml_list = mp;
	mlp->ml_total++;
	(void) mutex_unlock(&cp->cache_depot_lock);
}

/*
 * Update the working set statistics for cp's depot.
 */
static void
umem_depot_ws_update(umem_cache_t *cp)
{
	(void) mutex_lock(&cp->cache_depot_lock);
	cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
	cp->cache_full.ml_min = cp->cache_full.ml_total;
	cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
	cp->cache_empty.ml_min = cp->cache_empty.ml_total;
	(void) mutex_unlock(&cp->cache_depot_lock);
}

/*
 * Reap all magazines that have fallen out of the depot's working set.
 */
static void
umem_depot_ws_reap(umem_cache_t *cp)
{
	long reap;
	umem_magazine_t *mp;

	ASSERT(cp->cache_next == NULL || IN_REAP());

	reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL)
		umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);

	reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL)
		umem_magazine_destroy(cp, mp, 0);
}

static void
umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds)
{
	ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
	    (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
	ASSERT(ccp->cc_magsize > 0);

	ccp->cc_ploaded = ccp->cc_loaded;
	ccp->cc_prounds = ccp->cc_rounds;
	ccp->cc_loaded = mp;
	ccp->cc_rounds = rounds;
}

/*
 * Allocate a constructed object from cache cp.
 */
#pragma weak umem_cache_alloc = _umem_cache_alloc
void *
_umem_cache_alloc(umem_cache_t *cp, int umflag)
{
	umem_cpu_cache_t *ccp;
	umem_magazine_t *fmp;
	void *buf;
	int flags_nfatal;

retry:
	ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
	(void) mutex_lock(&ccp->cc_lock);
	for (;;) {
		/*
		 * If there's an object available in the current CPU's
		 * loaded magazine, just take it and return.
		 */
		if (ccp->cc_rounds > 0) {
			buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
			ccp->cc_alloc++;
			(void) mutex_unlock(&ccp->cc_lock);
			if ((ccp->cc_flags & UMF_BUFTAG) &&
			    umem_cache_alloc_debug(cp, buf, umflag) == -1) {
				if (umem_alloc_retry(cp, umflag)) {
					goto retry;
				}

				return (NULL);
			}
			return (buf);
		}

		/*
		 * The loaded magazine is empty.  If the previously loaded
		 * magazine was full, exchange them and try again.
		 */
		if (ccp->cc_prounds > 0) {
			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
			continue;
		}

		/*
		 * If the magazine layer is disabled, break out now.
		 */
		if (ccp->cc_magsize == 0)
			break;

		/*
		 * Try to get a full magazine from the depot.
		 */
		fmp = umem_depot_alloc(cp, &cp->cache_full);
		if (fmp != NULL) {
			if (ccp->cc_ploaded != NULL)
				umem_depot_free(cp, &cp->cache_empty,
				    ccp->cc_ploaded);
			umem_cpu_reload(ccp, fmp, ccp->cc_magsize);
			continue;
		}

		/*
		 * There are no full magazines in the depot,
		 * so fall through to the slab layer.
		 */
		break;
	}
	(void) mutex_unlock(&ccp->cc_lock);

	/*
	 * We couldn't allocate a constructed object from the magazine layer,
	 * so get a raw buffer from the slab layer and apply its constructor.
	 */
	buf = umem_slab_alloc(cp, umflag);

	if (buf == NULL) {
		if (cp == &umem_null_cache)
			return (NULL);
		if (umem_alloc_retry(cp, umflag)) {
			goto retry;
		}

		return (NULL);
	}

	if (cp->cache_flags & UMF_BUFTAG) {
		/*
		 * Let umem_cache_alloc_debug() apply the constructor for us.
		 */
		if (umem_cache_alloc_debug(cp, buf, umflag) == -1) {
			if (umem_alloc_retry(cp, umflag)) {
				goto retry;
			}
			return (NULL);
		}
		return (buf);
	}

	/*
	 * We do not pass fatal flags on to the constructor.  This prevents
	 * leaking buffers in the event of a subordinate constructor failing.
	 */
	flags_nfatal = UMEM_DEFAULT;
	if (cp->cache_constructor != NULL &&
	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) {
		atomic_add_64(&cp->cache_alloc_fail, 1);
		umem_slab_free(cp, buf);

		if (umem_alloc_retry(cp, umflag)) {
			goto retry;
		}
		return (NULL);
	}

	return (buf);
}

/*
 * Free a constructed object to cache cp.
 */
#pragma weak umem_cache_free = _umem_cache_free
void
_umem_cache_free(umem_cache_t *cp, void *buf)
{
	umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
	umem_magazine_t *emp;
	umem_magtype_t *mtp;

	if (ccp->cc_flags & UMF_BUFTAG)
		if (umem_cache_free_debug(cp, buf) == -1)
			return;

	(void) mutex_lock(&ccp->cc_lock);
	for (;;) {
		/*
		 * If there's a slot available in the current CPU's
		 * loaded magazine, just put the object there and return.
		 */
		if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
			ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
			ccp->cc_free++;
			(void) mutex_unlock(&ccp->cc_lock);
			return;
		}

		/*
		 * The loaded magazine is full.  If the previously loaded
		 * magazine was empty, exchange them and try again.
		 */
		if (ccp->cc_prounds == 0) {
			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
			continue;
		}

		/*
		 * If the magazine layer is disabled, break out now.
		 */
		if (ccp->cc_magsize == 0)
			break;

		/*
		 * Try to get an empty magazine from the depot.
		 */
		emp = umem_depot_alloc(cp, &cp->cache_empty);
		if (emp != NULL) {
			if (ccp->cc_ploaded != NULL)
				umem_depot_free(cp, &cp->cache_full,
				    ccp->cc_ploaded);
			umem_cpu_reload(ccp, emp, 0);
			continue;
		}

		/*
		 * There are no empty magazines in the depot,
		 * so try to allocate a new one.  We must drop all locks
		 * across umem_cache_alloc() because lower layers may
		 * attempt to allocate from this cache.
		 */
		mtp = cp->cache_magtype;
		(void) mutex_unlock(&ccp->cc_lock);
		emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT);
		(void) mutex_lock(&ccp->cc_lock);

		if (emp != NULL) {
			/*
			 * We successfully allocated an empty magazine.
			 * However, we had to drop ccp->cc_lock to do it,
			 * so the cache's magazine size may have changed.
			 * If so, free the magazine and try again.
			 */
			if (ccp->cc_magsize != mtp->mt_magsize) {
				(void) mutex_unlock(&ccp->cc_lock);
				_umem_cache_free(mtp->mt_cache, emp);
				(void) mutex_lock(&ccp->cc_lock);
				continue;
			}

			/*
			 * We got a magazine of the right size.  Add it to
			 * the depot and try the whole dance again.
			 */
			umem_depot_free(cp, &cp->cache_empty, emp);
			continue;
		}

		/*
		 * We couldn't allocate an empty magazine,
		 * so fall through to the slab layer.
		 */
		break;
	}
	(void) mutex_unlock(&ccp->cc_lock);

	/*
	 * We couldn't free our constructed object to the magazine layer,
	 * so apply its destructor and free it to the slab layer.
	 * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug()
	 * will have already applied the destructor.
	 */
	if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL)
		cp->cache_destructor(buf, cp->cache_private);

	umem_slab_free(cp, buf);
}

#pragma weak umem_zalloc = _umem_zalloc
void *
_umem_zalloc(size_t size, int umflag)
{
	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
	void *buf;

retry:
	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
		umem_cache_t *cp = umem_alloc_table[index];
		buf = _umem_cache_alloc(cp, umflag);
		if (buf != NULL) {
			if (cp->cache_flags & UMF_BUFTAG) {
				umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
				((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
				((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
			}
			bzero(buf, size);
		} else if (umem_alloc_retry(cp, umflag))
			goto retry;
	} else {
		buf = _umem_alloc(size, umflag);	/* handles failure */
		if (buf != NULL)
			bzero(buf, size);
	}
	return (buf);
}

#pragma weak umem_alloc = _umem_alloc
void *
_umem_alloc(size_t size, int umflag)
{
	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
	void *buf;
umem_alloc_retry:
	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
		umem_cache_t *cp = umem_alloc_table[index];
		buf = _umem_cache_alloc(cp, umflag);
		if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) {
			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
			((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
			((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
		}
		if (buf == NULL && umem_alloc_retry(cp, umflag))
			goto umem_alloc_retry;
		return (buf);
	}
	if (size == 0)
		return (NULL);
	if (umem_oversize_arena == NULL) {
		if (umem_init())
			ASSERT(umem_oversize_arena != NULL);
		else
			return (NULL);
	}
	buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag));
	if (buf == NULL) {
		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
		if (umem_alloc_retry(NULL, umflag))
			goto umem_alloc_retry;
	}
	return (buf);
}

#pragma weak umem_alloc_align = _umem_alloc_align
void *
_umem_alloc_align(size_t size, size_t align, int umflag)
{
	void *buf;

	if (size == 0)
		return (NULL);
	if ((align & (align - 1)) != 0)
		return (NULL);
	if (align < UMEM_ALIGN)
		align = UMEM_ALIGN;

umem_alloc_align_retry:
	if (umem_memalign_arena == NULL) {
		if (umem_init())
			ASSERT(umem_oversize_arena != NULL);
		else
			return (NULL);
	}
	buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL,
	    UMEM_VMFLAGS(umflag));
	if (buf == NULL) {
		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
		if (umem_alloc_retry(NULL, umflag))
			goto umem_alloc_align_retry;
	}
	return (buf);
}

#pragma weak umem_free = _umem_free
void
_umem_free(void *buf, size_t size)
{
	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;

	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
		umem_cache_t *cp = umem_alloc_table[index];
		if (cp->cache_flags & UMF_BUFTAG) {
			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
			uint32_t *ip = (uint32_t *)btp;
			if (ip[1] != UMEM_SIZE_ENCODE(size)) {
				if (*(uint64_t *)buf == UMEM_FREE_PATTERN) {
					umem_error(UMERR_DUPFREE, cp, buf);
					return;
				}
				if (UMEM_SIZE_VALID(ip[1])) {
					ip[0] = UMEM_SIZE_ENCODE(size);
					umem_error(UMERR_BADSIZE, cp, buf);
				} else {
					umem_error(UMERR_REDZONE, cp, buf);
				}
				return;
			}
			if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) {
				umem_error(UMERR_REDZONE, cp, buf);
				return;
			}
			btp->bt_redzone = UMEM_REDZONE_PATTERN;
		}
		_umem_cache_free(cp, buf);
	} else {
		if (buf == NULL && size == 0)
			return;
		vmem_free(umem_oversize_arena, buf, size);
	}
}

#pragma weak umem_free_align = _umem_free_align
void
_umem_free_align(void *buf, size_t size)
{
	if (buf == NULL && size == 0)
		return;
	vmem_xfree(umem_memalign_arena, buf, size);
}

static void *
umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
{
	size_t realsize = size + vmp->vm_quantum;

	/*
	 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
	 * vm_quantum will cause integer wraparound.  Check for this, and
	 * blow off the firewall page in this case.  Note that such a
	 * giant allocation (the entire address space) can never be
	 * satisfied, so it will either fail immediately (VM_NOSLEEP)
	 * or sleep forever (VM_SLEEP).  Thus, there is no need for a
	 * corresponding check in umem_firewall_va_free().
	 */
	if (realsize < size)
		realsize = size;

	return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT));
}

static void
umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
{
	vmem_free(vmp, addr, size + vmp->vm_quantum);
}

/*
 * Reclaim all unused memory from a cache.
 */
static void
umem_cache_reap(umem_cache_t *cp)
{
	/*
	 * Ask the cache's owner to free some memory if possible.
	 * The idea is to handle things like the inode cache, which
	 * typically sits on a bunch of memory that it doesn't truly
	 * *need*.  Reclaim policy is entirely up to the owner; this
	 * callback is just an advisory plea for help.
	 */
	if (cp->cache_reclaim != NULL)
		cp->cache_reclaim(cp->cache_private);

	umem_depot_ws_reap(cp);
}

/*
 * Purge all magazines from a cache and set its magazine limit to zero.
 * All calls are serialized by being done by the update thread, except for
 * the final call from umem_cache_destroy().
 */
static void
umem_cache_magazine_purge(umem_cache_t *cp)
{
	umem_cpu_cache_t *ccp;
	umem_magazine_t *mp, *pmp;
	int rounds, prounds, cpu_seqid;

	ASSERT(cp->cache_next == NULL || IN_UPDATE());

	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
		ccp = &cp->cache_cpu[cpu_seqid];

		(void) mutex_lock(&ccp->cc_lock);
		mp = ccp->cc_loaded;
		pmp = ccp->cc_ploaded;
		rounds = ccp->cc_rounds;
		prounds = ccp->cc_prounds;
		ccp->cc_loaded = NULL;
		ccp->cc_ploaded = NULL;
		ccp->cc_rounds = -1;
		ccp->cc_prounds = -1;
		ccp->cc_magsize = 0;
		(void) mutex_unlock(&ccp->cc_lock);

		if (mp)
			umem_magazine_destroy(cp, mp, rounds);
		if (pmp)
			umem_magazine_destroy(cp, pmp, prounds);
	}

	/*
	 * Updating the working set statistics twice in a row has the
	 * effect of setting the working set size to zero, so everything
	 * is eligible for reaping.
	 */
	umem_depot_ws_update(cp);
	umem_depot_ws_update(cp);

	umem_depot_ws_reap(cp);
}

/*
 * Enable per-cpu magazines on a cache.
 */
static void
umem_cache_magazine_enable(umem_cache_t *cp)
{
	int cpu_seqid;

	if (cp->cache_flags & UMF_NOMAGAZINE)
		return;

	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
		(void) mutex_lock(&ccp->cc_lock);
		ccp->cc_magsize = cp->cache_magtype->mt_magsize;
		(void) mutex_unlock(&ccp->cc_lock);
	}

}

/*
 * Recompute a cache's magazine size.  The trade-off is that larger magazines
 * provide a higher transfer rate with the depot, while smaller magazines
 * reduce memory consumption.  Magazine resizing is an expensive operation;
 * it should not be done frequently.
 *
 * Changes to the magazine size are serialized by only having one thread
 * doing updates. (the update thread)
 *
 * Note: at present this only grows the magazine size.  It might be useful
 * to allow shrinkage too.
 */
static void
umem_cache_magazine_resize(umem_cache_t *cp)
{
	umem_magtype_t *mtp = cp->cache_magtype;

	ASSERT(IN_UPDATE());

	if (cp->cache_chunksize < mtp->mt_maxbuf) {
		umem_cache_magazine_purge(cp);
		(void) mutex_lock(&cp->cache_depot_lock);
		cp->cache_magtype = ++mtp;
		cp->cache_depot_contention_prev =
		    cp->cache_depot_contention + INT_MAX;
		(void) mutex_unlock(&cp->cache_depot_lock);
		umem_cache_magazine_enable(cp);
	}
}

/*
 * Rescale a cache's hash table, so that the table size is roughly the
 * cache size.  We want the average lookup time to be extremely small.
 */
static void
umem_hash_rescale(umem_cache_t *cp)
{
	umem_bufctl_t **old_table, **new_table, *bcp;
	size_t old_size, new_size, h;

	ASSERT(IN_UPDATE());

	new_size = MAX(UMEM_HASH_INITIAL,
	    1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
	old_size = cp->cache_hash_mask + 1;

	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
		return;

	new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *),
	    VM_NOSLEEP);
	if (new_table == NULL)
		return;
	bzero(new_table, new_size * sizeof (void *));

	(void) mutex_lock(&cp->cache_lock);

	old_size = cp->cache_hash_mask + 1;
	old_table = cp->cache_hash_table;

	cp->cache_hash_mask = new_size - 1;
	cp->cache_hash_table = new_table;
	cp->cache_rescale++;

	for (h = 0; h < old_size; h++) {
		bcp = old_table[h];
		while (bcp != NULL) {
			void *addr = bcp->bc_addr;
			umem_bufctl_t *next_bcp = bcp->bc_next;
			umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr);
			bcp->bc_next = *hash_bucket;
			*hash_bucket = bcp;
			bcp = next_bcp;
		}
	}

	(void) mutex_unlock(&cp->cache_lock);

	vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *));
}

/*
 * Perform periodic maintenance on a cache: hash rescaling,
 * depot working-set update, and magazine resizing.
 */
void
umem_cache_update(umem_cache_t *cp)
{
	int update_flags = 0;

	ASSERT(MUTEX_HELD(&umem_cache_lock));

	/*
	 * If the cache has become much larger or smaller than its hash table,
	 * fire off a request to rescale the hash table.
	 */
	(void) mutex_lock(&cp->cache_lock);

	if ((cp->cache_flags & UMF_HASH) &&
	    (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
	    (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
	    cp->cache_hash_mask > UMEM_HASH_INITIAL)))
		update_flags |= UMU_HASH_RESCALE;

	(void) mutex_unlock(&cp->cache_lock);

	/*
	 * Update the depot working set statistics.
	 */
	umem_depot_ws_update(cp);

	/*
	 * If there's a lot of contention in the depot,
	 * increase the magazine size.
	 */
	(void) mutex_lock(&cp->cache_depot_lock);

	if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
	    (int)(cp->cache_depot_contention -
	    cp->cache_depot_contention_prev) > umem_depot_contention)
		update_flags |= UMU_MAGAZINE_RESIZE;

	cp->cache_depot_contention_prev = cp->cache_depot_contention;

	(void) mutex_unlock(&cp->cache_depot_lock);

	if (update_flags)
		umem_add_update(cp, update_flags);
}

/*
 * Runs all pending updates.
 *
 * The update lock must be held on entrance, and will be held on exit.
 */
void
umem_process_updates(void)
{
	ASSERT(MUTEX_HELD(&umem_update_lock));

	while (umem_null_cache.cache_unext != &umem_null_cache) {
		int notify = 0;
		umem_cache_t *cp = umem_null_cache.cache_unext;

		cp->cache_uprev->cache_unext = cp->cache_unext;
		cp->cache_unext->cache_uprev = cp->cache_uprev;
		cp->cache_uprev = cp->cache_unext = NULL;

		ASSERT(!(cp->cache_uflags & UMU_ACTIVE));

		while (cp->cache_uflags) {
			int uflags = (cp->cache_uflags |= UMU_ACTIVE);
			(void) mutex_unlock(&umem_update_lock);

			/*
			 * The order here is important.  Each step can speed up
			 * later steps.
			 */

			if (uflags & UMU_HASH_RESCALE)
				umem_hash_rescale(cp);

			if (uflags & UMU_MAGAZINE_RESIZE)
				umem_cache_magazine_resize(cp);

			if (uflags & UMU_REAP)
				umem_cache_reap(cp);

			(void) mutex_lock(&umem_update_lock);

			/*
			 * check if anyone has requested notification
			 */
			if (cp->cache_uflags & UMU_NOTIFY) {
				uflags |= UMU_NOTIFY;
				notify = 1;
			}
			cp->cache_uflags &= ~uflags;
		}
		if (notify)
			(void) cond_broadcast(&umem_update_cv);
	}
}

#ifndef UMEM_STANDALONE
static void
umem_st_update(void)
{
	ASSERT(MUTEX_HELD(&umem_update_lock));
	ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0);

	umem_st_update_thr = thr_self();

	(void) mutex_unlock(&umem_update_lock);

	vmem_update(NULL);
	umem_cache_applyall(umem_cache_update);

	(void) mutex_lock(&umem_update_lock);

	umem_process_updates();	/* does all of the requested work */

	umem_reap_next = gethrtime() +
	    (hrtime_t)umem_reap_interval * NANOSEC;

	umem_reaping = UMEM_REAP_DONE;

	umem_st_update_thr = 0;
}
#endif

/*
 * Reclaim all unused memory from all caches.  Called from vmem when memory
 * gets tight.  Must be called with no locks held.
 *
 * This just requests a reap on all caches, and notifies the update thread.
 */
void
umem_reap(void)
{
#ifndef UMEM_STANDALONE
	extern int __nthreads(void);
#endif

	if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE ||
	    gethrtime() < umem_reap_next)
		return;

	(void) mutex_lock(&umem_update_lock);

	if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) {
		(void) mutex_unlock(&umem_update_lock);
		return;
	}
	umem_reaping = UMEM_REAP_ADDING;	/* lock out other reaps */

	(void) mutex_unlock(&umem_update_lock);

	umem_updateall(UMU_REAP);

	(void) mutex_lock(&umem_update_lock);

	umem_reaping = UMEM_REAP_ACTIVE;

	/* Standalone is single-threaded */
#ifndef UMEM_STANDALONE
	if (umem_update_thr == 0) {
		/*
		 * The update thread does not exist.  If the process is
		 * multi-threaded, create it.  If not, or the creation fails,
		 * do the update processing inline.
		 */
		ASSERT(umem_st_update_thr == 0);

		if (__nthreads() <= 1 || umem_create_update_thread() == 0)
			umem_st_update();
	}

	(void) cond_broadcast(&umem_update_cv);	/* wake up the update thread */
#endif

	(void) mutex_unlock(&umem_update_lock);
}

umem_cache_t *
umem_cache_create(
	char *name,		/* descriptive name for this cache */
	size_t bufsize,		/* size of the objects it manages */
	size_t align,		/* required object alignment */
	umem_constructor_t *constructor, /* object constructor */
	umem_destructor_t *destructor, /* object destructor */
	umem_reclaim_t *reclaim, /* memory reclaim callback */
	void *private,		/* pass-thru arg for constr/destr/reclaim */
	vmem_t *vmp,		/* vmem source for slab allocation */
	int cflags)		/* cache creation flags */
{
	int cpu_seqid;
	size_t chunksize;
	umem_cache_t *cp, *cnext, *cprev;
	umem_magtype_t *mtp;
	size_t csize;
	size_t phase;

	/*
	 * The init thread is allowed to create internal and quantum caches.
	 *
	 * Other threads must wait until until initialization is complete.
	 */
	if (umem_init_thr == thr_self())
		ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0);
	else {
		ASSERT(!(cflags & UMC_INTERNAL));
		if (umem_ready != UMEM_READY && umem_init() == 0) {
			errno = EAGAIN;
			return (NULL);
		}
	}

	csize = UMEM_CACHE_SIZE(umem_max_ncpus);
	phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE);

	if (vmp == NULL)
		vmp = umem_default_arena;

	ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0);

	/*
	 * Check that the arguments are reasonable
	 */
	if ((align & (align - 1)) != 0 || align > vmp->vm_quantum ||
	    ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) ||
	    name == NULL || bufsize == 0) {
		errno = EINVAL;
		return (NULL);
	}

	/*
	 * If align == 0, we set it to the minimum required alignment.
	 *
	 * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless
	 * UMC_NOTOUCH was passed.
	 */
	if (align == 0) {
		if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN)
			align = UMEM_SECOND_ALIGN;
		else
			align = UMEM_ALIGN;
	} else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0)
		align = UMEM_ALIGN;


	/*
	 * Get a umem_cache structure.  We arrange that cp->cache_cpu[]
	 * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent
	 * false sharing of per-CPU data.
	 */
	cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase,
	    0, NULL, NULL, VM_NOSLEEP);

	if (cp == NULL) {
		errno = EAGAIN;
		return (NULL);
	}

	bzero(cp, csize);

	(void) mutex_lock(&umem_flags_lock);
	if (umem_flags & UMF_RANDOMIZE)
		umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) |
		    UMF_RANDOMIZE;
	cp->cache_flags = umem_flags | (cflags & UMF_DEBUG);
	(void) mutex_unlock(&umem_flags_lock);

	/*
	 * Make sure all the various flags are reasonable.
	 */
	if (cp->cache_flags & UMF_LITE) {
		if (bufsize >= umem_lite_minsize &&
		    align <= umem_lite_maxalign &&
		    P2PHASE(bufsize, umem_lite_maxalign) != 0) {
			cp->cache_flags |= UMF_BUFTAG;
			cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);
		} else {
			cp->cache_flags &= ~UMF_DEBUG;
		}
	}

	if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT))
		cp->cache_flags |= UMF_NOMAGAZINE;

	if (cflags & UMC_NODEBUG)
		cp->cache_flags &= ~UMF_DEBUG;

	if (cflags & UMC_NOTOUCH)
		cp->cache_flags &= ~UMF_TOUCH;

	if (cflags & UMC_NOHASH)
		cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);

	if (cflags & UMC_NOMAGAZINE)
		cp->cache_flags |= UMF_NOMAGAZINE;

	if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH))
		cp->cache_flags |= UMF_REDZONE;

	if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall &&
	    !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH))
		cp->cache_flags |= UMF_FIREWALL;

	if (vmp != umem_default_arena || umem_firewall_arena == NULL)
		cp->cache_flags &= ~UMF_FIREWALL;

	if (cp->cache_flags & UMF_FIREWALL) {
		cp->cache_flags &= ~UMF_BUFTAG;
		cp->cache_flags |= UMF_NOMAGAZINE;
		ASSERT(vmp == umem_default_arena);
		vmp = umem_firewall_arena;
	}

	/*
	 * Set cache properties.
	 */
	(void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1);
	cp->cache_bufsize = bufsize;
	cp->cache_align = align;
	cp->cache_constructor = constructor;
	cp->cache_destructor = destructor;
	cp->cache_reclaim = reclaim;
	cp->cache_private = private;
	cp->cache_arena = vmp;
	cp->cache_cflags = cflags;
	cp->cache_cpu_mask = umem_cpu_mask;

	/*
	 * Determine the chunk size.
	 */
	chunksize = bufsize;

	if (align >= UMEM_ALIGN) {
		chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN);
		cp->cache_bufctl = chunksize - UMEM_ALIGN;
	}

	if (cp->cache_flags & UMF_BUFTAG) {
		cp->cache_bufctl = chunksize;
		cp->cache_buftag = chunksize;
		chunksize += sizeof (umem_buftag_t);
	}

	if (cp->cache_flags & UMF_DEADBEEF) {
		cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify);
		if (cp->cache_flags & UMF_LITE)
			cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN);
	}

	cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave);

	cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);

	if (chunksize < bufsize) {
		errno = ENOMEM;
		goto fail;
	}

	/*
	 * Now that we know the chunk size, determine the optimal slab size.
	 */
	if (vmp == umem_firewall_arena) {
		cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
		cp->cache_mincolor = cp->cache_slabsize - chunksize;
		cp->cache_maxcolor = cp->cache_mincolor;
		cp->cache_flags |= UMF_HASH;
		ASSERT(!(cp->cache_flags & UMF_BUFTAG));
	} else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) &&
	    !(cp->cache_flags & UMF_AUDIT) &&
	    chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) {
		cp->cache_slabsize = vmp->vm_quantum;
		cp->cache_mincolor = 0;
		cp->cache_maxcolor =
		    (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize;

		if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) {
			errno = EINVAL;
			goto fail;
		}
		ASSERT(!(cp->cache_flags & UMF_AUDIT));
	} else {
		size_t chunks, waste, slabsize;
		size_t minwaste = LONG_MAX;
		size_t bestfit = SIZE_MAX;

		for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) {
			slabsize = P2ROUNDUP(chunksize * chunks,
			    vmp->vm_quantum);
			/*
			 * check for overflow
			 */
			if ((slabsize / chunks) < chunksize) {
				errno = ENOMEM;
				goto fail;
			}
			chunks = slabsize / chunksize;
			waste = (slabsize % chunksize) / chunks;
			if (waste < minwaste) {
				minwaste = waste;
				bestfit = slabsize;
			}
		}
		if (cflags & UMC_QCACHE)
			bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64);
		if (bestfit == SIZE_MAX) {
			errno = ENOMEM;
			goto fail;
		}
		cp->cache_slabsize = bestfit;
		cp->cache_mincolor = 0;
		cp->cache_maxcolor = bestfit % chunksize;
		cp->cache_flags |= UMF_HASH;
	}

	if (cp->cache_flags & UMF_HASH) {
		ASSERT(!(cflags & UMC_NOHASH));
		cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ?
		    umem_bufctl_audit_cache : umem_bufctl_cache;
	}

	if (cp->cache_maxcolor >= vmp->vm_quantum)
		cp->cache_maxcolor = vmp->vm_quantum - 1;

	cp->cache_color = cp->cache_mincolor;

	/*
	 * Initialize the rest of the slab layer.
	 */
	(void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL);

	cp->cache_freelist = &cp->cache_nullslab;
	cp->cache_nullslab.slab_cache = cp;
	cp->cache_nullslab.slab_refcnt = -1;
	cp->cache_nullslab.slab_next = &cp->cache_nullslab;
	cp->cache_nullslab.slab_prev = &cp->cache_nullslab;

	if (cp->cache_flags & UMF_HASH) {
		cp->cache_hash_table = vmem_alloc(umem_hash_arena,
		    UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP);
		if (cp->cache_hash_table == NULL) {
			errno = EAGAIN;
			goto fail_lock;
		}
		bzero(cp->cache_hash_table,
		    UMEM_HASH_INITIAL * sizeof (void *));
		cp->cache_hash_mask = UMEM_HASH_INITIAL - 1;
		cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
	}

	/*
	 * Initialize the depot.
	 */
	(void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL);

	for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
		continue;

	cp->cache_magtype = mtp;

	/*
	 * Initialize the CPU layer.
	 */
	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
		(void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL);
		ccp->cc_flags = cp->cache_flags;
		ccp->cc_rounds = -1;
		ccp->cc_prounds = -1;
	}

	/*
	 * Add the cache to the global list.  This makes it visible
	 * to umem_update(), so the cache must be ready for business.
	 */
	(void) mutex_lock(&umem_cache_lock);
	cp->cache_next = cnext = &umem_null_cache;
	cp->cache_prev = cprev = umem_null_cache.cache_prev;
	cnext->cache_prev = cp;
	cprev->cache_next = cp;
	(void) mutex_unlock(&umem_cache_lock);

	if (umem_ready == UMEM_READY)
		umem_cache_magazine_enable(cp);

	return (cp);

fail_lock:
	(void) mutex_destroy(&cp->cache_lock);
fail:
	vmem_xfree(umem_cache_arena, cp, csize);
	return (NULL);
}

void
umem_cache_destroy(umem_cache_t *cp)
{
	int cpu_seqid;

	/*
	 * Remove the cache from the global cache list so that no new updates
	 * will be scheduled on its behalf, wait for any pending tasks to
	 * complete, purge the cache, and then destroy it.
	 */
	(void) mutex_lock(&umem_cache_lock);
	cp->cache_prev->cache_next = cp->cache_next;
	cp->cache_next->cache_prev = cp->cache_prev;
	cp->cache_prev = cp->cache_next = NULL;
	(void) mutex_unlock(&umem_cache_lock);

	umem_remove_updates(cp);

	umem_cache_magazine_purge(cp);

	(void) mutex_lock(&cp->cache_lock);
	if (cp->cache_buftotal != 0)
		log_message("umem_cache_destroy: '%s' (%p) not empty\n",
		    cp->cache_name, (void *)cp);
	cp->cache_reclaim = NULL;
	/*
	 * The cache is now dead.  There should be no further activity.
	 * We enforce this by setting land mines in the constructor and
	 * destructor routines that induce a segmentation fault if invoked.
	 */
	cp->cache_constructor = (umem_constructor_t *)1;
	cp->cache_destructor = (umem_destructor_t *)2;
	(void) mutex_unlock(&cp->cache_lock);

	if (cp->cache_hash_table != NULL)
		vmem_free(umem_hash_arena, cp->cache_hash_table,
		    (cp->cache_hash_mask + 1) * sizeof (void *));

	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++)
		(void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);

	(void) mutex_destroy(&cp->cache_depot_lock);
	(void) mutex_destroy(&cp->cache_lock);

	vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus));
}

void
umem_alloc_sizes_clear(void)
{
	int i;

	umem_alloc_sizes[0] = UMEM_MAXBUF;
	for (i = 1; i < NUM_ALLOC_SIZES; i++)
		umem_alloc_sizes[i] = 0;
}

void
umem_alloc_sizes_add(size_t size_arg)
{
	int i, j;
	size_t size = size_arg;

	if (size == 0) {
		log_message("size_add: cannot add zero-sized cache\n",
		    size, UMEM_MAXBUF);
		return;
	}

	if (size > UMEM_MAXBUF) {
		log_message("size_add: %ld > %d, cannot add\n", size,
		    UMEM_MAXBUF);
		return;
	}

	if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) {
		log_message("size_add: no space in alloc_table for %d\n",
		    size);
		return;
	}

	if (P2PHASE(size, UMEM_ALIGN) != 0) {
		size = P2ROUNDUP(size, UMEM_ALIGN);
		log_message("size_add: rounding %d up to %d\n", size_arg,
		    size);
	}

	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
		int cur = umem_alloc_sizes[i];
		if (cur == size) {
			log_message("size_add: %ld already in table\n",
			    size);
			return;
		}
		if (cur > size)
			break;
	}

	for (j = NUM_ALLOC_SIZES - 1; j > i; j--)
		umem_alloc_sizes[j] = umem_alloc_sizes[j-1];
	umem_alloc_sizes[i] = size;
}

void
umem_alloc_sizes_remove(size_t size)
{
	int i;

	if (size == UMEM_MAXBUF) {
		log_message("size_remove: cannot remove %ld\n", size);
		return;
	}

	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
		int cur = umem_alloc_sizes[i];
		if (cur == size)
			break;
		else if (cur > size || cur == 0) {
			log_message("size_remove: %ld not found in table\n",
			    size);
			return;
		}
	}

	for (; i + 1 < NUM_ALLOC_SIZES; i++)
		umem_alloc_sizes[i] = umem_alloc_sizes[i+1];
	umem_alloc_sizes[i] = 0;
}

/*
 * We've been called back from libc to indicate that thread is terminating and
 * that it needs to release the per-thread memory that it has. We get to know
 * which entry in the thread's tmem array the allocation came from. Currently
 * this refers to first n umem_caches which makes this a pretty simple indexing
 * job.
 */
static void
umem_cache_tmem_cleanup(void *buf, int entry)
{
	size_t size;
	umem_cache_t *cp;

	size = umem_alloc_sizes[entry];
	cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT];
	_umem_cache_free(cp, buf);
}

static int
umem_cache_init(void)
{
	int i;
	size_t size, max_size;
	umem_cache_t *cp;
	umem_magtype_t *mtp;
	char name[UMEM_CACHE_NAMELEN + 1];
	umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES];

	for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) {
		mtp = &umem_magtype[i];
		(void) snprintf(name, sizeof (name), "umem_magazine_%d",
		    mtp->mt_magsize);
		mtp->mt_cache = umem_cache_create(name,
		    (mtp->mt_magsize + 1) * sizeof (void *),
		    mtp->mt_align, NULL, NULL, NULL, NULL,
		    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
		if (mtp->mt_cache == NULL)
			return (0);
	}

	umem_slab_cache = umem_cache_create("umem_slab_cache",
	    sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL,
	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);

	if (umem_slab_cache == NULL)
		return (0);

	umem_bufctl_cache = umem_cache_create("umem_bufctl_cache",
	    sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL,
	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);

	if (umem_bufctl_cache == NULL)
		return (0);

	/*
	 * The size of the umem_bufctl_audit structure depends upon
	 * umem_stack_depth.   See umem_impl.h for details on the size
	 * restrictions.
	 */

	size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
	max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE;

	if (size > max_size) {			/* too large -- truncate */
		int max_frames = UMEM_MAX_STACK_DEPTH;

		ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size);

		umem_stack_depth = max_frames;
		size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
	}

	umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache",
	    size, 0, NULL, NULL, NULL, NULL, umem_internal_arena,
	    UMC_NOHASH | UMC_INTERNAL);

	if (umem_bufctl_audit_cache == NULL)
		return (0);

	if (vmem_backend & VMEM_BACKEND_MMAP)
		umem_va_arena = vmem_create("umem_va",
		    NULL, 0, pagesize,
		    vmem_alloc, vmem_free, heap_arena,
		    8 * pagesize, VM_NOSLEEP);
	else
		umem_va_arena = heap_arena;

	if (umem_va_arena == NULL)
		return (0);

	umem_default_arena = vmem_create("umem_default",
	    NULL, 0, pagesize,
	    heap_alloc, heap_free, umem_va_arena,
	    0, VM_NOSLEEP);

	if (umem_default_arena == NULL)
		return (0);

	/*
	 * make sure the umem_alloc table initializer is correct
	 */
	i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table);
	ASSERT(umem_alloc_table[i - 1] == &umem_null_cache);

	/*
	 * Create the default caches to back umem_alloc()
	 */
	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
		size_t cache_size = umem_alloc_sizes[i];
		size_t align = 0;

		if (cache_size == 0)
			break;		/* 0 terminates the list */

		/*
		 * If they allocate a multiple of the coherency granularity,
		 * they get a coherency-granularity-aligned address.
		 */
		if (IS_P2ALIGNED(cache_size, 64))
			align = 64;
		if (IS_P2ALIGNED(cache_size, pagesize))
			align = pagesize;
		(void) snprintf(name, sizeof (name), "umem_alloc_%lu",
		    (long)cache_size);

		cp = umem_cache_create(name, cache_size, align,
		    NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL);
		if (cp == NULL)
			return (0);

		umem_alloc_caches[i] = cp;
	}

	umem_tmem_off = _tmem_get_base();
	_tmem_set_cleanup(umem_cache_tmem_cleanup);

#ifndef	UMEM_STANDALONE
	if (umem_genasm_supported && !(umem_flags & UMF_DEBUG) &&
	    !(umem_flags & UMF_NOMAGAZINE) &&
	    umem_ptc_size > 0) {
		umem_ptc_enabled = umem_genasm(umem_alloc_sizes,
		    umem_alloc_caches, i) ? 1 : 0;
	}
#else
	umem_ptc_enabled = 0;
#endif

	/*
	 * Initialization cannot fail at this point.  Make the caches
	 * visible to umem_alloc() and friends.
	 */
	size = UMEM_ALIGN;
	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
		size_t cache_size = umem_alloc_sizes[i];

		if (cache_size == 0)
			break;		/* 0 terminates the list */

		cp = umem_alloc_caches[i];

		while (size <= cache_size) {
			umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp;
			size += UMEM_ALIGN;
		}
	}
	ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF);
	return (1);
}

/*
 * umem_startup() is called early on, and must be called explicitly if we're
 * the standalone version.
 */
#ifdef UMEM_STANDALONE
void
#else
#pragma init(umem_startup)
static void
#endif
umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack,
    caddr_t maxstack)
{
#ifdef UMEM_STANDALONE
	int idx;
	/* Standalone doesn't fork */
#else
	umem_forkhandler_init(); /* register the fork handler */
#endif

#ifdef __lint
	/* make lint happy */
	minstack = maxstack;
#endif

#ifdef UMEM_STANDALONE
	umem_ready = UMEM_READY_STARTUP;
	umem_init_env_ready = 0;

	umem_min_stack = minstack;
	umem_max_stack = maxstack;

	nofail_callback = NULL;
	umem_slab_cache = NULL;
	umem_bufctl_cache = NULL;
	umem_bufctl_audit_cache = NULL;
	heap_arena = NULL;
	heap_alloc = NULL;
	heap_free = NULL;
	umem_internal_arena = NULL;
	umem_cache_arena = NULL;
	umem_hash_arena = NULL;
	umem_log_arena = NULL;
	umem_oversize_arena = NULL;
	umem_va_arena = NULL;
	umem_default_arena = NULL;
	umem_firewall_va_arena = NULL;
	umem_firewall_arena = NULL;
	umem_memalign_arena = NULL;
	umem_transaction_log = NULL;
	umem_content_log = NULL;
	umem_failure_log = NULL;
	umem_slab_log = NULL;
	umem_cpu_mask = 0;

	umem_cpus = &umem_startup_cpu;
	umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0);
	umem_startup_cpu.cpu_number = 0;

	bcopy(&umem_null_cache_template, &umem_null_cache,
	    sizeof (umem_cache_t));

	for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++)
		umem_alloc_table[idx] = &umem_null_cache;
#endif

	/*
	 * Perform initialization specific to the way we've been compiled
	 * (library or standalone)
	 */
	umem_type_init(start, len, pagesize);

	vmem_startup();
}

int
umem_init(void)
{
	size_t maxverify, minfirewall;
	size_t size;
	int idx;
	umem_cpu_t *new_cpus;

	vmem_t *memalign_arena, *oversize_arena;

	if (thr_self() != umem_init_thr) {
		/*
		 * The usual case -- non-recursive invocation of umem_init().
		 */
		(void) mutex_lock(&umem_init_lock);
		if (umem_ready != UMEM_READY_STARTUP) {
			/*
			 * someone else beat us to initializing umem.  Wait
			 * for them to complete, then return.
			 */
			while (umem_ready == UMEM_READY_INITING) {
				int cancel_state;

				(void) pthread_setcancelstate(
				    PTHREAD_CANCEL_DISABLE, &cancel_state);
				(void) cond_wait(&umem_init_cv,
				    &umem_init_lock);
				(void) pthread_setcancelstate(
				    cancel_state, NULL);
			}
			ASSERT(umem_ready == UMEM_READY ||
			    umem_ready == UMEM_READY_INIT_FAILED);
			(void) mutex_unlock(&umem_init_lock);
			return (umem_ready == UMEM_READY);
		}

		ASSERT(umem_ready == UMEM_READY_STARTUP);
		ASSERT(umem_init_env_ready == 0);

		umem_ready = UMEM_READY_INITING;
		umem_init_thr = thr_self();

		(void) mutex_unlock(&umem_init_lock);
		umem_setup_envvars(0);		/* can recurse -- see below */
		if (umem_init_env_ready) {
			/*
			 * initialization was completed already
			 */
			ASSERT(umem_ready == UMEM_READY ||
			    umem_ready == UMEM_READY_INIT_FAILED);
			ASSERT(umem_init_thr == 0);
			return (umem_ready == UMEM_READY);
		}
	} else if (!umem_init_env_ready) {
		/*
		 * The umem_setup_envvars() call (above) makes calls into
		 * the dynamic linker and directly into user-supplied code.
		 * Since we cannot know what that code will do, we could be
		 * recursively invoked (by, say, a malloc() call in the code
		 * itself, or in a (C++) _init section it causes to be fired).
		 *
		 * This code is where we end up if such recursion occurs.  We
		 * first clean up any partial results in the envvar code, then
		 * proceed to finish initialization processing in the recursive
		 * call.  The original call will notice this, and return
		 * immediately.
		 */
		umem_setup_envvars(1);		/* clean up any partial state */
	} else {
		umem_panic(
		    "recursive allocation while initializing umem\n");
	}
	umem_init_env_ready = 1;

	/*
	 * From this point until we finish, recursion into umem_init() will
	 * cause a umem_panic().
	 */
	maxverify = minfirewall = ULONG_MAX;

	/* LINTED constant condition */
	if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) {
		umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n",
		    sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE);
	}

	umem_max_ncpus = umem_get_max_ncpus();

	/*
	 * load tunables from environment
	 */
	umem_process_envvars();

	if (issetugid())
		umem_mtbf = 0;

	/*
	 * set up vmem
	 */
	if (!(umem_flags & UMF_AUDIT))
		vmem_no_debug();

	heap_arena = vmem_heap_arena(&heap_alloc, &heap_free);

	pagesize = heap_arena->vm_quantum;

	umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize,
	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);

	umem_default_arena = umem_internal_arena;

	if (umem_internal_arena == NULL)
		goto fail;

	umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN,
	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);

	umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN,
	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);

	umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN,
	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);

	umem_firewall_va_arena = vmem_create("umem_firewall_va",
	    NULL, 0, pagesize,
	    umem_firewall_va_alloc, umem_firewall_va_free, heap_arena,
	    0, VM_NOSLEEP);

	if (umem_cache_arena == NULL || umem_hash_arena == NULL ||
	    umem_log_arena == NULL || umem_firewall_va_arena == NULL)
		goto fail;

	umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize,
	    heap_alloc, heap_free, umem_firewall_va_arena, 0,
	    VM_NOSLEEP);

	if (umem_firewall_arena == NULL)
		goto fail;

	oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize,
	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);

	memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN,
	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);

	if (oversize_arena == NULL || memalign_arena == NULL)
		goto fail;

	if (umem_max_ncpus > CPUHINT_MAX())
		umem_max_ncpus = CPUHINT_MAX();

	while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0)
		umem_max_ncpus++;

	if (umem_max_ncpus == 0)
		umem_max_ncpus = 1;

	size = umem_max_ncpus * sizeof (umem_cpu_t);
	new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP);
	if (new_cpus == NULL)
		goto fail;

	bzero(new_cpus, size);
	for (idx = 0; idx < umem_max_ncpus; idx++) {
		new_cpus[idx].cpu_number = idx;
		new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx);
	}
	umem_cpus = new_cpus;
	umem_cpu_mask = (umem_max_ncpus - 1);

	if (umem_maxverify == 0)
		umem_maxverify = maxverify;

	if (umem_minfirewall == 0)
		umem_minfirewall = minfirewall;

	/*
	 * Set up updating and reaping
	 */
	umem_reap_next = gethrtime() + NANOSEC;

#ifndef UMEM_STANDALONE
	(void) gettimeofday(&umem_update_next, NULL);
#endif

	/*
	 * Set up logging -- failure here is okay, since it will just disable
	 * the logs
	 */
	if (umem_logging) {
		umem_transaction_log = umem_log_init(umem_transaction_log_size);
		umem_content_log = umem_log_init(umem_content_log_size);
		umem_failure_log = umem_log_init(umem_failure_log_size);
		umem_slab_log = umem_log_init(umem_slab_log_size);
	}

	/*
	 * Set up caches -- if successful, initialization cannot fail, since
	 * allocations from other threads can now succeed.
	 */
	if (umem_cache_init() == 0) {
		log_message("unable to create initial caches\n");
		goto fail;
	}
	umem_oversize_arena = oversize_arena;
	umem_memalign_arena = memalign_arena;

	umem_cache_applyall(umem_cache_magazine_enable);

	/*
	 * initialization done, ready to go
	 */
	(void) mutex_lock(&umem_init_lock);
	umem_ready = UMEM_READY;
	umem_init_thr = 0;
	(void) cond_broadcast(&umem_init_cv);
	(void) mutex_unlock(&umem_init_lock);
	return (1);

fail:
	log_message("umem initialization failed\n");

	(void) mutex_lock(&umem_init_lock);
	umem_ready = UMEM_READY_INIT_FAILED;
	umem_init_thr = 0;
	(void) cond_broadcast(&umem_init_cv);
	(void) mutex_unlock(&umem_init_lock);
	return (0);
}

void
umem_setmtbf(uint32_t mtbf)
{
	extern uint32_t vmem_mtbf;

	umem_mtbf = mtbf;
	vmem_mtbf = mtbf;
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _UMEM_H
#define	_UMEM_H

#include <sys/types.h>
#include <sys/vmem.h>
#include <stdlib.h>

#ifdef	__cplusplus
extern "C" {
#endif

#define	UMEM_DEFAULT	0x0000	/* normal -- may fail */
#define	UMEM_NOFAIL	0x0100	/* Never fails -- may call exit(2) */

#define	UMEM_FLAGS	0xffff	/* all settable umem flags */

extern void *umem_alloc(size_t, int);
extern void *umem_alloc_align(size_t, size_t, int);
extern void *umem_zalloc(size_t, int);
extern void umem_free(void *, size_t);
extern void umem_free_align(void *, size_t);

/*
 * Flags for umem_cache_create()
 */
#define	UMC_NOTOUCH	0x00010000
#define	UMC_NODEBUG	0x00020000
#define	UMC_NOMAGAZINE	0x00040000
#define	UMC_NOHASH	0x00080000

struct umem_cache;		/* cache structure is opaque to umem clients */

typedef struct umem_cache umem_cache_t;
typedef int umem_constructor_t(void *, void *, int);
typedef void umem_destructor_t(void *, void *);
typedef void umem_reclaim_t(void *);

typedef int umem_nofail_callback_t(void);
#define	UMEM_CALLBACK_RETRY		0
#define	UMEM_CALLBACK_EXIT(status)	(0x100 | ((status) & 0xFF))

extern void umem_nofail_callback(umem_nofail_callback_t *);

extern umem_cache_t *umem_cache_create(char *, size_t,
    size_t, umem_constructor_t *, umem_destructor_t *, umem_reclaim_t *,
    void *, vmem_t *, int);
extern void umem_cache_destroy(umem_cache_t *);

extern void *umem_cache_alloc(umem_cache_t *, int);
extern void umem_cache_free(umem_cache_t *, void *);

extern void umem_reap(void);

extern void umem_setmtbf(uint32_t);

#ifdef	__cplusplus
}
#endif

#endif	/* _UMEM_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "umem_base.h"

#define	AGENT_STACK_SIZE	4096

char __umem_agent_stack_beg[AGENT_STACK_SIZE];
char *__umem_agent_stack_end = __umem_agent_stack_beg + AGENT_STACK_SIZE;

void
__umem_agent_free_bp(umem_cache_t *cp, void *buf)
{
	extern void _breakpoint(void);			/* inline asm */

	_umem_cache_free(cp, buf);
	_breakpoint();
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2019, Joyent, Inc.
 */

#ifndef	_UMEM_BASE_H
#define	_UMEM_BASE_H

#include <umem_impl.h>

#ifdef	__cplusplus
extern "C" {
#endif

#include "misc.h"

extern size_t pagesize;
#undef PAGESIZE
#define	PAGESIZE	pagesize

/*
 * umem.c: non-tunables
 */
extern vmem_t		*umem_memalign_arena;

extern int umem_ready;
extern thread_t umem_init_thr;		/* the thread doing the init */

extern int umem_init(void);		/* do umem's initialization */

extern umem_log_header_t *umem_transaction_log;
extern umem_log_header_t *umem_content_log;
extern umem_log_header_t *umem_failure_log;
extern umem_log_header_t *umem_slab_log;

extern mutex_t umem_init_lock;

extern mutex_t umem_cache_lock;
extern umem_cache_t umem_null_cache;

extern mutex_t umem_flags_lock;

extern mutex_t umem_update_lock;
extern cond_t umem_update_cv;
extern volatile thread_t umem_st_update_thr;
extern thread_t umem_update_thr;
extern struct timeval	umem_update_next;

extern volatile hrtime_t umem_reap_next;
extern volatile uint32_t umem_reaping;
#define	UMEM_REAP_DONE		0x00000000	/* inactive */
#define	UMEM_REAP_ADDING	0x00000001	/* umem_reap() is active */
#define	UMEM_REAP_ACTIVE	0x00000002	/* update thread is reaping */

extern uintptr_t umem_tmem_off;

/*
 * umem.c: tunables
 */
extern uint32_t umem_max_ncpus;

extern uint32_t umem_stack_depth;
extern uint32_t umem_reap_interval;
extern uint32_t umem_update_interval;
extern uint32_t umem_depot_contention;
extern uint32_t umem_abort;
extern uint32_t umem_output;
extern uint32_t umem_logging;
extern uint32_t umem_mtbf;
extern size_t umem_transaction_log_size;
extern size_t umem_content_log_size;
extern size_t umem_failure_log_size;
extern size_t umem_slab_log_size;
extern size_t umem_content_maxsave;
extern size_t umem_lite_minsize;
extern size_t umem_lite_maxalign;
extern size_t umem_maxverify;
extern size_t umem_minfirewall;
extern size_t umem_ptc_size;

extern uint32_t umem_flags;

/*
 * umem.c: Internal aliases (to avoid PLTs)
 */
extern void *_umem_alloc(size_t size, int umflags);
extern void *_umem_zalloc(size_t size, int umflags);
extern void _umem_free(void *buf, size_t size);

extern void *_umem_cache_alloc(umem_cache_t *cache, int flags);
extern void _umem_cache_free(umem_cache_t *cache, void *buffer);

/*
 * umem.c: private interfaces
 */
extern void umem_type_init(caddr_t, size_t, size_t);
extern int umem_get_max_ncpus(void);
extern void umem_process_updates(void);
extern void umem_cache_applyall(void (*)(umem_cache_t *));
extern void umem_cache_update(umem_cache_t *);

extern void umem_alloc_sizes_add(size_t);
extern void umem_alloc_sizes_clear(void);
extern void umem_alloc_sizes_remove(size_t);

/*
 * umem_fork.c: private interfaces
 */
extern void umem_forkhandler_init(void);

/*
 * umem_update_thread.c
 */
extern int umem_create_update_thread(void);

/*
 * envvar.c:
 */
void umem_setup_envvars(int);
void umem_process_envvars(void);

/*
 * umem_genasm.c: private interfaces
 */
extern const int umem_genasm_supported;
extern boolean_t umem_genasm(int *, umem_cache_t **, int);

/*
 * malloc.c: traditional malloc/free interface for genasm
 */
extern void *umem_malloc(size_t);
extern void umem_malloc_free(void *);
extern void *_malloc(size_t);
extern void _free(void *);

#ifdef	__cplusplus
}
#endif

#endif	/* _UMEM_BASE_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Failure routines for libumem (not standalone)
 */

#include <sys/types.h>
#include <signal.h>
#include <stdarg.h>
#include <string.h>

#include "misc.h"

static volatile int umem_exiting = 0;
#define	UMEM_EXIT_ABORT	1

static mutex_t umem_exit_lock = DEFAULTMUTEX; /* protects umem_exiting */

static int
firstexit(int type)
{
	if (umem_exiting)
		return (0);

	(void) mutex_lock(&umem_exit_lock);
	if (umem_exiting) {
		(void) mutex_unlock(&umem_exit_lock);
		return (0);
	}
	umem_exiting = type;
	(void) mutex_unlock(&umem_exit_lock);

	return (1);
}

/*
 * We can't use abort(3C), since it closes all of the standard library
 * FILEs, which can call free().
 *
 * In addition, we can't just raise(SIGABRT), since the current handler
 * might do allocation.  We give them once chance, though.
 */
static void __NORETURN
umem_do_abort(void)
{
	if (firstexit(UMEM_EXIT_ABORT))
		(void) raise(SIGABRT);

	for (;;) {
		(void) signal(SIGABRT, SIG_DFL);
		(void) sigrelse(SIGABRT);
		(void) raise(SIGABRT);
	}
}

#define	SKIP_FRAMES		1	/* skip the panic frame */
#define	ERR_STACK_FRAMES	128

static void
print_stacktrace(void)
{
	uintptr_t cur_stack[ERR_STACK_FRAMES];

	/*
	 * if we are in a signal context, checking for it will recurse
	 */
	uint_t nframes = getpcstack(cur_stack, ERR_STACK_FRAMES, 0);
	uint_t idx;

	if (nframes > SKIP_FRAMES) {
		umem_printf("stack trace:\n");

		for (idx = SKIP_FRAMES; idx < nframes; idx++) {
			(void) print_sym((void *)cur_stack[idx]);
			umem_printf("\n");
		}
	}
}

void
umem_panic(const char *format, ...)
{
	va_list va;

	va_start(va, format);
	umem_vprintf(format, va);
	va_end(va);

	if (format[strlen(format)-1] != '\n')
		umem_error_enter("\n");

	print_stacktrace();

	umem_do_abort();
}

void
umem_err_recoverable(const char *format, ...)
{
	va_list va;

	va_start(va, format);
	umem_vprintf(format, va);
	va_end(va);

	if (format[strlen(format)-1] != '\n')
		umem_error_enter("\n");

	print_stacktrace();

	if (umem_abort > 0)
		umem_do_abort();
}

int
__umem_assert_failed(const char *assertion, const char *file, int line)
{
	umem_panic("Assertion failed: %s, file %s, line %d\n",
	    assertion, file, line);
	/*NOTREACHED*/
	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "umem_base.h"
#include "vmem_base.h"

#include <unistd.h>

/*
 * The following functions are for pre- and post-fork1(2) handling.  See
 * "Lock Ordering" in lib/libumem/common/umem.c for the lock ordering used.
 */

static void
umem_lockup_cache(umem_cache_t *cp)
{
	int idx;
	int ncpus = cp->cache_cpu_mask + 1;

	for (idx = 0; idx < ncpus; idx++)
		(void) mutex_lock(&cp->cache_cpu[idx].cc_lock);

	(void) mutex_lock(&cp->cache_depot_lock);
	(void) mutex_lock(&cp->cache_lock);
}

static void
umem_release_cache(umem_cache_t *cp)
{
	int idx;
	int ncpus = cp->cache_cpu_mask + 1;

	(void) mutex_unlock(&cp->cache_lock);
	(void) mutex_unlock(&cp->cache_depot_lock);

	for (idx = 0; idx < ncpus; idx++)
		(void) mutex_unlock(&cp->cache_cpu[idx].cc_lock);
}

static void
umem_lockup_log_header(umem_log_header_t *lhp)
{
	int idx;
	if (lhp == NULL)
		return;
	for (idx = 0; idx < umem_max_ncpus; idx++)
		(void) mutex_lock(&lhp->lh_cpu[idx].clh_lock);

	(void) mutex_lock(&lhp->lh_lock);
}

static void
umem_release_log_header(umem_log_header_t *lhp)
{
	int idx;
	if (lhp == NULL)
		return;

	(void) mutex_unlock(&lhp->lh_lock);

	for (idx = 0; idx < umem_max_ncpus; idx++)
		(void) mutex_unlock(&lhp->lh_cpu[idx].clh_lock);
}

static void
umem_lockup(void)
{
	umem_cache_t *cp;

	(void) mutex_lock(&umem_init_lock);
	/*
	 * If another thread is busy initializing the library, we must
	 * wait for it to complete (by calling umem_init()) before allowing
	 * the fork() to proceed.
	 */
	if (umem_ready == UMEM_READY_INITING && umem_init_thr != thr_self()) {
		(void) mutex_unlock(&umem_init_lock);
		(void) umem_init();
		(void) mutex_lock(&umem_init_lock);
	}

	vmem_lockup();
	vmem_sbrk_lockup();

	(void) mutex_lock(&umem_cache_lock);
	(void) mutex_lock(&umem_update_lock);
	(void) mutex_lock(&umem_flags_lock);

	umem_lockup_cache(&umem_null_cache);
	for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache;
	    cp = cp->cache_prev)
		umem_lockup_cache(cp);

	umem_lockup_log_header(umem_transaction_log);
	umem_lockup_log_header(umem_content_log);
	umem_lockup_log_header(umem_failure_log);
	umem_lockup_log_header(umem_slab_log);

	(void) cond_broadcast(&umem_update_cv);

}

static void
umem_do_release(int as_child)
{
	umem_cache_t *cp;
	int cleanup_update = 0;

	/*
	 * Clean up the update state if we are the child process and
	 * another thread was processing updates.
	 */
	if (as_child) {
		if (umem_update_thr != thr_self()) {
			umem_update_thr = 0;
			cleanup_update = 1;
		}
		if (umem_st_update_thr != thr_self()) {
			umem_st_update_thr = 0;
			cleanup_update = 1;
		}
	}

	if (cleanup_update) {
		umem_reaping = UMEM_REAP_DONE;

		for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
		    cp = cp->cache_next) {
			if (cp->cache_uflags & UMU_NOTIFY)
				cp->cache_uflags &= ~UMU_NOTIFY;

			/*
			 * If the cache is active, we just re-add it to
			 * the update list.  This will re-do any active
			 * updates on the cache, but that won't break
			 * anything.
			 *
			 * The worst that can happen is a cache has
			 * its magazines rescaled twice, instead of once.
			 */
			if (cp->cache_uflags & UMU_ACTIVE) {
				umem_cache_t *cnext, *cprev;

				ASSERT(cp->cache_unext == NULL &&
				    cp->cache_uprev == NULL);

				cp->cache_uflags &= ~UMU_ACTIVE;
				cp->cache_unext = cnext = &umem_null_cache;
				cp->cache_uprev = cprev =
				    umem_null_cache.cache_uprev;
				cnext->cache_uprev = cp;
				cprev->cache_unext = cp;
			}
		}
	}

	umem_release_log_header(umem_slab_log);
	umem_release_log_header(umem_failure_log);
	umem_release_log_header(umem_content_log);
	umem_release_log_header(umem_transaction_log);

	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
	    cp = cp->cache_next)
		umem_release_cache(cp);
	umem_release_cache(&umem_null_cache);

	(void) mutex_unlock(&umem_flags_lock);
	(void) mutex_unlock(&umem_update_lock);
	(void) mutex_unlock(&umem_cache_lock);

	vmem_sbrk_release();
	vmem_release();

	(void) mutex_unlock(&umem_init_lock);
}

static void
umem_release(void)
{
	umem_do_release(0);
}

static void
umem_release_child(void)
{
	umem_do_release(1);
}

void
umem_forkhandler_init(void)
{
	/*
	 * There is no way to unregister these atfork functions,
	 * but we don't need to.  The dynamic linker and libc take
	 * care of unregistering them if/when the library is unloaded.
	 */
	(void) pthread_atfork(umem_lockup, umem_release, umem_release_child);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
 * Copyright (c) 2015 by Delphix. All rights reserved.
 */

#ifndef _UMEM_IMPL_H
#define	_UMEM_IMPL_H

#include <umem.h>

#include <sys/sysmacros.h>
#include <sys/time.h>
#include <sys/vmem.h>
#include <thread.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * umem memory allocator: implementation-private data structures
 */

/*
 * Internal flags for umem_cache_create
 */
#define	UMC_QCACHE	0x00100000
#define	UMC_INTERNAL	0x80000000

/*
 * Cache flags
 */
#define	UMF_AUDIT	0x00000001	/* transaction auditing */
#define	UMF_DEADBEEF	0x00000002	/* deadbeef checking */
#define	UMF_REDZONE	0x00000004	/* redzone checking */
#define	UMF_CONTENTS	0x00000008	/* freed-buffer content logging */
#define	UMF_CHECKSIGNAL	0x00000010	/* abort when in signal context */
#define	UMF_NOMAGAZINE	0x00000020	/* disable per-cpu magazines */
#define	UMF_FIREWALL	0x00000040	/* put all bufs before unmapped pages */
#define	UMF_LITE	0x00000100	/* lightweight debugging */

#define	UMF_HASH	0x00000200	/* cache has hash table */
#define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
#define	UMF_PTC		0x00000800	/* cache has per-thread caching */

#define	UMF_CHECKNULL	0x00001000	/* heap exhaustion checking */

#define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
#define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
#define	UMF_RANDOM	(UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
#define	UMF_DEBUG	(UMF_RANDOM | UMF_FIREWALL)

#define	UMEM_STACK_DEPTH	umem_stack_depth

#define	UMEM_FREE_PATTERN		0xdeadbeefdeadbeefULL
#define	UMEM_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
#define	UMEM_REDZONE_PATTERN		0xfeedfacefeedfaceULL
#define	UMEM_REDZONE_BYTE		0xbb

#define	UMEM_FATAL_FLAGS	(UMEM_NOFAIL)
#define	UMEM_SLEEP_FLAGS	(0)

/*
 * Redzone size encodings for umem_alloc() / umem_free().  We encode the
 * allocation size, rather than storing it directly, so that umem_free()
 * can distinguish frees of the wrong size from redzone violations.
 */
#define	UMEM_SIZE_ENCODE(x)	(251 * (x) + 1)
#define	UMEM_SIZE_DECODE(x)	((x) / 251)
#define	UMEM_SIZE_VALID(x)	((x) % 251 == 1)

/*
 * The bufctl (buffer control) structure keeps some minimal information
 * about each buffer: its address, its slab, and its current linkage,
 * which is either on the slab's freelist (if the buffer is free), or
 * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
 * In the case of non-hashed, or "raw", caches (the common case), only
 * the freelist linkage is necessary: the buffer address is at a fixed
 * offset from the bufctl address, and the slab is at the end of the page.
 *
 * NOTE: bc_next must be the first field; raw buffers have linkage only.
 */
typedef struct umem_bufctl {
	struct umem_bufctl	*bc_next;	/* next bufctl struct */
	void			*bc_addr;	/* address of buffer */
	struct umem_slab	*bc_slab;	/* controlling slab */
} umem_bufctl_t;

/*
 * The UMF_AUDIT version of the bufctl structure.  The beginning of this
 * structure must be identical to the normal bufctl structure so that
 * pointers are interchangeable.
 */

#define	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
	((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))

/*
 * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
 * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
 * a single page.
 *
 * For ILP32, this is about 1000 frames.
 * For LP64, this is about 490 frames.
 */

#define	UMEM_BUFCTL_AUDIT_ALIGN	32

#define	UMEM_BUFCTL_AUDIT_MAX_SIZE					\
	(P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -			\
	    sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))

#define	UMEM_MAX_STACK_DEPTH						\
	((UMEM_BUFCTL_AUDIT_MAX_SIZE -					\
	    UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))

typedef struct umem_bufctl_audit {
	struct umem_bufctl	*bc_next;	/* next bufctl struct */
	void			*bc_addr;	/* address of buffer */
	struct umem_slab	*bc_slab;	/* controlling slab */
	umem_cache_t		*bc_cache;	/* controlling cache */
	hrtime_t		bc_timestamp;	/* transaction time */
	thread_t		bc_thread;	/* thread doing transaction */
	struct umem_bufctl	*bc_lastlog;	/* last log entry */
	void			*bc_contents;	/* contents at last free */
	int			bc_depth;	/* stack depth */
	uintptr_t		bc_stack[1];	/* pc stack */
} umem_bufctl_audit_t;

#define	UMEM_LOCAL_BUFCTL_AUDIT(bcpp)					\
		*(bcpp) = (umem_bufctl_audit_t *)			\
		    alloca(UMEM_BUFCTL_AUDIT_SIZE)

#define	UMEM_BUFCTL_AUDIT_SIZE						\
	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)

/*
 * A umem_buftag structure is appended to each buffer whenever any of the
 * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
 */
typedef struct umem_buftag {
	uint64_t		bt_redzone;	/* 64-bit redzone pattern */
	umem_bufctl_t		*bt_bufctl;	/* bufctl */
	intptr_t		bt_bxstat;	/* bufctl ^ (alloc/free) */
} umem_buftag_t;

#define	UMEM_BUFTAG(cp, buf)		\
	((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))

#define	UMEM_BUFCTL(cp, buf)		\
	((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))

#define	UMEM_BUF(cp, bcp)		\
	((void *)((char *)(bcp) - (cp)->cache_bufctl))

#define	UMEM_SLAB(cp, buf)		\
	((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)

#define	UMEM_CPU_CACHE(cp, cpu)		\
	(umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)

#define	UMEM_MAGAZINE_VALID(cp, mp)	\
	(((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
	    (cp)->cache_magtype->mt_cache)

#define	UMEM_SLAB_MEMBER(sp, buf)	\
	((size_t)(buf) - (size_t)(sp)->slab_base < \
	    (sp)->slab_cache->cache_slabsize)

#define	UMEM_BUFTAG_ALLOC	0xa110c8edUL
#define	UMEM_BUFTAG_FREE	0xf4eef4eeUL

typedef struct umem_slab {
	struct umem_cache	*slab_cache;	/* controlling cache */
	void			*slab_base;	/* base of allocated memory */
	struct umem_slab	*slab_next;	/* next slab on freelist */
	struct umem_slab	*slab_prev;	/* prev slab on freelist */
	struct umem_bufctl	*slab_head;	/* first free buffer */
	long			slab_refcnt;	/* outstanding allocations */
	long			slab_chunks;	/* chunks (bufs) in this slab */
} umem_slab_t;

#define	UMEM_HASH_INITIAL	64

#define	UMEM_HASH(cp, buf)	\
	((cp)->cache_hash_table +	\
	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))

typedef struct umem_magazine {
	void	*mag_next;
	void	*mag_round[1];		/* one or more rounds */
} umem_magazine_t;

/*
 * The magazine types for fast per-cpu allocation
 */
typedef struct umem_magtype {
	int		mt_magsize;	/* magazine size (number of rounds) */
	int		mt_align;	/* magazine alignment */
	size_t		mt_minbuf;	/* all smaller buffers qualify */
	size_t		mt_maxbuf;	/* no larger buffers qualify */
	umem_cache_t	*mt_cache;	/* magazine cache */
} umem_magtype_t;

#define	UMEM_CPU_CACHE_SIZE	64	/* must be power of 2 */
#define	UMEM_CPU_PAD		(UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
	2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
#define	UMEM_CACHE_SIZE(ncpus)	\
	((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))

typedef struct umem_cpu_cache {
	mutex_t		cc_lock;	/* protects this cpu's local cache */
	uint_t		cc_alloc;	/* allocations from this cpu */
	uint_t		cc_free;	/* frees to this cpu */
	umem_magazine_t	*cc_loaded;	/* the currently loaded magazine */
	umem_magazine_t	*cc_ploaded;	/* the previously loaded magazine */
	int		cc_rounds;	/* number of objects in loaded mag */
	int		cc_prounds;	/* number of objects in previous mag */
	int		cc_magsize;	/* number of rounds in a full mag */
	int		cc_flags;	/* CPU-local copy of cache_flags */
#ifndef _LP64
	char		cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
#endif
} umem_cpu_cache_t;

/*
 * The magazine lists used in the depot.
 */
typedef struct umem_maglist {
	umem_magazine_t	*ml_list;	/* magazine list */
	long		ml_total;	/* number of magazines */
	long		ml_min;		/* min since last update */
	long		ml_reaplimit;	/* max reapable magazines */
	uint64_t	ml_alloc;	/* allocations from this list */
} umem_maglist_t;

#define	UMEM_CACHE_NAMELEN	31

struct umem_cache {
	/*
	 * Statistics
	 */
	uint64_t	cache_slab_create;	/* slab creates */
	uint64_t	cache_slab_destroy;	/* slab destroys */
	uint64_t	cache_slab_alloc;	/* slab layer allocations */
	uint64_t	cache_slab_free;	/* slab layer frees */
	uint64_t	cache_alloc_fail;	/* total failed allocations */
	uint64_t	cache_buftotal;		/* total buffers */
	uint64_t	cache_bufmax;		/* max buffers ever */
	uint64_t	cache_rescale;		/* # of hash table rescales */
	uint64_t	cache_lookup_depth;	/* hash lookup depth */
	uint64_t	cache_depot_contention;	/* mutex contention count */
	uint64_t	cache_depot_contention_prev; /* previous snapshot */

	/*
	 * Cache properties
	 */
	char		cache_name[UMEM_CACHE_NAMELEN + 1];
	size_t		cache_bufsize;		/* object size */
	size_t		cache_align;		/* object alignment */
	umem_constructor_t *cache_constructor;
	umem_destructor_t *cache_destructor;
	umem_reclaim_t	*cache_reclaim;
	void		*cache_private;		/* opaque arg to callbacks */
	vmem_t		*cache_arena;		/* vmem source for slabs */
	int		cache_cflags;		/* cache creation flags */
	int		cache_flags;		/* various cache state info */
	int		cache_uflags;		/* UMU_* flags */
	uint32_t	cache_mtbf;		/* induced alloc failure rate */
	umem_cache_t	*cache_next;		/* forward cache linkage */
	umem_cache_t	*cache_prev;		/* backward cache linkage */
	umem_cache_t	*cache_unext;		/* next in update list */
	umem_cache_t	*cache_uprev;		/* prev in update list */
	uint32_t	cache_cpu_mask;		/* mask for cpu offset */

	/*
	 * Slab layer
	 */
	mutex_t		cache_lock;		/* protects slab layer */
	size_t		cache_chunksize;	/* buf + alignment [+ debug] */
	size_t		cache_slabsize;		/* size of a slab */
	size_t		cache_bufctl;		/* buf-to-bufctl distance */
	size_t		cache_buftag;		/* buf-to-buftag distance */
	size_t		cache_verify;		/* bytes to verify */
	size_t		cache_contents;		/* bytes of saved content */
	size_t		cache_color;		/* next slab color */
	size_t		cache_mincolor;		/* maximum slab color */
	size_t		cache_maxcolor;		/* maximum slab color */
	size_t		cache_hash_shift;	/* get to interesting bits */
	size_t		cache_hash_mask;	/* hash table mask */
	umem_slab_t	*cache_freelist;	/* slab free list */
	umem_slab_t	cache_nullslab;		/* end of freelist marker */
	umem_cache_t	*cache_bufctl_cache;	/* source of bufctls */
	umem_bufctl_t	**cache_hash_table;	/* hash table base */
	/*
	 * Depot layer
	 */
	mutex_t		cache_depot_lock;	/* protects depot */
	umem_magtype_t	*cache_magtype;		/* magazine type */
	umem_maglist_t	cache_full;		/* full magazines */
	umem_maglist_t	cache_empty;		/* empty magazines */

	/*
	 * Per-CPU layer
	 */
	umem_cpu_cache_t cache_cpu[1];		/* cache_cpu_mask + 1 entries */
};

typedef struct umem_cpu_log_header {
	mutex_t		clh_lock;
	char		*clh_current;
	size_t		clh_avail;
	int		clh_chunk;
	int		clh_hits;
	char		clh_pad[64 - sizeof (mutex_t) - sizeof (char *) -
				sizeof (size_t) - 2 * sizeof (int)];
} umem_cpu_log_header_t;

typedef struct umem_log_header {
	mutex_t		lh_lock;
	char		*lh_base;
	int		*lh_free;
	size_t		lh_chunksize;
	int		lh_nchunks;
	int		lh_head;
	int		lh_tail;
	int		lh_hits;
	umem_cpu_log_header_t lh_cpu[1];	/* actually umem_max_ncpus */
} umem_log_header_t;

typedef struct umem_cpu {
	uint32_t cpu_cache_offset;
	uint32_t cpu_number;
} umem_cpu_t;

#define	UMEM_MAXBUF	131072

#define	UMEM_ALIGN		8	/* min guaranteed alignment */
#define	UMEM_ALIGN_SHIFT	3	/* log2(UMEM_ALIGN) */
#define	UMEM_VOID_FRACTION	8	/* never waste more than 1/8 of slab */

/*
 * For 64 bits, buffers >= 16 bytes must be 16-byte aligned
 */
#ifdef _LP64
#define	UMEM_SECOND_ALIGN 16
#else
#define	UMEM_SECOND_ALIGN UMEM_ALIGN
#endif

#define	MALLOC_MAGIC			0x3a10c000 /* 8-byte tag */
#define	MEMALIGN_MAGIC			0x3e3a1000

#ifdef _LP64
#define	MALLOC_SECOND_MAGIC		0x16ba7000 /* 8-byte tag, 16-aligned */
#define	MALLOC_OVERSIZE_MAGIC		0x06e47000 /* 16-byte tag, _LP64 */
#endif

#define	UMEM_MALLOC_ENCODE(type, sz)	(uint32_t)((type) - (sz))
#define	UMEM_MALLOC_DECODE(stat, sz)	(uint32_t)((stat) + (sz))
#define	UMEM_FREE_PATTERN_32		(uint32_t)(UMEM_FREE_PATTERN)

#define	UMU_MAGAZINE_RESIZE	0x00000001
#define	UMU_HASH_RESCALE	0x00000002
#define	UMU_REAP		0x00000004
#define	UMU_NOTIFY		0x08000000
#define	UMU_ACTIVE		0x80000000

#define	UMEM_READY_INIT_FAILED		-1
#define	UMEM_READY_STARTUP		1
#define	UMEM_READY_INITING		2
#define	UMEM_READY			3

#ifdef UMEM_STANDALONE
extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
extern int umem_add(caddr_t, size_t);
#endif

/*
 * Private interface with libc for tcumem.
 */
extern uintptr_t _tmem_get_base(void);
extern int _tmem_get_nentries(void);
extern void _tmem_set_cleanup(void(*)(void *, int));

#ifdef	__cplusplus
}
#endif

#endif	/* _UMEM_IMPL_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "umem_base.h"
#include "vmem_base.h"

#include <sys/ccompile.h>

#include <signal.h>

/*ARGSUSED*/
static void * __NORETURN
umem_update_thread(void *arg)
{
	struct timeval now;
	int in_update = 0;

	(void) mutex_lock(&umem_update_lock);

	ASSERT(umem_update_thr == thr_self());
	ASSERT(umem_st_update_thr == 0);

	for (;;) {
		umem_process_updates();

		if (in_update) {
			in_update = 0;
			/*
			 * we wait until now to set the next update time
			 * so that the updates are self-throttling
			 */
			(void) gettimeofday(&umem_update_next, NULL);
			umem_update_next.tv_sec += umem_reap_interval;
		}

		switch (umem_reaping) {
		case UMEM_REAP_DONE:
		case UMEM_REAP_ADDING:
			break;

		case UMEM_REAP_ACTIVE:
			umem_reap_next = gethrtime() +
			    (hrtime_t)umem_reap_interval * NANOSEC;
			umem_reaping = UMEM_REAP_DONE;
			break;

		default:
			ASSERT(umem_reaping == UMEM_REAP_DONE ||
			    umem_reaping == UMEM_REAP_ADDING ||
			    umem_reaping == UMEM_REAP_ACTIVE);
			break;
		}

		(void) gettimeofday(&now, NULL);
		if (now.tv_sec > umem_update_next.tv_sec ||
		    (now.tv_sec == umem_update_next.tv_sec &&
		    now.tv_usec >= umem_update_next.tv_usec)) {
			/*
			 * Time to run an update
			 */
			(void) mutex_unlock(&umem_update_lock);

			vmem_update(NULL);
			/*
			 * umem_cache_update can use umem_add_update to
			 * request further work.  The update is not complete
			 * until all such work is finished.
			 */
			umem_cache_applyall(umem_cache_update);

			(void) mutex_lock(&umem_update_lock);
			in_update = 1;
			continue;	/* start processing immediately */
		}

		/*
		 * if there is no work to do, we wait until it is time for
		 * next update, or someone wakes us.
		 */
		if (umem_null_cache.cache_unext == &umem_null_cache) {
			int cancel_state;
			timespec_t abs_time;
			abs_time.tv_sec = umem_update_next.tv_sec;
			abs_time.tv_nsec = umem_update_next.tv_usec * 1000;

			(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
			    &cancel_state);
			(void) cond_timedwait(&umem_update_cv,
			    &umem_update_lock, &abs_time);
			(void) pthread_setcancelstate(cancel_state, NULL);
		}
	}
	/* LINTED no return statement */
}

int
umem_create_update_thread(void)
{
	sigset_t sigmask, oldmask;
	thread_t newthread;

	ASSERT(MUTEX_HELD(&umem_update_lock));
	ASSERT(umem_update_thr == 0);

	/*
	 * The update thread handles no signals
	 */
	(void) sigfillset(&sigmask);
	(void) thr_sigsetmask(SIG_BLOCK, &sigmask, &oldmask);

	/*
	 * drop the umem_update_lock; we cannot hold locks acquired in
	 * pre-fork handler while calling thr_create or thr_continue().
	 */

	(void) mutex_unlock(&umem_update_lock);

	if (thr_create(NULL, 0, umem_update_thread, NULL,
	    THR_BOUND | THR_DAEMON | THR_DETACHED | THR_SUSPENDED,
	    &newthread) == 0) {
		(void) thr_sigsetmask(SIG_SETMASK, &oldmask, NULL);
		(void) thr_setname(newthread, "umem_update");

		(void) mutex_lock(&umem_update_lock);
		/*
		 * due to the locking in umem_reap(), only one thread can
		 * ever call umem_create_update_thread() at a time.  This
		 * must be the case for this code to work.
		 */

		ASSERT(umem_update_thr == 0);
		umem_update_thr = newthread;
		(void) mutex_unlock(&umem_update_lock);
		(void) thr_continue(newthread);
		(void) mutex_lock(&umem_update_lock);

		return (1);
	} else { /* thr_create failed */
		(void) thr_sigsetmask(SIG_SETMASK, &oldmask, NULL);
		(void) mutex_lock(&umem_update_lock);
	}
	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2019, Joyent, Inc.
 * Copyright (c) 2017 by Delphix. All rights reserved.
 */

/*
 * For a more complete description of the main ideas, see:
 *
 *	Jeff Bonwick and Jonathan Adams,
 *
 *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
 *	Arbitrary Resources.
 *
 *	Proceedings of the 2001 Usenix Conference.
 *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
 *
 * For the "Big Theory Statement", see usr/src/uts/common/os/vmem.c
 *
 * 1. Overview of changes
 * ------------------------------
 * There have been a few changes to vmem in order to support umem.  The
 * main areas are:
 *
 *	* VM_SLEEP unsupported
 *
 *	* Reaping changes
 *
 *	* initialization changes
 *
 *	* _vmem_extend_alloc
 *
 *
 * 2. VM_SLEEP Removed
 * -------------------
 * Since VM_SLEEP allocations can hold locks (in vmem_populate()) for
 * possibly infinite amounts of time, they are not supported in this
 * version of vmem.  Sleep-like behavior can be achieved through
 * UMEM_NOFAIL umem allocations.
 *
 *
 * 3. Reaping changes
 * ------------------
 * Unlike kmem_reap(), which just asynchronously schedules work, umem_reap()
 * can do allocations and frees synchronously.  This is a problem if it
 * occurs during a vmem_populate() allocation.
 *
 * Instead, we delay reaps while populates are active.
 *
 *
 * 4. Initialization changes
 * -------------------------
 * In the kernel, vmem_init() allows you to create a single, top-level arena,
 * which has vmem_internal_arena as a child.  For umem, we want to be able
 * to extend arenas dynamically.  It is much easier to support this if we
 * allow a two-level "heap" arena:
 *
 *	+----------+
 *	|  "fake"  |
 *	+----------+
 *	      |
 *	+----------+
 *	|  "heap"  |
 *	+----------+
 *	  |    \ \
 *	  |     +-+-- ... <other children>
 *	  |
 *	+---------------+
 *	| vmem_internal |
 *	+---------------+
 *	    | | | |
 *	   <children>
 *
 * The new vmem_init() allows you to specify a "parent" of the heap, along
 * with allocation functions.
 *
 *
 * 5. _vmem_extend_alloc
 * ---------------------
 * The other part of extending is _vmem_extend_alloc.  This function allows
 * you to extend (expand current spans, if possible) an arena and allocate
 * a chunk of the newly extened span atomically.  This is needed to support
 * extending the heap while vmem_populate()ing it.
 *
 * In order to increase the usefulness of extending, non-imported spans are
 * sorted in address order.
 */

#include <sys/vmem_impl_user.h>
#include <alloca.h>
#include <sys/sysmacros.h>
#include <stdio.h>
#include <strings.h>
#include <atomic.h>

#include "vmem_base.h"
#include "umem_base.h"

#define	VMEM_INITIAL		6	/* early vmem arenas */
#define	VMEM_SEG_INITIAL	100	/* early segments */

/*
 * Adding a new span to an arena requires two segment structures: one to
 * represent the span, and one to represent the free segment it contains.
 */
#define	VMEM_SEGS_PER_SPAN_CREATE	2

/*
 * Allocating a piece of an existing segment requires 0-2 segment structures
 * depending on how much of the segment we're allocating.
 *
 * To allocate the entire segment, no new segment structures are needed; we
 * simply move the existing segment structure from the freelist to the
 * allocation hash table.
 *
 * To allocate a piece from the left or right end of the segment, we must
 * split the segment into two pieces (allocated part and remainder), so we
 * need one new segment structure to represent the remainder.
 *
 * To allocate from the middle of a segment, we need two new segment strucures
 * to represent the remainders on either side of the allocated part.
 */
#define	VMEM_SEGS_PER_EXACT_ALLOC	0
#define	VMEM_SEGS_PER_LEFT_ALLOC	1
#define	VMEM_SEGS_PER_RIGHT_ALLOC	1
#define	VMEM_SEGS_PER_MIDDLE_ALLOC	2

/*
 * vmem_populate() preallocates segment structures for vmem to do its work.
 * It must preallocate enough for the worst case, which is when we must import
 * a new span and then allocate from the middle of it.
 */
#define	VMEM_SEGS_PER_ALLOC_MAX		\
	(VMEM_SEGS_PER_SPAN_CREATE + VMEM_SEGS_PER_MIDDLE_ALLOC)

/*
 * The segment structures themselves are allocated from vmem_seg_arena, so
 * we have a recursion problem when vmem_seg_arena needs to populate itself.
 * We address this by working out the maximum number of segment structures
 * this act will require, and multiplying by the maximum number of threads
 * that we'll allow to do it simultaneously.
 *
 * The worst-case segment consumption to populate vmem_seg_arena is as
 * follows (depicted as a stack trace to indicate why events are occurring):
 *
 * vmem_alloc(vmem_seg_arena)		-> 2 segs (span create + exact alloc)
 *  vmem_alloc(vmem_internal_arena)	-> 2 segs (span create + exact alloc)
 *   heap_alloc(heap_arena)
 *    vmem_alloc(heap_arena)		-> 4 seg (span create + alloc)
 *     parent_alloc(parent_arena)
 *	_vmem_extend_alloc(parent_arena) -> 3 seg (span create + left alloc)
 *
 * Note:  The reservation for heap_arena must be 4, since vmem_xalloc()
 * is overly pessimistic on allocations where parent_arena has a stricter
 * alignment than heap_arena.
 *
 * The worst-case consumption for any arena is 4 segment structures.
 * For now, we only support VM_NOSLEEP allocations, so as long as we
 * serialize all vmem_populates, a 4-seg reserve is sufficient.
 */
#define	VMEM_POPULATE_SEGS_PER_ARENA	4
#define	VMEM_POPULATE_LOCKS		1

#define	VMEM_POPULATE_RESERVE		\
	(VMEM_POPULATE_SEGS_PER_ARENA * VMEM_POPULATE_LOCKS)

/*
 * vmem_populate() ensures that each arena has VMEM_MINFREE seg structures
 * so that it can satisfy the worst-case allocation *and* participate in
 * worst-case allocation from vmem_seg_arena.
 */
#define	VMEM_MINFREE	(VMEM_POPULATE_RESERVE + VMEM_SEGS_PER_ALLOC_MAX)

/* Don't assume new statics are zeroed - see vmem_startup() */
static vmem_t vmem0[VMEM_INITIAL];
static vmem_t *vmem_populator[VMEM_INITIAL];
static uint32_t vmem_id;
static uint32_t vmem_populators;
static vmem_seg_t vmem_seg0[VMEM_SEG_INITIAL];
static vmem_seg_t *vmem_segfree;
static mutex_t vmem_list_lock;
static mutex_t vmem_segfree_lock;
static vmem_populate_lock_t vmem_nosleep_lock;
#define	IN_POPULATE()	(vmem_nosleep_lock.vmpl_thr == thr_self())
static vmem_t *vmem_list;
static vmem_t *vmem_internal_arena;
static vmem_t *vmem_seg_arena;
static vmem_t *vmem_hash_arena;
static vmem_t *vmem_vmem_arena;

vmem_t *vmem_heap;
vmem_alloc_t *vmem_heap_alloc;
vmem_free_t *vmem_heap_free;

uint32_t vmem_mtbf;		/* mean time between failures [default: off] */
size_t vmem_seg_size = sizeof (vmem_seg_t);

/*
 * Insert/delete from arena list (type 'a') or next-of-kin list (type 'k').
 */
#define	VMEM_INSERT(vprev, vsp, type)					\
{									\
	vmem_seg_t *vnext = (vprev)->vs_##type##next;			\
	(vsp)->vs_##type##next = (vnext);				\
	(vsp)->vs_##type##prev = (vprev);				\
	(vprev)->vs_##type##next = (vsp);				\
	(vnext)->vs_##type##prev = (vsp);				\
}

#define	VMEM_DELETE(vsp, type)						\
{									\
	vmem_seg_t *vprev = (vsp)->vs_##type##prev;			\
	vmem_seg_t *vnext = (vsp)->vs_##type##next;			\
	(vprev)->vs_##type##next = (vnext);				\
	(vnext)->vs_##type##prev = (vprev);				\
}

/*
 * Get a vmem_seg_t from the global segfree list.
 */
static vmem_seg_t *
vmem_getseg_global(void)
{
	vmem_seg_t *vsp;

	(void) mutex_lock(&vmem_segfree_lock);
	if ((vsp = vmem_segfree) != NULL)
		vmem_segfree = vsp->vs_knext;
	(void) mutex_unlock(&vmem_segfree_lock);

	return (vsp);
}

/*
 * Put a vmem_seg_t on the global segfree list.
 */
static void
vmem_putseg_global(vmem_seg_t *vsp)
{
	(void) mutex_lock(&vmem_segfree_lock);
	vsp->vs_knext = vmem_segfree;
	vmem_segfree = vsp;
	(void) mutex_unlock(&vmem_segfree_lock);
}

/*
 * Get a vmem_seg_t from vmp's segfree list.
 */
static vmem_seg_t *
vmem_getseg(vmem_t *vmp)
{
	vmem_seg_t *vsp;

	ASSERT(vmp->vm_nsegfree > 0);

	vsp = vmp->vm_segfree;
	vmp->vm_segfree = vsp->vs_knext;
	vmp->vm_nsegfree--;

	return (vsp);
}

/*
 * Put a vmem_seg_t on vmp's segfree list.
 */
static void
vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp)
{
	vsp->vs_knext = vmp->vm_segfree;
	vmp->vm_segfree = vsp;
	vmp->vm_nsegfree++;
}

/*
 * Add vsp to the appropriate freelist.
 */
static void
vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp)
{
	vmem_seg_t *vprev;

	ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);

	vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1];
	vsp->vs_type = VMEM_FREE;
	vmp->vm_freemap |= VS_SIZE(vprev);
	VMEM_INSERT(vprev, vsp, k);

	(void) cond_broadcast(&vmp->vm_cv);
}

/*
 * Take vsp from the freelist.
 */
static void
vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp)
{
	ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);
	ASSERT(vsp->vs_type == VMEM_FREE);

	if (vsp->vs_knext->vs_start == 0 && vsp->vs_kprev->vs_start == 0) {
		/*
		 * The segments on both sides of 'vsp' are freelist heads,
		 * so taking vsp leaves the freelist at vsp->vs_kprev empty.
		 */
		ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev));
		vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev);
	}
	VMEM_DELETE(vsp, k);
}

/*
 * Add vsp to the allocated-segment hash table and update kstats.
 */
static void
vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp)
{
	vmem_seg_t **bucket;

	vsp->vs_type = VMEM_ALLOC;
	bucket = VMEM_HASH(vmp, vsp->vs_start);
	vsp->vs_knext = *bucket;
	*bucket = vsp;

	if (vmem_seg_size == sizeof (vmem_seg_t)) {
		vsp->vs_depth = (uint8_t)getpcstack(vsp->vs_stack,
		    VMEM_STACK_DEPTH, 0);
		vsp->vs_thread = thr_self();
		vsp->vs_timestamp = gethrtime();
	} else {
		vsp->vs_depth = 0;
	}

	vmp->vm_kstat.vk_alloc++;
	vmp->vm_kstat.vk_mem_inuse += VS_SIZE(vsp);
}

/*
 * Remove vsp from the allocated-segment hash table and update kstats.
 */
static vmem_seg_t *
vmem_hash_delete(vmem_t *vmp, uintptr_t addr, size_t size)
{
	vmem_seg_t *vsp, **prev_vspp;

	prev_vspp = VMEM_HASH(vmp, addr);
	while ((vsp = *prev_vspp) != NULL) {
		if (vsp->vs_start == addr) {
			*prev_vspp = vsp->vs_knext;
			break;
		}
		vmp->vm_kstat.vk_lookup++;
		prev_vspp = &vsp->vs_knext;
	}

	if (vsp == NULL) {
		umem_panic("vmem_hash_delete(%p, %lx, %lu): bad free",
		    vmp, addr, size);
	}
	if (VS_SIZE(vsp) != size) {
		umem_panic("vmem_hash_delete(%p, %lx, %lu): wrong size "
		    "(expect %lu)", vmp, addr, size, VS_SIZE(vsp));
	}

	vmp->vm_kstat.vk_free++;
	vmp->vm_kstat.vk_mem_inuse -= size;

	return (vsp);
}

/*
 * Create a segment spanning the range [start, end) and add it to the arena.
 */
static vmem_seg_t *
vmem_seg_create(vmem_t *vmp, vmem_seg_t *vprev, uintptr_t start, uintptr_t end)
{
	vmem_seg_t *newseg = vmem_getseg(vmp);

	newseg->vs_start = start;
	newseg->vs_end = end;
	newseg->vs_type = 0;
	newseg->vs_import = 0;

	VMEM_INSERT(vprev, newseg, a);

	return (newseg);
}

/*
 * Remove segment vsp from the arena.
 */
static void
vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp)
{
	ASSERT(vsp->vs_type != VMEM_ROTOR);
	VMEM_DELETE(vsp, a);

	vmem_putseg(vmp, vsp);
}

/*
 * Add the span [vaddr, vaddr + size) to vmp and update kstats.
 */
static vmem_seg_t *
vmem_span_create(vmem_t *vmp, void *vaddr, size_t size, uint8_t import)
{
	vmem_seg_t *knext;
	vmem_seg_t *newseg, *span;
	uintptr_t start = (uintptr_t)vaddr;
	uintptr_t end = start + size;

	knext = &vmp->vm_seg0;
	if (!import && vmp->vm_source_alloc == NULL) {
		vmem_seg_t *kend, *kprev;
		/*
		 * non-imported spans are sorted in address order.  This
		 * makes vmem_extend_unlocked() much more effective.
		 *
		 * We search in reverse order, since new spans are
		 * generally at higher addresses.
		 */
		kend = &vmp->vm_seg0;
		for (kprev = kend->vs_kprev; kprev != kend;
		    kprev = kprev->vs_kprev) {
			if (!kprev->vs_import && (kprev->vs_end - 1) < start)
				break;
		}
		knext = kprev->vs_knext;
	}

	ASSERT(MUTEX_HELD(&vmp->vm_lock));

	if ((start | end) & (vmp->vm_quantum - 1)) {
		umem_panic("vmem_span_create(%p, %p, %lu): misaligned",
		    vmp, vaddr, size);
	}

	span = vmem_seg_create(vmp, knext->vs_aprev, start, end);
	span->vs_type = VMEM_SPAN;
	span->vs_import = import;
	VMEM_INSERT(knext->vs_kprev, span, k);

	newseg = vmem_seg_create(vmp, span, start, end);
	vmem_freelist_insert(vmp, newseg);

	if (import)
		vmp->vm_kstat.vk_mem_import += size;
	vmp->vm_kstat.vk_mem_total += size;

	return (newseg);
}

/*
 * Remove span vsp from vmp and update kstats.
 */
static void
vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp)
{
	vmem_seg_t *span = vsp->vs_aprev;
	size_t size = VS_SIZE(vsp);

	ASSERT(MUTEX_HELD(&vmp->vm_lock));
	ASSERT(span->vs_type == VMEM_SPAN);

	if (span->vs_import)
		vmp->vm_kstat.vk_mem_import -= size;
	vmp->vm_kstat.vk_mem_total -= size;

	VMEM_DELETE(span, k);

	vmem_seg_destroy(vmp, vsp);
	vmem_seg_destroy(vmp, span);
}

/*
 * Allocate the subrange [addr, addr + size) from segment vsp.
 * If there are leftovers on either side, place them on the freelist.
 * Returns a pointer to the segment representing [addr, addr + size).
 */
static vmem_seg_t *
vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size)
{
	uintptr_t vs_start = vsp->vs_start;
	uintptr_t vs_end = vsp->vs_end;
	size_t vs_size = vs_end - vs_start;
	size_t realsize = P2ROUNDUP(size, vmp->vm_quantum);
	uintptr_t addr_end = addr + realsize;

	ASSERT(P2PHASE(vs_start, vmp->vm_quantum) == 0);
	ASSERT(P2PHASE(addr, vmp->vm_quantum) == 0);
	ASSERT(vsp->vs_type == VMEM_FREE);
	ASSERT(addr >= vs_start && addr_end - 1 <= vs_end - 1);
	ASSERT(addr - 1 <= addr_end - 1);

	/*
	 * If we're allocating from the start of the segment, and the
	 * remainder will be on the same freelist, we can save quite
	 * a bit of work.
	 */
	if (P2SAMEHIGHBIT(vs_size, vs_size - realsize) && addr == vs_start) {
		ASSERT(highbit(vs_size) == highbit(vs_size - realsize));
		vsp->vs_start = addr_end;
		vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size);
		vmem_hash_insert(vmp, vsp);
		return (vsp);
	}

	vmem_freelist_delete(vmp, vsp);

	if (vs_end != addr_end)
		vmem_freelist_insert(vmp,
		    vmem_seg_create(vmp, vsp, addr_end, vs_end));

	if (vs_start != addr)
		vmem_freelist_insert(vmp,
		    vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr));

	vsp->vs_start = addr;
	vsp->vs_end = addr + size;

	vmem_hash_insert(vmp, vsp);
	return (vsp);
}

/*
 * We cannot reap if we are in the middle of a vmem_populate().
 */
void
vmem_reap(void)
{
	if (!IN_POPULATE())
		umem_reap();
}

/*
 * Populate vmp's segfree list with VMEM_MINFREE vmem_seg_t structures.
 */
static int
vmem_populate(vmem_t *vmp, int vmflag)
{
	char *p;
	vmem_seg_t *vsp;
	ssize_t nseg;
	size_t size;
	vmem_populate_lock_t *lp;
	int i;

	while (vmp->vm_nsegfree < VMEM_MINFREE &&
	    (vsp = vmem_getseg_global()) != NULL)
		vmem_putseg(vmp, vsp);

	if (vmp->vm_nsegfree >= VMEM_MINFREE)
		return (1);

	/*
	 * If we're already populating, tap the reserve.
	 */
	if (vmem_nosleep_lock.vmpl_thr == thr_self()) {
		ASSERT(vmp->vm_cflags & VMC_POPULATOR);
		return (1);
	}

	(void) mutex_unlock(&vmp->vm_lock);

	ASSERT(vmflag & VM_NOSLEEP);	/* we do not allow sleep allocations */
	lp = &vmem_nosleep_lock;

	/*
	 * Cannot be just a mutex_lock(), since that has no effect if
	 * libthread is not linked.
	 */
	(void) mutex_lock(&lp->vmpl_mutex);
	ASSERT(lp->vmpl_thr == 0);
	lp->vmpl_thr = thr_self();

	nseg = VMEM_MINFREE + vmem_populators * VMEM_POPULATE_RESERVE;
	size = P2ROUNDUP(nseg * vmem_seg_size, vmem_seg_arena->vm_quantum);
	nseg = size / vmem_seg_size;

	/*
	 * The following vmem_alloc() may need to populate vmem_seg_arena
	 * and all the things it imports from.  When doing so, it will tap
	 * each arena's reserve to prevent recursion (see the block comment
	 * above the definition of VMEM_POPULATE_RESERVE).
	 *
	 * During this allocation, vmem_reap() is a no-op.  If the allocation
	 * fails, we call vmem_reap() after dropping the population lock.
	 */
	p = vmem_alloc(vmem_seg_arena, size, vmflag & VM_UMFLAGS);
	if (p == NULL) {
		lp->vmpl_thr = 0;
		(void) mutex_unlock(&lp->vmpl_mutex);
		vmem_reap();

		(void) mutex_lock(&vmp->vm_lock);
		vmp->vm_kstat.vk_populate_fail++;
		return (0);
	}
	/*
	 * Restock the arenas that may have been depleted during population.
	 */
	for (i = 0; i < vmem_populators; i++) {
		(void) mutex_lock(&vmem_populator[i]->vm_lock);
		while (vmem_populator[i]->vm_nsegfree < VMEM_POPULATE_RESERVE)
			vmem_putseg(vmem_populator[i],
			    (vmem_seg_t *)(p + --nseg * vmem_seg_size));
		(void) mutex_unlock(&vmem_populator[i]->vm_lock);
	}

	lp->vmpl_thr = 0;
	(void) mutex_unlock(&lp->vmpl_mutex);
	(void) mutex_lock(&vmp->vm_lock);

	/*
	 * Now take our own segments.
	 */
	ASSERT(nseg >= VMEM_MINFREE);
	while (vmp->vm_nsegfree < VMEM_MINFREE)
		vmem_putseg(vmp, (vmem_seg_t *)(p + --nseg * vmem_seg_size));

	/*
	 * Give the remainder to charity.
	 */
	while (nseg > 0)
		vmem_putseg_global((vmem_seg_t *)(p + --nseg * vmem_seg_size));

	return (1);
}

/*
 * Advance a walker from its previous position to 'afterme'.
 * Note: may drop and reacquire vmp->vm_lock.
 */
static void
vmem_advance(vmem_t *vmp, vmem_seg_t *walker, vmem_seg_t *afterme)
{
	vmem_seg_t *vprev = walker->vs_aprev;
	vmem_seg_t *vnext = walker->vs_anext;
	vmem_seg_t *vsp = NULL;

	VMEM_DELETE(walker, a);

	if (afterme != NULL)
		VMEM_INSERT(afterme, walker, a);

	/*
	 * The walker segment's presence may have prevented its neighbors
	 * from coalescing.  If so, coalesce them now.
	 */
	if (vprev->vs_type == VMEM_FREE) {
		if (vnext->vs_type == VMEM_FREE) {
			ASSERT(vprev->vs_end == vnext->vs_start);
			vmem_freelist_delete(vmp, vnext);
			vmem_freelist_delete(vmp, vprev);
			vprev->vs_end = vnext->vs_end;
			vmem_freelist_insert(vmp, vprev);
			vmem_seg_destroy(vmp, vnext);
		}
		vsp = vprev;
	} else if (vnext->vs_type == VMEM_FREE) {
		vsp = vnext;
	}

	/*
	 * vsp could represent a complete imported span,
	 * in which case we must return it to the source.
	 */
	if (vsp != NULL && vsp->vs_aprev->vs_import &&
	    vmp->vm_source_free != NULL &&
	    vsp->vs_aprev->vs_type == VMEM_SPAN &&
	    vsp->vs_anext->vs_type == VMEM_SPAN) {
		void *vaddr = (void *)vsp->vs_start;
		size_t size = VS_SIZE(vsp);
		ASSERT(size == VS_SIZE(vsp->vs_aprev));
		vmem_freelist_delete(vmp, vsp);
		vmem_span_destroy(vmp, vsp);
		(void) mutex_unlock(&vmp->vm_lock);
		vmp->vm_source_free(vmp->vm_source, vaddr, size);
		(void) mutex_lock(&vmp->vm_lock);
	}
}

/*
 * VM_NEXTFIT allocations deliberately cycle through all virtual addresses
 * in an arena, so that we avoid reusing addresses for as long as possible.
 * This helps to catch used-after-freed bugs.  It's also the perfect policy
 * for allocating things like process IDs, where we want to cycle through
 * all values in order.
 */
static void *
vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag)
{
	vmem_seg_t *vsp, *rotor;
	uintptr_t addr;
	size_t realsize = P2ROUNDUP(size, vmp->vm_quantum);
	size_t vs_size;

	(void) mutex_lock(&vmp->vm_lock);

	if (vmp->vm_nsegfree < VMEM_MINFREE && !vmem_populate(vmp, vmflag)) {
		(void) mutex_unlock(&vmp->vm_lock);
		return (NULL);
	}

	/*
	 * The common case is that the segment right after the rotor is free,
	 * and large enough that extracting 'size' bytes won't change which
	 * freelist it's on.  In this case we can avoid a *lot* of work.
	 * Instead of the normal vmem_seg_alloc(), we just advance the start
	 * address of the victim segment.  Instead of moving the rotor, we
	 * create the new segment structure *behind the rotor*, which has
	 * the same effect.  And finally, we know we don't have to coalesce
	 * the rotor's neighbors because the new segment lies between them.
	 */
	rotor = &vmp->vm_rotor;
	vsp = rotor->vs_anext;
	if (vsp->vs_type == VMEM_FREE && (vs_size = VS_SIZE(vsp)) > realsize &&
	    P2SAMEHIGHBIT(vs_size, vs_size - realsize)) {
		ASSERT(highbit(vs_size) == highbit(vs_size - realsize));
		addr = vsp->vs_start;
		vsp->vs_start = addr + realsize;
		vmem_hash_insert(vmp,
		    vmem_seg_create(vmp, rotor->vs_aprev, addr, addr + size));
		(void) mutex_unlock(&vmp->vm_lock);
		return ((void *)addr);
	}

	/*
	 * Starting at the rotor, look for a segment large enough to
	 * satisfy the allocation.
	 */
	for (;;) {
		vmp->vm_kstat.vk_search++;
		if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
			break;
		vsp = vsp->vs_anext;
		if (vsp == rotor) {
			int cancel_state;

			/*
			 * We've come full circle.  One possibility is that the
			 * there's actually enough space, but the rotor itself
			 * is preventing the allocation from succeeding because
			 * it's sitting between two free segments.  Therefore,
			 * we advance the rotor and see if that liberates a
			 * suitable segment.
			 */
			vmem_advance(vmp, rotor, rotor->vs_anext);
			vsp = rotor->vs_aprev;
			if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
				break;
			/*
			 * If there's a lower arena we can import from, or it's
			 * a VM_NOSLEEP allocation, let vmem_xalloc() handle it.
			 * Otherwise, wait until another thread frees something.
			 */
			if (vmp->vm_source_alloc != NULL ||
			    (vmflag & VM_NOSLEEP)) {
				(void) mutex_unlock(&vmp->vm_lock);
				return (vmem_xalloc(vmp, size, vmp->vm_quantum,
				    0, 0, NULL, NULL, vmflag & VM_UMFLAGS));
			}
			vmp->vm_kstat.vk_wait++;
			(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
			    &cancel_state);
			(void) cond_wait(&vmp->vm_cv, &vmp->vm_lock);
			(void) pthread_setcancelstate(cancel_state, NULL);
			vsp = rotor->vs_anext;
		}
	}

	/*
	 * We found a segment.  Extract enough space to satisfy the allocation.
	 */
	addr = vsp->vs_start;
	vsp = vmem_seg_alloc(vmp, vsp, addr, size);
	ASSERT(vsp->vs_type == VMEM_ALLOC &&
	    vsp->vs_start == addr && vsp->vs_end == addr + size);

	/*
	 * Advance the rotor to right after the newly-allocated segment.
	 * That's where the next VM_NEXTFIT allocation will begin searching.
	 */
	vmem_advance(vmp, rotor, vsp);
	(void) mutex_unlock(&vmp->vm_lock);
	return ((void *)addr);
}

/*
 * Allocate size bytes at offset phase from an align boundary such that the
 * resulting segment [addr, addr + size) is a subset of [minaddr, maxaddr)
 * that does not straddle a nocross-aligned boundary.
 */
void *
vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase,
    size_t nocross, void *minaddr, void *maxaddr, int vmflag)
{
	vmem_seg_t *vsp;
	vmem_seg_t *vbest = NULL;
	uintptr_t addr = 0, taddr, start, end;
	void *vaddr;
	int hb, flist, resv;
	uint32_t mtbf;

	if (phase > 0 && phase >= align)
		umem_panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
		    "invalid phase",
		    (void *)vmp, size, align, phase, nocross,
		    minaddr, maxaddr, vmflag);

	if (align == 0)
		align = vmp->vm_quantum;

	if ((align | phase | nocross) & (vmp->vm_quantum - 1)) {
		umem_panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
		    "parameters not vm_quantum aligned",
		    (void *)vmp, size, align, phase, nocross,
		    minaddr, maxaddr, vmflag);
	}

	if (nocross != 0 &&
	    (align > nocross || P2ROUNDUP(phase + size, align) > nocross)) {
		umem_panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
		    "overconstrained allocation",
		    (void *)vmp, size, align, phase, nocross,
		    minaddr, maxaddr, vmflag);
	}

	if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 &&
	    (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP)
		return (NULL);

	(void) mutex_lock(&vmp->vm_lock);
	for (;;) {
		int cancel_state;

		if (vmp->vm_nsegfree < VMEM_MINFREE &&
		    !vmem_populate(vmp, vmflag))
			break;

		/*
		 * highbit() returns the highest bit + 1, which is exactly
		 * what we want: we want to search the first freelist whose
		 * members are *definitely* large enough to satisfy our
		 * allocation.  However, there are certain cases in which we
		 * want to look at the next-smallest freelist (which *might*
		 * be able to satisfy the allocation):
		 *
		 * (1)	The size is exactly a power of 2, in which case
		 *	the smaller freelist is always big enough;
		 *
		 * (2)	All other freelists are empty;
		 *
		 * (3)	We're in the highest possible freelist, which is
		 *	always empty (e.g. the 4GB freelist on 32-bit systems);
		 *
		 * (4)	We're doing a best-fit or first-fit allocation.
		 */
		if ((size & (size - 1)) == 0) {
			flist = lowbit(P2ALIGN(vmp->vm_freemap, size));
		} else {
			hb = highbit(size);
			if ((vmp->vm_freemap >> hb) == 0 ||
			    hb == VMEM_FREELISTS ||
			    (vmflag & (VM_BESTFIT | VM_FIRSTFIT)))
				hb--;
			flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb));
		}

		for (vbest = NULL, vsp = (flist == 0) ? NULL :
		    vmp->vm_freelist[flist - 1].vs_knext;
		    vsp != NULL; vsp = vsp->vs_knext) {
			vmp->vm_kstat.vk_search++;
			if (vsp->vs_start == 0) {
				/*
				 * We're moving up to a larger freelist,
				 * so if we've already found a candidate,
				 * the fit can't possibly get any better.
				 */
				if (vbest != NULL)
					break;
				/*
				 * Find the next non-empty freelist.
				 */
				flist = lowbit(P2ALIGN(vmp->vm_freemap,
				    VS_SIZE(vsp)));
				if (flist-- == 0)
					break;
				vsp = (vmem_seg_t *)&vmp->vm_freelist[flist];
				ASSERT(vsp->vs_knext->vs_type == VMEM_FREE);
				continue;
			}
			if (vsp->vs_end - 1 < (uintptr_t)minaddr)
				continue;
			if (vsp->vs_start > (uintptr_t)maxaddr - 1)
				continue;
			start = MAX(vsp->vs_start, (uintptr_t)minaddr);
			end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1;
			taddr = P2PHASEUP(start, align, phase);
			if (P2BOUNDARY(taddr, size, nocross))
				taddr +=
				    P2ROUNDUP(P2NPHASE(taddr, nocross), align);
			if ((taddr - start) + size > end - start ||
			    (vbest != NULL && VS_SIZE(vsp) >= VS_SIZE(vbest)))
				continue;
			vbest = vsp;
			addr = taddr;
			if (!(vmflag & VM_BESTFIT) || VS_SIZE(vbest) == size)
				break;
		}
		if (vbest != NULL)
			break;
		if (size == 0)
			umem_panic("vmem_xalloc(): size == 0");
		if (vmp->vm_source_alloc != NULL && nocross == 0 &&
		    minaddr == NULL && maxaddr == NULL) {
			size_t asize = P2ROUNDUP(size + phase,
			    MAX(align, vmp->vm_source->vm_quantum));
			if (asize < size) {		/* overflow */
				(void) mutex_unlock(&vmp->vm_lock);
				if (vmflag & VM_NOSLEEP)
					return (NULL);

				umem_panic("vmem_xalloc(): "
				    "overflow on VM_SLEEP allocation");
			}
			/*
			 * Determine how many segment structures we'll consume.
			 * The calculation must be presise because if we're
			 * here on behalf of vmem_populate(), we are taking
			 * segments from a very limited reserve.
			 */
			resv = (size == asize) ?
			    VMEM_SEGS_PER_SPAN_CREATE +
			    VMEM_SEGS_PER_EXACT_ALLOC :
			    VMEM_SEGS_PER_ALLOC_MAX;
			ASSERT(vmp->vm_nsegfree >= resv);
			vmp->vm_nsegfree -= resv;	/* reserve our segs */
			(void) mutex_unlock(&vmp->vm_lock);
			vaddr = vmp->vm_source_alloc(vmp->vm_source, asize,
			    vmflag & VM_UMFLAGS);
			(void) mutex_lock(&vmp->vm_lock);
			vmp->vm_nsegfree += resv;	/* claim reservation */
			if (vaddr != NULL) {
				vbest = vmem_span_create(vmp, vaddr, asize, 1);
				addr = P2PHASEUP(vbest->vs_start, align, phase);
				break;
			}
		}
		(void) mutex_unlock(&vmp->vm_lock);
		vmem_reap();
		(void) mutex_lock(&vmp->vm_lock);
		if (vmflag & VM_NOSLEEP)
			break;
		vmp->vm_kstat.vk_wait++;
		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
		    &cancel_state);
		(void) cond_wait(&vmp->vm_cv, &vmp->vm_lock);
		(void) pthread_setcancelstate(cancel_state, NULL);
	}
	if (vbest != NULL) {
		if (addr == 0) {
			umem_panic("vmem_xalloc(): addr == 0");
		}
		ASSERT(vbest->vs_type == VMEM_FREE);
		ASSERT(vbest->vs_knext != vbest);
		(void) vmem_seg_alloc(vmp, vbest, addr, size);
		(void) mutex_unlock(&vmp->vm_lock);
		ASSERT(P2PHASE(addr, align) == phase);
		ASSERT(!P2BOUNDARY(addr, size, nocross));
		ASSERT(addr >= (uintptr_t)minaddr);
		ASSERT(addr + size - 1 <= (uintptr_t)maxaddr - 1);
		return ((void *)addr);
	}
	vmp->vm_kstat.vk_fail++;
	(void) mutex_unlock(&vmp->vm_lock);
	if (vmflag & VM_PANIC)
		umem_panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
		    "cannot satisfy mandatory allocation",
		    (void *)vmp, size, align, phase, nocross,
		    minaddr, maxaddr, vmflag);
	return (NULL);
}

/*
 * Free the segment [vaddr, vaddr + size), where vaddr was a constrained
 * allocation.  vmem_xalloc() and vmem_xfree() must always be paired because
 * both routines bypass the quantum caches.
 */
void
vmem_xfree(vmem_t *vmp, void *vaddr, size_t size)
{
	vmem_seg_t *vsp, *vnext, *vprev;

	(void) mutex_lock(&vmp->vm_lock);

	vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size);
	vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum);

	/*
	 * Attempt to coalesce with the next segment.
	 */
	vnext = vsp->vs_anext;
	if (vnext->vs_type == VMEM_FREE) {
		ASSERT(vsp->vs_end == vnext->vs_start);
		vmem_freelist_delete(vmp, vnext);
		vsp->vs_end = vnext->vs_end;
		vmem_seg_destroy(vmp, vnext);
	}

	/*
	 * Attempt to coalesce with the previous segment.
	 */
	vprev = vsp->vs_aprev;
	if (vprev->vs_type == VMEM_FREE) {
		ASSERT(vprev->vs_end == vsp->vs_start);
		vmem_freelist_delete(vmp, vprev);
		vprev->vs_end = vsp->vs_end;
		vmem_seg_destroy(vmp, vsp);
		vsp = vprev;
	}

	/*
	 * If the entire span is free, return it to the source.
	 */
	if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL &&
	    vsp->vs_aprev->vs_type == VMEM_SPAN &&
	    vsp->vs_anext->vs_type == VMEM_SPAN) {
		vaddr = (void *)vsp->vs_start;
		size = VS_SIZE(vsp);
		ASSERT(size == VS_SIZE(vsp->vs_aprev));
		vmem_span_destroy(vmp, vsp);
		(void) mutex_unlock(&vmp->vm_lock);
		vmp->vm_source_free(vmp->vm_source, vaddr, size);
	} else {
		vmem_freelist_insert(vmp, vsp);
		(void) mutex_unlock(&vmp->vm_lock);
	}
}

/*
 * Allocate size bytes from arena vmp.  Returns the allocated address
 * on success, NULL on failure.  vmflag specifies VM_SLEEP or VM_NOSLEEP,
 * and may also specify best-fit, first-fit, or next-fit allocation policy
 * instead of the default instant-fit policy.  VM_SLEEP allocations are
 * guaranteed to succeed.
 */
void *
vmem_alloc(vmem_t *vmp, size_t size, int vmflag)
{
	vmem_seg_t *vsp;
	uintptr_t addr;
	int hb;
	int flist = 0;
	uint32_t mtbf;
	vmflag |= vmem_allocator;

	if (size - 1 < vmp->vm_qcache_max) {
		ASSERT(vmflag & VM_NOSLEEP);
		return (_umem_cache_alloc(vmp->vm_qcache[(size - 1) >>
		    vmp->vm_qshift], UMEM_DEFAULT));
	}

	if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 &&
	    (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP)
		return (NULL);

	if (vmflag & VM_NEXTFIT)
		return (vmem_nextfit_alloc(vmp, size, vmflag));

	if (vmflag & (VM_BESTFIT | VM_FIRSTFIT))
		return (vmem_xalloc(vmp, size, vmp->vm_quantum, 0, 0,
		    NULL, NULL, vmflag));

	/*
	 * Unconstrained instant-fit allocation from the segment list.
	 */
	(void) mutex_lock(&vmp->vm_lock);

	if (vmp->vm_nsegfree >= VMEM_MINFREE || vmem_populate(vmp, vmflag)) {
		if ((size & (size - 1)) == 0)
			flist = lowbit(P2ALIGN(vmp->vm_freemap, size));
		else if ((hb = highbit(size)) < VMEM_FREELISTS)
			flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb));
	}

	if (flist-- == 0) {
		(void) mutex_unlock(&vmp->vm_lock);
		return (vmem_xalloc(vmp, size, vmp->vm_quantum,
		    0, 0, NULL, NULL, vmflag));
	}

	ASSERT(size <= (1UL << flist));
	vsp = vmp->vm_freelist[flist].vs_knext;
	addr = vsp->vs_start;
	(void) vmem_seg_alloc(vmp, vsp, addr, size);
	(void) mutex_unlock(&vmp->vm_lock);
	return ((void *)addr);
}

/*
 * Free the segment [vaddr, vaddr + size).
 */
void
vmem_free(vmem_t *vmp, void *vaddr, size_t size)
{
	if (size - 1 < vmp->vm_qcache_max)
		_umem_cache_free(vmp->vm_qcache[(size - 1) >> vmp->vm_qshift],
		    vaddr);
	else
		vmem_xfree(vmp, vaddr, size);
}

/*
 * Determine whether arena vmp contains the segment [vaddr, vaddr + size).
 */
int
vmem_contains(vmem_t *vmp, void *vaddr, size_t size)
{
	uintptr_t start = (uintptr_t)vaddr;
	uintptr_t end = start + size;
	vmem_seg_t *vsp;
	vmem_seg_t *seg0 = &vmp->vm_seg0;

	(void) mutex_lock(&vmp->vm_lock);
	vmp->vm_kstat.vk_contains++;
	for (vsp = seg0->vs_knext; vsp != seg0; vsp = vsp->vs_knext) {
		vmp->vm_kstat.vk_contains_search++;
		ASSERT(vsp->vs_type == VMEM_SPAN);
		if (start >= vsp->vs_start && end - 1 <= vsp->vs_end - 1)
			break;
	}
	(void) mutex_unlock(&vmp->vm_lock);
	return (vsp != seg0);
}

/*
 * Add the span [vaddr, vaddr + size) to arena vmp.
 */
void *
vmem_add(vmem_t *vmp, void *vaddr, size_t size, int vmflag)
{
	if (vaddr == NULL || size == 0) {
		umem_panic("vmem_add(%p, %p, %lu): bad arguments",
		    vmp, vaddr, size);
	}

	ASSERT(!vmem_contains(vmp, vaddr, size));

	(void) mutex_lock(&vmp->vm_lock);
	if (vmem_populate(vmp, vmflag))
		(void) vmem_span_create(vmp, vaddr, size, 0);
	else
		vaddr = NULL;
	(void) cond_broadcast(&vmp->vm_cv);
	(void) mutex_unlock(&vmp->vm_lock);
	return (vaddr);
}

/*
 * Adds the address range [addr, endaddr) to arena vmp, by either:
 *   1. joining two existing spans, [x, addr), and [endaddr, y) (which
 *      are in that order) into a single [x, y) span,
 *   2. expanding an existing [x, addr) span to [x, endaddr),
 *   3. expanding an existing [endaddr, x) span to [addr, x), or
 *   4. creating a new [addr, endaddr) span.
 *
 * Called with vmp->vm_lock held, and a successful vmem_populate() completed.
 * Cannot fail.  Returns the new segment.
 *
 * NOTE:  this algorithm is linear-time in the number of spans, but is
 *      constant-time when you are extending the last (highest-addressed)
 *      span.
 */
static vmem_seg_t *
vmem_extend_unlocked(vmem_t *vmp, uintptr_t addr, uintptr_t endaddr)
{
	vmem_seg_t *span;
	vmem_seg_t *vsp;

	vmem_seg_t *end = &vmp->vm_seg0;

	ASSERT(MUTEX_HELD(&vmp->vm_lock));

	/*
	 * the second "if" clause below relies on the direction of this search
	 */
	for (span = end->vs_kprev; span != end; span = span->vs_kprev) {
		if (span->vs_end == addr || span->vs_start == endaddr)
			break;
	}

	if (span == end)
		return (vmem_span_create(vmp, (void *)addr, endaddr - addr, 0));
	if (span->vs_kprev->vs_end == addr && span->vs_start == endaddr) {
		vmem_seg_t *prevspan = span->vs_kprev;
		vmem_seg_t *nextseg = span->vs_anext;
		vmem_seg_t *prevseg = span->vs_aprev;

		/*
		 * prevspan becomes the span marker for the full range
		 */
		prevspan->vs_end = span->vs_end;

		/*
		 * Notionally, span becomes a free segment representing
		 * [addr, endaddr).
		 *
		 * However, if either of its neighbors are free, we coalesce
		 * by destroying span and changing the free segment.
		 */
		if (prevseg->vs_type == VMEM_FREE &&
		    nextseg->vs_type == VMEM_FREE) {
			/*
			 * coalesce both ways
			 */
			ASSERT(prevseg->vs_end == addr &&
			    nextseg->vs_start == endaddr);

			vmem_freelist_delete(vmp, prevseg);
			prevseg->vs_end = nextseg->vs_end;

			vmem_freelist_delete(vmp, nextseg);
			VMEM_DELETE(span, k);
			vmem_seg_destroy(vmp, nextseg);
			vmem_seg_destroy(vmp, span);

			vsp = prevseg;
		} else if (prevseg->vs_type == VMEM_FREE) {
			/*
			 * coalesce left
			 */
			ASSERT(prevseg->vs_end == addr);

			VMEM_DELETE(span, k);
			vmem_seg_destroy(vmp, span);

			vmem_freelist_delete(vmp, prevseg);
			prevseg->vs_end = endaddr;

			vsp = prevseg;
		} else if (nextseg->vs_type == VMEM_FREE) {
			/*
			 * coalesce right
			 */
			ASSERT(nextseg->vs_start == endaddr);

			VMEM_DELETE(span, k);
			vmem_seg_destroy(vmp, span);

			vmem_freelist_delete(vmp, nextseg);
			nextseg->vs_start = addr;

			vsp = nextseg;
		} else {
			/*
			 * cannnot coalesce
			 */
			VMEM_DELETE(span, k);
			span->vs_start = addr;
			span->vs_end = endaddr;

			vsp = span;
		}
	} else if (span->vs_end == addr) {
		vmem_seg_t *oldseg = span->vs_knext->vs_aprev;
		span->vs_end = endaddr;

		ASSERT(oldseg->vs_type != VMEM_SPAN);
		if (oldseg->vs_type == VMEM_FREE) {
			ASSERT(oldseg->vs_end == addr);
			vmem_freelist_delete(vmp, oldseg);
			oldseg->vs_end = endaddr;
			vsp = oldseg;
		} else
			vsp = vmem_seg_create(vmp, oldseg, addr, endaddr);
	} else {
		vmem_seg_t *oldseg = span->vs_anext;
		ASSERT(span->vs_start == endaddr);
		span->vs_start = addr;

		ASSERT(oldseg->vs_type != VMEM_SPAN);
		if (oldseg->vs_type == VMEM_FREE) {
			ASSERT(oldseg->vs_start == endaddr);
			vmem_freelist_delete(vmp, oldseg);
			oldseg->vs_start = addr;
			vsp = oldseg;
		} else
			vsp = vmem_seg_create(vmp, span, addr, endaddr);
	}
	vmem_freelist_insert(vmp, vsp);
	vmp->vm_kstat.vk_mem_total += (endaddr - addr);
	return (vsp);
}

/*
 * Does some error checking, calls vmem_extend_unlocked to add
 * [vaddr, vaddr+size) to vmp, then allocates alloc bytes from the
 * newly merged segment.
 */
void *
_vmem_extend_alloc(vmem_t *vmp, void *vaddr, size_t size, size_t alloc,
    int vmflag)
{
	uintptr_t addr = (uintptr_t)vaddr;
	uintptr_t endaddr = addr + size;
	vmem_seg_t *vsp;

	ASSERT(vaddr != NULL && size != 0 && endaddr > addr);
	ASSERT(alloc <= size && alloc != 0);
	ASSERT(((addr | size | alloc) & (vmp->vm_quantum - 1)) == 0);

	ASSERT(!vmem_contains(vmp, vaddr, size));

	(void) mutex_lock(&vmp->vm_lock);
	if (!vmem_populate(vmp, vmflag)) {
		(void) mutex_unlock(&vmp->vm_lock);
		return (NULL);
	}
	/*
	 * if there is a source, we can't mess with the spans
	 */
	if (vmp->vm_source_alloc != NULL)
		vsp = vmem_span_create(vmp, vaddr, size, 0);
	else
		vsp = vmem_extend_unlocked(vmp, addr, endaddr);

	ASSERT(VS_SIZE(vsp) >= alloc);

	addr = vsp->vs_start;
	(void) vmem_seg_alloc(vmp, vsp, addr, alloc);
	vaddr = (void *)addr;

	(void) cond_broadcast(&vmp->vm_cv);
	(void) mutex_unlock(&vmp->vm_lock);

	return (vaddr);
}

/*
 * Walk the vmp arena, applying func to each segment matching typemask.
 * If VMEM_REENTRANT is specified, the arena lock is dropped across each
 * call to func(); otherwise, it is held for the duration of vmem_walk()
 * to ensure a consistent snapshot.  Note that VMEM_REENTRANT callbacks
 * are *not* necessarily consistent, so they may only be used when a hint
 * is adequate.
 */
void
vmem_walk(vmem_t *vmp, int typemask,
    void (*func)(void *, void *, size_t), void *arg)
{
	vmem_seg_t *vsp;
	vmem_seg_t *seg0 = &vmp->vm_seg0;
	vmem_seg_t walker;

	if (typemask & VMEM_WALKER)
		return;

	bzero(&walker, sizeof (walker));
	walker.vs_type = VMEM_WALKER;

	(void) mutex_lock(&vmp->vm_lock);
	VMEM_INSERT(seg0, &walker, a);
	for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) {
		if (vsp->vs_type & typemask) {
			void *start = (void *)vsp->vs_start;
			size_t size = VS_SIZE(vsp);
			if (typemask & VMEM_REENTRANT) {
				vmem_advance(vmp, &walker, vsp);
				(void) mutex_unlock(&vmp->vm_lock);
				func(arg, start, size);
				(void) mutex_lock(&vmp->vm_lock);
				vsp = &walker;
			} else {
				func(arg, start, size);
			}
		}
	}
	vmem_advance(vmp, &walker, NULL);
	(void) mutex_unlock(&vmp->vm_lock);
}

/*
 * Return the total amount of memory whose type matches typemask.  Thus:
 *
 *	typemask VMEM_ALLOC yields total memory allocated (in use).
 *	typemask VMEM_FREE yields total memory free (available).
 *	typemask (VMEM_ALLOC | VMEM_FREE) yields total arena size.
 */
size_t
vmem_size(vmem_t *vmp, int typemask)
{
	uint64_t size = 0;

	if (typemask & VMEM_ALLOC)
		size += vmp->vm_kstat.vk_mem_inuse;
	if (typemask & VMEM_FREE)
		size += vmp->vm_kstat.vk_mem_total -
		    vmp->vm_kstat.vk_mem_inuse;
	return ((size_t)size);
}

/*
 * Create an arena called name whose initial span is [base, base + size).
 * The arena's natural unit of currency is quantum, so vmem_alloc()
 * guarantees quantum-aligned results.  The arena may import new spans
 * by invoking afunc() on source, and may return those spans by invoking
 * ffunc() on source.  To make small allocations fast and scalable,
 * the arena offers high-performance caching for each integer multiple
 * of quantum up to qcache_max.
 */
vmem_t *
vmem_create(const char *name, void *base, size_t size, size_t quantum,
    vmem_alloc_t *afunc, vmem_free_t *ffunc, vmem_t *source,
    size_t qcache_max, int vmflag)
{
	int i;
	size_t nqcache;
	vmem_t *vmp, *cur, **vmpp;
	vmem_seg_t *vsp;
	vmem_freelist_t *vfp;
	uint32_t id = atomic_add_32_nv(&vmem_id, 1);

	if (vmem_vmem_arena != NULL) {
		vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t),
		    vmflag & VM_UMFLAGS);
	} else {
		ASSERT(id <= VMEM_INITIAL);
		vmp = &vmem0[id - 1];
	}

	if (vmp == NULL)
		return (NULL);
	bzero(vmp, sizeof (vmem_t));

	(void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name);
	(void) mutex_init(&vmp->vm_lock, USYNC_THREAD, NULL);
	(void) cond_init(&vmp->vm_cv, USYNC_THREAD, NULL);
	vmp->vm_cflags = vmflag;
	vmflag &= VM_UMFLAGS;

	vmp->vm_quantum = quantum;
	vmp->vm_qshift = highbit(quantum) - 1;
	nqcache = MIN(qcache_max >> vmp->vm_qshift, VMEM_NQCACHE_MAX);

	for (i = 0; i <= VMEM_FREELISTS; i++) {
		vfp = &vmp->vm_freelist[i];
		vfp->vs_end = 1UL << i;
		vfp->vs_knext = (vmem_seg_t *)(vfp + 1);
		vfp->vs_kprev = (vmem_seg_t *)(vfp - 1);
	}

	vmp->vm_freelist[0].vs_kprev = NULL;
	vmp->vm_freelist[VMEM_FREELISTS].vs_knext = NULL;
	vmp->vm_freelist[VMEM_FREELISTS].vs_end = 0;
	vmp->vm_hash_table = vmp->vm_hash0;
	vmp->vm_hash_mask = VMEM_HASH_INITIAL - 1;
	vmp->vm_hash_shift = highbit(vmp->vm_hash_mask);

	vsp = &vmp->vm_seg0;
	vsp->vs_anext = vsp;
	vsp->vs_aprev = vsp;
	vsp->vs_knext = vsp;
	vsp->vs_kprev = vsp;
	vsp->vs_type = VMEM_SPAN;

	vsp = &vmp->vm_rotor;
	vsp->vs_type = VMEM_ROTOR;
	VMEM_INSERT(&vmp->vm_seg0, vsp, a);

	vmp->vm_id = id;
	if (source != NULL)
		vmp->vm_kstat.vk_source_id = source->vm_id;
	vmp->vm_source = source;
	vmp->vm_source_alloc = afunc;
	vmp->vm_source_free = ffunc;

	if (nqcache != 0) {
		vmp->vm_qcache_max = nqcache << vmp->vm_qshift;
		for (i = 0; i < nqcache; i++) {
			char buf[VMEM_NAMELEN + 21];
			(void) snprintf(buf, sizeof (buf), "%s_%lu",
			    vmp->vm_name, (long)((i + 1) * quantum));
			vmp->vm_qcache[i] = umem_cache_create(buf,
			    (i + 1) * quantum, quantum, NULL, NULL, NULL,
			    NULL, vmp, UMC_QCACHE | UMC_NOTOUCH);
			if (vmp->vm_qcache[i] == NULL) {
				vmp->vm_qcache_max = i * quantum;
				break;
			}
		}
	}

	(void) mutex_lock(&vmem_list_lock);
	vmpp = &vmem_list;
	while ((cur = *vmpp) != NULL)
		vmpp = &cur->vm_next;
	*vmpp = vmp;
	(void) mutex_unlock(&vmem_list_lock);

	if (vmp->vm_cflags & VMC_POPULATOR) {
		uint_t pop_id = atomic_add_32_nv(&vmem_populators, 1);
		ASSERT(pop_id <= VMEM_INITIAL);
		vmem_populator[pop_id - 1] = vmp;
		(void) mutex_lock(&vmp->vm_lock);
		(void) vmem_populate(vmp, vmflag | VM_PANIC);
		(void) mutex_unlock(&vmp->vm_lock);
	}

	if ((base || size) && vmem_add(vmp, base, size, vmflag) == NULL) {
		vmem_destroy(vmp);
		return (NULL);
	}

	return (vmp);
}

/*
 * Destroy arena vmp.
 */
void
vmem_destroy(vmem_t *vmp)
{
	vmem_t *cur, **vmpp;
	vmem_seg_t *seg0 = &vmp->vm_seg0;
	vmem_seg_t *vsp;
	size_t leaked;
	int i;

	(void) mutex_lock(&vmem_list_lock);
	vmpp = &vmem_list;
	while ((cur = *vmpp) != vmp)
		vmpp = &cur->vm_next;
	*vmpp = vmp->vm_next;
	(void) mutex_unlock(&vmem_list_lock);

	for (i = 0; i < VMEM_NQCACHE_MAX; i++)
		if (vmp->vm_qcache[i])
			umem_cache_destroy(vmp->vm_qcache[i]);

	leaked = vmem_size(vmp, VMEM_ALLOC);
	if (leaked != 0)
		umem_printf("vmem_destroy('%s'): leaked %lu bytes",
		    vmp->vm_name, leaked);

	if (vmp->vm_hash_table != vmp->vm_hash0)
		vmem_free(vmem_hash_arena, vmp->vm_hash_table,
		    (vmp->vm_hash_mask + 1) * sizeof (void *));

	/*
	 * Give back the segment structures for anything that's left in the
	 * arena, e.g. the primary spans and their free segments.
	 */
	VMEM_DELETE(&vmp->vm_rotor, a);
	for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext)
		vmem_putseg_global(vsp);

	while (vmp->vm_nsegfree > 0)
		vmem_putseg_global(vmem_getseg(vmp));

	(void) mutex_destroy(&vmp->vm_lock);
	(void) cond_destroy(&vmp->vm_cv);
	vmem_free(vmem_vmem_arena, vmp, sizeof (vmem_t));
}

/*
 * Resize vmp's hash table to keep the average lookup depth near 1.0.
 */
static void
vmem_hash_rescale(vmem_t *vmp)
{
	vmem_seg_t **old_table, **new_table, *vsp;
	size_t old_size, new_size, h, nseg;

	nseg = (size_t)(vmp->vm_kstat.vk_alloc - vmp->vm_kstat.vk_free);

	new_size = MAX(VMEM_HASH_INITIAL, 1 << (highbit(3 * nseg + 4) - 2));
	old_size = vmp->vm_hash_mask + 1;

	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
		return;

	new_table = vmem_alloc(vmem_hash_arena, new_size * sizeof (void *),
	    VM_NOSLEEP);
	if (new_table == NULL)
		return;
	bzero(new_table, new_size * sizeof (void *));

	(void) mutex_lock(&vmp->vm_lock);

	old_size = vmp->vm_hash_mask + 1;
	old_table = vmp->vm_hash_table;

	vmp->vm_hash_mask = new_size - 1;
	vmp->vm_hash_table = new_table;
	vmp->vm_hash_shift = highbit(vmp->vm_hash_mask);

	for (h = 0; h < old_size; h++) {
		vsp = old_table[h];
		while (vsp != NULL) {
			uintptr_t addr = vsp->vs_start;
			vmem_seg_t *next_vsp = vsp->vs_knext;
			vmem_seg_t **hash_bucket = VMEM_HASH(vmp, addr);
			vsp->vs_knext = *hash_bucket;
			*hash_bucket = vsp;
			vsp = next_vsp;
		}
	}

	(void) mutex_unlock(&vmp->vm_lock);

	if (old_table != vmp->vm_hash0)
		vmem_free(vmem_hash_arena, old_table,
		    old_size * sizeof (void *));
}

/*
 * Perform periodic maintenance on all vmem arenas.
 */
/*ARGSUSED*/
void
vmem_update(void *dummy)
{
	vmem_t *vmp;

	(void) mutex_lock(&vmem_list_lock);
	for (vmp = vmem_list; vmp != NULL; vmp = vmp->vm_next) {
		/*
		 * If threads are waiting for resources, wake them up
		 * periodically so they can issue another vmem_reap()
		 * to reclaim resources cached by the slab allocator.
		 */
		(void) cond_broadcast(&vmp->vm_cv);

		/*
		 * Rescale the hash table to keep the hash chains short.
		 */
		vmem_hash_rescale(vmp);
	}
	(void) mutex_unlock(&vmem_list_lock);
}

/*
 * If vmem_init is called again, we need to be able to reset the world.
 * That includes resetting the statics back to their original values.
 */
void
vmem_startup(void)
{
#ifdef UMEM_STANDALONE
	vmem_id = 0;
	vmem_populators = 0;
	vmem_segfree = NULL;
	vmem_list = NULL;
	vmem_internal_arena = NULL;
	vmem_seg_arena = NULL;
	vmem_hash_arena = NULL;
	vmem_vmem_arena = NULL;
	vmem_heap = NULL;
	vmem_heap_alloc = NULL;
	vmem_heap_free = NULL;

	bzero(vmem0, sizeof (vmem0));
	bzero(vmem_populator, sizeof (vmem_populator));
	bzero(vmem_seg0, sizeof (vmem_seg0));
#endif
}

/*
 * Prepare vmem for use.
 */
vmem_t *
vmem_init(const char *parent_name, size_t parent_quantum,
    vmem_alloc_t *parent_alloc, vmem_free_t *parent_free,
    const char *heap_name, void *heap_start, size_t heap_size,
    size_t heap_quantum, vmem_alloc_t *heap_alloc, vmem_free_t *heap_free)
{
	uint32_t id;
	int nseg = VMEM_SEG_INITIAL;
	vmem_t *parent, *heap;

	ASSERT(vmem_internal_arena == NULL);

	while (--nseg >= 0)
		vmem_putseg_global(&vmem_seg0[nseg]);

	if (parent_name != NULL) {
		parent = vmem_create(parent_name,
		    heap_start, heap_size, parent_quantum,
		    NULL, NULL, NULL, 0,
		    VM_SLEEP | VMC_POPULATOR);
		heap_start = NULL;
		heap_size = 0;
	} else {
		ASSERT(parent_alloc == NULL && parent_free == NULL);
		parent = NULL;
	}

	heap = vmem_create(heap_name,
	    heap_start, heap_size, heap_quantum,
	    parent_alloc, parent_free, parent, 0,
	    VM_SLEEP | VMC_POPULATOR);

	vmem_heap = heap;
	vmem_heap_alloc = heap_alloc;
	vmem_heap_free = heap_free;

	vmem_internal_arena = vmem_create("vmem_internal",
	    NULL, 0, heap_quantum,
	    heap_alloc, heap_free, heap, 0,
	    VM_SLEEP | VMC_POPULATOR);

	vmem_seg_arena = vmem_create("vmem_seg",
	    NULL, 0, heap_quantum,
	    vmem_alloc, vmem_free, vmem_internal_arena, 0,
	    VM_SLEEP | VMC_POPULATOR);

	vmem_hash_arena = vmem_create("vmem_hash",
	    NULL, 0, 8,
	    vmem_alloc, vmem_free, vmem_internal_arena, 0,
	    VM_SLEEP);

	vmem_vmem_arena = vmem_create("vmem_vmem",
	    vmem0, sizeof (vmem0), 1,
	    vmem_alloc, vmem_free, vmem_internal_arena, 0,
	    VM_SLEEP);

	for (id = 0; id < vmem_id; id++)
		(void) vmem_xalloc(vmem_vmem_arena, sizeof (vmem_t),
		    1, 0, 0, &vmem0[id], &vmem0[id + 1],
		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

	return (heap);
}

void
vmem_no_debug(void)
{
	/*
	 * This size must be a multiple of the minimum required alignment,
	 * since vmem_populate allocates them compactly.
	 */
	vmem_seg_size = P2ROUNDUP(offsetof(vmem_seg_t, vs_thread),
	    sizeof (hrtime_t));
}

/*
 * Lockup and release, for fork1(2) handling.
 */
void
vmem_lockup(void)
{
	vmem_t *cur;

	(void) mutex_lock(&vmem_list_lock);
	(void) mutex_lock(&vmem_nosleep_lock.vmpl_mutex);

	/*
	 * Lock up and broadcast all arenas.
	 */
	for (cur = vmem_list; cur != NULL; cur = cur->vm_next) {
		(void) mutex_lock(&cur->vm_lock);
		(void) cond_broadcast(&cur->vm_cv);
	}

	(void) mutex_lock(&vmem_segfree_lock);
}

void
vmem_release(void)
{
	vmem_t *cur;

	(void) mutex_unlock(&vmem_nosleep_lock.vmpl_mutex);

	for (cur = vmem_list; cur != NULL; cur = cur->vm_next)
		(void) mutex_unlock(&cur->vm_lock);

	(void) mutex_unlock(&vmem_segfree_lock);
	(void) mutex_unlock(&vmem_list_lock);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2012 Joyent, Inc. All rights reserved.
 */

#include "vmem_base.h"
#include "umem_base.h"

uint_t vmem_backend = 0;
uint_t vmem_allocator = VM_BESTFIT;

vmem_t *
vmem_heap_arena(vmem_alloc_t **allocp, vmem_free_t **freep)
{
	static mutex_t arena_mutex = DEFAULTMUTEX;

	/*
	 * Allow the init thread through, block others until the init completes
	 */
	if (umem_ready != UMEM_READY && umem_init_thr != thr_self() &&
	    umem_init() == 0)
		return (NULL);

	(void) mutex_lock(&arena_mutex);
	if (vmem_heap == NULL)
		vmem_heap_init();
	(void) mutex_unlock(&arena_mutex);

	if (allocp != NULL)
		*allocp = vmem_heap_alloc;
	if (freep != NULL)
		*freep = vmem_heap_free;
	return (vmem_heap);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2012 Joyent, Inc. All rights reserved.
 */

#ifndef	_VMEM_BASE_H
#define	_VMEM_BASE_H

#include <sys/vmem.h>
#include <umem.h>

#ifdef	__cplusplus
extern "C" {
#endif

#include "misc.h"

extern void vmem_startup(void);
extern vmem_t *vmem_init(const char *parent_name, size_t parent_quantum,
	vmem_alloc_t *parent_alloc, vmem_free_t *parent_free,
	const char *heap_name,
	void *heap_start, size_t heap_size, size_t heap_quantum,
	vmem_alloc_t *heap_alloc, vmem_free_t *heap_free);

extern void *_vmem_extend_alloc(vmem_t *vmp, void *vaddr, size_t size,
	size_t alloc, int vmflag);

extern vmem_t *vmem_heap_arena(vmem_alloc_t **, vmem_free_t **);
extern void vmem_heap_init(void);

extern vmem_t *vmem_sbrk_arena(vmem_alloc_t **, vmem_free_t **);
extern vmem_t *vmem_mmap_arena(vmem_alloc_t **, vmem_free_t **);
extern vmem_t *vmem_stand_arena(vmem_alloc_t **, vmem_free_t **);

extern void vmem_update(void *);
extern void vmem_reap(void);		/* vmem_populate()-safe reap */

extern size_t pagesize;
extern size_t vmem_sbrk_pagesize;
extern size_t vmem_sbrk_minalloc;

extern uint_t vmem_backend;
#define	VMEM_BACKEND_SBRK	0x0000001
#define	VMEM_BACKEND_MMAP	0x0000002
#define	VMEM_BACKEND_STAND	0x0000003

extern uint_t vmem_allocator;

extern vmem_t *vmem_heap;
extern vmem_alloc_t *vmem_heap_alloc;
extern vmem_free_t *vmem_heap_free;

extern void vmem_lockup(void);
extern void vmem_release(void);

extern void vmem_sbrk_lockup(void);
extern void vmem_sbrk_release(void);

extern void vmem_no_debug(void);

#ifdef	__cplusplus
}
#endif

#endif	/* _VMEM_BASE_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/sysmacros.h>
#include "vmem_base.h"

#define	ALLOC_PROT	PROT_READ | PROT_WRITE | PROT_EXEC
#define	FREE_PROT	PROT_NONE

#define	ALLOC_FLAGS	MAP_PRIVATE | MAP_ANON
#define	FREE_FLAGS	MAP_PRIVATE | MAP_ANON | MAP_NORESERVE

#define	CHUNKSIZE	(64*1024)	/* 64 kilobytes */

static vmem_t *mmap_heap;

static void *
vmem_mmap_alloc(vmem_t *src, size_t size, int vmflags)
{
	void *ret;
	int old_errno = errno;

	ret = vmem_alloc(src, size, vmflags);
	if (ret != NULL &&
	    mmap(ret, size, ALLOC_PROT, ALLOC_FLAGS | MAP_FIXED, -1, 0) ==
	    MAP_FAILED) {
		vmem_free(src, ret, size);
		vmem_reap();

		ASSERT((vmflags & VM_NOSLEEP) == VM_NOSLEEP);
		errno = old_errno;
		return (NULL);
	}

	errno = old_errno;
	return (ret);
}

static void
vmem_mmap_free(vmem_t *src, void *addr, size_t size)
{
	int old_errno = errno;
	(void) mmap(addr, size, FREE_PROT, FREE_FLAGS | MAP_FIXED, -1, 0);
	vmem_free(src, addr, size);
	errno = old_errno;
}

static void *
vmem_mmap_top_alloc(vmem_t *src, size_t size, int vmflags)
{
	void *ret;
	void *buf;
	int old_errno = errno;

	ret = vmem_alloc(src, size, VM_NOSLEEP);

	if (ret) {
		errno = old_errno;
		return (ret);
	}
	/*
	 * Need to grow the heap
	 */
	buf = mmap((void *)CHUNKSIZE, size, FREE_PROT, FREE_FLAGS | MAP_ALIGN,
	    -1, 0);

	if (buf != MAP_FAILED) {
		ret = _vmem_extend_alloc(src, buf, size, size, vmflags);
		if (ret != NULL)
			return (ret);
		else {
			(void) munmap(buf, size);
			errno = old_errno;
			return (NULL);
		}
	} else {
		/*
		 * Growing the heap failed.  The allocation above will
		 * already have called umem_reap().
		 */
		ASSERT((vmflags & VM_NOSLEEP) == VM_NOSLEEP);

		errno = old_errno;
		return (NULL);
	}
}

vmem_t *
vmem_mmap_arena(vmem_alloc_t **a_out, vmem_free_t **f_out)
{
	size_t pagesize = sysconf(_SC_PAGESIZE);

	if (mmap_heap == NULL) {
		mmap_heap = vmem_init("mmap_top", CHUNKSIZE,
		    vmem_mmap_top_alloc, vmem_free,
		    "mmap_heap", NULL, 0, pagesize,
		    vmem_mmap_alloc, vmem_mmap_free);
	}

	if (a_out != NULL)
		*a_out = vmem_mmap_alloc;
	if (f_out != NULL)
		*f_out = vmem_mmap_free;

	return (mmap_heap);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * The structure of the sbrk backend:
 *
 * +-----------+
 * | sbrk_top  |
 * +-----------+
 *      | (vmem_sbrk_alloc(), vmem_free())
 *      |
 * +-----------+
 * | sbrk_heap |
 * +-----------+
 *   | | ... |  (vmem_alloc(), vmem_free())
 * <other arenas>
 *
 * The sbrk_top arena holds all controlled memory.  vmem_sbrk_alloc() handles
 * allocations from it, including growing the heap when we run low.
 *
 * Growing the heap is complicated by the fact that we have to extend the
 * sbrk_top arena (using _vmem_extend_alloc()), and that can fail.  Since
 * other threads may be actively allocating, we can't return the memory.
 *
 * Instead, we put it on a doubly-linked list, sbrk_fails, which we search
 * before calling sbrk().
 */

#include <errno.h>
#include <limits.h>
#include <sys/sysmacros.h>
#include <sys/mman.h>
#include <unistd.h>

#include "vmem_base.h"

#include "misc.h"

size_t vmem_sbrk_pagesize = 0; /* the preferred page size of the heap */

#define	VMEM_SBRK_MINALLOC	(64 * 1024)
size_t vmem_sbrk_minalloc = VMEM_SBRK_MINALLOC; /* minimum allocation */

static size_t real_pagesize;
static vmem_t *sbrk_heap;

typedef struct sbrk_fail {
	struct sbrk_fail *sf_next;
	struct sbrk_fail *sf_prev;
	void *sf_base;			/* == the sbrk_fail's address */
	size_t sf_size;			/* the size of this buffer */
} sbrk_fail_t;

static sbrk_fail_t sbrk_fails = {
	&sbrk_fails,
	&sbrk_fails,
	NULL,
	0
};

static mutex_t sbrk_faillock = DEFAULTMUTEX;

/*
 * Try to extend src with [pos, pos + size).
 *
 * If it fails, add the block to the sbrk_fails list.
 */
static void *
vmem_sbrk_extend_alloc(vmem_t *src, void *pos, size_t size, size_t alloc,
    int vmflags)
{
	sbrk_fail_t *fnext, *fprev, *fp;
	void *ret;

	ret = _vmem_extend_alloc(src, pos, size, alloc, vmflags);
	if (ret != NULL)
		return (ret);

	fp = (sbrk_fail_t *)pos;

	ASSERT(sizeof (sbrk_fail_t) <= size);

	fp->sf_base = pos;
	fp->sf_size = size;

	(void) mutex_lock(&sbrk_faillock);
	fp->sf_next = fnext = &sbrk_fails;
	fp->sf_prev = fprev = sbrk_fails.sf_prev;
	fnext->sf_prev = fp;
	fprev->sf_next = fp;
	(void) mutex_unlock(&sbrk_faillock);

	return (NULL);
}

/*
 * Try to add at least size bytes to src, using the sbrk_fails list
 */
static void *
vmem_sbrk_tryfail(vmem_t *src, size_t size, int vmflags)
{
	sbrk_fail_t *fp;

	(void) mutex_lock(&sbrk_faillock);
	for (fp = sbrk_fails.sf_next; fp != &sbrk_fails; fp = fp->sf_next) {
		if (fp->sf_size >= size) {
			fp->sf_next->sf_prev = fp->sf_prev;
			fp->sf_prev->sf_next = fp->sf_next;
			fp->sf_next = fp->sf_prev = NULL;
			break;
		}
	}
	(void) mutex_unlock(&sbrk_faillock);

	if (fp != &sbrk_fails) {
		ASSERT(fp->sf_base == (void *)fp);
		return (vmem_sbrk_extend_alloc(src, fp, fp->sf_size, size,
		    vmflags));
	}
	/*
	 * nothing of the right size on the freelist
	 */
	return (NULL);
}

static void *
vmem_sbrk_alloc(vmem_t *src, size_t size, int vmflags)
{
	extern void *_sbrk_grow_aligned(size_t min_size, size_t low_align,
	    size_t high_align, size_t *actual_size);

	void *ret;
	void *buf;
	size_t buf_size;

	int old_errno = errno;

	ret = vmem_alloc(src, size, VM_NOSLEEP);
	if (ret != NULL) {
		errno = old_errno;
		return (ret);
	}

	/*
	 * The allocation failed.  We need to grow the heap.
	 *
	 * First, try to use any buffers which failed earlier.
	 */
	if (sbrk_fails.sf_next != &sbrk_fails &&
	    (ret = vmem_sbrk_tryfail(src, size, vmflags)) != NULL)
		return (ret);

	buf_size = MAX(size, vmem_sbrk_minalloc);

	/*
	 * buf_size gets overwritten with the actual allocated size
	 */
	buf = _sbrk_grow_aligned(buf_size, real_pagesize, vmem_sbrk_pagesize,
	    &buf_size);

	if (buf != MAP_FAILED) {
		ret = vmem_sbrk_extend_alloc(src, buf, buf_size, size, vmflags);
		if (ret != NULL) {
			errno = old_errno;
			return (ret);
		}
	}

	/*
	 * Growing the heap failed. The vmem_alloc() above called umem_reap().
	 */
	ASSERT((vmflags & VM_NOSLEEP) == VM_NOSLEEP);

	errno = old_errno;
	return (NULL);
}

/*
 * fork1() support
 */
void
vmem_sbrk_lockup(void)
{
	(void) mutex_lock(&sbrk_faillock);
}

void
vmem_sbrk_release(void)
{
	(void) mutex_unlock(&sbrk_faillock);
}

vmem_t *
vmem_sbrk_arena(vmem_alloc_t **a_out, vmem_free_t **f_out)
{
	if (sbrk_heap == NULL) {
		size_t heap_size;

		real_pagesize = sysconf(_SC_PAGESIZE);

		heap_size = vmem_sbrk_pagesize;

		if (issetugid()) {
			heap_size = 0;
		} else if (heap_size != 0 && !ISP2(heap_size)) {
			heap_size = 0;
			log_message("ignoring bad pagesize: 0x%p\n", heap_size);
		}
		if (heap_size <= real_pagesize) {
			heap_size = real_pagesize;
		} else {
			struct memcntl_mha mha;
			mha.mha_cmd = MHA_MAPSIZE_BSSBRK;
			mha.mha_flags = 0;
			mha.mha_pagesize = heap_size;

			if (memcntl(NULL, 0, MC_HAT_ADVISE, (char *)&mha, 0, 0)
			    == -1) {
				log_message("unable to set MAPSIZE_BSSBRK to "
				    "0x%p\n", heap_size);
				heap_size = real_pagesize;
			}
		}
		vmem_sbrk_pagesize = heap_size;

		/* validate vmem_sbrk_minalloc */
		if (vmem_sbrk_minalloc < VMEM_SBRK_MINALLOC)
			vmem_sbrk_minalloc = VMEM_SBRK_MINALLOC;
		vmem_sbrk_minalloc = P2ROUNDUP(vmem_sbrk_minalloc, heap_size);

		sbrk_heap = vmem_init("sbrk_top", real_pagesize,
		    vmem_sbrk_alloc, vmem_free,
		    "sbrk_heap", NULL, 0, real_pagesize,
		    vmem_alloc, vmem_free);
	}

	if (a_out != NULL)
		*a_out = vmem_alloc;
	if (f_out != NULL)
		*f_out = vmem_free;

	return (sbrk_heap);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Standalone-specific vmem routines
 *
 * The standalone allocator operates on a pre-existing blob of memory, the
 * location and dimensions of which are set using vmem_stand_setsize().  We
 * then hand out CHUNKSIZE-sized pieces of this blob, until we run out.
 */

#define	DEF_CHUNKSIZE	(64 * 1024)	/* 64K */

#define	DEF_NREGIONS	2

#include <errno.h>
#include <limits.h>
#include <sys/sysmacros.h>
#include <sys/mman.h>
#include <unistd.h>
#include <strings.h>

#include "vmem_base.h"
#include "misc.h"

static vmem_t *stand_heap;

static size_t stand_chunksize;

typedef struct stand_region {
	caddr_t sr_base;
	caddr_t sr_curtop;
	size_t sr_left;
} stand_region_t;

static stand_region_t stand_regions[DEF_NREGIONS];
static int stand_nregions;

extern void membar_producer(void);

void
vmem_stand_init(void)
{
	stand_chunksize = MAX(DEF_CHUNKSIZE, pagesize);

	stand_nregions = 0;
}

int
vmem_stand_add(caddr_t base, size_t len)
{
	stand_region_t *sr = &stand_regions[stand_nregions];

	ASSERT(pagesize != 0);

	if (stand_nregions == DEF_NREGIONS) {
		errno = ENOSPC;
		return (-1); /* we don't have room -- throw it back */
	}

	/*
	 * We guarantee that only one call to `vmem_stand_add' will be
	 * active at a time, but we can't ensure that the allocator won't be
	 * in use while this function is being called.  As such, we have to
	 * ensure that sr is populated and visible to other processors before
	 * allowing the allocator to access the new region.
	 */
	sr->sr_base = base;
	sr->sr_curtop = (caddr_t)P2ROUNDUP((ulong_t)base, stand_chunksize);
	sr->sr_left = P2ALIGN(len - (size_t)(sr->sr_curtop - sr->sr_base),
	    stand_chunksize);
	membar_producer();

	stand_nregions++;

	return (0);
}

static void *
stand_parent_alloc(vmem_t *src, size_t size, int vmflags)
{
	int old_errno = errno;
	stand_region_t *sr;
	size_t chksize;
	void *ret;
	int i;

	if ((ret = vmem_alloc(src, size, VM_NOSLEEP)) != NULL) {
		errno = old_errno;
		return (ret);
	}

	/* We need to allocate another chunk */
	chksize = roundup(size, stand_chunksize);

	for (sr = stand_regions, i = 0; i < stand_nregions; i++, sr++) {
		if (sr->sr_left >= chksize)
			break;
	}

	if (i == stand_nregions) {
		/*
		 * We don't have enough in any of our regions to satisfy the
		 * request.
		 */
		errno = old_errno;
		return (NULL);
	}

	if ((ret = _vmem_extend_alloc(src, sr->sr_curtop, chksize, size,
	    vmflags)) == NULL) {
		errno = old_errno;
		return (NULL);
	}

	bzero(sr->sr_curtop, chksize);

	sr->sr_curtop += chksize;
	sr->sr_left -= chksize;

	return (ret);
}

vmem_t *
vmem_stand_arena(vmem_alloc_t **a_out, vmem_free_t **f_out)
{
	ASSERT(stand_nregions == 1);

	stand_heap = vmem_init("stand_parent", stand_chunksize,
	    stand_parent_alloc, vmem_free,
	    "stand_heap", NULL, 0, pagesize, vmem_alloc, vmem_free);

	if (a_out != NULL)
		*a_out = vmem_alloc;
	if (f_out != NULL)
		*f_out = vmem_free;

	return (stand_heap);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _VMEM_STAND_H
#define	_VMEM_STAND_H

/*
 * additional functions defined by the standalone backend
 */

#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#endif

extern void vmem_stand_init(void);
extern int vmem_stand_add(caddr_t, size_t);

#ifdef __cplusplus
}
#endif

#endif /* _VMEM_STAND_H */