softAESCrypt.c - zygaena/hammerhead

root / base / usr / src / lib / pkcs11 / pkcs11_softtoken / common / softAESCrypt.c
softAESCrypt.c C 1723 lines 48.1 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2017 Jason King.
 * Copyright 2025 RackTop Systems, Inc.
 */

#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/debug.h>
#include <sys/types.h>
#include <security/cryptoki.h>
#include <aes_impl.h>
#include <cryptoutil.h>
#include "softSession.h"
#include "softObject.h"
#include "softCrypt.h"
#include "softOps.h"

/*
 * Check that the mechanism parameter is present (if required)
 * and has the correct size. If so, allocate an AES context.
 *
 * Special handling for AES_GMAC paramters:
 *
 * The current PKCS#11 documentation for using AES_GMAC is:
 *	https://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/
 *	errata01/os/pkcs11-curr-v2.40-errata01-os-complete.html
 * According to that, the C_SignInit parameter for AES_GMAC is
 * just the 12-byte Initialization Vector (IV).  The passed IV
 * is converted to CK_AES_GMAC_PARAMS soft_aes_init_ctx().
 */
static CK_RV
soft_aes_check_mech_param(CK_MECHANISM_PTR mech, aes_ctx_t **ctxp)
{
	void *(*allocf)(int) = NULL;
	size_t param_len = 0;
	boolean_t param_req = B_TRUE;

	switch (mech->mechanism) {
	case CKM_AES_ECB:
		param_req = B_FALSE;
		allocf = ecb_alloc_ctx;
		break;
	case CKM_AES_CMAC:
		param_req = B_FALSE;
		allocf = cmac_alloc_ctx;
		break;
	case CKM_AES_CMAC_GENERAL:
		param_len = sizeof (CK_MAC_GENERAL_PARAMS);
		allocf = cmac_alloc_ctx;
		break;
	case CKM_AES_GMAC:
		/* See comment above this function. */
		param_len = AES_GMAC_IV_LEN;
		allocf = gmac_alloc_ctx;
		break;
	case CKM_AES_CBC:
	case CKM_AES_CBC_PAD:
		param_len = AES_BLOCK_LEN;
		allocf = cbc_alloc_ctx;
		break;
	case CKM_AES_CTR:
		param_len = sizeof (CK_AES_CTR_PARAMS);
		allocf = ctr_alloc_ctx;
		break;
	case CKM_AES_CCM:
		param_len = sizeof (CK_CCM_PARAMS);
		allocf = ccm_alloc_ctx;
		break;
	case CKM_AES_GCM:
		param_len = sizeof (CK_GCM_PARAMS);
		allocf = gcm_alloc_ctx;
		break;
	default:
		return (CKR_MECHANISM_INVALID);
	}

	if (param_req && (mech->pParameter == NULL ||
	    mech->ulParameterLen != param_len)) {
		return (CKR_MECHANISM_PARAM_INVALID);
	}

	*ctxp = allocf(0);
	if (*ctxp == NULL) {
		return (CKR_HOST_MEMORY);
	}

	return (CKR_OK);
}

/*
 * Create an AES key schedule for the given AES context from the given key.
 * If the key is not sensitive, cache a copy of the key schedule in the
 * key object and/or use the cached copy of the key schedule.
 *
 * Must be called before the init function for a given mode is called.
 */
static CK_RV
soft_aes_init_key(aes_ctx_t *aes_ctx, soft_object_t *key_p)
{
	void *ks = NULL;
	size_t size = 0;
	CK_RV rv = CKR_OK;

	(void) pthread_mutex_lock(&key_p->object_mutex);

	/*
	 * AES keys should be either 128, 192, or 256 bits long.
	 * soft_object_t stores the key size in bytes, so we check those sizes
	 * in bytes.
	 *
	 * While soft_build_secret_key_object() does these same validations for
	 * keys created by the user, it may be possible that a key loaded from
	 * disk could be invalid or corrupt.  We err on the side of caution
	 * and check again that it's the correct size before performing any
	 * AES operations.
	 */
	switch (OBJ_SEC_VALUE_LEN(key_p)) {
	case AES_MIN_KEY_BYTES:
	case AES_MAX_KEY_BYTES:
	case AES_192_KEY_BYTES:
		break;
	default:
		rv = CKR_KEY_SIZE_RANGE;
		goto done;
	}

	ks = aes_alloc_keysched(&size, 0);
	if (ks == NULL) {
		rv = CKR_HOST_MEMORY;
		goto done;
	}

	/* If this is a sensitive key, always expand the key schedule */
	if (key_p->bool_attr_mask & SENSITIVE_BOOL_ON) {
		/* aes_init_keysched() requires key length in bits.  */
#ifdef	__sparcv9
		/* LINTED */
		aes_init_keysched(OBJ_SEC_VALUE(key_p), (uint_t)
		    (OBJ_SEC_VALUE_LEN(key_p) * NBBY), ks);
#else	/* !__sparcv9 */
		aes_init_keysched(OBJ_SEC_VALUE(key_p),
		    (OBJ_SEC_VALUE_LEN(key_p) * NBBY), ks);
#endif	/* __sparcv9 */

		goto done;
	}

	/* If a non-sensitive key and doesn't have a key schedule, create it */
	if (OBJ_KEY_SCHED(key_p) == NULL) {
		void *obj_ks = NULL;

		obj_ks = aes_alloc_keysched(&size, 0);
		if (obj_ks == NULL) {
			rv = CKR_HOST_MEMORY;
			goto done;
		}

#ifdef	__sparcv9
		/* LINTED */
		aes_init_keysched(OBJ_SEC_VALUE(key_p),
		    (uint_t)(OBJ_SEC_VALUE_LEN(key_p) * 8), obj_ks);
#else	/* !__sparcv9 */
		aes_init_keysched(OBJ_SEC_VALUE(key_p),
		    (OBJ_SEC_VALUE_LEN(key_p) * 8), obj_ks);
#endif	/* __sparcv9 */

		OBJ_KEY_SCHED_LEN(key_p) = size;
		OBJ_KEY_SCHED(key_p) = obj_ks;
	}

	(void) memcpy(ks, OBJ_KEY_SCHED(key_p), OBJ_KEY_SCHED_LEN(key_p));

done:
	(void) pthread_mutex_unlock(&key_p->object_mutex);

	if (rv == CKR_OK) {
		aes_ctx->ac_keysched = ks;
		aes_ctx->ac_keysched_len = size;
	} else {
		freezero(ks, size);
	}

	return (rv);
}

/*
 * Initialize the AES context for the given mode, including allocating and
 * expanding the key schedule if required.
 */
static CK_RV
soft_aes_init_ctx(aes_ctx_t *aes_ctx, CK_MECHANISM_PTR mech_p,
    boolean_t encrypt)
{
	int rc = CRYPTO_SUCCESS;

	switch (mech_p->mechanism) {
	case CKM_AES_ECB:
		aes_ctx->ac_flags |= ECB_MODE;
		break;
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
		rc = cmac_init_ctx((cbc_ctx_t *)aes_ctx, AES_BLOCK_LEN);
		break;
	case CKM_AES_GMAC: {
		/*
		 * Given params are just the initialization vector.
		 * See comment above soft_aes_check_mech_param.
		 * Convert to CK_AES_GMAC_PARAMS for gmac_init_ctx.
		 */
		CK_AES_GMAC_PARAMS gmac_params = {
			.pIv = mech_p->pParameter,
			.pAAD = NULL,
			.ulAADLen = 0
		};
		rc = gmac_init_ctx((gcm_ctx_t *)aes_ctx, (char *)&gmac_params,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		break;
	}
	case CKM_AES_CBC:
	case CKM_AES_CBC_PAD:
		rc = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mech_p->pParameter,
		    mech_p->ulParameterLen, AES_BLOCK_LEN, aes_copy_block64);
		break;
	case CKM_AES_CTR:
	{
		/*
		 * soft_aes_check_param() verifies this is !NULL and is the
		 * correct size.
		 */
		CK_AES_CTR_PARAMS *pp = (CK_AES_CTR_PARAMS *)mech_p->pParameter;

		rc = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
		    pp->cb, aes_encrypt_block, aes_copy_block);
		break;
	}
	case CKM_AES_CCM: {
		CK_CCM_PARAMS *pp = (CK_CCM_PARAMS *)mech_p->pParameter;

		/*
		 * The illumos ccm mode implementation predates the PKCS#11
		 * version that specifies CK_CCM_PARAMS.  As a result, the order
		 * and names of the struct members are different, so we must
		 * translate.  ccm_init_ctx() does not store a ref ccm_params,
		 * so it is safe to allocate on the stack.
		 */
		CK_AES_CCM_PARAMS ccm_params = {
			.ulMACSize = pp->ulMACLen,
			.ulNonceSize = pp->ulNonceLen,
			.ulAuthDataSize = pp->ulAADLen,
			.ulDataSize = pp->ulDataLen,
			.nonce = pp->pNonce,
			.authData = pp->pAAD
		};

		rc = ccm_init_ctx((ccm_ctx_t *)aes_ctx, (char *)&ccm_params, 0,
		    encrypt, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		break;
	}
	case CKM_AES_GCM:
		/*
		 * Similar to the ccm mode implementation, the gcm mode also
		 * predates PKCS#11 2.40, however in this instance
		 * CK_AES_GCM_PARAMS and CK_GCM_PARAMS are identical except
		 * for the member names, so we can just pass it along.
		 */
		rc = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mech_p->pParameter,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		break;
	}

	return (crypto2pkcs11_error_number(rc));
}

/*
 * Allocate context for the active encryption or decryption operation, and
 * generate AES key schedule to speed up the operation.
 */
CK_RV
soft_aes_crypt_init_common(soft_session_t *session_p,
    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
    boolean_t encrypt)
{
	aes_ctx_t *aes_ctx = NULL;
	CK_RV rv = CKR_OK;

	if (key_p->key_type != CKK_AES)
		return (CKR_KEY_TYPE_INCONSISTENT);

	/* C_{Encrypt,Decrypt}Init() validate pMechanism != NULL */
	rv = soft_aes_check_mech_param(pMechanism, &aes_ctx);
	if (rv != CKR_OK) {
		goto done;
	}

	rv = soft_aes_init_key(aes_ctx, key_p);
	if (rv != CKR_OK) {
		goto done;
	}

	rv = soft_aes_init_ctx(aes_ctx, pMechanism, encrypt);
	if (rv != CKR_OK) {
		goto done;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	if (encrypt) {
		/* Called by C_EncryptInit. */
		session_p->encrypt.context = aes_ctx;
		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
	} else {
		/* Called by C_DecryptInit. */
		session_p->decrypt.context = aes_ctx;
		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);

done:
	if (rv != CKR_OK) {
		soft_aes_free_ctx(aes_ctx);
	}

	return (rv);
}


CK_RV
soft_aes_encrypt(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pEncryptedData,
    CK_ULONG_PTR pulEncryptedDataLen)
{
	aes_ctx_t *aes_ctx = session_p->encrypt.context;
	CK_MECHANISM_TYPE mech = session_p->encrypt.mech.mechanism;
	size_t length_needed;
	size_t remainder;
	int rc = CRYPTO_SUCCESS;
	CK_RV rv = CKR_OK;
	crypto_data_t out = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulEncryptedDataLen,
		.cd_raw.iov_base = (char *)pEncryptedData,
		.cd_raw.iov_len = *pulEncryptedDataLen
	};

	/*
	 * A bit unusual, but it's permissible for ccm and gcm modes to not
	 * encrypt any data.  This ends up being equivalent to CKM_AES_CMAC
	 * or CKM_AES_GMAC of the additional authenticated data (AAD).
	 */
	if ((pData == NULL || ulDataLen == 0) &&
	    !(aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|CMAC_MODE|GMAC_MODE))) {
		return (CKR_ARGUMENTS_BAD);
	}

	remainder = ulDataLen % AES_BLOCK_LEN;

	/*
	 * CTR, CCM, CMAC, and GCM modes do not require the plaintext
	 * to be a multiple of the AES block size. CKM_AES_CBC_PAD as the
	 * name suggests pads it's output, so it can also accept any
	 * size plaintext.
	 */
	switch (mech) {
	case CKM_AES_CBC_PAD:
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
	case CKM_AES_CTR:
	case CKM_AES_CCM:
	case CKM_AES_GCM:
	case CKM_AES_GMAC:
		break;
	default:
		if (remainder != 0) {
			rv = CKR_DATA_LEN_RANGE;
			goto cleanup;
		}
	}

	switch (mech) {
	case CKM_AES_CCM:
		length_needed = ulDataLen + aes_ctx->ac_mac_len;
		break;
	case CKM_AES_GCM:
		length_needed = ulDataLen + aes_ctx->ac_tag_len;
		break;
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
		length_needed = AES_BLOCK_LEN;
		break;
	case CKM_AES_GMAC:
		length_needed = AES_BLOCK_LEN;
		break;
	case CKM_AES_CBC_PAD:
		/* CKM_AES_CBC_PAD always adds 1..AES_BLOCK_LEN of padding */
		length_needed = ulDataLen + AES_BLOCK_LEN - remainder;
		break;
	default:
		length_needed = ulDataLen;
		break;
	}

	if (pEncryptedData == NULL) {
		/*
		 * The application can ask for the size of the output buffer
		 * with a NULL output buffer (pEncryptedData).
		 * C_Encrypt() guarantees pulEncryptedDataLen != NULL.
		 */
		*pulEncryptedDataLen = length_needed;
		return (CKR_OK);
	}

	if (*pulEncryptedDataLen < length_needed) {
		*pulEncryptedDataLen = length_needed;
		return (CKR_BUFFER_TOO_SMALL);
	}

	if (ulDataLen > 0) {
		rv = soft_aes_encrypt_update(session_p, pData, ulDataLen,
		    pEncryptedData, pulEncryptedDataLen);

		if (rv != CKR_OK) {
			rv = CKR_FUNCTION_FAILED;
			goto cleanup;
		}

		/*
		 * Some modes (e.g. CCM and GCM) will append data such as a MAC
		 * to the ciphertext after the plaintext has been encrypted.
		 * Update out to reflect the amount of data in pEncryptedData
		 * after encryption.
		 */
		out.cd_offset = *pulEncryptedDataLen;
	}

	switch (mech) {
	case CKM_AES_CBC_PAD: {
		/*
		 * aes_encrypt_contiguous_blocks() accumulates plaintext
		 * in aes_ctx until it has at least one full block of
		 * plaintext.  Any partial blocks of data remaining after
		 * encrypting are left for subsequent calls to
		 * aes_encrypt_contiguous_blocks().  If the input happened
		 * to be an exact multiple of AES_BLOCK_LEN, we must still
		 * append a block of padding (a full block in that case) so
		 * that the correct amount of padding to remove is known
		 * during decryption.
		 *
		 * soft_add_pkcs7_padding() is a bit overkill -- we just
		 * create a block filled with the pad amount using memset(),
		 * and encrypt 'amt' bytes of the block to pad out the input.
		 */
		char block[AES_BLOCK_LEN];
		size_t amt = AES_BLOCK_LEN - remainder;

		VERIFY3U(remainder, ==, aes_ctx->ac_remainder_len);

		(void) memset(block, amt & 0xff, sizeof (block));
		rc = aes_encrypt_contiguous_blocks(aes_ctx, block, amt, &out);
		rv = crypto2pkcs11_error_number(rc);
		explicit_bzero(block, sizeof (block));
		break;
	}
	case CKM_AES_CCM:
		rc = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
		break;
	case CKM_AES_GCM:
		rc = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
		break;
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
		rc = cmac_mode_final((cbc_ctx_t *)aes_ctx, &out,
		    aes_encrypt_block, aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
		aes_ctx->ac_remainder_len = 0;
		break;
	case CKM_AES_GMAC:
		rc = gmac_mode_final((gcm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
		break;
	case CKM_AES_CTR:
		/*
		 * As CKM_AES_CTR is a stream cipher, ctr_mode_final is always
		 * invoked in the xx_update() functions, so we do not need to
		 * call it again here.
		 */
		break;
	case CKM_AES_ECB:
	case CKM_AES_CBC:
		/*
		 * These mechanisms do not have nor require a xx_final function.
		 */
		break;
	default:
		rv = CKR_MECHANISM_INVALID;
		break;
	}

cleanup:
	switch (rv) {
	case CKR_OK:
		*pulEncryptedDataLen = out.cd_offset;
		break;
	case CKR_BUFFER_TOO_SMALL:
		/* *pulEncryptedDataLen was set earlier */
		break;
	default:
		/* something else failed */
		*pulEncryptedDataLen = 0;
		break;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	soft_aes_free_ctx(aes_ctx);
	session_p->encrypt.context = NULL;
	(void) pthread_mutex_unlock(&session_p->session_mutex);

	return (rv);
}

static CK_RV
soft_aes_cbc_pad_decrypt(aes_ctx_t *aes_ctx, CK_BYTE_PTR pEncryptedData,
    CK_ULONG ulEncryptedDataLen, crypto_data_t *out_orig)
{
	aes_ctx_t *ctx = aes_ctx;
	uint8_t *buf = NULL;
	uint8_t *outbuf = (uint8_t *)out_orig->cd_raw.iov_base;
	crypto_data_t out = *out_orig;
	size_t i;
	int rc;
	CK_RV rv = CKR_OK;
	uint8_t pad_len;
	boolean_t speculate = B_FALSE;

	/*
	 * Just a query for the output size.  When the output buffer is
	 * NULL, we are allowed to return a size slightly larger than
	 * necessary.  We know the output will never be larger than the
	 * input ciphertext, so we use that as an estimate.
	 */
	if (out_orig->cd_raw.iov_base == NULL) {
		out_orig->cd_length = ulEncryptedDataLen;
		return (CKR_OK);
	}

	/*
	 * The output plaintext size will be 1..AES_BLOCK_LEN bytes
	 * smaller than the input ciphertext.  However we cannot know
	 * exactly how much smaller until we decrypt the entire
	 * input ciphertext.  If we are unsure we have enough output buffer
	 * space, we have to allocate our own memory to hold the output,
	 * then see if we have enough room to hold the result.
	 *
	 * Unfortunately, having an output buffer that's too small does
	 * not terminate the operation, nor are we allowed to return
	 * partial results.  Therefore we must also duplicate the initial
	 * aes_ctx so that this can potentially be run again.
	 */
	if (out_orig->cd_length < ulEncryptedDataLen) {
		void *ks = malloc(aes_ctx->ac_keysched_len);

		ctx = malloc(sizeof (*aes_ctx));
		buf = malloc(ulEncryptedDataLen);
		if (ks == NULL || ctx == NULL || buf == NULL) {
			free(ks);
			free(ctx);
			free(buf);
			return (CKR_HOST_MEMORY);
		}

		bcopy(aes_ctx, ctx, sizeof (*ctx));
		bcopy(aes_ctx->ac_keysched, ks, aes_ctx->ac_keysched_len);
		ctx->ac_keysched = ks;

		out.cd_length = ulEncryptedDataLen;
		out.cd_raw.iov_base = (char *)buf;
		out.cd_raw.iov_len = ulEncryptedDataLen;
		outbuf = buf;

		speculate = B_TRUE;
	}

	rc = aes_decrypt_contiguous_blocks(ctx, (char *)pEncryptedData,
	    ulEncryptedDataLen, &out);
	if (rc != CRYPTO_SUCCESS) {
		out_orig->cd_offset = 0;
		rv = CKR_FUNCTION_FAILED;
		goto done;
	}

	/*
	 * RFC5652 6.3 The amount of padding must be
	 * block_sz - (len mod block_size).  This means
	 * the amount of padding must always be in the
	 * range [1..block_size].
	 */
	pad_len = outbuf[ulEncryptedDataLen - 1];
	if (pad_len == 0 || pad_len > AES_BLOCK_LEN) {
		rv = CKR_ENCRYPTED_DATA_INVALID;
		goto done;
	}
	out.cd_offset -= pad_len;

	/*
	 * Verify pad values, trying to do so in as close to constant
	 * time as possible.
	 */
	for (i = ulEncryptedDataLen - pad_len; i < ulEncryptedDataLen; i++) {
		if (outbuf[i] != pad_len) {
			rv = CKR_ENCRYPTED_DATA_INVALID;
		}
	}
	if (rv != CKR_OK) {
		goto done;
	}

	if (speculate) {
		if (out.cd_offset <= out_orig->cd_length) {
			bcopy(out.cd_raw.iov_base, out_orig->cd_raw.iov_base,
			    out.cd_offset);
		} else {
			rv = CKR_BUFFER_TOO_SMALL;
		}
	}

	/*
	 * No matter what, we report the exact size required.
	 */
	out_orig->cd_offset = out.cd_offset;

done:
	freezero(buf, ulEncryptedDataLen);
	if (ctx != aes_ctx) {
		VERIFY(speculate);
		soft_aes_free_ctx(ctx);
	}

	return (rv);
}

CK_RV
soft_aes_decrypt(soft_session_t *session_p, CK_BYTE_PTR pEncryptedData,
    CK_ULONG ulEncryptedDataLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
{
	aes_ctx_t *aes_ctx = session_p->decrypt.context;
	CK_MECHANISM_TYPE mech = session_p->decrypt.mech.mechanism;
	size_t length_needed;
	size_t remainder;
	int rc = CRYPTO_SUCCESS;
	CK_RV rv = CKR_OK;
	crypto_data_t out = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulDataLen,
		.cd_raw.iov_base = (char *)pData,
		.cd_raw.iov_len = *pulDataLen
	};

	/*
	 * A bit unusual, but it's permissible for ccm and gcm modes to not
	 * decrypt any data.  This ends up being equivalent to CKM_AES_CMAC
	 * or CKM_AES_GMAC of the additional authenticated data (AAD).
	 */
	if ((pEncryptedData == NULL || ulEncryptedDataLen == 0) &&
	    !(aes_ctx->ac_flags & (CCM_MODE|GCM_MODE))) {
		return (CKR_ARGUMENTS_BAD);
	}

	remainder = ulEncryptedDataLen % AES_BLOCK_LEN;

	/*
	 * CTR, CCM, CMAC, and GCM modes do not require the ciphertext
	 * to be a multiple of the AES block size.  Note that while
	 * CKM_AES_CBC_PAD accepts an arbitrary sized plaintext, the
	 * ciphertext is always a multiple of the AES block size
	 */
	switch (mech) {
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
	case CKM_AES_CTR:
	case CKM_AES_CCM:
	case CKM_AES_GCM:
	case CKM_AES_GMAC:
		break;
	default:
		if (remainder != 0) {
			rv = CKR_DATA_LEN_RANGE;
			goto cleanup;
		}
	}

	if (mech == CKM_AES_CBC_PAD) {
		rv = soft_aes_cbc_pad_decrypt(aes_ctx, pEncryptedData,
		    ulEncryptedDataLen, &out);
		if (pData == NULL || rv == CKR_BUFFER_TOO_SMALL) {
			*pulDataLen = out.cd_offset;
			return (rv);
		}
		goto cleanup;
	}

	switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE)) {
	case CCM_MODE:
		length_needed = aes_ctx->ac_processed_data_len;
		break;
	case GCM_MODE:
		length_needed = ulEncryptedDataLen - aes_ctx->ac_tag_len;
		break;
	default:
		/*
		 * Note: for CKM_AES_CBC_PAD, we cannot know exactly how much
		 * space is needed for the plaintext until after we decrypt it.
		 * However, it is permissible to return a value 'somewhat'
		 * larger than necessary (PKCS#11 Base Specification, sec 5.2).
		 *
		 * Since CKM_AES_CBC_PAD adds at most AES_BLOCK_LEN bytes to
		 * the plaintext, we report the ciphertext length as the
		 * required plaintext length.  This means we specify at most
		 * AES_BLOCK_LEN additional bytes of memory for the plaintext.
		 *
		 * This behavior is slightly different from the earlier
		 * version of this code which returned the value of
		 * (ulEncryptedDataLen - AES_BLOCK_LEN), which was only ever
		 * correct when the original plaintext was already a multiple
		 * of AES_BLOCK_LEN (i.e. when AES_BLOCK_LEN of padding was
		 * added).  This should not be a concern for existing
		 * consumers -- if they were previously using the value of
		 * *pulDataLen to size the outbut buffer, the resulting
		 * plaintext would be truncated anytime the original plaintext
		 * wasn't a multiple of AES_BLOCK_LEN.  No consumer should
		 * be relying on such wrong behavior.  More likely they are
		 * using the size of the ciphertext or larger for the
		 * buffer to hold the decrypted plaintext (which is always
		 * acceptable).
		 */
		length_needed = ulEncryptedDataLen;
	}

	if (pData == NULL) {
		/*
		 * The application can ask for the size of the output buffer
		 * with a NULL output buffer (pData).
		 * C_Decrypt() guarantees pulDataLen != NULL.
		 */
		*pulDataLen = length_needed;
		return (CKR_OK);
	}

	if (*pulDataLen < length_needed) {
		*pulDataLen = length_needed;
		return (CKR_BUFFER_TOO_SMALL);
	}

	if (ulEncryptedDataLen > 0) {
		rv = soft_aes_decrypt_update(session_p, pEncryptedData,
		    ulEncryptedDataLen, pData, pulDataLen);
	}

	if (rv != CKR_OK) {
		rv = CKR_FUNCTION_FAILED;
		goto cleanup;
	}

	/*
	 * Some modes (e.g. CCM and GCM) will output additional data
	 * after the plaintext (such as the MAC).  Update out to
	 * reflect the amount of data in pData for the _final() functions.
	 */
	out.cd_offset = *pulDataLen;

	/*
	 * As CKM_AES_CTR is a stream cipher, ctr_mode_final is always
	 * invoked in the _update() functions, so we do not need to call it
	 * here.
	 */
	if (aes_ctx->ac_flags & CCM_MODE) {
		ASSERT3U(aes_ctx->ac_processed_data_len, ==,
		    aes_ctx->ac_data_len);
		ASSERT3U(aes_ctx->ac_processed_mac_len, ==,
		    aes_ctx->ac_mac_len);

		rc = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
	} else if (aes_ctx->ac_flags & GCM_MODE) {
		rc = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		rv = crypto2pkcs11_error_number(rc);
	}

cleanup:
	if (rv == CKR_OK) {
		*pulDataLen = out.cd_offset;
	} else {
		*pulDataLen = 0;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	soft_aes_free_ctx(aes_ctx);
	session_p->decrypt.context = NULL;
	(void) pthread_mutex_unlock(&session_p->session_mutex);

	return (rv);
}

CK_RV
soft_aes_encrypt_update(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pEncryptedData,
    CK_ULONG_PTR pulEncryptedDataLen)
{
	aes_ctx_t *aes_ctx = session_p->encrypt.context;
	crypto_data_t out = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulEncryptedDataLen,
		.cd_raw.iov_base = (char *)pEncryptedData,
		.cd_raw.iov_len = *pulEncryptedDataLen
	};
	CK_MECHANISM_TYPE mech = session_p->encrypt.mech.mechanism;
	CK_RV rv = CKR_OK;
	size_t out_len;
	int rc;

	/*
	 * If pData is NULL, we should have zero bytes to process, and
	 * the aes_encrypt_contiguous_blocks() call will be an effective no-op.
	 */
	IMPLY(pData == NULL, ulDataLen == 0);

	/* Check size of the output buffer */
	switch (mech) {
	case CKM_AES_CMAC:
		/*
		 * The underlying CMAC implementation handles the storing of
		 * extra bytes and does not output any data until *_final,
		 * so do not bother looking at the size of the output
		 * buffer at this time.
		 */
		out_len = 0;
		break;
	case CKM_AES_GMAC:
		out_len = 0;
		break;
	case CKM_AES_CTR:
		/*
		 * CTR mode is a stream cipher, so we always output exactly as
		 * much ciphertext as input plaintext
		 */
		out_len = ulDataLen;
		break;
	default:
		out_len = aes_ctx->ac_remainder_len + ulDataLen;

		/*
		 * The number of complete blocks we can encrypt right now.
		 * The underlying implementation will buffer any remaining data
		 * until the next *_update call.
		 */
		out_len &= ~(AES_BLOCK_LEN - 1);
		break;
	}

	if (pEncryptedData == NULL) {
		*pulEncryptedDataLen = out_len;
		return (CKR_OK);
	}

	if (*pulEncryptedDataLen < out_len) {
		*pulEncryptedDataLen = out_len;
		return (CKR_BUFFER_TOO_SMALL);
	}

	rc = aes_encrypt_contiguous_blocks(aes_ctx, (char *)pData, ulDataLen,
	    &out);

	/*
	 * Since out.cd_offset is set to 0 initially and the underlying
	 * implementation increments out.cd_offset by the amount of output
	 * written, so we can just use the value as the amount written.
	 */
	*pulEncryptedDataLen = out.cd_offset;

	if (rc != CRYPTO_SUCCESS) {
		return (CKR_FUNCTION_FAILED);
	}

	rv = crypto2pkcs11_error_number(rc);

	return (rv);
}

CK_RV
soft_aes_decrypt_update(soft_session_t *session_p, CK_BYTE_PTR pEncryptedData,
    CK_ULONG ulEncryptedDataLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
{
	aes_ctx_t *aes_ctx = session_p->decrypt.context;
	uint8_t *buffer_block = NULL;
	crypto_data_t out = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulDataLen,
		.cd_raw.iov_base = (char *)pData,
		.cd_raw.iov_len = *pulDataLen
	};
	CK_MECHANISM_TYPE mech = session_p->decrypt.mech.mechanism;
	CK_RV rv = CKR_OK;
	size_t in_len = ulEncryptedDataLen;
	size_t out_len;
	int rc = CRYPTO_SUCCESS;

	switch (mech) {
	case CKM_AES_CCM:
	case CKM_AES_GCM:
		out_len = 0;
		break;
	case CKM_AES_CBC_PAD:
		/*
		 * For CKM_AES_CBC_PAD, we use the existing code for CBC
		 * mode in libsoftcrypto (which itself uses the code in
		 * usr/src/common/crypto/modes for CBC mode).  For
		 * non-padding AES CBC mode, aes_decrypt_contiguous_blocks()
		 * will accumulate ciphertext in aes_ctx->ac_remainder until
		 * there is at least AES_BLOCK_LEN bytes of ciphertext available
		 * to decrypt.  At that point, as many blocks of AES_BLOCK_LEN
		 * sized ciphertext blocks are decrypted.  Any remainder is
		 * copied into aes_ctx->ac_remainder for decryption in
		 * subsequent calls to aes_decrypt_contiguous_blocks().
		 *
		 * When PKCS#7 padding is used, the buffering
		 * aes_decrypt_contigous_blocks() performs is insufficient.
		 * PKCS#7 padding always adds [1..AES_BLOCK_LEN] bytes of
		 * padding to plaintext, so the resulting ciphertext is always
		 * larger than the input plaintext.  However we cannot know
		 * which block is the final block (and needs its padding
		 * stripped) until C_DecryptFinal() is called.  Additionally,
		 * it is permissible for a caller to use buffers sized to the
		 * output plaintext -- i.e. smaller than the input ciphertext.
		 * This leads to a more complicated buffering/accumulation
		 * strategy than what aes_decrypt_contiguous_blocks() provides
		 * us.
		 *
		 * Our buffering strategy works as follows:
		 *  For each call to C_DecryptUpdate, we calculate the
		 *  total amount of ciphertext available (buffered plus what's
		 *  passed in) as the initial output size (out_len). Based
		 *  on the value of out_len, there are three possibilties:
		 *
		 *  1. We have less than AES_BLOCK_LEN + 1 bytes of
		 *  ciphertext available. Accumulate the ciphertext in
		 *  aes_ctx->ac_remainder. Note that while we could let
		 *  aes_decrypt_contiguous_blocks() buffer the input for us
		 *  when we have less than AES_BLOCK_LEN bytes, we would still
		 *  need to buffer when we have exactly AES_BLOCK_LEN
		 *  bytes available, so we just handle both situations with
		 *  one if clause.
		 *
		 *  2. We have at least AES_BLOCK_LEN + 1 bytes of
		 *  ciphertext, and the total amount available is also an
		 *  exact multiple of AES_BLOCK_LEN. We cannot know if the
		 *  last block of input is the final block (yet), but we
		 *  are an exact multiple of AES_BLOCK_LEN, and we have
		 *  at least AES_BLOCK_LEN + 1 bytes available, therefore
		 *  there must be at least 2 * AES_BLOCK_LEN bytes of input
		 *  ciphertext available. It also means there's at least one
		 *  full block of input ciphertext that can be decrypted. We
		 *  reduce the size of the input (in_len) given to
		 *  aes_decrypt_contiguous_bytes() by AES_BLOCK_LEN to prevent
		 *  it from decrypting the last full block of data.
		 *  aes_decrypt_contiguous_blocks() will when decrypt any
		 *  buffered data in aex_ctx->ac_remainder, and then any
		 *  input data passed. Since we have an exact multiple of
		 *  AES_BLOCK_LEN, aes_ctx->ac_remainder will be empty
		 *  (aes_ctx->ac_remainder_len == 0), once
		 *  aes_decrypt_contiguout_block() completes, and we can
		 *  copy the last block of data into aes_ctx->ac_remainder.
		 *
		 *  3. We have at least AES_BLOCK_LEN + 1 bytes of
		 *  ciphertext, but the total amount available is not an
		 *  exact multiple of AES_BLOCK_LEN. We decrypt all of
		 *  full blocks of data we have. The remainder will be
		 *  less than AES_BLOCK_LEN bytes. We let
		 *  aes_decrypt_contiguous_blocks() buffer the remainder
		 *  for us since it would normally do this anyway. Since there
		 *  is a remainder, the full blocks that are present cannot
		 *  be the last block, so we can safey decrypt all of them.
		 *
		 * Some things to note:
		 *  - The above semantics will cause aes_ctx->ac_remainder to
		 *  never accumulate more than AES_BLOCK_LEN bytes of
		 *  ciphertext. Once we reach at least AES_BLOCK_LEN + 1 bytes,
		 *  we will decrypt the contents of aes_ctx->ac_remainder by one
		 *  of the last two scenarios described above.
		 *
		 *  - We must always end up with AES_BLOCK_LEN bytes of data
		 *  in aes_ctx->ac_remainder when C_DecryptFinal() is called.
		 *  The first and third scenarios above may leave
		 *  aes_ctx->ac_remainder with less than AES_BLOCK_LEN bytes,
		 *  however the total size of the input ciphertext that's
		 *  been decrypted must end up a multiple of AES_BLOCK_LEN.
		 *  Therefore, we can always assume when there is a
		 *  remainder that more data is coming.  If we do end up
		 *  with a remainder that's not AES_BLOCK_LEN bytes long
		 *  when C_DecryptFinal() is called, the input is assumed
		 *  invalid and we return CKR_DATA_LEN_RANGE (see
		 *  soft_aes_decrypt_final()).
		 */

		VERIFY3U(aes_ctx->ac_remainder_len, <=, AES_BLOCK_LEN);
		if (in_len >= SIZE_MAX - AES_BLOCK_LEN)
			return (CKR_ENCRYPTED_DATA_LEN_RANGE);

		out_len = aes_ctx->ac_remainder_len + in_len;

		if (out_len <= AES_BLOCK_LEN) {
			/*
			 * The first scenario detailed above, accumulate
			 * ciphertext in ac_remainder_len and return.
			 */
			uint8_t *dest = (uint8_t *)aes_ctx->ac_remainder +
			    aes_ctx->ac_remainder_len;

			bcopy(pEncryptedData, dest, in_len);
			aes_ctx->ac_remainder_len += in_len;
			*pulDataLen = 0;

			/*
			 * Since we aren't writing an output, and are returning
			 * here, we don't need to adjust out_len -- we never
			 * reach the output buffer size checks after the
			 * switch statement.
			 */
			return (CKR_OK);
		} else if (out_len % AES_BLOCK_LEN == 0) {
			/*
			 * The second scenario decribed above. The total amount
			 * available is a multiple of AES_BLOCK_LEN, and
			 * we have more than one block.  We reduce the
			 * input size (in_len) by AES_BLOCK_LEN. We also
			 * reduce the output size (out_len) by AES_BLOCK_LEN
			 * for the output buffer size checks that follow
			 * the switch statement. In certain situations,
			 * PKCS#11 requires this to be an exact value, so
			 * the size check cannot occur for CKM_AES_CBC_PAD
			 * until after we've determine which scenario we
			 * have.
			 *
			 * Because we never accumulate more than AES_BLOCK_LEN
			 * bytes in aes_ctx->ac_remainder, when we are in
			 * this scenario, the following VERIFYs should always
			 * be true (and serve as a final safeguard against
			 * underflow).
			 */
			VERIFY3U(in_len, >=, AES_BLOCK_LEN);

			buffer_block = pEncryptedData + in_len - AES_BLOCK_LEN;

			in_len -= AES_BLOCK_LEN;

			/*
			 * This else clause explicity checks
			 * out_len > AES_BLOCK_LEN, so this is also safe.
			 */
			out_len -= AES_BLOCK_LEN;
		} else {
			/*
			 * The third scenario above.  We have at least
			 * AES_BLOCK_LEN + 1 bytes, but the total amount of
			 * input ciphertext available is not an exact
			 * multiple of AES_BLOCK_LEN.  Let
			 * aes_decrypt_contiguous_blocks() handle the
			 * buffering of the remainder.  Update the
			 * output size to reflect the actual amount of output
			 * we want to emit for the checks after the switch
			 * statement.
			 */
			out_len &= ~(AES_BLOCK_LEN - 1);
		}
		break;
	case CKM_AES_CTR:
		/*
		 * CKM_AES_CTR is a stream cipher, so we always output
		 * exactly as much output plaintext as input ciphertext
		 */
		out_len = in_len;
		break;
	default:
		out_len = aes_ctx->ac_remainder_len + in_len;
		out_len &= ~(AES_BLOCK_LEN - 1);
		break;
	}

	/*
	 * C_DecryptUpdate() verifies that pulDataLen is not NULL prior
	 * to calling soft_decrypt_common() (which calls us).
	 */

	if (pData == NULL) {
		/*
		 * If the output buffer (pData) is NULL, that means the
		 * caller is inquiring about the size buffer needed to
		 * complete the C_DecryptUpdate() request.  While we are
		 * permitted to set *pulDataLen to an estimated value that can
		 * be 'slightly' larger than the actual value required,
		 * since we know the exact size we need, we stick with the
		 * exact size.
		 */
		*pulDataLen = out_len;
		return (CKR_OK);
	}

	if (*pulDataLen < out_len) {
		/*
		 * Not an inquiry, but the output buffer isn't large enough.
		 * PKCS#11 requires that this scenario not fail fatally (as
		 * well as return a different error value). This situation
		 * also requires us to set *pulDataLen to the _exact_ size
		 * required.
		 */
		*pulDataLen = out_len;
		return (CKR_BUFFER_TOO_SMALL);
	}

	rc = aes_decrypt_contiguous_blocks(aes_ctx, (char *)pEncryptedData,
	    in_len, &out);

	if (rc != CRYPTO_SUCCESS) {
		rv = CKR_FUNCTION_FAILED;
		goto done;
	}

	*pulDataLen = out.cd_offset;

	switch (mech) {
	case CKM_AES_CBC_PAD:
		if (buffer_block == NULL) {
			break;
		}

		VERIFY0(aes_ctx->ac_remainder_len);

		/*
		 * We had multiple blocks of data to decrypt with nothing
		 * left over and deferred decrypting the last block of data.
		 * Copy it into aes_ctx->ac_remainder to decrypt on the
		 * next update call (or final).
		 */
		bcopy(buffer_block, aes_ctx->ac_remainder, AES_BLOCK_LEN);
		aes_ctx->ac_remainder_len = AES_BLOCK_LEN;
		break;
	}

done:
	return (rv);
}

CK_RV
soft_aes_encrypt_final(soft_session_t *session_p,
    CK_BYTE_PTR pLastEncryptedPart, CK_ULONG_PTR pulLastEncryptedPartLen)
{
	aes_ctx_t *aes_ctx = session_p->encrypt.context;
	crypto_data_t data = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulLastEncryptedPartLen,
		.cd_raw.iov_base = (char *)pLastEncryptedPart,
		.cd_raw.iov_len = *pulLastEncryptedPartLen
	};
	CK_MECHANISM_TYPE mech = session_p->encrypt.mech.mechanism;
	CK_RV rv = CKR_OK;
	size_t out_len;
	int rc = CRYPTO_SUCCESS;

	switch (mech) {
	case CKM_AES_CBC_PAD:
		/*
		 * We always add 1..AES_BLOCK_LEN of padding to the input
		 * plaintext to round up to a multiple of AES_BLOCK_LEN.
		 * During encryption, we never output a partially encrypted
		 * block (that is the amount encrypted by each call of
		 * C_EncryptUpdate() is always either 0 or n * AES_BLOCK_LEN).
		 * As a result, at the end of the encryption operation, we
		 * output AES_BLOCK_LEN bytes of data -- this could be a full
		 * block of padding, or a combination of data + padding.
		 */
		out_len = AES_BLOCK_LEN;
		break;
	case CKM_AES_CTR:
		/*
		 * Since CKM_AES_CTR is a stream cipher, we never buffer any
		 * input, so we always have 0 remaining bytes of output.
		 */
		out_len = 0;
		break;
	case CKM_AES_CCM:
		out_len = aes_ctx->ac_remainder_len +
		    aes_ctx->acu.acu_ccm.ccm_mac_len;
		break;
	case CKM_AES_GCM:
		out_len = aes_ctx->ac_remainder_len +
		    aes_ctx->acu.acu_gcm.gcm_tag_len;
		break;
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
		out_len = AES_BLOCK_LEN;
		break;
	case CKM_AES_GMAC:
		out_len = AES_BLOCK_LEN;
		break;
	default:
		/*
		 * Everything other AES mechansism requires full blocks of
		 * input.  If the input was not an exact multiple of
		 * AES_BLOCK_LEN, it is a fatal error.
		 */
		if (aes_ctx->ac_remainder_len > 0) {
			rv = CKR_DATA_LEN_RANGE;
			goto done;
		}
		out_len = 0;
	}

	if (*pulLastEncryptedPartLen < out_len || pLastEncryptedPart == NULL) {
		*pulLastEncryptedPartLen = out_len;
		return ((pLastEncryptedPart == NULL) ?
		    CKR_OK : CKR_BUFFER_TOO_SMALL);
	}

	switch (mech) {
	case CKM_AES_CBC_PAD: {
		char block[AES_BLOCK_LEN] = { 0 };
		size_t padlen = AES_BLOCK_LEN - aes_ctx->ac_remainder_len;

		if (padlen == 0) {
			padlen = AES_BLOCK_LEN;
		}

		(void) memset(block, padlen & 0xff, sizeof (block));
		rc = aes_encrypt_contiguous_blocks(aes_ctx, block,
		    padlen, &data);
		explicit_bzero(block, sizeof (block));
		break;
	}
	case CKM_AES_CTR:
		/*
		 * Since CKM_AES_CTR is a stream cipher, we never
		 * buffer any data, and thus have no remaining data
		 * to output at the end
		 */
		break;
	case CKM_AES_CCM:
		rc = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, &data,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		break;
	case CKM_AES_GCM:
		rc = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, &data,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		break;
	case CKM_AES_CMAC:
	case CKM_AES_CMAC_GENERAL:
		rc = cmac_mode_final((cbc_ctx_t *)aes_ctx, &data,
		    aes_encrypt_block, aes_xor_block);
		break;
	case CKM_AES_GMAC:
		rc = gmac_mode_final((gcm_ctx_t *)aes_ctx, &data,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		break;
	default:
		break;
	}
	rv = crypto2pkcs11_error_number(rc);

done:
	if (rv == CKR_OK) {
		*pulLastEncryptedPartLen = data.cd_offset;
	}

	soft_aes_free_ctx(aes_ctx);
	session_p->encrypt.context = NULL;
	return (rv);
}

CK_RV
soft_aes_decrypt_final(soft_session_t *session_p, CK_BYTE_PTR pLastPart,
    CK_ULONG_PTR pulLastPartLen)
{
	aes_ctx_t *aes_ctx = session_p->decrypt.context;
	CK_MECHANISM_TYPE mech = session_p->decrypt.mech.mechanism;
	CK_RV rv = CKR_OK;
	int rc = CRYPTO_SUCCESS;
	size_t out_len;
	crypto_data_t out = {
		.cd_format = CRYPTO_DATA_RAW,
		.cd_offset = 0,
		.cd_length = *pulLastPartLen,
		.cd_raw.iov_base = (char *)pLastPart,
		.cd_raw.iov_len = *pulLastPartLen
	};

	switch (mech) {
	case CKM_AES_CBC_PAD:
		/*
		 * PKCS#11 requires that a caller can discover the size of
		 * the output buffer required by calling
		 * C_DecryptFinal(hSession, NULL, &len) which sets
		 * *pulLastPartLen to the size required.  However, it also
		 * allows if one calls C_DecryptFinal with a buffer (i.e.
		 * pLastPart != NULL) that is too small, to return
		 * CKR_BUFFER_TOO_SMALL with *pulLastPartLen set to the
		 * _exact_ size required (when pLastPart is NULL, the
		 * implementation is allowed to set a 'sightly' larger
		 * value than is strictly necessary.  In either case, the
		 * caller is allowed to retry the operation (the operation
		 * is not terminated).
		 *
		 * With PKCS#7 padding, we cannot determine the exact size of
		 * the output until we decrypt the final block.  As such, the
		 * first time for a given decrypt operation we are called,
		 * we decrypt the final block and stash it in the aes_ctx
		 * remainder block.  On any subsequent calls in the
		 * current decrypt operation, we then can use the decrypted
		 * block as necessary to provide the correct semantics.
		 *
		 * The cleanup of aes_ctx when the operation terminates
		 * will take care of clearing out aes_ctx->ac_remainder_len.
		 */
		if ((aes_ctx->ac_flags & P11_DECRYPTED) == 0) {
			uint8_t block[AES_BLOCK_LEN] = { 0 };
			crypto_data_t block_out = {
				.cd_format = CRYPTO_DATA_RAW,
				.cd_offset = 0,
				.cd_length = sizeof (block),
				.cd_raw.iov_base = (char *)block,
				.cd_raw.iov_len = sizeof (block)
			};
			size_t amt, i;
			uint8_t pad_len;

			if (aes_ctx->ac_remainder_len != AES_BLOCK_LEN) {
				return (CKR_DATA_LEN_RANGE);
			}

			rc = aes_decrypt_contiguous_blocks(aes_ctx,
			    (char *)block, 0, &block_out);
			if (rc != CRYPTO_SUCCESS) {
				explicit_bzero(block, sizeof (block));
				return (CKR_FUNCTION_FAILED);
			}

			pad_len = block[AES_BLOCK_LEN - 1];

			/*
			 * RFC5652 6.3 The amount of padding must be
			 * block_sz - (len mod block_size).  This means
			 * the amount of padding must always be in the
			 * range [1..block_size].
			 */
			if (pad_len == 0 || pad_len > AES_BLOCK_LEN) {
				rv = CKR_ENCRYPTED_DATA_INVALID;
				explicit_bzero(block, sizeof (block));
				goto done;
			}
			amt = AES_BLOCK_LEN - pad_len;

			/*
			 * Verify the padding is correct.  Try to do so
			 * in as constant a time as possible.
			 */
			for (i = amt; i < AES_BLOCK_LEN; i++) {
				if (block[i] != pad_len) {
					rv = CKR_ENCRYPTED_DATA_INVALID;
				}
			}
			if (rv != CKR_OK) {
				explicit_bzero(block, sizeof (block));
				goto done;
			}

			bcopy(block, aes_ctx->ac_remainder, amt);
			explicit_bzero(block, sizeof (block));

			aes_ctx->ac_flags |= P11_DECRYPTED;
			aes_ctx->ac_remainder_len = amt;
		}

		out_len = aes_ctx->ac_remainder_len;
		break;
	case CKM_AES_CTR:
		/*
		 * Since CKM_AES_CTR is a stream cipher, we never have
		 * any remaining bytes to output.
		 */
		out_len = 0;
		break;
	case CKM_AES_CCM:
		out_len = aes_ctx->ac_data_len;
		break;
	case CKM_AES_GCM:
		out_len = aes_ctx->acu.acu_gcm.gcm_processed_data_len -
		    aes_ctx->acu.acu_gcm.gcm_tag_len;
		break;
	default:
		/*
		 * The remaining mechanims require an exact multiple of
		 * AES_BLOCK_LEN of ciphertext.  Any other value is an error.
		 */
		if (aes_ctx->ac_remainder_len > 0) {
			rv = CKR_DATA_LEN_RANGE;
			goto done;
		}
		out_len = 0;
		break;
	}

	if (*pulLastPartLen < out_len || pLastPart == NULL) {
		*pulLastPartLen = out_len;
		return ((pLastPart == NULL) ? CKR_OK : CKR_BUFFER_TOO_SMALL);
	}

	switch (mech) {
	case CKM_AES_CBC_PAD:
		*pulLastPartLen = out_len;
		if (out_len == 0) {
			break;
		}
		bcopy(aes_ctx->ac_remainder, pLastPart, out_len);
		out.cd_offset += out_len;
		break;
	case CKM_AES_CCM:
		ASSERT3U(aes_ctx->ac_processed_data_len, ==, out_len);
		ASSERT3U(aes_ctx->ac_processed_mac_len, ==,
		    aes_ctx->ac_mac_len);

		rc = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
		    aes_xor_block);
		break;
	case CKM_AES_GCM:
		rc = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, &out,
		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
		break;
	default:
		break;
	}

	VERIFY3U(out.cd_offset, ==, out_len);
	rv = crypto2pkcs11_error_number(rc);

done:
	if (rv == CKR_OK) {
		*pulLastPartLen = out.cd_offset;
	}

	soft_aes_free_ctx(aes_ctx);
	session_p->decrypt.context = NULL;

	return (rv);
}

/*
 * Allocate and initialize AES contexts for sign and verify operations
 * (including the underlying encryption context needed to sign or verify) --
 * called by C_SignInit() and C_VerifyInit() to perform the CKM_AES_* MAC
 * mechanisms. For general-length AES MAC, also validate the MAC length.
 */
CK_RV
soft_aes_sign_verify_init_common(soft_session_t *session_p,
    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p, boolean_t sign_op)
{
	soft_aes_sign_ctx_t	*ctx = NULL;
	size_t		mac_len = AES_BLOCK_LEN;

	/* For AES CMAC the IV is always 0 */
	CK_BYTE		iv_zero[AES_BLOCK_LEN] = { 0 };
	CK_MECHANISM	encrypt_mech = { 0 };
	CK_RV		rv;

	switch (pMechanism->mechanism) {
	case CKM_AES_CMAC_GENERAL:
		if (pMechanism->pParameter == NULL) {
			return (CKR_MECHANISM_PARAM_INVALID);
		}
		mac_len = *(CK_MAC_GENERAL_PARAMS *)pMechanism->pParameter;
		if (mac_len > AES_BLOCK_LEN) {
			return (CKR_MECHANISM_PARAM_INVALID);
		}
		encrypt_mech.mechanism = CKM_AES_CMAC;
		encrypt_mech.pParameter = iv_zero;
		encrypt_mech.ulParameterLen = AES_BLOCK_LEN;
		break;

	case CKM_AES_CMAC:
		encrypt_mech.mechanism = CKM_AES_CMAC;
		encrypt_mech.pParameter = iv_zero;
		encrypt_mech.ulParameterLen = AES_BLOCK_LEN;
		break;

	case CKM_AES_GMAC:
		/* See soft_aes_init_ctx */
		if (pMechanism->pParameter == NULL ||
		    pMechanism->ulParameterLen != AES_GMAC_IV_LEN)
			return (CKR_MECHANISM_PARAM_INVALID);
		encrypt_mech.mechanism = CKM_AES_GMAC;
		encrypt_mech.pParameter = pMechanism->pParameter;
		encrypt_mech.ulParameterLen = pMechanism->ulParameterLen;
		break;

	default:
		return (CKR_MECHANISM_INVALID);
	}

	if (key_p->key_type != CKK_AES)
		return (CKR_KEY_TYPE_INCONSISTENT);

	ctx = calloc(1, sizeof (*ctx));
	if (ctx == NULL) {
		return (CKR_HOST_MEMORY);
	}

	rv = soft_aes_check_mech_param(pMechanism, &ctx->aes_ctx);
	if (rv != CKR_OK) {
		soft_aes_free_ctx(ctx->aes_ctx);
		goto done;
	}

	/* See soft_aes_crypt_init_common */
	if ((rv = soft_encrypt_init_internal(session_p, &encrypt_mech,
	    key_p)) != CKR_OK) {
		soft_aes_free_ctx(ctx->aes_ctx);
		goto done;
	}

	ctx->mac_len = mac_len;

	(void) pthread_mutex_lock(&session_p->session_mutex);

	if (sign_op) {
		session_p->sign.context = ctx;
		session_p->sign.mech.mechanism = pMechanism->mechanism;
	} else {
		session_p->verify.context = ctx;
		session_p->verify.mech.mechanism = pMechanism->mechanism;
	}

	(void) pthread_mutex_unlock(&session_p->session_mutex);

done:
	if (rv != CKR_OK) {
		soft_aes_free_ctx(ctx->aes_ctx);
		free(ctx);
	}

	return (rv);
}

CK_RV
soft_aes_sign_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned, CK_ULONG_PTR pulSignedLen,
    boolean_t sign_op, boolean_t Final)
{
	soft_aes_sign_ctx_t	*soft_aes_ctx_sign_verify;
	CK_RV			rv;
	CK_BYTE			*pEncrypted = NULL;
	CK_ULONG		ulEncryptedLen = AES_BLOCK_LEN;
	CK_BYTE			last_block[AES_BLOCK_LEN];

	if (sign_op) {
		soft_aes_ctx_sign_verify =
		    (soft_aes_sign_ctx_t *)session_p->sign.context;

		if (soft_aes_ctx_sign_verify->mac_len == 0) {
			*pulSignedLen = 0;
			goto clean_exit;
		}

		/* Application asks for the length of the output buffer. */
		if (pSigned == NULL) {
			*pulSignedLen = soft_aes_ctx_sign_verify->mac_len;
			return (CKR_OK);
		}

		/* Is the application-supplied buffer large enough? */
		if (*pulSignedLen < soft_aes_ctx_sign_verify->mac_len) {
			*pulSignedLen = soft_aes_ctx_sign_verify->mac_len;
			return (CKR_BUFFER_TOO_SMALL);
		}
	} else {
		soft_aes_ctx_sign_verify =
		    (soft_aes_sign_ctx_t *)session_p->verify.context;
	}

	if (Final) {
		rv = soft_encrypt_final(session_p, last_block,
		    &ulEncryptedLen);
	} else {
		rv = soft_encrypt(session_p, pData, ulDataLen,
		    last_block, &ulEncryptedLen);
	}

	if (rv == CKR_OK) {
		*pulSignedLen = soft_aes_ctx_sign_verify->mac_len;

		/* the leftmost mac_len bytes of last_block is our MAC */
		(void) memcpy(pSigned, last_block, *pulSignedLen);
	}

clean_exit:

	(void) pthread_mutex_lock(&session_p->session_mutex);

	/* soft_encrypt_common() has freed the encrypt context */
	if (sign_op) {
		free(session_p->sign.context);
		session_p->sign.context = NULL;
	} else {
		free(session_p->verify.context);
		session_p->verify.context = NULL;
	}
	session_p->encrypt.flags = 0;

	(void) pthread_mutex_unlock(&session_p->session_mutex);

	if (pEncrypted) {
		free(pEncrypted);
	}

	return (rv);
}

/*
 * Called by soft_sign_update()
 */
CK_RV
soft_aes_mac_sign_verify_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
    CK_ULONG ulPartLen)
{
	CK_BYTE		buf[AES_BLOCK_LEN];
	CK_ULONG	ulEncryptedLen = AES_BLOCK_LEN;
	CK_RV		rv;

	rv = soft_encrypt_update(session_p, pPart, ulPartLen,
	    buf, &ulEncryptedLen);
	explicit_bzero(buf, sizeof (buf));

	return (rv);
}

/*
 * Compare with crypto_free_mode_ctx()
 * in $SRC/common/crypto/modes/modes.c
 */
void
soft_aes_free_ctx(aes_ctx_t *ctx)
{
	size_t len = 0;

	if (ctx == NULL)
		return;

	if (ctx->ac_flags & ECB_MODE) {
		len = sizeof (ecb_ctx_t);
	} else if (ctx->ac_flags & (CBC_MODE|CMAC_MODE)) {
		len = sizeof (cbc_ctx_t);
	} else if (ctx->ac_flags & CTR_MODE) {
		len = sizeof (ctr_ctx_t);
	} else if (ctx->ac_flags & CCM_MODE) {
		ccm_ctx_t *ccm_ctx = &ctx->acu.acu_ccm;
		if (ccm_ctx->ccm_pt_buf != NULL) {
			freezero(ccm_ctx->ccm_pt_buf,
			    ccm_ctx->ccm_data_len);
			ccm_ctx->ccm_pt_buf = NULL;
		}
		len = sizeof (ccm_ctx_t);
	} else if (ctx->ac_flags & GMAC_MODE) {
		len = sizeof (gcm_ctx_t);
	} else if (ctx->ac_flags & GCM_MODE) {
		gcm_ctx_t *gcm_ctx = &ctx->acu.acu_gcm;
		if (gcm_ctx->gcm_pt_buf != NULL) {
			freezero(gcm_ctx->gcm_pt_buf,
			    gcm_ctx->gcm_pt_buf_len);
			gcm_ctx->gcm_pt_buf = NULL;
		}
		len = sizeof (gcm_ctx_t);
	}

	freezero(ctx->ac_keysched, ctx->ac_keysched_len);
	freezero(ctx, len);
}