modes - zygaena/hammerhead

root / base / usr / src / common / crypto / modes
modes Plain Text 3958 lines 105.2 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009 Intel Corporation
 * All Rights Reserved.
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
 * instructions.  This file contains an accelerated
 * Galois Field Multiplication implementation.
 *
 * PCLMULQDQ is used to accelerate the most time-consuming part of GHASH,
 * carry-less multiplication. More information about PCLMULQDQ can be
 * found at:
 * http://software.intel.com/en-us/articles/
 * carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
 *
 */

/*
 * ====================================================================
 * OpenSolaris OS modifications
 *
 * This source originates as file galois_hash_asm.c from
 * Intel Corporation dated September 21, 2009.
 *
 * This OpenSolaris version has these major changes from the original source:
 *
 * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
 * /usr/include/sys/asm_linkage.h, lint(1B) guards, and a dummy C function
 * definition for lint.
 *
 * 2. Formatted code, added comments, and added #includes and #defines.
 *
 * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
 * calling kpreempt_disable() and kpreempt_enable().
 * If the TS bit is not set, Save and restore %xmm registers at the beginning
 * and end of function calls (%xmm* registers are not saved and restored by
 * during kernel thread preemption).
 *
 * 4. Removed code to perform hashing.  This is already done with C macro
 * GHASH in gcm.c.  For better performance, this removed code should be
 * reintegrated in the future to replace the C GHASH macro.
 *
 * 5. Added code to byte swap 16-byte input and output.
 *
 * 6. Folded in comments from the original C source with embedded assembly
 * (SB_w_shift_xor.c)
 *
 * 7. Renamed function and reordered parameters to match OpenSolaris:
 * Intel interface:
 *	void galois_hash_asm(unsigned char *hk, unsigned char *s,
 *		unsigned char *d, int length)
 * OpenSolaris OS interface:
 *	void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
 * ====================================================================
 */


#if defined(lint) || defined(__lint)

#include <sys/types.h>

/* ARGSUSED */
void
gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) {
}

#else	/* lint */

#include <sys/asm_linkage.h>
#include <sys/controlregs.h>
#ifdef _KERNEL
#include <sys/machprivregs.h>
#endif

#ifdef _KERNEL
	/*
	 * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv.  That is,
	 * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
	 * uses it to pass P2 to syscall.
	 * This also occurs with the STTS macro, but we don't care if
	 * P2 (%rsi) is modified just before function exit.
	 * The CLTS and STTS macros push and pop P1 (%rdi) already.
	 */
#ifdef __xpv
#define	PROTECTED_CLTS \
	push	%rsi; \
	CLTS; \
	pop	%rsi
#else
#define	PROTECTED_CLTS \
	CLTS
#endif	/* __xpv */

	/*
	 * If CR0_TS is not set, align stack (with push %rbp) and push
	 * %xmm0 - %xmm10 on stack, otherwise clear CR0_TS
	 */
#define	CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) \
	push	%rbp; \
	mov	%rsp, %rbp; \
	movq	%cr0, tmpreg; \
	testq	$CR0_TS, tmpreg; \
	jnz	1f; \
	and	$-XMM_ALIGN, %rsp; \
	sub	$[XMM_SIZE * 11], %rsp; \
	movaps	%xmm0, 160(%rsp); \
	movaps	%xmm1, 144(%rsp); \
	movaps	%xmm2, 128(%rsp); \
	movaps	%xmm3, 112(%rsp); \
	movaps	%xmm4, 96(%rsp); \
	movaps	%xmm5, 80(%rsp); \
	movaps	%xmm6, 64(%rsp); \
	movaps	%xmm7, 48(%rsp); \
	movaps	%xmm8, 32(%rsp); \
	movaps	%xmm9, 16(%rsp); \
	movaps	%xmm10, (%rsp); \
	jmp	2f; \
1: \
	PROTECTED_CLTS; \
2:


	/*
	 * If CR0_TS was not set above, pop %xmm0 - %xmm10 off stack,
	 * otherwise set CR0_TS.
	 */
#define	SET_TS_OR_POP_XMM_REGISTERS(tmpreg) \
	testq	$CR0_TS, tmpreg; \
	jnz	1f; \
	movaps	(%rsp), %xmm10; \
	movaps	16(%rsp), %xmm9; \
	movaps	32(%rsp), %xmm8; \
	movaps	48(%rsp), %xmm7; \
	movaps	64(%rsp), %xmm6; \
	movaps	80(%rsp), %xmm5; \
	movaps	96(%rsp), %xmm4; \
	movaps	112(%rsp), %xmm3; \
	movaps	128(%rsp), %xmm2; \
	movaps	144(%rsp), %xmm1; \
	movaps	160(%rsp), %xmm0; \
	jmp	2f; \
1: \
	STTS(tmpreg); \
2: \
	mov	%rbp, %rsp; \
	pop	%rbp


#else
#define	PROTECTED_CLTS
#define	CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg)
#define	SET_TS_OR_POP_XMM_REGISTERS(tmpreg)
#endif	/* _KERNEL */

/*
 * Use this mask to byte-swap a 16-byte integer with the pshufb instruction
 */

// static uint8_t byte_swap16_mask[] = {
//	 15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 };
.text
.align XMM_ALIGN
.Lbyte_swap16_mask:
	.byte	15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0



/*
 * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
 *
 * Perform a carry-less multiplication (that is, use XOR instead of the
 * multiply operator) on P1 and P2 and place the result in P3.
 *
 * Byte swap the input and the output.
 *
 * Note: x_in, y, and res all point to a block of 20-byte numbers
 * (an array of two 64-bit integers).
 *
 * Note2: For kernel code, caller is responsible for ensuring
 * kpreempt_disable() has been called.  This is because %xmm registers are
 * not saved/restored.  Clear and set the CR0.TS bit on entry and exit,
 * respectively, if TS is set on entry.  Otherwise, if TS is not set,
 * save and restore %xmm registers on the stack.
 *
 * Note3: Original Intel definition:
 * void galois_hash_asm(unsigned char *hk, unsigned char *s,
 *	unsigned char *d, int length)
 *
 * Note4: Register/parameter mapping:
 * Intel:
 *	Parameter 1: %rcx (copied to %xmm0)	hk or x_in
 *	Parameter 2: %rdx (copied to %xmm1)	s or y
 *	Parameter 3: %rdi (result)		d or res
 * OpenSolaris:
 *	Parameter 1: %rdi (copied to %xmm0)	x_in
 *	Parameter 2: %rsi (copied to %xmm1)	y
 *	Parameter 3: %rdx (result)		res
 */

ENTRY_NP(gcm_mul_pclmulqdq)
	CLEAR_TS_OR_PUSH_XMM_REGISTERS(%r10)

	//
	// Copy Parameters
	//
	movdqu	(%rdi), %xmm0	// P1
	movdqu	(%rsi), %xmm1	// P2

	//
	// Byte swap 16-byte input
	//
	lea	.Lbyte_swap16_mask(%rip), %rax
	movaps	(%rax), %xmm10
	pshufb	%xmm10, %xmm0
	pshufb	%xmm10, %xmm1


	//
	// Multiply with the hash key
	//
	movdqu	%xmm0, %xmm3
	pclmulqdq $0, %xmm1, %xmm3	// xmm3 holds a0*b0

	movdqu	%xmm0, %xmm4
	pclmulqdq $16, %xmm1, %xmm4	// xmm4 holds a0*b1

	movdqu	%xmm0, %xmm5
	pclmulqdq $1, %xmm1, %xmm5	// xmm5 holds a1*b0
	movdqu	%xmm0, %xmm6
	pclmulqdq $17, %xmm1, %xmm6	// xmm6 holds a1*b1

	pxor	%xmm5, %xmm4	// xmm4 holds a0*b1 + a1*b0

	movdqu	%xmm4, %xmm5	// move the contents of xmm4 to xmm5
	psrldq	$8, %xmm4	// shift by xmm4 64 bits to the right
	pslldq	$8, %xmm5	// shift by xmm5 64 bits to the left
	pxor	%xmm5, %xmm3
	pxor	%xmm4, %xmm6	// Register pair <xmm6:xmm3> holds the result
				// of the carry-less multiplication of
				// xmm0 by xmm1.

	// We shift the result of the multiplication by one bit position
	// to the left to cope for the fact that the bits are reversed.
	movdqu	%xmm3, %xmm7
	movdqu	%xmm6, %xmm8
	pslld	$1, %xmm3
	pslld	$1, %xmm6
	psrld	$31, %xmm7
	psrld	$31, %xmm8
	movdqu	%xmm7, %xmm9
	pslldq	$4, %xmm8
	pslldq	$4, %xmm7
	psrldq	$12, %xmm9
	por	%xmm7, %xmm3
	por	%xmm8, %xmm6
	por	%xmm9, %xmm6

	//
	// First phase of the reduction
	//
	// Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
	// independently.
	movdqu	%xmm3, %xmm7
	movdqu	%xmm3, %xmm8
	movdqu	%xmm3, %xmm9
	pslld	$31, %xmm7	// packed right shift shifting << 31
	pslld	$30, %xmm8	// packed right shift shifting << 30
	pslld	$25, %xmm9	// packed right shift shifting << 25
	pxor	%xmm8, %xmm7	// xor the shifted versions
	pxor	%xmm9, %xmm7
	movdqu	%xmm7, %xmm8
	pslldq	$12, %xmm7
	psrldq	$4, %xmm8
	pxor	%xmm7, %xmm3	// first phase of the reduction complete

	//
	// Second phase of the reduction
	//
	// Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
	// shift operations.
	movdqu	%xmm3, %xmm2
	movdqu	%xmm3, %xmm4	// packed left shifting >> 1
	movdqu	%xmm3, %xmm5
	psrld	$1, %xmm2
	psrld	$2, %xmm4	// packed left shifting >> 2
	psrld	$7, %xmm5	// packed left shifting >> 7
	pxor	%xmm4, %xmm2	// xor the shifted versions
	pxor	%xmm5, %xmm2
	pxor	%xmm8, %xmm2
	pxor	%xmm2, %xmm3
	pxor	%xmm3, %xmm6	// the result is in xmm6

	//
	// Byte swap 16-byte result
	//
	pshufb	%xmm10, %xmm6	// %xmm10 has the swap mask

	//
	// Store the result
	//
	movdqu	%xmm6, (%rdx)	// P3


	//
	// Cleanup and Return
	//
	SET_TS_OR_POP_XMM_REGISTERS(%r10)
	ret
	SET_SIZE(gcm_mul_pclmulqdq)

#endif	/* lint || __lint */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
 */

#ifndef _KERNEL
#include <strings.h>
#include <limits.h>
#include <assert.h>
#include <security/cryptoki.h>
#endif

#include <sys/debug.h>
#include <sys/types.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>
#include <aes/aes_impl.h>

/* These are the CMAC Rb constants from NIST SP 800-38B */
#define	CONST_RB_128	0x87
#define	CONST_RB_64	0x1B

/*
 * Algorithm independent CBC functions.
 */
int
cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*encrypt)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	uint8_t *lastp;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;

	if (length + ctx->cbc_remainder_len < ctx->max_remain) {
		/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
		    length);
		ctx->cbc_remainder_len += length;
		ctx->cbc_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	lastp = (uint8_t *)ctx->cbc_iv;
	if (out != NULL)
		crypto_init_ptrs(out, &iov_or_mp, &offset);

	do {
		/* Unprocessed data from last call. */
		if (ctx->cbc_remainder_len > 0) {
			need = block_size - ctx->cbc_remainder_len;

			if (need > remainder)
				return (CRYPTO_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
			    [ctx->cbc_remainder_len], need);

			blockp = (uint8_t *)ctx->cbc_remainder;
		} else {
			blockp = datap;
		}

		if (out == NULL) {
			/*
			 * XOR the previous cipher block or IV with the
			 * current clear block.
			 */
			xor_block(lastp, blockp);
			encrypt(ctx->cbc_keysched, blockp, blockp);

			ctx->cbc_lastp = blockp;
			lastp = blockp;

			if ((ctx->cbc_flags & CMAC_MODE) == 0 &&
			    ctx->cbc_remainder_len > 0) {
				bcopy(blockp, ctx->cbc_copy_to,
				    ctx->cbc_remainder_len);
				bcopy(blockp + ctx->cbc_remainder_len, datap,
				    need);
			}
		} else {
			/*
			 * XOR the previous cipher block or IV with the
			 * current clear block.
			 */
			xor_block(blockp, lastp);
			encrypt(ctx->cbc_keysched, lastp, lastp);

			/*
			 * CMAC doesn't output until encrypt_final
			 */
			if ((ctx->cbc_flags & CMAC_MODE) == 0) {
				crypto_get_ptrs(out, &iov_or_mp, &offset,
				    &out_data_1, &out_data_1_len,
				    &out_data_2, block_size);

				/* copy block to where it belongs */
				if (out_data_1_len == block_size) {
					copy_block(lastp, out_data_1);
				} else {
					bcopy(lastp, out_data_1,
					    out_data_1_len);
					if (out_data_2 != NULL) {
						bcopy(lastp + out_data_1_len,
						    out_data_2,
						    block_size -
						    out_data_1_len);
					}
				}
				/* update offset */
				out->cd_offset += block_size;
			}
		}

		/* Update pointer to next block of data to be processed. */
		if (ctx->cbc_remainder_len != 0) {
			datap += need;
			ctx->cbc_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block. */
		if (remainder > 0 && remainder < ctx->max_remain) {
			bcopy(datap, ctx->cbc_remainder, remainder);
			ctx->cbc_remainder_len = remainder;
			ctx->cbc_copy_to = datap;
			goto out;
		}
		ctx->cbc_copy_to = NULL;

	} while (remainder > 0);

out:
	/*
	 * Save the last encrypted block in the context.
	 */
	if (ctx->cbc_lastp != NULL) {
		copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv);
		ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
	}

	return (CRYPTO_SUCCESS);
}

#define	OTHER(a, ctx) \
	(((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock)

/* ARGSUSED */
int
cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*decrypt)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	uint8_t *lastp;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;

	if (length + ctx->cbc_remainder_len < block_size) {
		/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
		    length);
		ctx->cbc_remainder_len += length;
		ctx->cbc_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	lastp = ctx->cbc_lastp;
	if (out != NULL)
		crypto_init_ptrs(out, &iov_or_mp, &offset);

	do {
		/* Unprocessed data from last call. */
		if (ctx->cbc_remainder_len > 0) {
			need = block_size - ctx->cbc_remainder_len;

			if (need > remainder)
				return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
			    [ctx->cbc_remainder_len], need);

			blockp = (uint8_t *)ctx->cbc_remainder;
		} else {
			blockp = datap;
		}

		/* LINTED: pointer alignment */
		copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx));

		if (out != NULL) {
			decrypt(ctx->cbc_keysched, blockp,
			    (uint8_t *)ctx->cbc_remainder);
			blockp = (uint8_t *)ctx->cbc_remainder;
		} else {
			decrypt(ctx->cbc_keysched, blockp, blockp);
		}

		/*
		 * XOR the previous cipher block or IV with the
		 * currently decrypted block.
		 */
		xor_block(lastp, blockp);

		/* LINTED: pointer alignment */
		lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx);

		if (out != NULL) {
			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
			    &out_data_1_len, &out_data_2, block_size);

			bcopy(blockp, out_data_1, out_data_1_len);
			if (out_data_2 != NULL) {
				bcopy(blockp + out_data_1_len, out_data_2,
				    block_size - out_data_1_len);
			}

			/* update offset */
			out->cd_offset += block_size;

		} else if (ctx->cbc_remainder_len > 0) {
			/* copy temporary block to where it belongs */
			bcopy(blockp, ctx->cbc_copy_to, ctx->cbc_remainder_len);
			bcopy(blockp + ctx->cbc_remainder_len, datap, need);
		}

		/* Update pointer to next block of data to be processed. */
		if (ctx->cbc_remainder_len != 0) {
			datap += need;
			ctx->cbc_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block. */
		if (remainder > 0 && remainder < block_size) {
			bcopy(datap, ctx->cbc_remainder, remainder);
			ctx->cbc_remainder_len = remainder;
			ctx->cbc_lastp = lastp;
			ctx->cbc_copy_to = datap;
			return (CRYPTO_SUCCESS);
		}
		ctx->cbc_copy_to = NULL;

	} while (remainder > 0);

	ctx->cbc_lastp = lastp;
	return (CRYPTO_SUCCESS);
}

int
cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len,
    size_t block_size, void (*copy_block)(uint8_t *, uint64_t *))
{
	/*
	 * Copy IV into context.
	 *
	 * If cm_param == NULL then the IV comes from the
	 * cd_miscdata field in the crypto_data structure.
	 */
	if (param != NULL) {
#ifdef _KERNEL
		ASSERT(param_len == block_size);
#else
		assert(param_len == block_size);
#endif
		copy_block((uchar_t *)param, cbc_ctx->cbc_iv);
	}

	cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0];
	cbc_ctx->cbc_flags |= CBC_MODE;
	cbc_ctx->max_remain = block_size;
	return (CRYPTO_SUCCESS);
}

/* ARGSUSED */
static void *
cbc_cmac_alloc_ctx(int kmflag, uint32_t mode)
{
	cbc_ctx_t *cbc_ctx;
	uint32_t modeval = mode & (CBC_MODE|CMAC_MODE);

	/* Only one of the two modes can be set */
	VERIFY(modeval == CBC_MODE || modeval == CMAC_MODE);

#ifdef _KERNEL
	if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL)
#else
	if ((cbc_ctx = calloc(1, sizeof (cbc_ctx_t))) == NULL)
#endif
		return (NULL);

	cbc_ctx->cbc_flags = mode;
	return (cbc_ctx);
}

void *
cbc_alloc_ctx(int kmflag)
{
	return (cbc_cmac_alloc_ctx(kmflag, CBC_MODE));
}

/*
 * Algorithms for supporting AES-CMAC
 * NOTE: CMAC is generally just a wrapper for CBC
 */

void *
cmac_alloc_ctx(int kmflag)
{
	return (cbc_cmac_alloc_ctx(kmflag, CMAC_MODE));
}


/*
 * Typically max_remain is set to block_size - 1, since we usually
 * will process the data once we have a full block.  However with CMAC,
 * we must preprocess the final block of data.  Since we cannot know
 * when we've received the final block of data until the _final() method
 * is called, we must not process the last block of data until we know
 * it is the last block, or we receive a new block of data.  As such,
 * max_remain for CMAC is block_size + 1.
 */
int
cmac_init_ctx(cbc_ctx_t *cbc_ctx, size_t block_size)
{
	/*
	 * CMAC is only approved for block sizes 64 and 128 bits /
	 * 8 and 16 bytes.
	 */

	if (block_size != 16 && block_size != 8)
		return (CRYPTO_INVALID_CONTEXT);

	/*
	 * For CMAC, cbc_iv is always 0.
	 */

	cbc_ctx->cbc_iv[0] = 0;
	cbc_ctx->cbc_iv[1] = 0;

	cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0];
	cbc_ctx->cbc_flags |= CMAC_MODE;

	cbc_ctx->max_remain = block_size + 1;
	return (CRYPTO_SUCCESS);
}

/*
 * Left shifts blocks by one and returns the leftmost bit
 */
static uint8_t
cmac_left_shift_block_by1(uint8_t *block, size_t block_size)
{
	uint8_t carry = 0, old;
	size_t i;
	for (i = block_size; i > 0; i--) {
		old = carry;
		carry = (block[i - 1] & 0x80) ? 1 : 0;
		block[i - 1] = (block[i - 1] << 1) | old;
	}
	return (carry);
}

/*
 * Generate subkeys to preprocess the last block according to RFC 4493.
 * Store the final block_size MAC generated in 'out'.
 */
int
cmac_mode_final(cbc_ctx_t *cbc_ctx, crypto_data_t *out,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t buf[AES_BLOCK_LEN] = {0};
	uint8_t *M_last = (uint8_t *)cbc_ctx->cbc_remainder;
	size_t length = cbc_ctx->cbc_remainder_len;
	size_t block_size = cbc_ctx->max_remain - 1;
	uint8_t const_rb;

	if (length > block_size)
		return (CRYPTO_INVALID_CONTEXT);

	if (out->cd_length < block_size)
		return (CRYPTO_DATA_LEN_RANGE);

	if (block_size == 16)
		const_rb = CONST_RB_128;
	else if (block_size == 8)
		const_rb = CONST_RB_64;
	else
		return (CRYPTO_INVALID_CONTEXT);

	/* k_0 = E_k(0) */
	encrypt_block(cbc_ctx->cbc_keysched, buf, buf);

	if (cmac_left_shift_block_by1(buf, block_size))
		buf[block_size - 1] ^= const_rb;

	if (length == block_size) {
		/* Last block complete, so m_n = k_1 + m_n' */
		xor_block(buf, M_last);
		xor_block(cbc_ctx->cbc_lastp, M_last);
		encrypt_block(cbc_ctx->cbc_keysched, M_last, M_last);
	} else {
		/* Last block incomplete, so m_n = k_2 + (m_n' | 100...0_bin) */
		if (cmac_left_shift_block_by1(buf, block_size))
			buf[block_size - 1] ^= const_rb;

		M_last[length] = 0x80;
		bzero(M_last + length + 1, block_size - length - 1);
		xor_block(buf, M_last);
		xor_block(cbc_ctx->cbc_lastp, M_last);
		encrypt_block(cbc_ctx->cbc_keysched, M_last, M_last);
	}

	/*
	 * zero out the sub-key.
	 */
#ifndef _KERNEL
	explicit_bzero(&buf, sizeof (buf));
#else
	bzero(&buf, sizeof (buf));
#endif
	return (crypto_put_output_data(M_last, out, block_size));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#ifndef _KERNEL
#include <strings.h>
#include <limits.h>
#include <assert.h>
#include <security/cryptoki.h>
#endif

#include <sys/types.h>
#include <sys/kmem.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>
#include <sys/byteorder.h>

#if defined(__i386) || defined(__amd64)
#define	UNALIGNED_POINTERS_PERMITTED
#endif

/*
 * Encrypt multiple blocks of data in CCM mode.  Decrypt for CCM mode
 * is done in another function.
 */
int
ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	uint8_t *lastp;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;
	uint64_t counter;
	uint8_t *mac_buf;

	if (length + ctx->ccm_remainder_len < block_size) {
		/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
		    length);
		ctx->ccm_remainder_len += length;
		ctx->ccm_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	lastp = (uint8_t *)ctx->ccm_cb;
	if (out != NULL)
		crypto_init_ptrs(out, &iov_or_mp, &offset);

	mac_buf = (uint8_t *)ctx->ccm_mac_buf;

	do {
		/* Unprocessed data from last call. */
		if (ctx->ccm_remainder_len > 0) {
			need = block_size - ctx->ccm_remainder_len;

			if (need > remainder)
				return (CRYPTO_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
			    [ctx->ccm_remainder_len], need);

			blockp = (uint8_t *)ctx->ccm_remainder;
		} else {
			blockp = datap;
		}

		/*
		 * do CBC MAC
		 *
		 * XOR the previous cipher block current clear block.
		 * mac_buf always contain previous cipher block.
		 */
		xor_block(blockp, mac_buf);
		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);

		/* ccm_cb is the counter block */
		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb,
		    (uint8_t *)ctx->ccm_tmp);

		lastp = (uint8_t *)ctx->ccm_tmp;

		/*
		 * Increment counter. Counter bits are confined
		 * to the bottom 64 bits of the counter block.
		 */
		counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
		counter = htonll(counter + 1);
		counter &= ctx->ccm_counter_mask;
		ctx->ccm_cb[1] =
		    (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;

		/*
		 * XOR encrypted counter block with the current clear block.
		 */
		xor_block(blockp, lastp);

		ctx->ccm_processed_data_len += block_size;

		if (out == NULL) {
			if (ctx->ccm_remainder_len > 0) {
				bcopy(blockp, ctx->ccm_copy_to,
				    ctx->ccm_remainder_len);
				bcopy(blockp + ctx->ccm_remainder_len, datap,
				    need);
			}
		} else {
			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
			    &out_data_1_len, &out_data_2, block_size);

			/* copy block to where it belongs */
			if (out_data_1_len == block_size) {
				copy_block(lastp, out_data_1);
			} else {
				bcopy(lastp, out_data_1, out_data_1_len);
				if (out_data_2 != NULL) {
					bcopy(lastp + out_data_1_len,
					    out_data_2,
					    block_size - out_data_1_len);
				}
			}
			/* update offset */
			out->cd_offset += block_size;
		}

		/* Update pointer to next block of data to be processed. */
		if (ctx->ccm_remainder_len != 0) {
			datap += need;
			ctx->ccm_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block. */
		if (remainder > 0 && remainder < block_size) {
			bcopy(datap, ctx->ccm_remainder, remainder);
			ctx->ccm_remainder_len = remainder;
			ctx->ccm_copy_to = datap;
			goto out;
		}
		ctx->ccm_copy_to = NULL;

	} while (remainder > 0);

out:
	return (CRYPTO_SUCCESS);
}

void
calculate_ccm_mac(ccm_ctx_t *ctx, uint8_t *ccm_mac,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
{
	uint64_t counter;
	uint8_t *counterp, *mac_buf;
	int i;

	mac_buf = (uint8_t *)ctx->ccm_mac_buf;

	/* first counter block start with index 0 */
	counter = 0;
	ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;

	counterp = (uint8_t *)ctx->ccm_tmp;
	encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);

	/* calculate XOR of MAC with first counter block */
	for (i = 0; i < ctx->ccm_mac_len; i++) {
		ccm_mac[i] = mac_buf[i] ^ counterp[i];
	}
}

/* ARGSUSED */
int
ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;
	int i;

	if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) {
		return (CRYPTO_DATA_LEN_RANGE);
	}

	/*
	 * When we get here, the number of bytes of payload processed
	 * plus whatever data remains, if any,
	 * should be the same as the number of bytes that's being
	 * passed in the argument during init time.
	 */
	if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len)
	    != (ctx->ccm_data_len)) {
		return (CRYPTO_DATA_LEN_RANGE);
	}

	mac_buf = (uint8_t *)ctx->ccm_mac_buf;

	if (ctx->ccm_remainder_len > 0) {

		/* ccm_mac_input_buf is not used for encryption */
		macp = (uint8_t *)ctx->ccm_mac_input_buf;
		bzero(macp, block_size);

		/* copy remainder to temporary buffer */
		bcopy(ctx->ccm_remainder, macp, ctx->ccm_remainder_len);

		/* calculate the CBC MAC */
		xor_block(macp, mac_buf);
		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);

		/* calculate the counter mode */
		lastp = (uint8_t *)ctx->ccm_tmp;
		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, lastp);

		/* XOR with counter block */
		for (i = 0; i < ctx->ccm_remainder_len; i++) {
			macp[i] ^= lastp[i];
		}
		ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
	}

	/* Calculate the CCM MAC */
	ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
	calculate_ccm_mac(ctx, ccm_mac_p, encrypt_block);

	crypto_init_ptrs(out, &iov_or_mp, &offset);
	crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
	    &out_data_1_len, &out_data_2,
	    ctx->ccm_remainder_len + ctx->ccm_mac_len);

	if (ctx->ccm_remainder_len > 0) {

		/* copy temporary block to where it belongs */
		if (out_data_2 == NULL) {
			/* everything will fit in out_data_1 */
			bcopy(macp, out_data_1, ctx->ccm_remainder_len);
			bcopy(ccm_mac_p, out_data_1 + ctx->ccm_remainder_len,
			    ctx->ccm_mac_len);
		} else {

			if (out_data_1_len < ctx->ccm_remainder_len) {

				size_t data_2_len_used;

				bcopy(macp, out_data_1, out_data_1_len);

				data_2_len_used = ctx->ccm_remainder_len
				    - out_data_1_len;

				bcopy((uint8_t *)macp + out_data_1_len,
				    out_data_2, data_2_len_used);
				bcopy(ccm_mac_p, out_data_2 + data_2_len_used,
				    ctx->ccm_mac_len);
			} else {
				bcopy(macp, out_data_1, out_data_1_len);
				if (out_data_1_len == ctx->ccm_remainder_len) {
					/* mac will be in out_data_2 */
					bcopy(ccm_mac_p, out_data_2,
					    ctx->ccm_mac_len);
				} else {
					size_t len_not_used = out_data_1_len -
					    ctx->ccm_remainder_len;
					/*
					 * part of mac in will be in
					 * out_data_1, part of the mac will be
					 * in out_data_2
					 */
					bcopy(ccm_mac_p,
					    out_data_1 + ctx->ccm_remainder_len,
					    len_not_used);
					bcopy(ccm_mac_p + len_not_used,
					    out_data_2,
					    ctx->ccm_mac_len - len_not_used);

				}
			}
		}
	} else {
		/* copy block to where it belongs */
		bcopy(ccm_mac_p, out_data_1, out_data_1_len);
		if (out_data_2 != NULL) {
			bcopy(ccm_mac_p + out_data_1_len, out_data_2,
			    block_size - out_data_1_len);
		}
	}
	out->cd_offset += ctx->ccm_remainder_len + ctx->ccm_mac_len;
	ctx->ccm_remainder_len = 0;
	return (CRYPTO_SUCCESS);
}

/*
 * This will only deal with decrypting the last block of the input that
 * might not be a multiple of block length.
 */
void
ccm_decrypt_incomplete_block(ccm_ctx_t *ctx,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
{
	uint8_t *datap, *outp, *counterp;
	int i;

	datap = (uint8_t *)ctx->ccm_remainder;
	outp = &((ctx->ccm_pt_buf)[ctx->ccm_processed_data_len]);

	counterp = (uint8_t *)ctx->ccm_tmp;
	encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);

	/* XOR with counter block */
	for (i = 0; i < ctx->ccm_remainder_len; i++) {
		outp[i] = datap[i] ^ counterp[i];
	}
}

/*
 * This will decrypt the cipher text.  However, the plaintext won't be
 * returned to the caller.  It will be returned when decrypt_final() is
 * called if the MAC matches
 */
/* ARGSUSED */
int
ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	uint8_t *cbp;
	uint64_t counter;
	size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len;
	uint8_t *resultp;


	pm_len = ctx->ccm_processed_mac_len;

	if (pm_len > 0) {
		uint8_t *tmp;
		/*
		 * all ciphertext has been processed, just waiting for
		 * part of the value of the mac
		 */
		if ((pm_len + length) > ctx->ccm_mac_len) {
			return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
		}
		tmp = (uint8_t *)ctx->ccm_mac_input_buf;

		bcopy(datap, tmp + pm_len, length);

		ctx->ccm_processed_mac_len += length;
		return (CRYPTO_SUCCESS);
	}

	/*
	 * If we decrypt the given data, what total amount of data would
	 * have been decrypted?
	 */
	pd_len = ctx->ccm_processed_data_len;
	total_decrypted_len = pd_len + length + ctx->ccm_remainder_len;

	if (total_decrypted_len >
	    (ctx->ccm_data_len + ctx->ccm_mac_len)) {
		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
	}

	pt_len = ctx->ccm_data_len;

	if (total_decrypted_len > pt_len) {
		/*
		 * part of the input will be the MAC, need to isolate that
		 * to be dealt with later.  The left-over data in
		 * ccm_remainder_len from last time will not be part of the
		 * MAC.  Otherwise, it would have already been taken out
		 * when this call is made last time.
		 */
		size_t pt_part = pt_len - pd_len - ctx->ccm_remainder_len;

		mac_len = length - pt_part;

		ctx->ccm_processed_mac_len = mac_len;
		bcopy(data + pt_part, ctx->ccm_mac_input_buf, mac_len);

		if (pt_part + ctx->ccm_remainder_len < block_size) {
			/*
			 * since this is last of the ciphertext, will
			 * just decrypt with it here
			 */
			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
			    [ctx->ccm_remainder_len], pt_part);
			ctx->ccm_remainder_len += pt_part;
			ccm_decrypt_incomplete_block(ctx, encrypt_block);
			ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
			ctx->ccm_remainder_len = 0;
			return (CRYPTO_SUCCESS);
		} else {
			/* let rest of the code handle this */
			length = pt_part;
		}
	} else if (length + ctx->ccm_remainder_len < block_size) {
			/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
		    length);
		ctx->ccm_remainder_len += length;
		ctx->ccm_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	do {
		/* Unprocessed data from last call. */
		if (ctx->ccm_remainder_len > 0) {
			need = block_size - ctx->ccm_remainder_len;

			if (need > remainder)
				return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
			    [ctx->ccm_remainder_len], need);

			blockp = (uint8_t *)ctx->ccm_remainder;
		} else {
			blockp = datap;
		}

		/* Calculate the counter mode, ccm_cb is the counter block */
		cbp = (uint8_t *)ctx->ccm_tmp;
		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, cbp);

		/*
		 * Increment counter.
		 * Counter bits are confined to the bottom 64 bits
		 */
		counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
		counter = htonll(counter + 1);
		counter &= ctx->ccm_counter_mask;
		ctx->ccm_cb[1] =
		    (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;

		/* XOR with the ciphertext */
		xor_block(blockp, cbp);

		/* Copy the plaintext to the "holding buffer" */
		resultp = (uint8_t *)ctx->ccm_pt_buf +
		    ctx->ccm_processed_data_len;
		copy_block(cbp, resultp);

		ctx->ccm_processed_data_len += block_size;

		ctx->ccm_lastp = blockp;

		/* Update pointer to next block of data to be processed. */
		if (ctx->ccm_remainder_len != 0) {
			datap += need;
			ctx->ccm_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block */
		if (remainder > 0 && remainder < block_size) {
			bcopy(datap, ctx->ccm_remainder, remainder);
			ctx->ccm_remainder_len = remainder;
			ctx->ccm_copy_to = datap;
			if (ctx->ccm_processed_mac_len > 0) {
				/*
				 * not expecting anymore ciphertext, just
				 * compute plaintext for the remaining input
				 */
				ccm_decrypt_incomplete_block(ctx,
				    encrypt_block);
				ctx->ccm_processed_data_len += remainder;
				ctx->ccm_remainder_len = 0;
			}
			goto out;
		}
		ctx->ccm_copy_to = NULL;

	} while (remainder > 0);

out:
	return (CRYPTO_SUCCESS);
}

int
ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t mac_remain, pt_len;
	uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
	int rv;

	pt_len = ctx->ccm_data_len;

	/* Make sure output buffer can fit all of the plaintext */
	if (out->cd_length < pt_len) {
		return (CRYPTO_DATA_LEN_RANGE);
	}

	pt = ctx->ccm_pt_buf;
	mac_remain = ctx->ccm_processed_data_len;
	mac_buf = (uint8_t *)ctx->ccm_mac_buf;

	macp = (uint8_t *)ctx->ccm_tmp;

	while (mac_remain > 0) {

		if (mac_remain < block_size) {
			bzero(macp, block_size);
			bcopy(pt, macp, mac_remain);
			mac_remain = 0;
		} else {
			copy_block(pt, macp);
			mac_remain -= block_size;
			pt += block_size;
		}

		/* calculate the CBC MAC */
		xor_block(macp, mac_buf);
		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
	}

	/* Calculate the CCM MAC */
	ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
	calculate_ccm_mac((ccm_ctx_t *)ctx, ccm_mac_p, encrypt_block);

	/* compare the input CCM MAC value with what we calculated */
	if (bcmp(ctx->ccm_mac_input_buf, ccm_mac_p, ctx->ccm_mac_len)) {
		/* They don't match */
		return (CRYPTO_INVALID_MAC);
	} else {
		rv = crypto_put_output_data(ctx->ccm_pt_buf, out, pt_len);
		if (rv != CRYPTO_SUCCESS)
			return (rv);
		out->cd_offset += pt_len;
	}
	return (CRYPTO_SUCCESS);
}

int
ccm_validate_args(CK_AES_CCM_PARAMS *ccm_param, boolean_t is_encrypt_init)
{
	size_t macSize, nonceSize;
	uint8_t q;
	uint64_t maxValue;

	/*
	 * Check the length of the MAC.  The only valid
	 * lengths for the MAC are: 4, 6, 8, 10, 12, 14, 16
	 */
	macSize = ccm_param->ulMACSize;
	if ((macSize < 4) || (macSize > 16) || ((macSize % 2) != 0)) {
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}

	/* Check the nonce length.  Valid values are 7, 8, 9, 10, 11, 12, 13 */
	nonceSize = ccm_param->ulNonceSize;
	if ((nonceSize < 7) || (nonceSize > 13)) {
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}

	/* q is the length of the field storing the length, in bytes */
	q = (uint8_t)((15 - nonceSize) & 0xFF);


	/*
	 * If it is decrypt, need to make sure size of ciphertext is at least
	 * bigger than MAC len
	 */
	if ((!is_encrypt_init) && (ccm_param->ulDataSize < macSize)) {
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}

	/*
	 * Check to make sure the length of the payload is within the
	 * range of values allowed by q
	 */
	if (q < 8) {
		maxValue = (1ULL << (q * 8)) - 1;
	} else {
		maxValue = ULONG_MAX;
	}

	if (ccm_param->ulDataSize > maxValue) {
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}
	return (CRYPTO_SUCCESS);
}

/*
 * Format the first block used in CBC-MAC (B0) and the initial counter
 * block based on formatting functions and counter generation functions
 * specified in RFC 3610 and NIST publication 800-38C, appendix A
 *
 * b0 is the first block used in CBC-MAC
 * cb0 is the first counter block
 *
 * It's assumed that the arguments b0 and cb0 are preallocated AES blocks
 *
 */
static void
ccm_format_initial_blocks(uchar_t *nonce, ulong_t nonceSize,
    ulong_t authDataSize, uint8_t *b0, ccm_ctx_t *aes_ctx)
{
	uint64_t payloadSize;
	uint8_t t, q, have_adata = 0;
	size_t limit;
	int i, j, k;
	uint64_t mask = 0;
	uint8_t *cb;

	q = (uint8_t)((15 - nonceSize) & 0xFF);
	t = (uint8_t)((aes_ctx->ccm_mac_len) & 0xFF);

	/* Construct the first octet of b0 */
	if (authDataSize > 0) {
		have_adata = 1;
	}
	b0[0] = (have_adata << 6) | (((t - 2)  / 2) << 3) | (q - 1);

	/* copy the nonce value into b0 */
	bcopy(nonce, &(b0[1]), nonceSize);

	/* store the length of the payload into b0 */
	bzero(&(b0[1+nonceSize]), q);

	payloadSize = aes_ctx->ccm_data_len;
	limit = 8 < q ? 8 : q;

	for (i = 0, j = 0, k = 15; i < limit; i++, j += 8, k--) {
		b0[k] = (uint8_t)((payloadSize >> j) & 0xFF);
	}

	/* format the counter block */

	cb = (uint8_t *)aes_ctx->ccm_cb;

	cb[0] = 0x07 & (q-1); /* first byte */

	/* copy the nonce value into the counter block */
	bcopy(nonce, &(cb[1]), nonceSize);

	bzero(&(cb[1+nonceSize]), q);

	/* Create the mask for the counter field based on the size of nonce */
	q <<= 3;
	while (q-- > 0) {
		mask |= (1ULL << q);
	}

	aes_ctx->ccm_counter_mask = htonll(mask);

	/*
	 * During calculation, we start using counter block 1, we will
	 * set it up right here.
	 * We can just set the last byte to have the value 1, because
	 * even with the biggest nonce of 13, the last byte of the
	 * counter block will be used for the counter value.
	 */
	cb[15] = 0x01;
}

/*
 * Encode the length of the associated data as
 * specified in RFC 3610 and NIST publication 800-38C, appendix A
 */
static void
encode_adata_len(ulong_t auth_data_len, uint8_t *encoded, size_t *encoded_len)
{
#ifdef UNALIGNED_POINTERS_PERMITTED
	uint32_t	*lencoded_ptr;
#ifdef _LP64
	uint64_t	*llencoded_ptr;
#endif
#endif	/* UNALIGNED_POINTERS_PERMITTED */

	if (auth_data_len < ((1ULL<<16) - (1ULL<<8))) {
		/* 0 < a < (2^16-2^8) */
		*encoded_len = 2;
		encoded[0] = (auth_data_len & 0xff00) >> 8;
		encoded[1] = auth_data_len & 0xff;

	} else if ((auth_data_len >= ((1ULL<<16) - (1ULL<<8))) &&
	    (auth_data_len < (1ULL << 31))) {
		/* (2^16-2^8) <= a < 2^32 */
		*encoded_len = 6;
		encoded[0] = 0xff;
		encoded[1] = 0xfe;
#ifdef UNALIGNED_POINTERS_PERMITTED
		lencoded_ptr = (uint32_t *)(void *)&encoded[2];
		*lencoded_ptr = htonl(auth_data_len);
#else
		encoded[2] = (auth_data_len & 0xff000000) >> 24;
		encoded[3] = (auth_data_len & 0xff0000) >> 16;
		encoded[4] = (auth_data_len & 0xff00) >> 8;
		encoded[5] = auth_data_len & 0xff;
#endif	/* UNALIGNED_POINTERS_PERMITTED */

#ifdef _LP64
	} else {
		/* 2^32 <= a < 2^64 */
		*encoded_len = 10;
		encoded[0] = 0xff;
		encoded[1] = 0xff;
#ifdef UNALIGNED_POINTERS_PERMITTED
		llencoded_ptr = (uint64_t *)(void *)&encoded[2];
		*llencoded_ptr = htonl(auth_data_len);
#else
		encoded[2] = (auth_data_len & 0xff00000000000000) >> 56;
		encoded[3] = (auth_data_len & 0xff000000000000) >> 48;
		encoded[4] = (auth_data_len & 0xff0000000000) >> 40;
		encoded[5] = (auth_data_len & 0xff00000000) >> 32;
		encoded[6] = (auth_data_len & 0xff000000) >> 24;
		encoded[7] = (auth_data_len & 0xff0000) >> 16;
		encoded[8] = (auth_data_len & 0xff00) >> 8;
		encoded[9] = auth_data_len & 0xff;
#endif	/* UNALIGNED_POINTERS_PERMITTED */
#endif	/* _LP64 */
	}
}

/*
 * The following function should be call at encrypt or decrypt init time
 * for AES CCM mode.
 */
int
ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
    unsigned char *auth_data, size_t auth_data_len, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *mac_buf, *datap, *ivp, *authp;
	size_t remainder, processed;
	uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */
	size_t encoded_a_len = 0;

	mac_buf = (uint8_t *)&(ctx->ccm_mac_buf);

	/*
	 * Format the 1st block for CBC-MAC and construct the
	 * 1st counter block.
	 *
	 * aes_ctx->ccm_iv is used for storing the counter block
	 * mac_buf will store b0 at this time.
	 */
	ccm_format_initial_blocks(nonce, nonce_len,
	    auth_data_len, mac_buf, ctx);

	/* The IV for CBC MAC for AES CCM mode is always zero */
	ivp = (uint8_t *)ctx->ccm_tmp;
	bzero(ivp, block_size);

	xor_block(ivp, mac_buf);

	/* encrypt the nonce */
	encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);

	/* take care of the associated data, if any */
	if (auth_data_len == 0) {
		return (CRYPTO_SUCCESS);
	}

	encode_adata_len(auth_data_len, encoded_a, &encoded_a_len);

	remainder = auth_data_len;

	/* 1st block: it contains encoded associated data, and some data */
	authp = (uint8_t *)ctx->ccm_tmp;
	bzero(authp, block_size);
	bcopy(encoded_a, authp, encoded_a_len);
	processed = block_size - encoded_a_len;
	if (processed > auth_data_len) {
		/* in case auth_data is very small */
		processed = auth_data_len;
	}
	bcopy(auth_data, authp+encoded_a_len, processed);
	/* xor with previous buffer */
	xor_block(authp, mac_buf);
	encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
	remainder -= processed;
	if (remainder == 0) {
		/* a small amount of associated data, it's all done now */
		return (CRYPTO_SUCCESS);
	}

	do {
		if (remainder < block_size) {
			/*
			 * There's not a block full of data, pad rest of
			 * buffer with zero
			 */
			bzero(authp, block_size);
			bcopy(&(auth_data[processed]), authp, remainder);
			datap = (uint8_t *)authp;
			remainder = 0;
		} else {
			datap = (uint8_t *)(&(auth_data[processed]));
			processed += block_size;
			remainder -= block_size;
		}

		xor_block(datap, mac_buf);
		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);

	} while (remainder > 0);

	return (CRYPTO_SUCCESS);
}

int
ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
    boolean_t is_encrypt_init, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	int rv;
	CK_AES_CCM_PARAMS *ccm_param;

	if (param != NULL) {
		ccm_param = (CK_AES_CCM_PARAMS *)(void *)param;

		if ((rv = ccm_validate_args(ccm_param,
		    is_encrypt_init)) != 0) {
			return (rv);
		}

		ccm_ctx->ccm_mac_len = ccm_param->ulMACSize;
		if (is_encrypt_init) {
			ccm_ctx->ccm_data_len = ccm_param->ulDataSize;
		} else {
			ccm_ctx->ccm_data_len =
			    ccm_param->ulDataSize - ccm_ctx->ccm_mac_len;
			ccm_ctx->ccm_processed_mac_len = 0;
		}
		ccm_ctx->ccm_processed_data_len = 0;

		ccm_ctx->ccm_flags |= CCM_MODE;
	} else {
		rv = CRYPTO_MECHANISM_PARAM_INVALID;
		goto out;
	}

	if (ccm_init(ccm_ctx, ccm_param->nonce, ccm_param->ulNonceSize,
	    ccm_param->authData, ccm_param->ulAuthDataSize, block_size,
	    encrypt_block, xor_block) != 0) {
		rv = CRYPTO_MECHANISM_PARAM_INVALID;
		goto out;
	}
	if (!is_encrypt_init && ccm_ctx->ccm_data_len != 0) {
		/* allocate buffer for storing decrypted plaintext */
#ifdef _KERNEL
		ccm_ctx->ccm_pt_buf = kmem_alloc(ccm_ctx->ccm_data_len,
		    kmflag);
#else
		ccm_ctx->ccm_pt_buf = malloc(ccm_ctx->ccm_data_len);
#endif
		if (ccm_ctx->ccm_pt_buf == NULL) {
			rv = CRYPTO_HOST_MEMORY;
		}
	}
out:
	return (rv);
}

void *
ccm_alloc_ctx(int kmflag)
{
	ccm_ctx_t *ccm_ctx;

#ifdef _KERNEL
	if ((ccm_ctx = kmem_zalloc(sizeof (ccm_ctx_t), kmflag)) == NULL)
#else
	if ((ccm_ctx = calloc(1, sizeof (ccm_ctx_t))) == NULL)
#endif
		return (NULL);

	ccm_ctx->ccm_flags = CCM_MODE;
	return (ccm_ctx);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2019 Joyent, Inc.
 */

#ifndef _KERNEL
#include <strings.h>
#include <limits.h>
#include <assert.h>
#include <security/cryptoki.h>
#endif

#include <sys/debug.h>
#include <sys/types.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>
#include <sys/byteorder.h>

/*
 * CTR (counter mode) is a stream cipher.  That is, it generates a
 * pseudo-random keystream that is used to XOR with the input to
 * encrypt or decrypt.  The pseudo-random keystream is generated by
 * concatenating a nonce (supplied during initialzation) and with a
 * counter (initialized to zero) to form an input block to the cipher
 * mechanism.  The resulting output of the cipher is used as a chunk
 * of the pseudo-random keystream.  Once all of the bytes of the
 * keystream block have been used, the counter is incremented and
 * the process repeats.
 *
 * Since this is a stream cipher, we do not accumulate input cipher
 * text like we do for block modes.  Instead we use ctr_ctx_t->ctr_offset
 * to track the amount of bytes used in the current keystream block.
 */

static void
ctr_new_keyblock(ctr_ctx_t *ctx,
    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
{
	uint64_t lower_counter, upper_counter;

	/* increment the counter */
	lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask);
	lower_counter = htonll(lower_counter + 1);
	lower_counter &= ctx->ctr_lower_mask;
	ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) |
	    lower_counter;

	/* wrap around */
	if (lower_counter == 0) {
		upper_counter = ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask);
		upper_counter = htonll(upper_counter + 1);
		upper_counter &= ctx->ctr_upper_mask;
		ctx->ctr_cb[0] = (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) |
		    upper_counter;
	}

	/* generate the new keyblock */
	cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
	    (uint8_t *)ctx->ctr_keystream);
	ctx->ctr_offset = 0;
}

/*
 * XOR the input with the keystream and write the result to out.
 * This requires that the amount of data in 'in' is >= outlen
 * (ctr_mode_contiguous_blocks() guarantees this for us before we are
 * called).  As CTR mode is a stream cipher, we cannot use a cipher's
 * xxx_xor_block function (e.g. aes_xor_block()) as we must handle
 * arbitrary lengths of input and should not buffer/accumulate partial blocks
 * between calls.
 */
static void
ctr_xor(ctr_ctx_t *ctx, const uint8_t *in, uint8_t *out, size_t outlen,
    size_t block_size,
    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
{
	const uint8_t *keyp;
	size_t keyamt;

	while (outlen > 0) {
		/*
		 * This occurs once we've consumed all the bytes in the
		 * current block of the keystream. ctr_init_ctx() creates
		 * the initial block of the keystream, so we always start
		 * with a full block of key data.
		 */
		if (ctx->ctr_offset == block_size) {
			ctr_new_keyblock(ctx, cipher);
		}

		keyp = (uint8_t *)ctx->ctr_keystream + ctx->ctr_offset;
		keyamt = block_size - ctx->ctr_offset;

		/*
		 * xor a byte at a time (while we have data and output
		 * space) and try to get in, out, and keyp 32-bit aligned.
		 * If in, out, and keyp all do become 32-bit aligned,
		 * we switch to xor-ing 32-bits at a time until we run out
		 * of 32-bit chunks, then switch back to xor-ing a byte at
		 * a time for any remainder.
		 */
		while (keyamt > 0 && outlen > 0 &&
		    !IS_P2ALIGNED(in, sizeof (uint32_t)) &&
		    !IS_P2ALIGNED(out, sizeof (uint32_t)) &&
		    !IS_P2ALIGNED(keyp, sizeof (uint32_t))) {
			*out++ = *in++ ^ *keyp++;
			keyamt--;
			outlen--;
		}

		if (keyamt > 3 && outlen > 3 &&
		    IS_P2ALIGNED(in, sizeof (uint32_t)) &&
		    IS_P2ALIGNED(out, sizeof (uint32_t)) &&
		    IS_P2ALIGNED(keyp, sizeof (uint32_t))) {
			const uint32_t *key32 = (const uint32_t *)keyp;
			const uint32_t *in32 = (const uint32_t *)in;
			uint32_t *out32 = (uint32_t *)out;

			do {
				*out32++ = *in32++ ^ *key32++;
				keyamt -= sizeof (uint32_t);
				outlen -= sizeof (uint32_t);
			} while (keyamt > 3 && outlen > 3);

			keyp = (const uint8_t *)key32;
			in = (const uint8_t *)in32;
			out = (uint8_t *)out32;
		}

		while (keyamt > 0 && outlen > 0) {
			*out++ = *in++ ^ *keyp++;
			keyamt--;
			outlen--;
		}

		ctx->ctr_offset = block_size - keyamt;
	}
}

/*
 * Encrypt and decrypt multiple blocks of data in counter mode.
 */
int
ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *in, size_t in_length,
    crypto_data_t *out, size_t block_size,
    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
{
	size_t in_remainder = in_length;
	uint8_t *inp = (uint8_t *)in;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data;
	uint8_t *out_data_remainder;
	size_t out_data_len;

	if (block_size > sizeof (ctx->ctr_keystream))
		return (CRYPTO_ARGUMENTS_BAD);

	if (out == NULL)
		return (CRYPTO_ARGUMENTS_BAD);

	/* Make sure 'out->cd_offset + in_length' doesn't overflow. */
	if (out->cd_offset < 0)
		return (CRYPTO_DATA_LEN_RANGE);
	if (SIZE_MAX - in_length < (size_t)out->cd_offset)
		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);

	/*
	 * This check guarantees 'out' contains sufficient space for
	 * the resulting output.
	 */
	if (out->cd_offset + in_length > out->cd_length)
		return (CRYPTO_BUFFER_TOO_SMALL);

	crypto_init_ptrs(out, &iov_or_mp, &offset);

	/* Now XOR the output with the keystream */
	while (in_remainder > 0) {
		/*
		 * If out is a uio_t or an mblk_t, in_remainder might be
		 * larger than an individual iovec_t or mblk_t in out.
		 * crypto_get_ptrs uses the value of offset to set the
		 * the value of out_data to the correct address for writing
		 * and sets out_data_len to reflect the largest amount of data
		 * (up to in_remainder) that can be written to out_data. It
		 * also increments offset by out_data_len. out_data_remainder
		 * is set to the start of the next segment for writing, however
		 * it is not used here since the updated value of offset
		 * will be used in the next loop iteration to locate the
		 * next mblk_t/iovec_t. Since the sum of the size of all data
		 * buffers in 'out' (out->cd_length) was checked immediately
		 * prior to starting the loop, we should always terminate
		 * the loop.
		 */
		crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data,
		    &out_data_len, &out_data_remainder, in_remainder);

		/*
		 * crypto_get_ptrs() should guarantee these, but act as a
		 * safeguard in case the behavior ever changes.
		 */
		ASSERT3U(out_data_len, <=, in_remainder);
		ASSERT3U(out_data_len, >, 0);

		ctr_xor(ctx, inp, out_data, out_data_len, block_size, cipher);

		inp += out_data_len;
		in_remainder -= out_data_len;
	}

	out->cd_offset += in_length;

	return (CRYPTO_SUCCESS);
}

int
ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
    void (*copy_block)(uint8_t *, uint8_t *))
{
	uint64_t upper_mask = 0;
	uint64_t lower_mask = 0;

	if (count == 0 || count > 128) {
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}
	/* upper 64 bits of the mask */
	if (count >= 64) {
		count -= 64;
		upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
		lower_mask = UINT64_MAX;
	} else {
		/* now the lower 63 bits */
		lower_mask = (1ULL << count) - 1;
	}
	ctr_ctx->ctr_lower_mask = htonll(lower_mask);
	ctr_ctx->ctr_upper_mask = htonll(upper_mask);

	copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);
	ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0];

	/* Generate the first block of the keystream */
	cipher(ctr_ctx->ctr_keysched, (uint8_t *)ctr_ctx->ctr_cb,
	    (uint8_t *)ctr_ctx->ctr_keystream);

	ctr_ctx->ctr_flags |= CTR_MODE;
	return (CRYPTO_SUCCESS);
}

/* ARGSUSED */
void *
ctr_alloc_ctx(int kmflag)
{
	ctr_ctx_t *ctr_ctx;

#ifdef _KERNEL
	if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL)
#else
	if ((ctr_ctx = calloc(1, sizeof (ctr_ctx_t))) == NULL)
#endif
		return (NULL);

	ctr_ctx->ctr_flags = CTR_MODE;
	return (ctr_ctx);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _KERNEL
#include <strings.h>
#include <limits.h>
#include <assert.h>
#include <security/cryptoki.h>
#endif

#include <sys/types.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>

/*
 * Algorithm independent ECB functions.
 */
int
ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	uint8_t *lastp;
	void *iov_or_mp;
	offset_t offset;
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;

	if (length + ctx->ecb_remainder_len < block_size) {
		/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len,
		    length);
		ctx->ecb_remainder_len += length;
		ctx->ecb_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	lastp = (uint8_t *)ctx->ecb_iv;
	if (out != NULL)
		crypto_init_ptrs(out, &iov_or_mp, &offset);

	do {
		/* Unprocessed data from last call. */
		if (ctx->ecb_remainder_len > 0) {
			need = block_size - ctx->ecb_remainder_len;

			if (need > remainder)
				return (CRYPTO_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->ecb_remainder)
			    [ctx->ecb_remainder_len], need);

			blockp = (uint8_t *)ctx->ecb_remainder;
		} else {
			blockp = datap;
		}

		if (out == NULL) {
			cipher(ctx->ecb_keysched, blockp, blockp);

			ctx->ecb_lastp = blockp;
			lastp = blockp;

			if (ctx->ecb_remainder_len > 0) {
				bcopy(blockp, ctx->ecb_copy_to,
				    ctx->ecb_remainder_len);
				bcopy(blockp + ctx->ecb_remainder_len, datap,
				    need);
			}
		} else {
			cipher(ctx->ecb_keysched, blockp, lastp);
			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
			    &out_data_1_len, &out_data_2, block_size);

			/* copy block to where it belongs */
			bcopy(lastp, out_data_1, out_data_1_len);
			if (out_data_2 != NULL) {
				bcopy(lastp + out_data_1_len, out_data_2,
				    block_size - out_data_1_len);
			}
			/* update offset */
			out->cd_offset += block_size;
		}

		/* Update pointer to next block of data to be processed. */
		if (ctx->ecb_remainder_len != 0) {
			datap += need;
			ctx->ecb_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block. */
		if (remainder > 0 && remainder < block_size) {
			bcopy(datap, ctx->ecb_remainder, remainder);
			ctx->ecb_remainder_len = remainder;
			ctx->ecb_copy_to = datap;
			goto out;
		}
		ctx->ecb_copy_to = NULL;

	} while (remainder > 0);

out:
	return (CRYPTO_SUCCESS);
}

/* ARGSUSED */
void *
ecb_alloc_ctx(int kmflag)
{
	ecb_ctx_t *ecb_ctx;

#ifdef _KERNEL
	if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL)
#else
	if ((ecb_ctx = calloc(1, sizeof (ecb_ctx_t))) == NULL)
#endif
		return (NULL);

	ecb_ctx->ecb_flags = ECB_MODE;
	return (ecb_ctx);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2018, Joyent, Inc.
 * Copyright 2023-2026 RackTop Systems, Inc.
 */

/*
 * This file implements GCM and GMAC, as decribed in
 *	NIST Special Publication 800-38D
 *	Recommendation for Block Cipher Modes of Operation:
 *	Galois/Counter Mode (GCM) and GMAC
 *
 * Briefly, GMAC uses GCM just for "authentication" (sign/verify),
 * discarding the ouptut data (cipher/clear) that GCM would produce.
 *
 * Some functions below serve both GCM and GMAC, adjusting behavior
 * based on (ctx->gcm_flags & GMAC_MODE) to skip output production
 * or actions needed only when actually doing encrypt or decrypt.
 *
 * Some non-obvious things to note:
 *
 * The struct member gcm_len_a_len_c[] is an array of two uint64_t
 * (AAD length and input data length, in that order, in BITS).
 * The values are needed in that form for a hash computation that
 * happens in the "final" function for GCM or GMAC.  Just before the
 * "final" hash computation, the values are converted to big-endian
 * form as required by the altgorithm specification. Before that
 * point those values are in host order (always BITS).
 *
 * The calling framework (one of uts/common/crypto/io/aes.c
 * or lib/pkcs11/pkcs11_softtoken/common/softAESCrypt.c)
 * uses different "alloc", "init", and "final" functions
 * for GCM vs GMAC.  See calls to:
 *	gcm_alloc_ctx, gmac_alloc_ctx,
 *	gcm_init_ctx,  gmac_init_ctx,
 *	gcm_encrypt_final, gmac_mode_final
 * Operation of the GCM vs GMAC varints of those functions are
 * similar other than encrypt/decrypt in GCM, skipped in GMAC.
 */

#ifndef _KERNEL
#include <strings.h>
#include <limits.h>
#include <security/cryptoki.h>
#endif	/* _KERNEL */

#include <sys/debug.h>
#include <sys/types.h>
#include <sys/kmem.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>
#include <sys/byteorder.h>

#ifdef __amd64

#ifdef _KERNEL
#include <sys/cpuvar.h>		/* cpu_t, CPU */
#include <sys/x86_archext.h>	/* x86_featureset, X86FSET_*, CPUID_* */
#include <sys/disp.h>		/* kpreempt_disable(), kpreempt_enable */
/* Workaround for no XMM kernel thread save/restore */
#define	KPREEMPT_DISABLE	kpreempt_disable()
#define	KPREEMPT_ENABLE		kpreempt_enable()

#else
#include <sys/auxv.h>		/* getisax() */
#include <sys/auxv_386.h>	/* AV_386_PCLMULQDQ bit */
#define	KPREEMPT_DISABLE
#define	KPREEMPT_ENABLE
#endif	/* _KERNEL */

extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
static int intel_pclmulqdq_instruction_present(void);
#endif	/* __amd64 */

struct aes_block {
	uint64_t a;
	uint64_t b;
};


/*
 * gcm_mul()
 * Perform a carry-less multiplication (that is, use XOR instead of the
 * multiply operator) on *x_in and *y and place the result in *res.
 *
 * Byte swap the input (*x_in and *y) and the output (*res).
 *
 * Note: x_in, y, and res all point to 16-byte numbers (an array of two
 * 64-bit integers).
 */
void
gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
{
#ifdef __amd64
	if (intel_pclmulqdq_instruction_present()) {
		KPREEMPT_DISABLE;
		gcm_mul_pclmulqdq(x_in, y, res);
		KPREEMPT_ENABLE;
	} else
#endif	/* __amd64 */
	{
		static const uint64_t R = 0xe100000000000000ULL;
		struct aes_block z = {0, 0};
		struct aes_block v;
		uint64_t x;
		int i, j;

		v.a = ntohll(y[0]);
		v.b = ntohll(y[1]);

		for (j = 0; j < 2; j++) {
			x = ntohll(x_in[j]);
			for (i = 0; i < 64; i++, x <<= 1) {
				if (x & 0x8000000000000000ULL) {
					z.a ^= v.a;
					z.b ^= v.b;
				}
				if (v.b & 1ULL) {
					v.b = (v.a << 63)|(v.b >> 1);
					v.a = (v.a >> 1) ^ R;
				} else {
					v.b = (v.a << 63)|(v.b >> 1);
					v.a = v.a >> 1;
				}
			}
		}
		res[0] = htonll(z.a);
		res[1] = htonll(z.b);
	}
}


#define	GHASH(c, d, t) \
	xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
	gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
	(uint64_t *)(void *)(t));

/*
 * helper factored out of gcm_mode_encrypt_contiguous_blocks
 */
static inline void
gcm_encrypt_block(gcm_ctx_t *ctx, uint8_t *datap, crypto_data_t *out,
    size_t block_size, uint8_t *blockp, void *iov_or_mp, offset_t *offset,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *out_data_1;
	uint8_t *out_data_2;
	size_t out_data_1_len;
	uint64_t counter;
	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);

	/*
	 * Increment counter. Counter bits are confined
	 * to the bottom 32 bits of the counter block.
	 */
	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
	counter = htonll(counter + 1);
	counter &= counter_mask;
	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;

	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
	    (uint8_t *)ctx->gcm_tmp);
	xor_block(blockp, (uint8_t *)ctx->gcm_tmp);

	if (out == NULL) {
		if (ctx->gcm_remainder_len > 0) {
			bcopy(blockp, ctx->gcm_copy_to,
			    ctx->gcm_remainder_len);
			bcopy(blockp + ctx->gcm_remainder_len, datap,
			    block_size - ctx->gcm_remainder_len);
		}
	} else {
		uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;
		crypto_get_ptrs(out, iov_or_mp, offset, &out_data_1,
		    &out_data_1_len, &out_data_2, block_size);

		/* copy block to where it belongs */
		if (out_data_1_len == block_size) {
			copy_block(tmpp, out_data_1);
		} else {
			bcopy(tmpp, out_data_1, out_data_1_len);
			if (out_data_2 != NULL) {
				bcopy(tmpp + out_data_1_len,
				    out_data_2,
				    block_size - out_data_1_len);
			}
		}
		/* update offset */
		out->cd_offset += block_size;
	}
}

/*
 * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
 * is done in another function: gcm_mode_decrypt_contiguous_blocks().
 *
 * When doing GCM, gcm_processed_data_len is advanced (which is the
 * encrypted/decrypted data bytes, excluding AAD).  When this is doing
 * GMAC (serving C_Sign) it advances the "input" pointers instead:
 * gcm_len_a_len_c[0] is the ADD input length, and
 * gcm_len_a_len_c[1] is the data input length.
 * (Details at the top of this file).
 */
int
gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t remainder = length;
	size_t need;
	uint8_t *datap = (uint8_t *)data;
	uint8_t *blockp;
	void *iov_or_mp;
	offset_t offset;

	if (length + ctx->gcm_remainder_len < block_size) {
		/* accumulate bytes here and return */
		bcopy(datap,
		    (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len,
		    length);
		ctx->gcm_remainder_len += length;
		ctx->gcm_copy_to = datap;
		return (CRYPTO_SUCCESS);
	}

	if (out != NULL)
		crypto_init_ptrs(out, &iov_or_mp, &offset);

	do {
		/* Unprocessed data from last call. */
		if (ctx->gcm_remainder_len > 0) {
			need = block_size - ctx->gcm_remainder_len;

			if (need > remainder)
				return (CRYPTO_DATA_LEN_RANGE);

			bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
			    [ctx->gcm_remainder_len], need);

			blockp = (uint8_t *)ctx->gcm_remainder;
		} else {
			blockp = datap;
		}

		if ((ctx->gcm_flags & GMAC_MODE) != 0) {
			/* add AAD to the hash */
			ctx->gcm_len_a_len_c[0] +=
			    CRYPTO_BYTES2BITS(block_size);
			GHASH(ctx, blockp, ctx->gcm_ghash);
		} else {
			gcm_encrypt_block(ctx, datap, out, block_size, blockp,
			    &iov_or_mp, &offset, encrypt_block, copy_block,
			    xor_block);
			/* add ciphertext to the hash */
			ctx->gcm_processed_data_len += block_size;
			GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
		}

		/* Update pointer to next block of data to be processed. */
		if (ctx->gcm_remainder_len != 0) {
			datap += need;
			ctx->gcm_remainder_len = 0;
		} else {
			datap += block_size;
		}

		remainder = (size_t)&data[length] - (size_t)datap;

		/* Incomplete last block. */
		if (remainder > 0 && remainder < block_size) {
			bcopy(datap, ctx->gcm_remainder, remainder);
			ctx->gcm_remainder_len = remainder;
			ctx->gcm_copy_to = datap;
			goto out;
		}
		ctx->gcm_copy_to = NULL;

	} while (remainder > 0);

out:
	return (CRYPTO_SUCCESS);
}

/* ARGSUSED */
int
gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
	uint8_t *ghash, *macp;
	int i, rv;

	if (out->cd_length <
	    (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
		return (CRYPTO_DATA_LEN_RANGE);
	}

	ghash = (uint8_t *)ctx->gcm_ghash;

	if (ctx->gcm_remainder_len > 0) {
		uint64_t counter;
		uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;

		/*
		 * Here is where we deal with data that is not a
		 * multiple of the block size.
		 */

		/*
		 * Increment counter.
		 */
		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
		counter = htonll(counter + 1);
		counter &= counter_mask;
		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;

		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
		    (uint8_t *)ctx->gcm_tmp);

		macp = (uint8_t *)ctx->gcm_remainder;
		bzero(macp + ctx->gcm_remainder_len,
		    block_size - ctx->gcm_remainder_len);

		/* XOR with counter block */
		for (i = 0; i < ctx->gcm_remainder_len; i++) {
			macp[i] ^= tmpp[i];
		}

		/* add ciphertext to the hash */
		GHASH(ctx, macp, ghash);

		ctx->gcm_processed_data_len += ctx->gcm_remainder_len;
	}

	/*
	 * The gcm_len_a_len_c values are in host order until final,
	 * where we convert them to network order before GHASH
	 */
	ctx->gcm_len_a_len_c[0] = htonll(ctx->gcm_len_a_len_c[0]);
	ctx->gcm_len_a_len_c[1] =
	    htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
	    (uint8_t *)ctx->gcm_J0);
	xor_block((uint8_t *)ctx->gcm_J0, ghash);

	if (ctx->gcm_remainder_len > 0) {
		rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
		if (rv != CRYPTO_SUCCESS)
			return (rv);
	}
	out->cd_offset += ctx->gcm_remainder_len;
	ctx->gcm_remainder_len = 0;
	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
	if (rv != CRYPTO_SUCCESS)
		return (rv);
	out->cd_offset += ctx->gcm_tag_len;

	return (CRYPTO_SUCCESS);
}

/*
 * This is used in the AES encrypt operations when we're using them
 * for MAC computations. In these cases encrypted data is discarded
 * and we keep only the final data block (used as the MAC).
 */
int
gmac_mode_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *ghash;
	int rv;

	/* Unlike encrypt_final, this has no output but the tag. */
	if (out->cd_length < ctx->gcm_tag_len)
		return (CRYPTO_DATA_LEN_RANGE);

	ghash = (uint8_t *)ctx->gcm_ghash;

	if (ctx->gcm_remainder_len > 0) {
		uint8_t *macp;

		/*
		 * Here is where we deal with data that is not a
		 * multiple of the block size.
		 *
		 * Not encrypting, so no counter, gcm_cb[].
		 */

		macp = (uint8_t *)ctx->gcm_remainder;
		bzero(macp + ctx->gcm_remainder_len,
		    block_size - ctx->gcm_remainder_len);

		ctx->gcm_len_a_len_c[0] +=
		    CRYPTO_BYTES2BITS(ctx->gcm_remainder_len);
		ctx->gcm_remainder_len = 0;
		/* add AAD to the hash */
		GHASH(ctx, macp, ghash);
	}

	/*
	 * We've stored the total auth data in bits here, but before we
	 * add it to the hash, we need to convert to network order.
	 * GMAC keeps gcm_len_a_len_c[1] = 0.
	 */
	ctx->gcm_len_a_len_c[0] = htonll(ctx->gcm_len_a_len_c[0]);
	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
	    (uint8_t *)ctx->gcm_J0);
	xor_block((uint8_t *)ctx->gcm_J0, ghash);

	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
	if (rv != CRYPTO_SUCCESS)
		return (rv);
	out->cd_offset += ctx->gcm_tag_len;

	return (CRYPTO_SUCCESS);
}

/*
 * This will only deal with decrypting the last block of the input that
 * might not be a multiple of block length.
 */
static void
gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *datap, *outp, *counterp;
	uint64_t counter;
	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
	int i;

	/*
	 * Increment counter.
	 * Counter bits are confined to the bottom 32 bits
	 */
	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
	counter = htonll(counter + 1);
	counter &= counter_mask;
	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;

	datap = (uint8_t *)ctx->gcm_remainder;
	outp = &((ctx->gcm_pt_buf)[index]);
	counterp = (uint8_t *)ctx->gcm_tmp;

	/* authentication tag */
	bzero((uint8_t *)ctx->gcm_tmp, block_size);
	bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);

	/* add ciphertext to the hash */
	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);

	/* decrypt remaining ciphertext */
	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);

	/* XOR with counter block */
	for (i = 0; i < ctx->gcm_remainder_len; i++) {
		outp[i] = datap[i] ^ counterp[i];
	}
}

/*
 * See notes above gcm_mode_encrypt_contiguous_blocks for GMAC
 * cases (serving C_Verify here) -- same applies here.
 */
int
gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
    crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t new_len;
	uint8_t *new;

	if ((ctx->gcm_flags & GMAC_MODE) != 0 &&
	    ctx->gcm_remainder_len != 0) {
		/*
		 * For GMAC we need to hash the AAD as we go because
		 * we don't save the data for _final like GCM does.
		 */
		uint8_t *macp, *ghash;

		macp = (uint8_t *)ctx->gcm_remainder;
		ghash = (uint8_t *)ctx->gcm_ghash;

		bzero(macp + ctx->gcm_remainder_len,
		    block_size - ctx->gcm_remainder_len);

		/* remainder AAD len in bits */
		ctx->gcm_len_a_len_c[0] +=
		    CRYPTO_BYTES2BITS(ctx->gcm_remainder_len);
		/* add AAD to the hash */
		GHASH(ctx, macp, ghash);
	}

	/*
	 * Copy contiguous ciphertext input blocks to plaintext buffer.
	 * Ciphertext will be decrypted in the final.
	 */
	if (length > 0) {
		new_len = ctx->gcm_pt_buf_len + length;
#ifdef _KERNEL
		new = kmem_alloc(new_len, ctx->gcm_kmflag);
		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
		kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
#else
		new = malloc(new_len);
		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
		free(ctx->gcm_pt_buf);
#endif
		if (new == NULL)
			return (CRYPTO_HOST_MEMORY);

		ctx->gcm_pt_buf = new;
		ctx->gcm_pt_buf_len = new_len;
		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
		    length);
		ctx->gcm_processed_data_len += length;
	}

	ctx->gcm_remainder_len = 0;
	return (CRYPTO_SUCCESS);
}

int
gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	size_t pt_len;
	size_t remainder;
	uint8_t *ghash;
	uint8_t *blockp;
	uint8_t *cbp;
	uint64_t counter;
	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
	int processed = 0, rv;

	ASSERT3U(ctx->gcm_processed_data_len, ==, ctx->gcm_pt_buf_len);

	pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
	ghash = (uint8_t *)ctx->gcm_ghash;
	blockp = ctx->gcm_pt_buf;
	remainder = pt_len;

	if ((ctx->gcm_flags & GMAC_MODE) != 0) {
		ASSERT3U(remainder, ==, 0);
	}

	while (remainder > 0) {
		/* Incomplete last block */
		if (remainder < block_size) {
			bcopy(blockp, ctx->gcm_remainder, remainder);
			ctx->gcm_remainder_len = remainder;
			/*
			 * not expecting anymore ciphertext, just
			 * compute plaintext for the remaining input
			 */
			gcm_decrypt_incomplete_block(ctx, block_size,
			    processed, encrypt_block, xor_block);
			ctx->gcm_remainder_len = 0;
			goto out;
		}
		/* add ciphertext to the hash */
		GHASH(ctx, blockp, ghash);

		/*
		 * Increment counter.
		 * Counter bits are confined to the bottom 32 bits
		 */
		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
		counter = htonll(counter + 1);
		counter &= counter_mask;
		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;

		cbp = (uint8_t *)ctx->gcm_tmp;
		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);

		/* XOR with ciphertext */
		xor_block(cbp, blockp);

		processed += block_size;
		blockp += block_size;
		remainder -= block_size;
	}

out:
	/*
	 * We've stored the total auth data in bits here, but before we
	 * add it to the hash, we need to change byte order.
	 */
	ctx->gcm_len_a_len_c[0] = htonll(ctx->gcm_len_a_len_c[0]);
	ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
	    (uint8_t *)ctx->gcm_J0);
	xor_block((uint8_t *)ctx->gcm_J0, ghash);

	/* compare the input authentication tag with what we calculated */
	if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
		/* They don't match */
		return (CRYPTO_INVALID_MAC);
	} else {
		rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
		if (rv != CRYPTO_SUCCESS)
			return (rv);
		out->cd_offset += pt_len;
	}
	return (CRYPTO_SUCCESS);
}

static int
gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param)
{
	size_t tag_len;

	/*
	 * Check the length of the authentication tag (in bits).
	 */
	tag_len = gcm_param->ulTagBits;
	switch (tag_len) {
	case 32:
	case 64:
	case 96:
	case 104:
	case 112:
	case 120:
	case 128:
		break;
	default:
		return (CRYPTO_MECHANISM_PARAM_INVALID);
	}

	if (gcm_param->ulIvLen == 0)
		return (CRYPTO_MECHANISM_PARAM_INVALID);

	return (CRYPTO_SUCCESS);
}

static void
gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
    gcm_ctx_t *ctx, size_t block_size,
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *cb;
	ulong_t remainder = iv_len;
	ulong_t processed = 0;
	uint8_t *datap, *ghash;
	uint64_t len_a_len_c[2];

	ghash = (uint8_t *)ctx->gcm_ghash;
	cb = (uint8_t *)ctx->gcm_cb;
	if (iv_len == 12) {
		bcopy(iv, cb, 12);
		cb[12] = 0;
		cb[13] = 0;
		cb[14] = 0;
		cb[15] = 1;
		/* J0 will be used again in the final */
		copy_block(cb, (uint8_t *)ctx->gcm_J0);
	} else {
		/* GHASH the IV */
		do {
			if (remainder < block_size) {
				bzero(cb, block_size);
				bcopy(&(iv[processed]), cb, remainder);
				datap = (uint8_t *)cb;
				remainder = 0;
			} else {
				datap = (uint8_t *)(&(iv[processed]));
				processed += block_size;
				remainder -= block_size;
			}
			GHASH(ctx, datap, ghash);
		} while (remainder > 0);

		len_a_len_c[0] = 0;
		len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len));
		GHASH(ctx, len_a_len_c, ctx->gcm_J0);

		/* J0 will be used again in the final */
		copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb);
	}
}

/*
 * The following function is called at encrypt or decrypt init time
 * for AES GCM mode.
 */
int
gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
    unsigned char *auth_data, size_t auth_data_len, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	uint8_t *ghash, *datap, *authp;
	size_t remainder, processed;

	/* encrypt zero block to get subkey H */
	bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
	    (uint8_t *)ctx->gcm_H);

	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
	    copy_block, xor_block);

	authp = (uint8_t *)ctx->gcm_tmp;
	ghash = (uint8_t *)ctx->gcm_ghash;
	bzero(authp, block_size);
	bzero(ghash, block_size);

	processed = 0;
	remainder = auth_data_len;
	do {
		if (remainder < block_size) {
			if ((ctx->gcm_flags & GMAC_MODE) != 0) {
				/*
				 * GMAC does not encrypt or decrypt, and
				 * therefore doesn't keep any out buffer,
				 * so gcm_remainder holds any remainder
				 * that GMAC needs to handle.
				 */
				bcopy(&(auth_data[processed]),
				    ctx->gcm_remainder, remainder);
				ctx->gcm_remainder_len = remainder;
				break;
			}
			/*
			 * There's not a block full of data, pad rest of
			 * buffer with zero
			 */
			bzero(authp, block_size);
			bcopy(&(auth_data[processed]), authp, remainder);
			datap = (uint8_t *)authp;
			remainder = 0;
		} else {
			datap = (uint8_t *)(&(auth_data[processed]));
			processed += block_size;
			remainder -= block_size;
		}

		/* add auth data to the hash */
		GHASH(ctx, datap, ghash);

	} while (remainder > 0);

	if ((ctx->gcm_flags & GMAC_MODE) != 0) {
		ctx->gcm_len_a_len_c[0] =
		    CRYPTO_BYTES2BITS(auth_data_len - remainder);
	}

	return (CRYPTO_SUCCESS);
}

int
gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	int rv;
	CK_AES_GCM_PARAMS *gcm_param;

	if (param != NULL) {
		gcm_param = (CK_AES_GCM_PARAMS *)(void *)param;

		if ((rv = gcm_validate_args(gcm_param)) != 0) {
			return (rv);
		}

		gcm_ctx->gcm_tag_len = gcm_param->ulTagBits;
		gcm_ctx->gcm_tag_len >>= 3;
		gcm_ctx->gcm_processed_data_len = 0;

		/* these values are in bits */
		gcm_ctx->gcm_len_a_len_c[0] =
		    CRYPTO_BYTES2BITS(gcm_param->ulAADLen);

		rv = CRYPTO_SUCCESS;
		gcm_ctx->gcm_flags |= GCM_MODE;
	} else {
		rv = CRYPTO_MECHANISM_PARAM_INVALID;
		goto out;
	}

	if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen,
	    gcm_param->pAAD, gcm_param->ulAADLen, block_size,
	    encrypt_block, copy_block, xor_block) != 0) {
		rv = CRYPTO_MECHANISM_PARAM_INVALID;
	}
out:
	return (rv);
}

int
gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *))
{
	int rv;
	CK_AES_GMAC_PARAMS *gmac_param;

	if (param == NULL)
		return (CRYPTO_MECHANISM_PARAM_INVALID);

	gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param;

	gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS);
	gcm_ctx->gcm_processed_data_len = 0;

	/* these values are in bits */
	gcm_ctx->gcm_len_a_len_c[0] = 0;
	gcm_ctx->gcm_len_a_len_c[1] = 0;

	rv = CRYPTO_SUCCESS;
	gcm_ctx->gcm_flags |= GMAC_MODE;

	if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN,
	    gmac_param->pAAD, gmac_param->ulAADLen, block_size,
	    encrypt_block, copy_block, xor_block) != 0) {
		rv = CRYPTO_MECHANISM_PARAM_INVALID;
	}

	return (rv);
}

void *
gcm_alloc_ctx(int kmflag)
{
	gcm_ctx_t *gcm_ctx;

	/* Free in crypto_free_mode_ctx() */
#ifdef _KERNEL
	if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
#else
	if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL)
#endif
		return (NULL);

	gcm_ctx->gcm_flags = GCM_MODE;
	return (gcm_ctx);
}

void *
gmac_alloc_ctx(int kmflag)
{
	gcm_ctx_t *gcm_ctx;

	/* Free in crypto_free_mode_ctx() */
#ifdef _KERNEL
	if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
#else
	if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL)
#endif
		return (NULL);

	gcm_ctx->gcm_flags = GMAC_MODE;
	return (gcm_ctx);
}

void
gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag)
{
	ctx->gcm_kmflag = kmflag;
}


#ifdef __amd64
/*
 * Return 1 if executing on Intel with PCLMULQDQ instructions,
 * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64).
 * Cache the result, as the CPU can't change.
 *
 * Note: the userland version uses getisax().  The kernel version uses
 * is_x86_featureset().
 */
static int
intel_pclmulqdq_instruction_present(void)
{
	static int	cached_result = -1;

	if (cached_result == -1) { /* first time */
#ifdef _KERNEL
		cached_result =
		    is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ);
#else
		uint_t		ui = 0;

		(void) getisax(&ui, 1);
		cached_result = (ui & AV_386_PCLMULQDQ) != 0;
#endif	/* _KERNEL */
	}

	return (cached_result);
}
#endif	/* __amd64 */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 */

#ifndef _KERNEL
#include <stdlib.h>
#include <assert.h>
#include <strings.h>
#endif

#include <sys/strsun.h>
#include <sys/types.h>
#include <modes/modes.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>

/*
 * Initialize by setting iov_or_mp to point to the current iovec or mp,
 * and by setting current_offset to an offset within the current iovec or mp.
 */
void
crypto_init_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset)
{
	offset_t offset;

	switch (out->cd_format) {
	case CRYPTO_DATA_RAW:
		*current_offset = out->cd_offset;
		break;

	case CRYPTO_DATA_UIO: {
		uio_t *uiop = out->cd_uio;
		uintptr_t vec_idx;

		offset = out->cd_offset;
		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
		    offset >= uiop->uio_iov[vec_idx].iov_len;
		    offset -= uiop->uio_iov[vec_idx++].iov_len)
			;

		*current_offset = offset;
		*iov_or_mp = (void *)vec_idx;
		break;
	}

	case CRYPTO_DATA_MBLK: {
		mblk_t *mp;

		offset = out->cd_offset;
		for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp);
		    offset -= MBLKL(mp), mp = mp->b_cont)
			;

		*current_offset = offset;
		*iov_or_mp = mp;
		break;

	}
	} /* end switch */
}

/*
 * Get pointers for where in the output to copy a block of encrypted or
 * decrypted data.  The iov_or_mp argument stores a pointer to the current
 * iovec or mp, and offset stores an offset into the current iovec or mp.
 */
void
crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
    uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
    size_t amt)
{
	offset_t offset;

	switch (out->cd_format) {
	case CRYPTO_DATA_RAW: {
		iovec_t *iov;

		offset = *current_offset;
		iov = &out->cd_raw;
		if ((offset + amt) <= iov->iov_len) {
			/* one block fits */
			*out_data_1 = (uint8_t *)iov->iov_base + offset;
			*out_data_1_len = amt;
			*out_data_2 = NULL;
			*current_offset = offset + amt;
		}
		break;
	}

	case CRYPTO_DATA_UIO: {
		uio_t *uio = out->cd_uio;
		iovec_t *iov;
		offset_t offset;
		uintptr_t vec_idx;
		uint8_t *p;

		offset = *current_offset;
		vec_idx = (uintptr_t)(*iov_or_mp);
		iov = &uio->uio_iov[vec_idx];
		p = (uint8_t *)iov->iov_base + offset;
		*out_data_1 = p;

		if (offset + amt <= iov->iov_len) {
			/* can fit one block into this iov */
			*out_data_1_len = amt;
			*out_data_2 = NULL;
			*current_offset = offset + amt;
		} else {
			/* one block spans two iovecs */
			*out_data_1_len = iov->iov_len - offset;
			if (vec_idx == uio->uio_iovcnt)
				return;
			vec_idx++;
			iov = &uio->uio_iov[vec_idx];
			*out_data_2 = (uint8_t *)iov->iov_base;
			*current_offset = amt - *out_data_1_len;
		}
		*iov_or_mp = (void *)vec_idx;
		break;
	}

	case CRYPTO_DATA_MBLK: {
		mblk_t *mp;
		uint8_t *p;

		offset = *current_offset;
		mp = (mblk_t *)*iov_or_mp;
		p = mp->b_rptr + offset;
		*out_data_1 = p;
		if ((p + amt) <= mp->b_wptr) {
			/* can fit one block into this mblk */
			*out_data_1_len = amt;
			*out_data_2 = NULL;
			*current_offset = offset + amt;
		} else {
			/* one block spans two mblks */
			*out_data_1_len = _PTRDIFF(mp->b_wptr, p);
			if ((mp = mp->b_cont) == NULL)
				return;
			*out_data_2 = mp->b_rptr;
			*current_offset = (amt - *out_data_1_len);
		}
		*iov_or_mp = mp;
		break;
	}
	} /* end switch */
}

void
crypto_free_mode_ctx(void *ctx)
{
	common_ctx_t *common_ctx = (common_ctx_t *)ctx;

	switch (common_ctx->cc_flags & (ECB_MODE|CBC_MODE|CMAC_MODE|CTR_MODE|
	    CCM_MODE|GCM_MODE|GMAC_MODE)) {
	case ECB_MODE:
#ifdef _KERNEL
		kmem_free(common_ctx, sizeof (ecb_ctx_t));
#else
		free(common_ctx);
#endif
		break;

	case CBC_MODE:
	case CMAC_MODE:
#ifdef _KERNEL
		kmem_free(common_ctx, sizeof (cbc_ctx_t));
#else
		free(common_ctx);
#endif
		break;

	case CTR_MODE:
#ifdef _KERNEL
		kmem_free(common_ctx, sizeof (ctr_ctx_t));
#else
		free(common_ctx);
#endif
		break;

	case CCM_MODE:
#ifdef _KERNEL
		if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL)
			kmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf,
			    ((ccm_ctx_t *)ctx)->ccm_data_len);

		kmem_free(ctx, sizeof (ccm_ctx_t));
#else
		if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL)
			free(((ccm_ctx_t *)ctx)->ccm_pt_buf);
		free(ctx);
#endif
		break;

	case GCM_MODE:
	case GMAC_MODE:
#ifdef _KERNEL
		if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
			kmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf,
			    ((gcm_ctx_t *)ctx)->gcm_pt_buf_len);

		kmem_free(ctx, sizeof (gcm_ctx_t));
#else
		if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
			free(((gcm_ctx_t *)ctx)->gcm_pt_buf);
		free(ctx);
#endif
	}
}

/*
 * Utility routine to apply the command, 'cmd', to the
 * data in the uio structure.
 */
int
crypto_uio_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd,
    void *digest_ctx, void (*update)())
{
	uio_t *uiop = data->cd_uio;
	off_t offset = data->cd_offset;
	size_t length = len;
	uint_t vec_idx;
	size_t cur_len;
	uchar_t *datap;

#ifdef _KERNEL
	ASSERT3U(data->cd_format, ==, CRYPTO_DATA_UIO);
#else
	assert(data->cd_format == CRYPTO_DATA_UIO);
#endif
	if (uiop->uio_segflg != UIO_SYSSPACE) {
		return (CRYPTO_ARGUMENTS_BAD);
	}

	/*
	 * Jump to the first iovec containing data to be
	 * processed.
	 */
	for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
	    offset >= uiop->uio_iov[vec_idx].iov_len;
	    offset -= uiop->uio_iov[vec_idx++].iov_len)
		;

	if (vec_idx == uiop->uio_iovcnt && length > 0) {
		/*
		 * The caller specified an offset that is larger than
		 * the total size of the buffers it provided.
		 */
		return (CRYPTO_DATA_LEN_RANGE);
	}

	while (vec_idx < uiop->uio_iovcnt && length > 0) {
		cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
		    offset, length);

		datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
		    offset);
		switch (cmd) {
		case COPY_FROM_DATA:
			bcopy(datap, buf, cur_len);
			buf += cur_len;
			break;
		case COPY_TO_DATA:
			bcopy(buf, datap, cur_len);
			buf += cur_len;
			break;
		case COMPARE_TO_DATA:
			if (bcmp(datap, buf, cur_len))
				return (CRYPTO_SIGNATURE_INVALID);
			buf += cur_len;
			break;
		case MD5_DIGEST_DATA:
		case SHA1_DIGEST_DATA:
		case SHA2_DIGEST_DATA:
		case GHASH_DATA:
			update(digest_ctx, datap, cur_len);
			break;
		}

		length -= cur_len;
		vec_idx++;
		offset = 0;
	}

	if (vec_idx == uiop->uio_iovcnt && length > 0) {
		/*
		 * The end of the specified iovec's was reached but
		 * the length requested could not be processed.
		 */
		switch (cmd) {
		case COPY_TO_DATA:
			data->cd_length = len;
			return (CRYPTO_BUFFER_TOO_SMALL);
		default:
			return (CRYPTO_DATA_LEN_RANGE);
		}
	}

	return (CRYPTO_SUCCESS);
}

/*
 * Utility routine to apply the command, 'cmd', to the
 * data in the mblk structure.
 */
int
crypto_mblk_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd,
    void *digest_ctx, void (*update)())
{
	off_t offset = data->cd_offset;
	size_t length = len;
	mblk_t *mp;
	size_t cur_len;
	uchar_t *datap;

#ifdef _KERNEL
	ASSERT3U(data->cd_format, ==, CRYPTO_DATA_MBLK);
#else
	assert(data->cd_format == CRYPTO_DATA_MBLK);
#endif
	/*
	 * Jump to the first mblk_t containing data to be processed.
	 */
	for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
	    offset -= MBLKL(mp), mp = mp->b_cont)
		;
	if (mp == NULL) {
		/*
		 * The caller specified an offset that is larger
		 * than the total size of the buffers it provided.
		 */
		return (CRYPTO_DATA_LEN_RANGE);
	}

	/*
	 * Now do the processing on the mblk chain.
	 */
	while (mp != NULL && length > 0) {
		cur_len = MIN(MBLKL(mp) - offset, length);

		datap = (uchar_t *)(mp->b_rptr + offset);
		switch (cmd) {
		case COPY_FROM_DATA:
			bcopy(datap, buf, cur_len);
			buf += cur_len;
			break;
		case COPY_TO_DATA:
			bcopy(buf, datap, cur_len);
			buf += cur_len;
			break;
		case COMPARE_TO_DATA:
			if (bcmp(datap, buf, cur_len))
				return (CRYPTO_SIGNATURE_INVALID);
			buf += cur_len;
			break;
		case MD5_DIGEST_DATA:
		case SHA1_DIGEST_DATA:
		case SHA2_DIGEST_DATA:
		case GHASH_DATA:
			update(digest_ctx, datap, cur_len);
			break;
		}

		length -= cur_len;
		offset = 0;
		mp = mp->b_cont;
	}

	if (mp == NULL && length > 0) {
		/*
		 * The end of the mblk was reached but the length
		 * requested could not be processed.
		 */
		switch (cmd) {
		case COPY_TO_DATA:
			data->cd_length = len;
			return (CRYPTO_BUFFER_TOO_SMALL);
		default:
			return (CRYPTO_DATA_LEN_RANGE);
		}
	}

	return (CRYPTO_SUCCESS);
}

/*
 * Utility routine to copy a buffer to a crypto_data structure.
 */
int
crypto_put_output_data(uchar_t *buf, crypto_data_t *output, int len)
{
	switch (output->cd_format) {
	case CRYPTO_DATA_RAW:
		if (MAXOFF_T - output->cd_offset < (off_t)len) {
			return (CRYPTO_ARGUMENTS_BAD);
		}
		if (output->cd_raw.iov_len < len + output->cd_offset) {
			output->cd_length = len;
			return (CRYPTO_BUFFER_TOO_SMALL);
		}
		bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
		    output->cd_offset), len);
		break;

	case CRYPTO_DATA_UIO:
		return (crypto_uio_data(output, buf, len,
		    COPY_TO_DATA, NULL, NULL));

	case CRYPTO_DATA_MBLK:
		return (crypto_mblk_data(output, buf, len,
		    COPY_TO_DATA, NULL, NULL));

	default:
		return (CRYPTO_ARGUMENTS_BAD);
	}

	return (CRYPTO_SUCCESS);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2023 RackTop Systems, Inc.
 */

#ifndef	_COMMON_CRYPTO_MODES_H
#define	_COMMON_CRYPTO_MODES_H

#ifdef	__cplusplus
extern "C" {
#endif

#include <sys/strsun.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/rwlock.h>
#include <sys/kmem.h>
#include <sys/crypto/common.h>
#include <sys/crypto/impl.h>

#define	ECB_MODE			0x00000002
#define	CBC_MODE			0x00000004
#define	CTR_MODE			0x00000008
#define	CCM_MODE			0x00000010
#define	GCM_MODE			0x00000020
#define	GMAC_MODE			0x00000040
#define	CMAC_MODE			0x00000080

/* Private flag for pkcs11_softtoken */
#define	P11_DECRYPTED			0x80000000

/*
 * cc_keysched:		Pointer to key schedule.
 *
 * cc_keysched_len:	Length of the key schedule.
 *
 * cc_remainder:	This is for residual data, i.e. data that can't
 *			be processed because there are too few bytes.
 *			Must wait until more data arrives.
 *
 * cc_remainder_len:	Number of bytes in cc_remainder.
 *
 * cc_iv:		Scratch buffer that sometimes contains the IV.
 *
 * cc_lastp:		Pointer to previous block of ciphertext.
 *
 * cc_copy_to:		Pointer to where encrypted residual data needs
 *			to be copied.
 *
 * cc_flags:		PROVIDER_OWNS_KEY_SCHEDULE
 *			When a context is freed, it is necessary
 *			to know whether the key schedule was allocated
 *			by the caller, or internally, e.g. an init routine.
 *			If allocated by the latter, then it needs to be freed.
 *
 *			ECB_MODE, CBC_MODE, CTR_MODE, or CCM_MODE
 */
struct common_ctx {
	void *cc_keysched;
	size_t cc_keysched_len;
	uint64_t cc_iv[2];
	uint64_t cc_remainder[2];
	size_t cc_remainder_len;
	uint8_t *cc_lastp;
	uint8_t *cc_copy_to;
	uint32_t cc_flags;
};

typedef struct common_ctx common_ctx_t;

typedef struct ecb_ctx {
	struct common_ctx ecb_common;
	uint64_t ecb_lastblock[2];
} ecb_ctx_t;

#define	ecb_keysched		ecb_common.cc_keysched
#define	ecb_keysched_len	ecb_common.cc_keysched_len
#define	ecb_iv			ecb_common.cc_iv
#define	ecb_remainder		ecb_common.cc_remainder
#define	ecb_remainder_len	ecb_common.cc_remainder_len
#define	ecb_lastp		ecb_common.cc_lastp
#define	ecb_copy_to		ecb_common.cc_copy_to
#define	ecb_flags		ecb_common.cc_flags

/*
 * max_remain			max bytes in cbc_remainder
 */
typedef struct cbc_ctx {
	struct common_ctx cbc_common;
	uint64_t cbc_lastblock[2];
	size_t max_remain;
} cbc_ctx_t;

#define	cbc_keysched		cbc_common.cc_keysched
#define	cbc_keysched_len	cbc_common.cc_keysched_len
#define	cbc_iv			cbc_common.cc_iv
#define	cbc_remainder		cbc_common.cc_remainder
#define	cbc_remainder_len	cbc_common.cc_remainder_len
#define	cbc_lastp		cbc_common.cc_lastp
#define	cbc_copy_to		cbc_common.cc_copy_to
#define	cbc_flags		cbc_common.cc_flags

/*
 * ctr_lower_mask		Bit-mask for lower 8 bytes of counter block.
 * ctr_upper_mask		Bit-mask for upper 8 bytes of counter block.
 */
typedef struct ctr_ctx {
	struct common_ctx ctr_common;
	uint64_t ctr_lower_mask;
	uint64_t ctr_upper_mask;
	size_t ctr_offset;
	uint32_t ctr_keystream[4];
} ctr_ctx_t;

/*
 * ctr_cb			Counter block.
 */
#define	ctr_keysched		ctr_common.cc_keysched
#define	ctr_keysched_len	ctr_common.cc_keysched_len
#define	ctr_cb			ctr_common.cc_iv
#define	ctr_remainder		ctr_common.cc_remainder
#define	ctr_remainder_len	ctr_common.cc_remainder_len
#define	ctr_lastp		ctr_common.cc_lastp
#define	ctr_copy_to		ctr_common.cc_copy_to
#define	ctr_flags		ctr_common.cc_flags

/*
 *
 * ccm_mac_len:		Stores length of the MAC in CCM mode.
 * ccm_mac_buf:		Stores the intermediate value for MAC in CCM encrypt.
 *			In CCM decrypt, stores the input MAC value.
 * ccm_data_len:	Length of the plaintext for CCM mode encrypt, or
 *			length of the ciphertext for CCM mode decrypt.
 * ccm_processed_data_len:
 *			Length of processed plaintext in CCM mode encrypt,
 *			or length of processed ciphertext for CCM mode decrypt.
 * ccm_processed_mac_len:
 *			Length of MAC data accumulated in CCM mode decrypt.
 *
 * ccm_pt_buf:		Only used in CCM mode decrypt.  It stores the
 *			decrypted plaintext to be returned when
 *			MAC verification succeeds in decrypt_final.
 *			Memory for this should be allocated in the AES module.
 *
 */
typedef struct ccm_ctx {
	struct common_ctx ccm_common;
	uint32_t ccm_tmp[4];
	size_t ccm_mac_len;
	uint64_t ccm_mac_buf[2];
	size_t ccm_data_len;
	size_t ccm_processed_data_len;
	size_t ccm_processed_mac_len;
	uint8_t *ccm_pt_buf;
	uint64_t ccm_mac_input_buf[2];
	uint64_t ccm_counter_mask;
} ccm_ctx_t;

#define	ccm_keysched		ccm_common.cc_keysched
#define	ccm_keysched_len	ccm_common.cc_keysched_len
#define	ccm_cb			ccm_common.cc_iv
#define	ccm_remainder		ccm_common.cc_remainder
#define	ccm_remainder_len	ccm_common.cc_remainder_len
#define	ccm_lastp		ccm_common.cc_lastp
#define	ccm_copy_to		ccm_common.cc_copy_to
#define	ccm_flags		ccm_common.cc_flags

/*
 * gcm_tag_len:		Length of authentication tag.
 *
 * gcm_ghash:		Stores output from the GHASH function.
 *
 * gcm_processed_data_len:
 *			Length of processed plaintext (encrypt) or
 *			length of processed ciphertext (decrypt).
 *
 * gcm_pt_buf:		Stores the decrypted plaintext returned by
 *			decrypt_final when the computed authentication
 *			tag matches the	user supplied tag.
 *
 * gcm_pt_buf_len:	Length of the plaintext buffer.
 *
 * gcm_H:		Subkey.
 *
 * gcm_J0:		Pre-counter block generated from the IV.
 *
 * gcm_len_a_len_c:	64-bit representations of the bit lengths of
 *			AAD and ciphertext.
 *
 * gcm_kmflag:		Current value of kmflag. Used only for allocating
 *			the plaintext buffer during decryption.
 */
typedef struct gcm_ctx {
	struct common_ctx gcm_common;
	size_t gcm_tag_len;
	size_t gcm_processed_data_len;
	size_t gcm_pt_buf_len;
	uint32_t gcm_tmp[4];
	uint64_t gcm_ghash[2];
	uint64_t gcm_H[2];
	uint64_t gcm_J0[2];
	uint64_t gcm_len_a_len_c[2];
	uint8_t *gcm_pt_buf;
	int gcm_kmflag;
} gcm_ctx_t;

#define	gcm_keysched		gcm_common.cc_keysched
#define	gcm_keysched_len	gcm_common.cc_keysched_len
#define	gcm_cb			gcm_common.cc_iv
#define	gcm_remainder		gcm_common.cc_remainder
#define	gcm_remainder_len	gcm_common.cc_remainder_len
#define	gcm_lastp		gcm_common.cc_lastp
#define	gcm_copy_to		gcm_common.cc_copy_to
#define	gcm_flags		gcm_common.cc_flags

#define	AES_GMAC_IV_LEN		12
#define	AES_GMAC_TAG_BITS	128

typedef struct aes_ctx {
	union {
		ecb_ctx_t acu_ecb;
		cbc_ctx_t acu_cbc;
		ctr_ctx_t acu_ctr;
		ccm_ctx_t acu_ccm;
		gcm_ctx_t acu_gcm;
	} acu;
} aes_ctx_t;

#define	ac_flags		acu.acu_ecb.ecb_common.cc_flags
#define	ac_remainder_len	acu.acu_ecb.ecb_common.cc_remainder_len
#define	ac_remainder		acu.acu_ecb.ecb_common.cc_remainder
#define	ac_keysched		acu.acu_ecb.ecb_common.cc_keysched
#define	ac_keysched_len		acu.acu_ecb.ecb_common.cc_keysched_len
#define	ac_iv			acu.acu_ecb.ecb_common.cc_iv
#define	ac_lastp		acu.acu_ecb.ecb_common.cc_lastp
#define	ac_pt_buf		acu.acu_ccm.ccm_pt_buf
#define	ac_mac_len		acu.acu_ccm.ccm_mac_len
#define	ac_data_len		acu.acu_ccm.ccm_data_len
#define	ac_processed_mac_len	acu.acu_ccm.ccm_processed_mac_len
#define	ac_processed_data_len	acu.acu_ccm.ccm_processed_data_len
#define	ac_tag_len		acu.acu_gcm.gcm_tag_len

typedef struct blowfish_ctx {
	union {
		ecb_ctx_t bcu_ecb;
		cbc_ctx_t bcu_cbc;
	} bcu;
} blowfish_ctx_t;

#define	bc_flags		bcu.bcu_ecb.ecb_common.cc_flags
#define	bc_remainder_len	bcu.bcu_ecb.ecb_common.cc_remainder_len
#define	bc_keysched		bcu.bcu_ecb.ecb_common.cc_keysched
#define	bc_keysched_len		bcu.bcu_ecb.ecb_common.cc_keysched_len
#define	bc_iv			bcu.bcu_ecb.ecb_common.cc_iv
#define	bc_lastp		bcu.bcu_ecb.ecb_common.cc_lastp

typedef struct des_ctx {
	union {
		ecb_ctx_t dcu_ecb;
		cbc_ctx_t dcu_cbc;
	} dcu;
} des_ctx_t;

#define	dc_flags		dcu.dcu_ecb.ecb_common.cc_flags
#define	dc_remainder_len	dcu.dcu_ecb.ecb_common.cc_remainder_len
#define	dc_keysched		dcu.dcu_ecb.ecb_common.cc_keysched
#define	dc_keysched_len		dcu.dcu_ecb.ecb_common.cc_keysched_len
#define	dc_iv			dcu.dcu_ecb.ecb_common.cc_iv
#define	dc_lastp		dcu.dcu_ecb.ecb_common.cc_lastp

extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t,
    crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *,
    uint8_t *));

extern int cbc_encrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*encrypt)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int cbc_decrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*decrypt)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int ctr_mode_contiguous_blocks(ctr_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*cipher)(const void *, const uint8_t *, uint8_t *));

extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
    crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

int gmac_mode_final(gcm_ctx_t *, crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int cmac_mode_final(cbc_ctx_t *, crypto_data_t *,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int cbc_init_ctx(cbc_ctx_t *, char *, size_t, size_t,
    void (*copy_block)(uint8_t *, uint64_t *));

extern int cmac_init_ctx(cbc_ctx_t *, size_t);

extern int ctr_init_ctx(ctr_ctx_t *, ulong_t, uint8_t *,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *));

extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern int gmac_init_ctx(gcm_ctx_t *, char *, size_t,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
    void (*copy_block)(uint8_t *, uint8_t *),
    void (*xor_block)(uint8_t *, uint8_t *));

extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *,
    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *));

extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *);

extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *);
extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *,
    uint8_t **, size_t *, uint8_t **, size_t);

extern void *ecb_alloc_ctx(int);
extern void *cbc_alloc_ctx(int);
extern void *cmac_alloc_ctx(int);
extern void *ctr_alloc_ctx(int);
extern void *ccm_alloc_ctx(int);
extern void *gcm_alloc_ctx(int);
extern void *gmac_alloc_ctx(int);
extern void crypto_free_mode_ctx(void *);
extern void gcm_set_kmflag(gcm_ctx_t *, int);
extern int crypto_put_output_data(uchar_t *, crypto_data_t *, int);

#ifdef	__cplusplus
}
#endif

#endif	/* _COMMON_CRYPTO_MODES_H */