sha2 - zygaena/hammerhead

root / base / usr / src / common / crypto / sha2
sha2 Plain Text 1852 lines 57.8 KB
Intel SHA Extensions optimized implementation of a SHA-256 update function

BSD LICENSE

Copyright(c) 2015 Intel Corporation.
Copyright (c) 2018, Joyent, Inc.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

	* Redistributions of source code must retain the above copyright
	  notice, this list of conditions and the following disclaimer.
	* Redistributions in binary form must reproduce the above copyright
	  notice, this list of conditions and the following disclaimer in
	  the documentation and/or other materials provided with the
	  distribution.
	* Neither the name of Intel Corporation nor the names of its
	  contributors may be used to endorse or promote products derived
	  from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
PORTIONS OF SHA2 FUNCTIONALITY
/*
 * Intel SHA Extensions optimized implementation of a SHA-256 update function
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * Copyright(c) 2015 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * Contact Information:
 * 	Sean Gulley <sean.m.gulley@intel.com>
 * 	Tim Chen <tim.c.chen@linux.intel.com>
 *
 * BSD LICENSE
 *
 * Copyright(c) 2015 Intel Corporation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 	* Redistributions of source code must retain the above copyright
 * 	  notice, this list of conditions and the following disclaimer.
 * 	* Redistributions in binary form must reproduce the above copyright
 * 	  notice, this list of conditions and the following disclaimer in
 * 	  the documentation and/or other materials provided with the
 * 	  distribution.
 * 	* Neither the name of Intel Corporation nor the names of its
 * 	  contributors may be used to endorse or promote products derived
 * 	  from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 */

/*
 * illumos uses this file under the terms of the BSD license.
 *
 * The following are a series of changes that we have made to this code:
 *
 *  o Changed the include to be sys/asm_linkage.h.
 *  o Use the sys/asm_linkage.h prototypes for assembly functions.
 *  o Renamed the function from sha256_ni_transform to SHA256TransformBlocks to
 *    match the illumos name for the function.
 *  o The illumos SHA256_CTX does not have the digest as the first member of its
 *    context struct. As such, an offset has to be added to the digest argument
 *    to make sure that we get to the actual digest.
 *  o Update the function prototype block comment to reflect that we are
 *    passing the context and not the direct digest.
 */

#include <sys/asm_linkage.h>

#define DIGEST_PTR	%rdi	/* 1st arg */
#define DATA_PTR	%rsi	/* 2nd arg */
#define NUM_BLKS	%rdx	/* 3rd arg */

#define SHA256CONSTANTS	%rax

#define MSG		%xmm0
#define STATE0		%xmm1
#define STATE1		%xmm2
#define MSGTMP0		%xmm3
#define MSGTMP1		%xmm4
#define MSGTMP2		%xmm5
#define MSGTMP3		%xmm6
#define MSGTMP4		%xmm7

#define SHUF_MASK	%xmm8

#define ABEF_SAVE	%xmm9
#define CDGH_SAVE	%xmm10

/*
 * Intel SHA Extensions optimized implementation of a SHA-256 update function
 *
 * The function takes a pointer to the current hash values, a pointer to the
 * input data, and a number of 64 byte blocks to process.  Once all blocks have
 * been processed, the digest pointer is  updated with the resulting hash value.
 * The function only processes complete blocks, there is no functionality to
 * store partial blocks.  All message padding and hash value initialization must
 * be done outside the update function.
 *
 * The indented lines in the loop are instructions related to rounds processing.
 * The non-indented lines are instructions related to the message schedule.
 *
 * void SHA256TransformBlocks(SHA256_CTX *ctx, const void *data,
		uint32_t numBlocks);
 * digest : pointer to digest
 * data: pointer to input data
 * numBlocks: Number of blocks to process
 */

.text
.align 32
ENTRY_NP(SHA256TransformBlocks)

	shl		$6, NUM_BLKS		/* convert to bytes */
	jz		.Ldone_hash
	add		DATA_PTR, NUM_BLKS	/* pointer to end of data */

	/*
	 * load initial hash values
	 * Need to reorder these appropriately
	 * DCBA, HGFE -> ABEF, CDGH
	 *
	 * Offset DIGEST_PTR to account for the algorithm in the context.
	 */
	addq		$8, DIGEST_PTR
	movdqu		0*16(DIGEST_PTR), STATE0
	movdqu		1*16(DIGEST_PTR), STATE1

	pshufd		$0xB1, STATE0,  STATE0		/* CDAB */
	pshufd		$0x1B, STATE1,  STATE1		/* EFGH */
	movdqa		STATE0, MSGTMP4
	palignr		$8, STATE1,  STATE0		/* ABEF */
	pblendw		$0xF0, MSGTMP4, STATE1		/* CDGH */

	movdqa		PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
	lea		K256(%rip), SHA256CONSTANTS

.Lloop0:
	/* Save hash values for addition after rounds */
	movdqa		STATE0, ABEF_SAVE
	movdqa		STATE1, CDGH_SAVE

	/* Rounds 0-3 */
	movdqu		0*16(DATA_PTR), MSG
	pshufb		SHUF_MASK, MSG
	movdqa		MSG, MSGTMP0
		paddd		0*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0

	/* Rounds 4-7 */
	movdqu		1*16(DATA_PTR), MSG
	pshufb		SHUF_MASK, MSG
	movdqa		MSG, MSGTMP1
		paddd		1*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP1, MSGTMP0

	/* Rounds 8-11 */
	movdqu		2*16(DATA_PTR), MSG
	pshufb		SHUF_MASK, MSG
	movdqa		MSG, MSGTMP2
		paddd		2*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP2, MSGTMP1

	/* Rounds 12-15 */
	movdqu		3*16(DATA_PTR), MSG
	pshufb		SHUF_MASK, MSG
	movdqa		MSG, MSGTMP3
		paddd		3*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP3, MSGTMP4
	palignr		$4, MSGTMP2, MSGTMP4
	paddd		MSGTMP4, MSGTMP0
	sha256msg2	MSGTMP3, MSGTMP0
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP3, MSGTMP2

	/* Rounds 16-19 */
	movdqa		MSGTMP0, MSG
		paddd		4*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP0, MSGTMP4
	palignr		$4, MSGTMP3, MSGTMP4
	paddd		MSGTMP4, MSGTMP1
	sha256msg2	MSGTMP0, MSGTMP1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP0, MSGTMP3

	/* Rounds 20-23 */
	movdqa		MSGTMP1, MSG
		paddd		5*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP1, MSGTMP4
	palignr		$4, MSGTMP0, MSGTMP4
	paddd		MSGTMP4, MSGTMP2
	sha256msg2	MSGTMP1, MSGTMP2
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP1, MSGTMP0

	/* Rounds 24-27 */
	movdqa		MSGTMP2, MSG
		paddd		6*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP2, MSGTMP4
	palignr		$4, MSGTMP1, MSGTMP4
	paddd		MSGTMP4, MSGTMP3
	sha256msg2	MSGTMP2, MSGTMP3
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP2, MSGTMP1

	/* Rounds 28-31 */
	movdqa		MSGTMP3, MSG
		paddd		7*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP3, MSGTMP4
	palignr		$4, MSGTMP2, MSGTMP4
	paddd		MSGTMP4, MSGTMP0
	sha256msg2	MSGTMP3, MSGTMP0
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP3, MSGTMP2

	/* Rounds 32-35 */
	movdqa		MSGTMP0, MSG
		paddd		8*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP0, MSGTMP4
	palignr		$4, MSGTMP3, MSGTMP4
	paddd		MSGTMP4, MSGTMP1
	sha256msg2	MSGTMP0, MSGTMP1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP0, MSGTMP3

	/* Rounds 36-39 */
	movdqa		MSGTMP1, MSG
		paddd		9*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP1, MSGTMP4
	palignr		$4, MSGTMP0, MSGTMP4
	paddd		MSGTMP4, MSGTMP2
	sha256msg2	MSGTMP1, MSGTMP2
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP1, MSGTMP0

	/* Rounds 40-43 */
	movdqa		MSGTMP2, MSG
		paddd		10*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP2, MSGTMP4
	palignr		$4, MSGTMP1, MSGTMP4
	paddd		MSGTMP4, MSGTMP3
	sha256msg2	MSGTMP2, MSGTMP3
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP2, MSGTMP1

	/* Rounds 44-47 */
	movdqa		MSGTMP3, MSG
		paddd		11*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP3, MSGTMP4
	palignr		$4, MSGTMP2, MSGTMP4
	paddd		MSGTMP4, MSGTMP0
	sha256msg2	MSGTMP3, MSGTMP0
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP3, MSGTMP2

	/* Rounds 48-51 */
	movdqa		MSGTMP0, MSG
		paddd		12*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP0, MSGTMP4
	palignr		$4, MSGTMP3, MSGTMP4
	paddd		MSGTMP4, MSGTMP1
	sha256msg2	MSGTMP0, MSGTMP1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0
	sha256msg1	MSGTMP0, MSGTMP3

	/* Rounds 52-55 */
	movdqa		MSGTMP1, MSG
		paddd		13*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP1, MSGTMP4
	palignr		$4, MSGTMP0, MSGTMP4
	paddd		MSGTMP4, MSGTMP2
	sha256msg2	MSGTMP1, MSGTMP2
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0

	/* Rounds 56-59 */
	movdqa		MSGTMP2, MSG
		paddd		14*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
	movdqa		MSGTMP2, MSGTMP4
	palignr		$4, MSGTMP1, MSGTMP4
	paddd		MSGTMP4, MSGTMP3
	sha256msg2	MSGTMP2, MSGTMP3
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0

	/* Rounds 60-63 */
	movdqa		MSGTMP3, MSG
		paddd		15*16(SHA256CONSTANTS), MSG
		sha256rnds2	STATE0, STATE1
		pshufd 		$0x0E, MSG, MSG
		sha256rnds2	STATE1, STATE0

	/* Add current hash values with previously saved */
	paddd		ABEF_SAVE, STATE0
	paddd		CDGH_SAVE, STATE1

	/* Increment data pointer and loop if more to process */
	add		$64, DATA_PTR
	cmp		NUM_BLKS, DATA_PTR
	jne		.Lloop0

	/* Write hash values back in the correct order */
	pshufd		$0x1B, STATE0,  STATE0		/* FEBA */
	pshufd		$0xB1, STATE1,  STATE1		/* DCHG */
	movdqa		STATE0, MSGTMP4
	pblendw		$0xF0, STATE1,  STATE0		/* DCBA */
	palignr		$8, MSGTMP4, STATE1		/* HGFE */

	movdqu		STATE0, 0*16(DIGEST_PTR)
	movdqu		STATE1, 1*16(DIGEST_PTR)

.Ldone_hash:

	ret
SET_SIZE(SHA256TransformBlocks)

.section	.rodata.cst256.K256, "aM", @progbits, 256
.align 64
K256:
	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2

.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
.align 16
PSHUFFLE_BYTE_FLIP_MASK:
	.octa 0x0c0d0e0f08090a0b0405060700010203
#!/usr/bin/env perl
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# project. Rights for redistribution and usage in source and binary
# forms are granted according to the OpenSSL license.
# ====================================================================
#
# sha256/512_block procedure for x86_64.
#
# 40% improvement over compiler-generated code on Opteron. On EM64T
# sha256 was observed to run >80% faster and sha512 - >40%. No magical
# tricks, just straight implementation... I really wonder why gcc
# [being armed with inline assembler] fails to generate as fast code.
# The only thing which is cool about this module is that it's very
# same instruction sequence used for both SHA-256 and SHA-512. In
# former case the instructions operate on 32-bit operands, while in
# latter - on 64-bit ones. All I had to do is to get one flavor right,
# the other one passed the test right away:-)
#
# sha256_block runs in ~1005 cycles on Opteron, which gives you
# asymptotic performance of 64*1000/1005=63.7MBps times CPU clock
# frequency in GHz. sha512_block runs in ~1275 cycles, which results
# in 128*1000/1275=100MBps per GHz. Is there room for improvement?
# Well, if you compare it to IA-64 implementation, which maintains
# X[16] in register bank[!], tends to 4 instructions per CPU clock
# cycle and runs in 1003 cycles, 1275 is very good result for 3-way
# issue Opteron pipeline and X[16] maintained in memory. So that *if*
# there is a way to improve it, *then* the only way would be to try to
# offload X[16] updates to SSE unit, but that would require "deeper"
# loop unroll, which in turn would naturally cause size blow-up, not
# to mention increased complexity! And once again, only *if* it's
# actually possible to noticeably improve overall ILP, instruction
# level parallelism, on a given CPU implementation in this case.
#
# Special note on Intel EM64T. While Opteron CPU exhibits perfect
# perfromance ratio of 1.5 between 64- and 32-bit flavors [see above],
# [currently available] EM64T CPUs apparently are far from it. On the
# contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit
# sha256_block:-( This is presumably because 64-bit shifts/rotates
# apparently are not atomic instructions, but implemented in microcode.

#
# OpenSolaris OS modifications
#
# Sun elects to use this software under the BSD license.
#
# This source originates from OpenSSL file sha512-x86_64.pl at
# ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz
# (presumably for future OpenSSL release 0.9.8h), with these changes:
#
# 1. Added perl "use strict" and declared variables.
#
# 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
# /usr/include/sys/asm_linkage.h, .ident keywords, and lint(1B) guards.
#
# 3. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1)
# assemblers).  Replaced the .picmeup macro with assembler code.
#
# 4. Added 8 to $ctx, as OpenSolaris OS has an extra 4-byte field, "algotype",
# at the beginning of SHA2_CTX (the next field is 8-byte aligned).
#

use strict;
my ($code, $func, $TABLE, $SZ, @Sigma0, @Sigma1, @sigma0, @sigma1, $rounds,
	@ROT, $A, $B, $C, $D, $E, $F, $G, $H, $T1, $a0, $a1, $a2, $i,
	$ctx, $round, $inp, $Tbl, $_ctx, $_inp, $_end, $_rsp, $framesz);
my $output = shift;
open STDOUT,">$output";

#
# OpenSSL library:
# void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num);
# void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num);
#
# OpenSolaris OS:
# void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
# void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
# Note: the OpenSolaris SHA2 structure has an extra 8 byte field at the
# beginning (over OpenSSL's SHA512 or SHA256 structure).
#

if ($output =~ /512/) {
	$func="SHA512TransformBlocks";
	$TABLE="K512";
	$SZ=8;
	@ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%rax","%rbx","%rcx","%rdx",
					"%r8", "%r9", "%r10","%r11");
	($T1,$a0,$a1,$a2)=("%r12","%r13","%r14","%r15");
	@Sigma0=(28,34,39);
	@Sigma1=(14,18,41);
	@sigma0=(1,  8, 7);
	@sigma1=(19,61, 6);
	$rounds=80;
} else {
	$func="SHA256TransformBlocks";
	$TABLE="K256";
	$SZ=4;
	@ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx",
					"%r8d","%r9d","%r10d","%r11d");
	($T1,$a0,$a1,$a2)=("%r12d","%r13d","%r14d","%r15d");
	@Sigma0=( 2,13,22);
	@Sigma1=( 6,11,25);
	@sigma0=( 7,18, 3);
	@sigma1=(17,19,10);
	$rounds=64;
}

$ctx="%rdi";	# 1st arg
$round="%rdi";	# zaps $ctx
$inp="%rsi";	# 2nd arg
$Tbl="%rbp";

$_ctx="16*$SZ+0*8(%rsp)";
$_inp="16*$SZ+1*8(%rsp)";
$_end="16*$SZ+2*8(%rsp)";
$_rsp="16*$SZ+3*8(%rsp)";
$framesz="16*$SZ+4*8";


sub ROUND_00_15()
{ my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___;
	mov	$e,$a0
	mov	$e,$a1
	mov	$f,$a2

	ror	\$$Sigma1[0],$a0
	ror	\$$Sigma1[1],$a1
	xor	$g,$a2			# f^g

	xor	$a1,$a0
	ror	\$`$Sigma1[2]-$Sigma1[1]`,$a1
	and	$e,$a2			# (f^g)&e
	mov	$T1,`$SZ*($i&0xf)`(%rsp)

	xor	$a1,$a0			# Sigma1(e)
	xor	$g,$a2			# Ch(e,f,g)=((f^g)&e)^g
	add	$h,$T1			# T1+=h

	mov	$a,$h
	add	$a0,$T1			# T1+=Sigma1(e)

	add	$a2,$T1			# T1+=Ch(e,f,g)
	mov	$a,$a0
	mov	$a,$a1

	ror	\$$Sigma0[0],$h
	ror	\$$Sigma0[1],$a0
	mov	$a,$a2
	add	($Tbl,$round,$SZ),$T1	# T1+=K[round]

	xor	$a0,$h
	ror	\$`$Sigma0[2]-$Sigma0[1]`,$a0
	or	$c,$a1			# a|c

	xor	$a0,$h			# h=Sigma0(a)
	and	$c,$a2			# a&c
	add	$T1,$d			# d+=T1

	and	$b,$a1			# (a|c)&b
	add	$T1,$h			# h+=T1

	or	$a2,$a1			# Maj(a,b,c)=((a|c)&b)|(a&c)
	lea	1($round),$round	# round++

	add	$a1,$h			# h+=Maj(a,b,c)
___
}

sub ROUND_16_XX()
{ my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___;
	mov	`$SZ*(($i+1)&0xf)`(%rsp),$a0
	mov	`$SZ*(($i+14)&0xf)`(%rsp),$T1

	mov	$a0,$a2

	shr	\$$sigma0[2],$a0
	ror	\$$sigma0[0],$a2

	xor	$a2,$a0
	ror	\$`$sigma0[1]-$sigma0[0]`,$a2

	xor	$a2,$a0			# sigma0(X[(i+1)&0xf])
	mov	$T1,$a1

	shr	\$$sigma1[2],$T1
	ror	\$$sigma1[0],$a1

	xor	$a1,$T1
	ror	\$`$sigma1[1]-$sigma1[0]`,$a1

	xor	$a1,$T1			# sigma1(X[(i+14)&0xf])

	add	$a0,$T1

	add	`$SZ*(($i+9)&0xf)`(%rsp),$T1

	add	`$SZ*($i&0xf)`(%rsp),$T1
___
	&ROUND_00_15(@_);
}

#
# Execution begins here
#

$code=<<___;
#if defined(lint) || defined(__lint)
#include <sys/stdint.h>
#include <sys/sha2.h>

/* ARGSUSED */
void
$func(SHA2_CTX *ctx, const void *in, size_t num)
{
}


#else
#include <sys/asm_linkage.h>

ENTRY_NP($func)
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	mov	%rsp,%rbp		# copy %rsp
	shl	\$4,%rdx		# num*16
	sub	\$$framesz,%rsp
	lea	($inp,%rdx,$SZ),%rdx	# inp+num*16*$SZ
	and	\$-64,%rsp		# align stack frame
	add	\$8,$ctx		# Skip OpenSolaris field, "algotype"
	mov	$ctx,$_ctx		# save ctx, 1st arg
	mov	$inp,$_inp		# save inp, 2nd arg
	mov	%rdx,$_end		# save end pointer, "3rd" arg
	mov	%rbp,$_rsp		# save copy of %rsp

	/.picmeup $Tbl
	/ The .picmeup pseudo-directive, from perlasm/x86_64_xlate.pl, puts
	/ the address of the "next" instruction into the target register
	/ ($Tbl).  This generates these 2 instructions:
	lea	.Llea(%rip),$Tbl
	/nop	/ .picmeup generates a nop for mod 8 alignment--not needed here

.Llea:
	lea	$TABLE-.($Tbl),$Tbl

	mov	$SZ*0($ctx),$A
	mov	$SZ*1($ctx),$B
	mov	$SZ*2($ctx),$C
	mov	$SZ*3($ctx),$D
	mov	$SZ*4($ctx),$E
	mov	$SZ*5($ctx),$F
	mov	$SZ*6($ctx),$G
	mov	$SZ*7($ctx),$H
	jmp	.Lloop

.align	16
.Lloop:
	xor	$round,$round
___
	for($i=0;$i<16;$i++) {
		$code.="	mov	$SZ*$i($inp),$T1\n";
		$code.="	bswap	$T1\n";
		&ROUND_00_15($i,@ROT);
		unshift(@ROT,pop(@ROT));
	}
$code.=<<___;
	jmp	.Lrounds_16_xx
.align	16
.Lrounds_16_xx:
___
	for(;$i<32;$i++) {
		&ROUND_16_XX($i,@ROT);
		unshift(@ROT,pop(@ROT));
	}

$code.=<<___;
	cmp	\$$rounds,$round
	jb	.Lrounds_16_xx

	mov	$_ctx,$ctx
	lea	16*$SZ($inp),$inp

	add	$SZ*0($ctx),$A
	add	$SZ*1($ctx),$B
	add	$SZ*2($ctx),$C
	add	$SZ*3($ctx),$D
	add	$SZ*4($ctx),$E
	add	$SZ*5($ctx),$F
	add	$SZ*6($ctx),$G
	add	$SZ*7($ctx),$H

	cmp	$_end,$inp

	mov	$A,$SZ*0($ctx)
	mov	$B,$SZ*1($ctx)
	mov	$C,$SZ*2($ctx)
	mov	$D,$SZ*3($ctx)
	mov	$E,$SZ*4($ctx)
	mov	$F,$SZ*5($ctx)
	mov	$G,$SZ*6($ctx)
	mov	$H,$SZ*7($ctx)
	jb	.Lloop

	mov	$_rsp,%rsp
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx

	ret
SET_SIZE($func)

___

if ($SZ==4) {
# SHA256
$code.=<<___;
.align	64
.type	$TABLE,\@object
$TABLE:
	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
___
} else {
# SHA512
$code.=<<___;
.align	64
.type	$TABLE,\@object
$TABLE:
	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
	.quad	0x3956c25bf348b538,0x59f111f1b605d019
	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
	.quad	0xd807aa98a3030242,0x12835b0145706fbe
	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
	.quad	0x06ca6351e003826f,0x142929670a0e6e70
	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
	.quad	0x81c2c92e47edaee6,0x92722c851482353b
	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
	.quad	0xd192e819d6ef5218,0xd69906245565a910
	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
	.quad	0x90befffa23631e28,0xa4506cebde82bde9
	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
	.quad	0xca273eceea26619c,0xd186b8c721c0c207
	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
	.quad	0x113f9804bef90dae,0x1b710b35131c471b
	.quad	0x28db77f523047d84,0x32caab7b40c72493
	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
___
}
$code.=<<___;
#endif /* !lint && !__lint */
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT;
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2013 Saso Kiselkov.  All rights reserved.
 * Copyright 2024 Bill Sommerfeld <sommerfeld@hamachi.org>
 */

/*
 * The basic framework for this code came from the reference
 * implementation for MD5.  That implementation is Copyright (C)
 * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
 *
 * License to copy and use this software is granted provided that it
 * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
 * Algorithm" in all material mentioning or referencing this software
 * or this function.
 *
 * License is also granted to make and use derivative works provided
 * that such works are identified as "derived from the RSA Data
 * Security, Inc. MD5 Message-Digest Algorithm" in all material
 * mentioning or referencing the derived work.
 *
 * RSA Data Security, Inc. makes no representations concerning either
 * the merchantability of this software or the suitability of this
 * software for any particular purpose. It is provided "as is"
 * without express or implied warranty of any kind.
 *
 * These notices must be retained in any copies of any part of this
 * documentation and/or software.
 *
 * NOTE: Cleaned-up and optimized, version of SHA2, based on the FIPS 180-2
 * standard, available at
 * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
 * Not as fast as one would like -- further optimizations are encouraged
 * and appreciated.
 */

#ifndef _KERNEL
#include <stdint.h>
#include <strings.h>
#include <stdlib.h>
#include <errno.h>
#endif /* _KERNEL */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#define	_SHA2_IMPL
#include <sys/sha2.h>
#include <sys/sha2_consts.h>

#ifdef _KERNEL
#include <sys/cmn_err.h>

#else
#pragma weak SHA256Update = SHA2Update
#pragma weak SHA384Update = SHA2Update
#pragma weak SHA512Update = SHA2Update

#pragma weak SHA256Final = SHA2Final
#pragma weak SHA384Final = SHA2Final
#pragma weak SHA512Final = SHA2Final

#endif	/* _KERNEL */

#ifdef _LITTLE_ENDIAN
#include <sys/byteorder.h>
#define	HAVE_HTONL
#endif

static void Encode(uint8_t *, uint32_t *, size_t);
static void Encode64(uint8_t *, uint64_t *, size_t);

#if	defined(__amd64)
#define	SHA512Transform(ctx, in) SHA512TransformBlocks((ctx), (in), 1)
#define	SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1)

void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);

#else
static void SHA256Transform(SHA2_CTX *, const uint8_t *);
static void SHA512Transform(SHA2_CTX *, const uint8_t *);
#endif	/* __amd64 */

static uint8_t PADDING[128] = { 0x80, /* all zeros */ };

/* Ch and Maj are the basic SHA2 functions. */
#define	Ch(b, c, d)	(((b) & (c)) ^ ((~b) & (d)))
#define	Maj(b, c, d)	(((b) & (c)) ^ ((b) & (d)) ^ ((c) & (d)))

/* Rotates x right n bits. */
#define	ROTR(x, n)	\
	(((x) >> (n)) | ((x) << ((sizeof (x) * NBBY)-(n))))

/* Shift x right n bits */
#define	SHR(x, n)	((x) >> (n))

/* SHA256 Functions */
#define	BIGSIGMA0_256(x)	(ROTR((x), 2) ^ ROTR((x), 13) ^ ROTR((x), 22))
#define	BIGSIGMA1_256(x)	(ROTR((x), 6) ^ ROTR((x), 11) ^ ROTR((x), 25))
#define	SIGMA0_256(x)		(ROTR((x), 7) ^ ROTR((x), 18) ^ SHR((x), 3))
#define	SIGMA1_256(x)		(ROTR((x), 17) ^ ROTR((x), 19) ^ SHR((x), 10))

#define	SHA256ROUND(a, b, c, d, e, f, g, h, i, w)			\
	T1 = h + BIGSIGMA1_256(e) + Ch(e, f, g) + SHA256_CONST(i) + w;	\
	d += T1;							\
	T2 = BIGSIGMA0_256(a) + Maj(a, b, c);				\
	h = T1 + T2

/* SHA384/512 Functions */
#define	BIGSIGMA0(x)	(ROTR((x), 28) ^ ROTR((x), 34) ^ ROTR((x), 39))
#define	BIGSIGMA1(x)	(ROTR((x), 14) ^ ROTR((x), 18) ^ ROTR((x), 41))
#define	SIGMA0(x)	(ROTR((x), 1) ^ ROTR((x), 8) ^ SHR((x), 7))
#define	SIGMA1(x)	(ROTR((x), 19) ^ ROTR((x), 61) ^ SHR((x), 6))
#define	SHA512ROUND(a, b, c, d, e, f, g, h, i, w)			\
	T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + SHA512_CONST(i) + w;	\
	d += T1;							\
	T2 = BIGSIGMA0(a) + Maj(a, b, c);				\
	h = T1 + T2

/*
 * sparc optimization:
 *
 * on the sparc, we can load big endian 32-bit data easily.  note that
 * special care must be taken to ensure the address is 32-bit aligned.
 * in the interest of speed, we don't check to make sure, since
 * careful programming can guarantee this for us.
 */

#if	defined(_BIG_ENDIAN)
#define	LOAD_BIG_32(addr)	(*(uint32_t *)(addr))
#define	LOAD_BIG_64(addr)	(*(uint64_t *)(addr))

#elif	defined(HAVE_HTONL)
#define	LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr)))
#define	LOAD_BIG_64(addr) htonll(*((uint64_t *)(addr)))

#else
/* little endian -- will work on big endian, but slowly */
#define	LOAD_BIG_32(addr)	\
	(((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3])
#define	LOAD_BIG_64(addr)	\
	(((uint64_t)(addr)[0] << 56) | ((uint64_t)(addr)[1] << 48) |	\
	    ((uint64_t)(addr)[2] << 40) | ((uint64_t)(addr)[3] << 32) |	\
	    ((uint64_t)(addr)[4] << 24) | ((uint64_t)(addr)[5] << 16) |	\
	    ((uint64_t)(addr)[6] << 8) | (uint64_t)(addr)[7])
#endif	/* _BIG_ENDIAN */


#if	!defined(__amd64)
/* SHA256 Transform */

static void
SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk)
{
	uint32_t a = ctx->state.s32[0];
	uint32_t b = ctx->state.s32[1];
	uint32_t c = ctx->state.s32[2];
	uint32_t d = ctx->state.s32[3];
	uint32_t e = ctx->state.s32[4];
	uint32_t f = ctx->state.s32[5];
	uint32_t g = ctx->state.s32[6];
	uint32_t h = ctx->state.s32[7];

	uint32_t w0, w1, w2, w3, w4, w5, w6, w7;
	uint32_t w8, w9, w10, w11, w12, w13, w14, w15;
	uint32_t T1, T2;

#if	defined(__sparc)
	static const uint32_t sha256_consts[] = {
		SHA256_CONST_0, SHA256_CONST_1, SHA256_CONST_2,
		SHA256_CONST_3, SHA256_CONST_4, SHA256_CONST_5,
		SHA256_CONST_6, SHA256_CONST_7, SHA256_CONST_8,
		SHA256_CONST_9, SHA256_CONST_10, SHA256_CONST_11,
		SHA256_CONST_12, SHA256_CONST_13, SHA256_CONST_14,
		SHA256_CONST_15, SHA256_CONST_16, SHA256_CONST_17,
		SHA256_CONST_18, SHA256_CONST_19, SHA256_CONST_20,
		SHA256_CONST_21, SHA256_CONST_22, SHA256_CONST_23,
		SHA256_CONST_24, SHA256_CONST_25, SHA256_CONST_26,
		SHA256_CONST_27, SHA256_CONST_28, SHA256_CONST_29,
		SHA256_CONST_30, SHA256_CONST_31, SHA256_CONST_32,
		SHA256_CONST_33, SHA256_CONST_34, SHA256_CONST_35,
		SHA256_CONST_36, SHA256_CONST_37, SHA256_CONST_38,
		SHA256_CONST_39, SHA256_CONST_40, SHA256_CONST_41,
		SHA256_CONST_42, SHA256_CONST_43, SHA256_CONST_44,
		SHA256_CONST_45, SHA256_CONST_46, SHA256_CONST_47,
		SHA256_CONST_48, SHA256_CONST_49, SHA256_CONST_50,
		SHA256_CONST_51, SHA256_CONST_52, SHA256_CONST_53,
		SHA256_CONST_54, SHA256_CONST_55, SHA256_CONST_56,
		SHA256_CONST_57, SHA256_CONST_58, SHA256_CONST_59,
		SHA256_CONST_60, SHA256_CONST_61, SHA256_CONST_62,
		SHA256_CONST_63
	};
#endif	/* __sparc */

	if ((uintptr_t)blk & 0x3) {		/* not 4-byte aligned? */
		bcopy(blk, ctx->buf_un.buf32,  sizeof (ctx->buf_un.buf32));
		blk = (uint8_t *)ctx->buf_un.buf32;
	}

	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w0 =  LOAD_BIG_32(blk + 4 * 0);
	SHA256ROUND(a, b, c, d, e, f, g, h, 0, w0);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w1 =  LOAD_BIG_32(blk + 4 * 1);
	SHA256ROUND(h, a, b, c, d, e, f, g, 1, w1);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w2 =  LOAD_BIG_32(blk + 4 * 2);
	SHA256ROUND(g, h, a, b, c, d, e, f, 2, w2);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w3 =  LOAD_BIG_32(blk + 4 * 3);
	SHA256ROUND(f, g, h, a, b, c, d, e, 3, w3);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w4 =  LOAD_BIG_32(blk + 4 * 4);
	SHA256ROUND(e, f, g, h, a, b, c, d, 4, w4);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w5 =  LOAD_BIG_32(blk + 4 * 5);
	SHA256ROUND(d, e, f, g, h, a, b, c, 5, w5);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w6 =  LOAD_BIG_32(blk + 4 * 6);
	SHA256ROUND(c, d, e, f, g, h, a, b, 6, w6);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w7 =  LOAD_BIG_32(blk + 4 * 7);
	SHA256ROUND(b, c, d, e, f, g, h, a, 7, w7);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w8 =  LOAD_BIG_32(blk + 4 * 8);
	SHA256ROUND(a, b, c, d, e, f, g, h, 8, w8);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w9 =  LOAD_BIG_32(blk + 4 * 9);
	SHA256ROUND(h, a, b, c, d, e, f, g, 9, w9);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w10 =  LOAD_BIG_32(blk + 4 * 10);
	SHA256ROUND(g, h, a, b, c, d, e, f, 10, w10);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w11 =  LOAD_BIG_32(blk + 4 * 11);
	SHA256ROUND(f, g, h, a, b, c, d, e, 11, w11);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w12 =  LOAD_BIG_32(blk + 4 * 12);
	SHA256ROUND(e, f, g, h, a, b, c, d, 12, w12);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w13 =  LOAD_BIG_32(blk + 4 * 13);
	SHA256ROUND(d, e, f, g, h, a, b, c, 13, w13);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w14 =  LOAD_BIG_32(blk + 4 * 14);
	SHA256ROUND(c, d, e, f, g, h, a, b, 14, w14);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w15 =  LOAD_BIG_32(blk + 4 * 15);
	SHA256ROUND(b, c, d, e, f, g, h, a, 15, w15);

	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
	SHA256ROUND(a, b, c, d, e, f, g, h, 16, w0);
	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
	SHA256ROUND(h, a, b, c, d, e, f, g, 17, w1);
	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
	SHA256ROUND(g, h, a, b, c, d, e, f, 18, w2);
	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
	SHA256ROUND(f, g, h, a, b, c, d, e, 19, w3);
	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
	SHA256ROUND(e, f, g, h, a, b, c, d, 20, w4);
	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
	SHA256ROUND(d, e, f, g, h, a, b, c, 21, w5);
	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
	SHA256ROUND(c, d, e, f, g, h, a, b, 22, w6);
	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
	SHA256ROUND(b, c, d, e, f, g, h, a, 23, w7);
	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
	SHA256ROUND(a, b, c, d, e, f, g, h, 24, w8);
	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
	SHA256ROUND(h, a, b, c, d, e, f, g, 25, w9);
	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
	SHA256ROUND(g, h, a, b, c, d, e, f, 26, w10);
	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
	SHA256ROUND(f, g, h, a, b, c, d, e, 27, w11);
	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
	SHA256ROUND(e, f, g, h, a, b, c, d, 28, w12);
	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
	SHA256ROUND(d, e, f, g, h, a, b, c, 29, w13);
	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
	SHA256ROUND(c, d, e, f, g, h, a, b, 30, w14);
	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
	SHA256ROUND(b, c, d, e, f, g, h, a, 31, w15);

	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
	SHA256ROUND(a, b, c, d, e, f, g, h, 32, w0);
	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
	SHA256ROUND(h, a, b, c, d, e, f, g, 33, w1);
	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
	SHA256ROUND(g, h, a, b, c, d, e, f, 34, w2);
	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
	SHA256ROUND(f, g, h, a, b, c, d, e, 35, w3);
	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
	SHA256ROUND(e, f, g, h, a, b, c, d, 36, w4);
	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
	SHA256ROUND(d, e, f, g, h, a, b, c, 37, w5);
	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
	SHA256ROUND(c, d, e, f, g, h, a, b, 38, w6);
	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
	SHA256ROUND(b, c, d, e, f, g, h, a, 39, w7);
	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
	SHA256ROUND(a, b, c, d, e, f, g, h, 40, w8);
	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
	SHA256ROUND(h, a, b, c, d, e, f, g, 41, w9);
	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
	SHA256ROUND(g, h, a, b, c, d, e, f, 42, w10);
	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
	SHA256ROUND(f, g, h, a, b, c, d, e, 43, w11);
	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
	SHA256ROUND(e, f, g, h, a, b, c, d, 44, w12);
	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
	SHA256ROUND(d, e, f, g, h, a, b, c, 45, w13);
	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
	SHA256ROUND(c, d, e, f, g, h, a, b, 46, w14);
	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
	SHA256ROUND(b, c, d, e, f, g, h, a, 47, w15);

	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
	SHA256ROUND(a, b, c, d, e, f, g, h, 48, w0);
	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
	SHA256ROUND(h, a, b, c, d, e, f, g, 49, w1);
	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
	SHA256ROUND(g, h, a, b, c, d, e, f, 50, w2);
	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
	SHA256ROUND(f, g, h, a, b, c, d, e, 51, w3);
	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
	SHA256ROUND(e, f, g, h, a, b, c, d, 52, w4);
	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
	SHA256ROUND(d, e, f, g, h, a, b, c, 53, w5);
	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
	SHA256ROUND(c, d, e, f, g, h, a, b, 54, w6);
	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
	SHA256ROUND(b, c, d, e, f, g, h, a, 55, w7);
	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
	SHA256ROUND(a, b, c, d, e, f, g, h, 56, w8);
	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
	SHA256ROUND(h, a, b, c, d, e, f, g, 57, w9);
	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
	SHA256ROUND(g, h, a, b, c, d, e, f, 58, w10);
	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
	SHA256ROUND(f, g, h, a, b, c, d, e, 59, w11);
	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
	SHA256ROUND(e, f, g, h, a, b, c, d, 60, w12);
	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
	SHA256ROUND(d, e, f, g, h, a, b, c, 61, w13);
	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
	SHA256ROUND(c, d, e, f, g, h, a, b, 62, w14);
	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
	SHA256ROUND(b, c, d, e, f, g, h, a, 63, w15);

	ctx->state.s32[0] += a;
	ctx->state.s32[1] += b;
	ctx->state.s32[2] += c;
	ctx->state.s32[3] += d;
	ctx->state.s32[4] += e;
	ctx->state.s32[5] += f;
	ctx->state.s32[6] += g;
	ctx->state.s32[7] += h;
}


/* SHA384 and SHA512 Transform */

static void
SHA512Transform(SHA2_CTX *ctx, const uint8_t *blk)
{

	uint64_t a = ctx->state.s64[0];
	uint64_t b = ctx->state.s64[1];
	uint64_t c = ctx->state.s64[2];
	uint64_t d = ctx->state.s64[3];
	uint64_t e = ctx->state.s64[4];
	uint64_t f = ctx->state.s64[5];
	uint64_t g = ctx->state.s64[6];
	uint64_t h = ctx->state.s64[7];

	uint64_t w0, w1, w2, w3, w4, w5, w6, w7;
	uint64_t w8, w9, w10, w11, w12, w13, w14, w15;
	uint64_t T1, T2;

#if	defined(__sparc)
	static const uint64_t sha512_consts[] = {
		SHA512_CONST_0, SHA512_CONST_1, SHA512_CONST_2,
		SHA512_CONST_3, SHA512_CONST_4, SHA512_CONST_5,
		SHA512_CONST_6, SHA512_CONST_7, SHA512_CONST_8,
		SHA512_CONST_9, SHA512_CONST_10, SHA512_CONST_11,
		SHA512_CONST_12, SHA512_CONST_13, SHA512_CONST_14,
		SHA512_CONST_15, SHA512_CONST_16, SHA512_CONST_17,
		SHA512_CONST_18, SHA512_CONST_19, SHA512_CONST_20,
		SHA512_CONST_21, SHA512_CONST_22, SHA512_CONST_23,
		SHA512_CONST_24, SHA512_CONST_25, SHA512_CONST_26,
		SHA512_CONST_27, SHA512_CONST_28, SHA512_CONST_29,
		SHA512_CONST_30, SHA512_CONST_31, SHA512_CONST_32,
		SHA512_CONST_33, SHA512_CONST_34, SHA512_CONST_35,
		SHA512_CONST_36, SHA512_CONST_37, SHA512_CONST_38,
		SHA512_CONST_39, SHA512_CONST_40, SHA512_CONST_41,
		SHA512_CONST_42, SHA512_CONST_43, SHA512_CONST_44,
		SHA512_CONST_45, SHA512_CONST_46, SHA512_CONST_47,
		SHA512_CONST_48, SHA512_CONST_49, SHA512_CONST_50,
		SHA512_CONST_51, SHA512_CONST_52, SHA512_CONST_53,
		SHA512_CONST_54, SHA512_CONST_55, SHA512_CONST_56,
		SHA512_CONST_57, SHA512_CONST_58, SHA512_CONST_59,
		SHA512_CONST_60, SHA512_CONST_61, SHA512_CONST_62,
		SHA512_CONST_63, SHA512_CONST_64, SHA512_CONST_65,
		SHA512_CONST_66, SHA512_CONST_67, SHA512_CONST_68,
		SHA512_CONST_69, SHA512_CONST_70, SHA512_CONST_71,
		SHA512_CONST_72, SHA512_CONST_73, SHA512_CONST_74,
		SHA512_CONST_75, SHA512_CONST_76, SHA512_CONST_77,
		SHA512_CONST_78, SHA512_CONST_79
	};
#endif	/* __sparc */


	if ((uintptr_t)blk & 0x7) {		/* not 8-byte aligned? */
		bcopy(blk, ctx->buf_un.buf64,  sizeof (ctx->buf_un.buf64));
		blk = (uint8_t *)ctx->buf_un.buf64;
	}

	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w0 =  LOAD_BIG_64(blk + 8 * 0);
	SHA512ROUND(a, b, c, d, e, f, g, h, 0, w0);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w1 =  LOAD_BIG_64(blk + 8 * 1);
	SHA512ROUND(h, a, b, c, d, e, f, g, 1, w1);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w2 =  LOAD_BIG_64(blk + 8 * 2);
	SHA512ROUND(g, h, a, b, c, d, e, f, 2, w2);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w3 =  LOAD_BIG_64(blk + 8 * 3);
	SHA512ROUND(f, g, h, a, b, c, d, e, 3, w3);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w4 =  LOAD_BIG_64(blk + 8 * 4);
	SHA512ROUND(e, f, g, h, a, b, c, d, 4, w4);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w5 =  LOAD_BIG_64(blk + 8 * 5);
	SHA512ROUND(d, e, f, g, h, a, b, c, 5, w5);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w6 =  LOAD_BIG_64(blk + 8 * 6);
	SHA512ROUND(c, d, e, f, g, h, a, b, 6, w6);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w7 =  LOAD_BIG_64(blk + 8 * 7);
	SHA512ROUND(b, c, d, e, f, g, h, a, 7, w7);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w8 =  LOAD_BIG_64(blk + 8 * 8);
	SHA512ROUND(a, b, c, d, e, f, g, h, 8, w8);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w9 =  LOAD_BIG_64(blk + 8 * 9);
	SHA512ROUND(h, a, b, c, d, e, f, g, 9, w9);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w10 =  LOAD_BIG_64(blk + 8 * 10);
	SHA512ROUND(g, h, a, b, c, d, e, f, 10, w10);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w11 =  LOAD_BIG_64(blk + 8 * 11);
	SHA512ROUND(f, g, h, a, b, c, d, e, 11, w11);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w12 =  LOAD_BIG_64(blk + 8 * 12);
	SHA512ROUND(e, f, g, h, a, b, c, d, 12, w12);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w13 =  LOAD_BIG_64(blk + 8 * 13);
	SHA512ROUND(d, e, f, g, h, a, b, c, 13, w13);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w14 =  LOAD_BIG_64(blk + 8 * 14);
	SHA512ROUND(c, d, e, f, g, h, a, b, 14, w14);
	/* LINTED E_BAD_PTR_CAST_ALIGN */
	w15 =  LOAD_BIG_64(blk + 8 * 15);
	SHA512ROUND(b, c, d, e, f, g, h, a, 15, w15);

	w0 = SIGMA1(w14) + w9 + SIGMA0(w1) + w0;
	SHA512ROUND(a, b, c, d, e, f, g, h, 16, w0);
	w1 = SIGMA1(w15) + w10 + SIGMA0(w2) + w1;
	SHA512ROUND(h, a, b, c, d, e, f, g, 17, w1);
	w2 = SIGMA1(w0) + w11 + SIGMA0(w3) + w2;
	SHA512ROUND(g, h, a, b, c, d, e, f, 18, w2);
	w3 = SIGMA1(w1) + w12 + SIGMA0(w4) + w3;
	SHA512ROUND(f, g, h, a, b, c, d, e, 19, w3);
	w4 = SIGMA1(w2) + w13 + SIGMA0(w5) + w4;
	SHA512ROUND(e, f, g, h, a, b, c, d, 20, w4);
	w5 = SIGMA1(w3) + w14 + SIGMA0(w6) + w5;
	SHA512ROUND(d, e, f, g, h, a, b, c, 21, w5);
	w6 = SIGMA1(w4) + w15 + SIGMA0(w7) + w6;
	SHA512ROUND(c, d, e, f, g, h, a, b, 22, w6);
	w7 = SIGMA1(w5) + w0 + SIGMA0(w8) + w7;
	SHA512ROUND(b, c, d, e, f, g, h, a, 23, w7);
	w8 = SIGMA1(w6) + w1 + SIGMA0(w9) + w8;
	SHA512ROUND(a, b, c, d, e, f, g, h, 24, w8);
	w9 = SIGMA1(w7) + w2 + SIGMA0(w10) + w9;
	SHA512ROUND(h, a, b, c, d, e, f, g, 25, w9);
	w10 = SIGMA1(w8) + w3 + SIGMA0(w11) + w10;
	SHA512ROUND(g, h, a, b, c, d, e, f, 26, w10);
	w11 = SIGMA1(w9) + w4 + SIGMA0(w12) + w11;
	SHA512ROUND(f, g, h, a, b, c, d, e, 27, w11);
	w12 = SIGMA1(w10) + w5 + SIGMA0(w13) + w12;
	SHA512ROUND(e, f, g, h, a, b, c, d, 28, w12);
	w13 = SIGMA1(w11) + w6 + SIGMA0(w14) + w13;
	SHA512ROUND(d, e, f, g, h, a, b, c, 29, w13);
	w14 = SIGMA1(w12) + w7 + SIGMA0(w15) + w14;
	SHA512ROUND(c, d, e, f, g, h, a, b, 30, w14);
	w15 = SIGMA1(w13) + w8 + SIGMA0(w0) + w15;
	SHA512ROUND(b, c, d, e, f, g, h, a, 31, w15);

	w0 = SIGMA1(w14) + w9 + SIGMA0(w1) + w0;
	SHA512ROUND(a, b, c, d, e, f, g, h, 32, w0);
	w1 = SIGMA1(w15) + w10 + SIGMA0(w2) + w1;
	SHA512ROUND(h, a, b, c, d, e, f, g, 33, w1);
	w2 = SIGMA1(w0) + w11 + SIGMA0(w3) + w2;
	SHA512ROUND(g, h, a, b, c, d, e, f, 34, w2);
	w3 = SIGMA1(w1) + w12 + SIGMA0(w4) + w3;
	SHA512ROUND(f, g, h, a, b, c, d, e, 35, w3);
	w4 = SIGMA1(w2) + w13 + SIGMA0(w5) + w4;
	SHA512ROUND(e, f, g, h, a, b, c, d, 36, w4);
	w5 = SIGMA1(w3) + w14 + SIGMA0(w6) + w5;
	SHA512ROUND(d, e, f, g, h, a, b, c, 37, w5);
	w6 = SIGMA1(w4) + w15 + SIGMA0(w7) + w6;
	SHA512ROUND(c, d, e, f, g, h, a, b, 38, w6);
	w7 = SIGMA1(w5) + w0 + SIGMA0(w8) + w7;
	SHA512ROUND(b, c, d, e, f, g, h, a, 39, w7);
	w8 = SIGMA1(w6) + w1 + SIGMA0(w9) + w8;
	SHA512ROUND(a, b, c, d, e, f, g, h, 40, w8);
	w9 = SIGMA1(w7) + w2 + SIGMA0(w10) + w9;
	SHA512ROUND(h, a, b, c, d, e, f, g, 41, w9);
	w10 = SIGMA1(w8) + w3 + SIGMA0(w11) + w10;
	SHA512ROUND(g, h, a, b, c, d, e, f, 42, w10);
	w11 = SIGMA1(w9) + w4 + SIGMA0(w12) + w11;
	SHA512ROUND(f, g, h, a, b, c, d, e, 43, w11);
	w12 = SIGMA1(w10) + w5 + SIGMA0(w13) + w12;
	SHA512ROUND(e, f, g, h, a, b, c, d, 44, w12);
	w13 = SIGMA1(w11) + w6 + SIGMA0(w14) + w13;
	SHA512ROUND(d, e, f, g, h, a, b, c, 45, w13);
	w14 = SIGMA1(w12) + w7 + SIGMA0(w15) + w14;
	SHA512ROUND(c, d, e, f, g, h, a, b, 46, w14);
	w15 = SIGMA1(w13) + w8 + SIGMA0(w0) + w15;
	SHA512ROUND(b, c, d, e, f, g, h, a, 47, w15);

	w0 = SIGMA1(w14) + w9 + SIGMA0(w1) + w0;
	SHA512ROUND(a, b, c, d, e, f, g, h, 48, w0);
	w1 = SIGMA1(w15) + w10 + SIGMA0(w2) + w1;
	SHA512ROUND(h, a, b, c, d, e, f, g, 49, w1);
	w2 = SIGMA1(w0) + w11 + SIGMA0(w3) + w2;
	SHA512ROUND(g, h, a, b, c, d, e, f, 50, w2);
	w3 = SIGMA1(w1) + w12 + SIGMA0(w4) + w3;
	SHA512ROUND(f, g, h, a, b, c, d, e, 51, w3);
	w4 = SIGMA1(w2) + w13 + SIGMA0(w5) + w4;
	SHA512ROUND(e, f, g, h, a, b, c, d, 52, w4);
	w5 = SIGMA1(w3) + w14 + SIGMA0(w6) + w5;
	SHA512ROUND(d, e, f, g, h, a, b, c, 53, w5);
	w6 = SIGMA1(w4) + w15 + SIGMA0(w7) + w6;
	SHA512ROUND(c, d, e, f, g, h, a, b, 54, w6);
	w7 = SIGMA1(w5) + w0 + SIGMA0(w8) + w7;
	SHA512ROUND(b, c, d, e, f, g, h, a, 55, w7);
	w8 = SIGMA1(w6) + w1 + SIGMA0(w9) + w8;
	SHA512ROUND(a, b, c, d, e, f, g, h, 56, w8);
	w9 = SIGMA1(w7) + w2 + SIGMA0(w10) + w9;
	SHA512ROUND(h, a, b, c, d, e, f, g, 57, w9);
	w10 = SIGMA1(w8) + w3 + SIGMA0(w11) + w10;
	SHA512ROUND(g, h, a, b, c, d, e, f, 58, w10);
	w11 = SIGMA1(w9) + w4 + SIGMA0(w12) + w11;
	SHA512ROUND(f, g, h, a, b, c, d, e, 59, w11);
	w12 = SIGMA1(w10) + w5 + SIGMA0(w13) + w12;
	SHA512ROUND(e, f, g, h, a, b, c, d, 60, w12);
	w13 = SIGMA1(w11) + w6 + SIGMA0(w14) + w13;
	SHA512ROUND(d, e, f, g, h, a, b, c, 61, w13);
	w14 = SIGMA1(w12) + w7 + SIGMA0(w15) + w14;
	SHA512ROUND(c, d, e, f, g, h, a, b, 62, w14);
	w15 = SIGMA1(w13) + w8 + SIGMA0(w0) + w15;
	SHA512ROUND(b, c, d, e, f, g, h, a, 63, w15);

	w0 = SIGMA1(w14) + w9 + SIGMA0(w1) + w0;
	SHA512ROUND(a, b, c, d, e, f, g, h, 64, w0);
	w1 = SIGMA1(w15) + w10 + SIGMA0(w2) + w1;
	SHA512ROUND(h, a, b, c, d, e, f, g, 65, w1);
	w2 = SIGMA1(w0) + w11 + SIGMA0(w3) + w2;
	SHA512ROUND(g, h, a, b, c, d, e, f, 66, w2);
	w3 = SIGMA1(w1) + w12 + SIGMA0(w4) + w3;
	SHA512ROUND(f, g, h, a, b, c, d, e, 67, w3);
	w4 = SIGMA1(w2) + w13 + SIGMA0(w5) + w4;
	SHA512ROUND(e, f, g, h, a, b, c, d, 68, w4);
	w5 = SIGMA1(w3) + w14 + SIGMA0(w6) + w5;
	SHA512ROUND(d, e, f, g, h, a, b, c, 69, w5);
	w6 = SIGMA1(w4) + w15 + SIGMA0(w7) + w6;
	SHA512ROUND(c, d, e, f, g, h, a, b, 70, w6);
	w7 = SIGMA1(w5) + w0 + SIGMA0(w8) + w7;
	SHA512ROUND(b, c, d, e, f, g, h, a, 71, w7);
	w8 = SIGMA1(w6) + w1 + SIGMA0(w9) + w8;
	SHA512ROUND(a, b, c, d, e, f, g, h, 72, w8);
	w9 = SIGMA1(w7) + w2 + SIGMA0(w10) + w9;
	SHA512ROUND(h, a, b, c, d, e, f, g, 73, w9);
	w10 = SIGMA1(w8) + w3 + SIGMA0(w11) + w10;
	SHA512ROUND(g, h, a, b, c, d, e, f, 74, w10);
	w11 = SIGMA1(w9) + w4 + SIGMA0(w12) + w11;
	SHA512ROUND(f, g, h, a, b, c, d, e, 75, w11);
	w12 = SIGMA1(w10) + w5 + SIGMA0(w13) + w12;
	SHA512ROUND(e, f, g, h, a, b, c, d, 76, w12);
	w13 = SIGMA1(w11) + w6 + SIGMA0(w14) + w13;
	SHA512ROUND(d, e, f, g, h, a, b, c, 77, w13);
	w14 = SIGMA1(w12) + w7 + SIGMA0(w15) + w14;
	SHA512ROUND(c, d, e, f, g, h, a, b, 78, w14);
	w15 = SIGMA1(w13) + w8 + SIGMA0(w0) + w15;
	SHA512ROUND(b, c, d, e, f, g, h, a, 79, w15);

	ctx->state.s64[0] += a;
	ctx->state.s64[1] += b;
	ctx->state.s64[2] += c;
	ctx->state.s64[3] += d;
	ctx->state.s64[4] += e;
	ctx->state.s64[5] += f;
	ctx->state.s64[6] += g;
	ctx->state.s64[7] += h;

}
#endif	/* !__amd64 */


/*
 * Encode()
 *
 * purpose: to convert a list of numbers from little endian to big endian
 *   input: uint8_t *	: place to store the converted big endian numbers
 *	    uint32_t *	: place to get numbers to convert from
 *          size_t	: the length of the input in bytes
 *  output: void
 */

static void
Encode(uint8_t *_RESTRICT_KYWD output, uint32_t *_RESTRICT_KYWD input,
    size_t len)
{
	size_t		i, j;

#if	defined(__sparc)
	if (IS_P2ALIGNED(output, sizeof (uint32_t))) {
		for (i = 0, j = 0; j < len; i++, j += 4) {
			/* LINTED E_BAD_PTR_CAST_ALIGN */
			*((uint32_t *)(output + j)) = input[i];
		}
	} else {
#endif	/* little endian -- will work on big endian, but slowly */
		for (i = 0, j = 0; j < len; i++, j += 4) {
			output[j]	= (input[i] >> 24) & 0xff;
			output[j + 1]	= (input[i] >> 16) & 0xff;
			output[j + 2]	= (input[i] >>  8) & 0xff;
			output[j + 3]	= input[i] & 0xff;
		}
#if	defined(__sparc)
	}
#endif
}

static void
Encode64(uint8_t *_RESTRICT_KYWD output, uint64_t *_RESTRICT_KYWD input,
    size_t len)
{
	size_t		i, j;

#if	defined(__sparc)
	if (IS_P2ALIGNED(output, sizeof (uint64_t))) {
		for (i = 0, j = 0; j < len; i++, j += 8) {
			/* LINTED E_BAD_PTR_CAST_ALIGN */
			*((uint64_t *)(output + j)) = input[i];
		}
	} else {
#endif	/* little endian -- will work on big endian, but slowly */
		for (i = 0, j = 0; j < len; i++, j += 8) {

			output[j]	= (input[i] >> 56) & 0xff;
			output[j + 1]	= (input[i] >> 48) & 0xff;
			output[j + 2]	= (input[i] >> 40) & 0xff;
			output[j + 3]	= (input[i] >> 32) & 0xff;
			output[j + 4]	= (input[i] >> 24) & 0xff;
			output[j + 5]	= (input[i] >> 16) & 0xff;
			output[j + 6]	= (input[i] >>  8) & 0xff;
			output[j + 7]	= input[i] & 0xff;
		}
#if	defined(__sparc)
	}
#endif
}


void
SHA2Init(uint64_t mech, SHA2_CTX *ctx)
{

	switch (mech) {
	case SHA256_MECH_INFO_TYPE:
	case SHA256_HMAC_MECH_INFO_TYPE:
	case SHA256_HMAC_GEN_MECH_INFO_TYPE:
		ctx->state.s32[0] = 0x6a09e667U;
		ctx->state.s32[1] = 0xbb67ae85U;
		ctx->state.s32[2] = 0x3c6ef372U;
		ctx->state.s32[3] = 0xa54ff53aU;
		ctx->state.s32[4] = 0x510e527fU;
		ctx->state.s32[5] = 0x9b05688cU;
		ctx->state.s32[6] = 0x1f83d9abU;
		ctx->state.s32[7] = 0x5be0cd19U;
		break;
	case SHA384_MECH_INFO_TYPE:
	case SHA384_HMAC_MECH_INFO_TYPE:
	case SHA384_HMAC_GEN_MECH_INFO_TYPE:
		ctx->state.s64[0] = 0xcbbb9d5dc1059ed8ULL;
		ctx->state.s64[1] = 0x629a292a367cd507ULL;
		ctx->state.s64[2] = 0x9159015a3070dd17ULL;
		ctx->state.s64[3] = 0x152fecd8f70e5939ULL;
		ctx->state.s64[4] = 0x67332667ffc00b31ULL;
		ctx->state.s64[5] = 0x8eb44a8768581511ULL;
		ctx->state.s64[6] = 0xdb0c2e0d64f98fa7ULL;
		ctx->state.s64[7] = 0x47b5481dbefa4fa4ULL;
		break;
	case SHA512_MECH_INFO_TYPE:
	case SHA512_HMAC_MECH_INFO_TYPE:
	case SHA512_HMAC_GEN_MECH_INFO_TYPE:
		ctx->state.s64[0] = 0x6a09e667f3bcc908ULL;
		ctx->state.s64[1] = 0xbb67ae8584caa73bULL;
		ctx->state.s64[2] = 0x3c6ef372fe94f82bULL;
		ctx->state.s64[3] = 0xa54ff53a5f1d36f1ULL;
		ctx->state.s64[4] = 0x510e527fade682d1ULL;
		ctx->state.s64[5] = 0x9b05688c2b3e6c1fULL;
		ctx->state.s64[6] = 0x1f83d9abfb41bd6bULL;
		ctx->state.s64[7] = 0x5be0cd19137e2179ULL;
		break;
	case SHA512_224_MECH_INFO_TYPE:
		ctx->state.s64[0] = 0x8C3D37C819544DA2ULL;
		ctx->state.s64[1] = 0x73E1996689DCD4D6ULL;
		ctx->state.s64[2] = 0x1DFAB7AE32FF9C82ULL;
		ctx->state.s64[3] = 0x679DD514582F9FCFULL;
		ctx->state.s64[4] = 0x0F6D2B697BD44DA8ULL;
		ctx->state.s64[5] = 0x77E36F7304C48942ULL;
		ctx->state.s64[6] = 0x3F9D85A86A1D36C8ULL;
		ctx->state.s64[7] = 0x1112E6AD91D692A1ULL;
		break;
	case SHA512_256_MECH_INFO_TYPE:
		ctx->state.s64[0] = 0x22312194FC2BF72CULL;
		ctx->state.s64[1] = 0x9F555FA3C84C64C2ULL;
		ctx->state.s64[2] = 0x2393B86B6F53B151ULL;
		ctx->state.s64[3] = 0x963877195940EABDULL;
		ctx->state.s64[4] = 0x96283EE2A88EFFE3ULL;
		ctx->state.s64[5] = 0xBE5E1E2553863992ULL;
		ctx->state.s64[6] = 0x2B0199FC2C85B8AAULL;
		ctx->state.s64[7] = 0x0EB72DDC81C52CA2ULL;
		break;
#ifdef _KERNEL
	default:
		cmn_err(CE_PANIC,
		    "sha2_init: failed to find a supported algorithm: 0x%x",
		    (uint32_t)mech);

#endif /* _KERNEL */
	}

	ctx->algotype = (uint32_t)mech;
	ctx->count.c64[0] = ctx->count.c64[1] = 0;
}

#ifndef _KERNEL

void
SHA256Init(SHA256_CTX *ctx)
{
	SHA2Init(SHA256, ctx);
}

void
SHA384Init(SHA384_CTX *ctx)
{
	SHA2Init(SHA384, ctx);
}

void
SHA512Init(SHA512_CTX *ctx)
{
	SHA2Init(SHA512, ctx);
}

#endif /* _KERNEL */

/*
 * SHA2Update()
 *
 * purpose: continues an sha2 digest operation, using the message block
 *          to update the context.
 *   input: SHA2_CTX *	: the context to update
 *          void *	: the message block
 *          size_t      : the length of the message block, in bytes
 *  output: void
 */

void
SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
{
	size_t		i, buf_index, buf_len, buf_limit;
	const uint8_t	*input = inptr;
	uint32_t	algotype = ctx->algotype;
#if defined(__amd64)
	size_t		block_count;
#endif	/* !__amd64 */


	/* check for noop */
	if (input_len == 0)
		return;

	if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
		/*
		 * Extract low 32 bits of input_len; when we adjust
		 * count.c32[0] we must fold in the carry from the
		 * addition of the low bits along with the nonzero
		 * upper bits (if any) from input_len.
		 */
		uint32_t il = input_len & UINT32_MAX;

		il = il << 3;
		buf_limit = 64;

		/* compute number of bytes mod 64 */
		buf_index = (ctx->count.c32[1] >> 3) & 0x3F;

		/* update number of bits */
		if ((ctx->count.c32[1] += il) < il)
			ctx->count.c32[0]++;

		ctx->count.c32[0] += (input_len >> 29);

	} else {
		uint64_t il = input_len;

		il = il << 3;
		buf_limit = 128;

		/* compute number of bytes mod 128 */
		buf_index = (ctx->count.c64[1] >> 3) & 0x7F;

		/* update number of bits */
		if ((ctx->count.c64[1] += il) < il)
			ctx->count.c64[0]++;

		ctx->count.c64[0] += ((uintmax_t)input_len >> 61);
	}

	buf_len = buf_limit - buf_index;

	/* transform as many times as possible */
	i = 0;
	if (input_len >= buf_len) {

		/*
		 * general optimization:
		 *
		 * only do initial bcopy() and SHA2Transform() if
		 * buf_index != 0.  if buf_index == 0, we're just
		 * wasting our time doing the bcopy() since there
		 * wasn't any data left over from a previous call to
		 * SHA2Update().
		 */
		if (buf_index) {
			bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
			if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE)
				SHA256Transform(ctx, ctx->buf_un.buf8);
			else
				SHA512Transform(ctx, ctx->buf_un.buf8);

			i = buf_len;
		}

#if !defined(__amd64)
		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
			for (; i + buf_limit - 1 < input_len; i += buf_limit) {
				SHA256Transform(ctx, &input[i]);
			}
		} else {
			for (; i + buf_limit - 1 < input_len; i += buf_limit) {
				SHA512Transform(ctx, &input[i]);
			}
		}

#else
		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
			block_count = (input_len - i) >> 6;
			if (block_count > 0) {
				SHA256TransformBlocks(ctx, &input[i],
				    block_count);
				i += block_count << 6;
			}
		} else {
			block_count = (input_len - i) >> 7;
			if (block_count > 0) {
				SHA512TransformBlocks(ctx, &input[i],
				    block_count);
				i += block_count << 7;
			}
		}
#endif	/* !__amd64 */

		/*
		 * general optimization:
		 *
		 * if i and input_len are the same, return now instead
		 * of calling bcopy(), since the bcopy() in this case
		 * will be an expensive noop.
		 */

		if (input_len == i)
			return;

		buf_index = 0;
	}

	/* buffer remaining input */
	bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i);
}


/*
 * SHA2Final()
 *
 * purpose: ends an sha2 digest operation, finalizing the message digest and
 *          zeroing the context.
 *   input: uchar_t *	: a buffer to store the digest
 *			: The function actually uses void* because many
 *			: callers pass things other than uchar_t here.
 *          SHA2_CTX *  : the context to finalize, save, and zero
 *  output: void
 */

void
SHA2Final(void *digest, SHA2_CTX *ctx)
{
	uint8_t		bitcount_be[sizeof (ctx->count.c32)];
	uint8_t		bitcount_be64[sizeof (ctx->count.c64)];
	uint32_t	index;
	uint32_t	algotype = ctx->algotype;

	if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
		index  = (ctx->count.c32[1] >> 3) & 0x3f;
		Encode(bitcount_be, ctx->count.c32, sizeof (bitcount_be));
		SHA2Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index);
		SHA2Update(ctx, bitcount_be, sizeof (bitcount_be));
		Encode(digest, ctx->state.s32, sizeof (ctx->state.s32));
	} else {
		index  = (ctx->count.c64[1] >> 3) & 0x7f;
		Encode64(bitcount_be64, ctx->count.c64,
		    sizeof (bitcount_be64));
		SHA2Update(ctx, PADDING, ((index < 112) ? 112 : 240) - index);
		SHA2Update(ctx, bitcount_be64, sizeof (bitcount_be64));
		if (algotype <= SHA384_HMAC_GEN_MECH_INFO_TYPE) {
			ctx->state.s64[6] = ctx->state.s64[7] = 0;
			Encode64(digest, ctx->state.s64,
			    sizeof (uint64_t) * 6);
		} else if (algotype == SHA512_224_MECH_INFO_TYPE) {
			uint8_t last[sizeof (uint64_t)];
			/*
			 * Since SHA-512/224 doesn't align well to 64-bit
			 * boundaries, we must do the encoding in three steps:
			 * 1) encode the three 64-bit words that fit neatly
			 * 2) encode the last 64-bit word to a temp buffer
			 * 3) chop out the lower 32-bits from the temp buffer
			 *    and append them to the digest
			 */
			Encode64(digest, ctx->state.s64, sizeof (uint64_t) * 3);
			Encode64(last, &ctx->state.s64[3], sizeof (uint64_t));
			bcopy(last, (uint8_t *)digest + 24, 4);
		} else if (algotype == SHA512_256_MECH_INFO_TYPE) {
			Encode64(digest, ctx->state.s64, sizeof (uint64_t) * 4);
		} else {
			Encode64(digest, ctx->state.s64,
			    sizeof (ctx->state.s64));
		}
	}

	/* zeroize sensitive information */
	bzero(ctx, sizeof (*ctx));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SHA2_IMPL_H
#define	_SHA2_IMPL_H

#ifdef __cplusplus
extern "C" {
#endif

typedef enum {
	SHA1_TYPE,
	SHA256_TYPE,
	SHA384_TYPE,
	SHA512_TYPE
} sha2_mech_t;

#ifdef _KERNEL

/*
 * Context for SHA2 mechanism.
 */
typedef struct sha2_ctx {
	sha2_mech_type_t	sc_mech_type;	/* type of context */
	SHA2_CTX		sc_sha2_ctx;	/* SHA2 context */
} sha2_ctx_t;

/*
 * Context for SHA2 HMAC and HMAC GENERAL mechanisms.
 */
typedef struct sha2_hmac_ctx {
	sha2_mech_type_t	hc_mech_type;	/* type of context */
	uint32_t		hc_digest_len;	/* digest len in bytes */
	SHA2_CTX		hc_icontext;	/* inner SHA2 context */
	SHA2_CTX		hc_ocontext;	/* outer SHA2 context */
} sha2_hmac_ctx_t;

#endif

#ifdef	__cplusplus
}
#endif

#endif /* _SHA2_IMPL_H */