sha256_cryptopp: Add crypto++ 32-bit assembly implementation

This commit is contained in:
Jeff Garzik 2010-12-06 20:14:58 -05:00 committed by Jeff Garzik
parent 1a1a018627
commit f1fcd76ba7
3 changed files with 518 additions and 2 deletions

View file

@ -39,7 +39,8 @@ enum sha256_algos {
ALGO_C, /* plain C */ ALGO_C, /* plain C */
ALGO_4WAY, /* parallel SSE2 */ ALGO_4WAY, /* parallel SSE2 */
ALGO_VIA, /* VIA padlock */ ALGO_VIA, /* VIA padlock */
ALGO_CRYPTOPP, /* Crypto++ */ ALGO_CRYPTOPP, /* Crypto++ (C) */
ALGO_CRYPTOPP_ASM32, /* Crypto++ 32-bit assembly */
}; };
static const char *algo_names[] = { static const char *algo_names[] = {
@ -51,6 +52,9 @@ static const char *algo_names[] = {
[ALGO_VIA] = "via", [ALGO_VIA] = "via",
#endif #endif
[ALGO_CRYPTOPP] = "cryptopp", [ALGO_CRYPTOPP] = "cryptopp",
#ifdef WANT_CRYPTOPP_ASM32
[ALGO_CRYPTOPP_ASM32] = "cryptopp_asm32",
#endif
}; };
bool opt_debug = false; bool opt_debug = false;
@ -82,6 +86,9 @@ static struct option_help options_help[] = {
"\n\tvia\t\tVIA padlock implementation" "\n\tvia\t\tVIA padlock implementation"
#endif #endif
"\n\tcryptopp\tCrypto++ library implementation (EXPERIMENTAL)" "\n\tcryptopp\tCrypto++ library implementation (EXPERIMENTAL)"
#ifdef WANT_CRYPTOPP_ASM32
"\n\tcryptopp_asm32\tCrypto++ library implementation (EXPERIMENTAL)"
#endif
}, },
{ "debug", { "debug",
@ -294,6 +301,16 @@ static void *miner_thread(void *thr_id_int)
work.hash1, work.hash, &hashes_done); work.hash1, work.hash, &hashes_done);
break; break;
#ifdef WANT_CRYPTOPP_ASM32
case ALGO_CRYPTOPP_ASM32:
rc = scanhash_asm32(work.midstate, work.data + 64,
work.hash1, work.hash, &hashes_done);
break;
#endif
default:
/* should never happen */
return NULL;
} }
hashmeter(thr_id, &tv_start, hashes_done); hashmeter(thr_id, &tv_start, hashes_done);

View file

@ -14,6 +14,10 @@
#define WANT_VIA_PADLOCK 1 #define WANT_VIA_PADLOCK 1
#endif #endif
#if defined(__i386__)
#define WANT_CRYPTOPP_ASM32
#endif
#ifndef ARRAY_SIZE #ifndef ARRAY_SIZE
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
#endif #endif
@ -40,6 +44,9 @@ extern bool scanhash_c(const unsigned char *midstate, unsigned char *data,
extern bool scanhash_cryptopp(const unsigned char *midstate,unsigned char *data, extern bool scanhash_cryptopp(const unsigned char *midstate,unsigned char *data,
unsigned char *hash1, unsigned char *hash, unsigned char *hash1, unsigned char *hash,
unsigned long *hashes_done); unsigned long *hashes_done);
extern bool scanhash_asm32(const unsigned char *midstate,unsigned char *data,
unsigned char *hash1, unsigned char *hash,
unsigned long *hashes_done);
extern int extern int
timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y); timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y);

View file

@ -59,7 +59,7 @@ static const word32 SHA256_K[64] = {
static void SHA256_Transform(word32 *state, const word32 *data) static void SHA256_Transform(word32 *state, const word32 *data)
{ {
word32 W[16]; word32 W[16] = { };
word32 T[8]; word32 T[8];
unsigned int j; unsigned int j;
@ -131,3 +131,495 @@ bool scanhash_cryptopp(const unsigned char *midstate, unsigned char *data,
} }
} }
#if defined(WANT_CRYPTOPP_ASM32)
#define CRYPTOPP_FASTCALL
#define CRYPTOPP_BOOL_X86 1
#define CRYPTOPP_BOOL_X64 0
#define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0
#ifdef CRYPTOPP_GENERATE_X64_MASM
#define AS1(x) x*newline*
#define AS2(x, y) x, y*newline*
#define AS3(x, y, z) x, y, z*newline*
#define ASS(x, y, a, b, c, d) x, y, a*64+b*16+c*4+d*newline*
#define ASL(x) label##x:*newline*
#define ASJ(x, y, z) x label##y*newline*
#define ASC(x, y) x label##y*newline*
#define AS_HEX(y) 0##y##h
#elif defined(_MSC_VER) || defined(__BORLANDC__)
#define CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
#define AS1(x) __asm {x}
#define AS2(x, y) __asm {x, y}
#define AS3(x, y, z) __asm {x, y, z}
#define ASS(x, y, a, b, c, d) __asm {x, y, (a)*64+(b)*16+(c)*4+(d)}
#define ASL(x) __asm {label##x:}
#define ASJ(x, y, z) __asm {x label##y}
#define ASC(x, y) __asm {x label##y}
#define CRYPTOPP_NAKED __declspec(naked)
#define AS_HEX(y) 0x##y
#else
#define CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
// define these in two steps to allow arguments to be expanded
#define GNU_AS1(x) #x ";"
#define GNU_AS2(x, y) #x ", " #y ";"
#define GNU_AS3(x, y, z) #x ", " #y ", " #z ";"
#define GNU_ASL(x) "\n" #x ":"
#define GNU_ASJ(x, y, z) #x " " #y #z ";"
#define AS1(x) GNU_AS1(x)
#define AS2(x, y) GNU_AS2(x, y)
#define AS3(x, y, z) GNU_AS3(x, y, z)
#define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";"
#define ASL(x) GNU_ASL(x)
#define ASJ(x, y, z) GNU_ASJ(x, y, z)
#define ASC(x, y) #x " " #y ";"
#define CRYPTOPP_NAKED
#define AS_HEX(y) 0x##y
#endif
#define IF0(y)
#define IF1(y) y
#ifdef CRYPTOPP_GENERATE_X64_MASM
#define ASM_MOD(x, y) ((x) MOD (y))
#define XMMWORD_PTR XMMWORD PTR
#else
// GNU assembler doesn't seem to have mod operator
#define ASM_MOD(x, y) ((x)-((x)/(y))*(y))
// GAS 2.15 doesn't support XMMWORD PTR. it seems necessary only for MASM
#define XMMWORD_PTR
#endif
#if CRYPTOPP_BOOL_X86
#define AS_REG_1 ecx
#define AS_REG_2 edx
#define AS_REG_3 esi
#define AS_REG_4 edi
#define AS_REG_5 eax
#define AS_REG_6 ebx
#define AS_REG_7 ebp
#define AS_REG_1d ecx
#define AS_REG_2d edx
#define AS_REG_3d esi
#define AS_REG_4d edi
#define AS_REG_5d eax
#define AS_REG_6d ebx
#define AS_REG_7d ebp
#define WORD_SZ 4
#define WORD_REG(x) e##x
#define WORD_PTR DWORD PTR
#define AS_PUSH_IF86(x) AS1(push e##x)
#define AS_POP_IF86(x) AS1(pop e##x)
#define AS_JCXZ jecxz
#elif CRYPTOPP_BOOL_X64
#ifdef CRYPTOPP_GENERATE_X64_MASM
#define AS_REG_1 rcx
#define AS_REG_2 rdx
#define AS_REG_3 r8
#define AS_REG_4 r9
#define AS_REG_5 rax
#define AS_REG_6 r10
#define AS_REG_7 r11
#define AS_REG_1d ecx
#define AS_REG_2d edx
#define AS_REG_3d r8d
#define AS_REG_4d r9d
#define AS_REG_5d eax
#define AS_REG_6d r10d
#define AS_REG_7d r11d
#else
#define AS_REG_1 rdi
#define AS_REG_2 rsi
#define AS_REG_3 rdx
#define AS_REG_4 rcx
#define AS_REG_5 r8
#define AS_REG_6 r9
#define AS_REG_7 r10
#define AS_REG_1d edi
#define AS_REG_2d esi
#define AS_REG_3d edx
#define AS_REG_4d ecx
#define AS_REG_5d r8d
#define AS_REG_6d r9d
#define AS_REG_7d r10d
#endif
#define WORD_SZ 8
#define WORD_REG(x) r##x
#define WORD_PTR QWORD PTR
#define AS_PUSH_IF86(x)
#define AS_POP_IF86(x)
#define AS_JCXZ jrcxz
#endif
static void CRYPTOPP_FASTCALL X86_SHA256_HashBlocks(word32 *state, const word32 *data, size_t len
#if defined(_MSC_VER) && (_MSC_VER == 1200)
, ... // VC60 workaround: prevent VC 6 from inlining this function
#endif
)
{
#if defined(_MSC_VER) && (_MSC_VER == 1200)
AS2(mov ecx, [state])
AS2(mov edx, [data])
#endif
#define LOCALS_SIZE 8*4 + 16*4 + 4*WORD_SZ
#define H(i) [BASE+ASM_MOD(1024+7-(i),8)*4]
#define G(i) H(i+1)
#define F(i) H(i+2)
#define E(i) H(i+3)
#define D(i) H(i+4)
#define C(i) H(i+5)
#define B(i) H(i+6)
#define A(i) H(i+7)
#define Wt(i) BASE+8*4+ASM_MOD(1024+15-(i),16)*4
#define Wt_2(i) Wt((i)-2)
#define Wt_15(i) Wt((i)-15)
#define Wt_7(i) Wt((i)-7)
#define K_END [BASE+8*4+16*4+0*WORD_SZ]
#define STATE_SAVE [BASE+8*4+16*4+1*WORD_SZ]
#define DATA_SAVE [BASE+8*4+16*4+2*WORD_SZ]
#define DATA_END [BASE+8*4+16*4+3*WORD_SZ]
#define Kt(i) WORD_REG(si)+(i)*4
#if CRYPTOPP_BOOL_X86
#define BASE esp+4
#elif defined(__GNUC__)
#define BASE r8
#else
#define BASE rsp
#endif
#define RA0(i, edx, edi) \
AS2( add edx, [Kt(i)] )\
AS2( add edx, [Wt(i)] )\
AS2( add edx, H(i) )\
#define RA1(i, edx, edi)
#define RB0(i, edx, edi)
#define RB1(i, edx, edi) \
AS2( mov AS_REG_7d, [Wt_2(i)] )\
AS2( mov edi, [Wt_15(i)])\
AS2( mov ebx, AS_REG_7d )\
AS2( shr AS_REG_7d, 10 )\
AS2( ror ebx, 17 )\
AS2( xor AS_REG_7d, ebx )\
AS2( ror ebx, 2 )\
AS2( xor ebx, AS_REG_7d )/* s1(W_t-2) */\
AS2( add ebx, [Wt_7(i)])\
AS2( mov AS_REG_7d, edi )\
AS2( shr AS_REG_7d, 3 )\
AS2( ror edi, 7 )\
AS2( add ebx, [Wt(i)])/* s1(W_t-2) + W_t-7 + W_t-16 */\
AS2( xor AS_REG_7d, edi )\
AS2( add edx, [Kt(i)])\
AS2( ror edi, 11 )\
AS2( add edx, H(i) )\
AS2( xor AS_REG_7d, edi )/* s0(W_t-15) */\
AS2( add AS_REG_7d, ebx )/* W_t = s1(W_t-2) + W_t-7 + s0(W_t-15) W_t-16*/\
AS2( mov [Wt(i)], AS_REG_7d)\
AS2( add edx, AS_REG_7d )\
#define ROUND(i, r, eax, ecx, edi, edx)\
/* in: edi = E */\
/* unused: eax, ecx, temp: ebx, AS_REG_7d, out: edx = T1 */\
AS2( mov edx, F(i) )\
AS2( xor edx, G(i) )\
AS2( and edx, edi )\
AS2( xor edx, G(i) )/* Ch(E,F,G) = (G^(E&(F^G))) */\
AS2( mov AS_REG_7d, edi )\
AS2( ror edi, 6 )\
AS2( ror AS_REG_7d, 25 )\
RA##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\
AS2( xor AS_REG_7d, edi )\
AS2( ror edi, 5 )\
AS2( xor AS_REG_7d, edi )/* S1(E) */\
AS2( add edx, AS_REG_7d )/* T1 = S1(E) + Ch(E,F,G) + H + Wt + Kt */\
RB##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\
/* in: ecx = A, eax = B^C, edx = T1 */\
/* unused: edx, temp: ebx, AS_REG_7d, out: eax = A, ecx = B^C, edx = E */\
AS2( mov ebx, ecx )\
AS2( xor ecx, B(i) )/* A^B */\
AS2( and eax, ecx )\
AS2( xor eax, B(i) )/* Maj(A,B,C) = B^((A^B)&(B^C) */\
AS2( mov AS_REG_7d, ebx )\
AS2( ror ebx, 2 )\
AS2( add eax, edx )/* T1 + Maj(A,B,C) */\
AS2( add edx, D(i) )\
AS2( mov D(i), edx )\
AS2( ror AS_REG_7d, 22 )\
AS2( xor AS_REG_7d, ebx )\
AS2( ror ebx, 11 )\
AS2( xor AS_REG_7d, ebx )\
AS2( add eax, AS_REG_7d )/* T1 + S0(A) + Maj(A,B,C) */\
AS2( mov H(i), eax )\
#define SWAP_COPY(i) \
AS2( mov WORD_REG(bx), [WORD_REG(dx)+i*WORD_SZ])\
AS1( bswap WORD_REG(bx))\
AS2( mov [Wt(i*(1+CRYPTOPP_BOOL_X64)+CRYPTOPP_BOOL_X64)], WORD_REG(bx))
#if defined(__GNUC__)
#if CRYPTOPP_BOOL_X64
FixedSizeAlignedSecBlock<byte, LOCALS_SIZE> workspace;
#endif
__asm__ __volatile__
(
#if CRYPTOPP_BOOL_X64
"lea %4, %%r8;"
#endif
".intel_syntax noprefix;"
#elif defined(CRYPTOPP_GENERATE_X64_MASM)
ALIGN 8
X86_SHA256_HashBlocks PROC FRAME
rex_push_reg rsi
push_reg rdi
push_reg rbx
push_reg rbp
alloc_stack(LOCALS_SIZE+8)
.endprolog
mov rdi, r8
lea rsi, [?SHA256_K@CryptoPP@@3QBIB + 48*4]
#endif
#if CRYPTOPP_BOOL_X86
#ifndef __GNUC__
AS2( mov edi, [len])
AS2( lea WORD_REG(si), [SHA256_K+48*4])
#endif
#if !defined(_MSC_VER) || (_MSC_VER < 1400)
AS_PUSH_IF86(bx)
#endif
AS_PUSH_IF86(bp)
AS2( mov ebx, esp)
AS2( and esp, -16)
AS2( sub WORD_REG(sp), LOCALS_SIZE)
AS_PUSH_IF86(bx)
#endif
AS2( mov STATE_SAVE, WORD_REG(cx))
AS2( mov DATA_SAVE, WORD_REG(dx))
AS2( lea WORD_REG(ax), [WORD_REG(di) + WORD_REG(dx)])
AS2( mov DATA_END, WORD_REG(ax))
AS2( mov K_END, WORD_REG(si))
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
#if CRYPTOPP_BOOL_X86
AS2( test edi, 1)
ASJ( jnz, 2, f)
AS1( dec DWORD PTR K_END)
#endif
AS2( movdqa xmm0, XMMWORD_PTR [WORD_REG(cx)+0*16])
AS2( movdqa xmm1, XMMWORD_PTR [WORD_REG(cx)+1*16])
#endif
#if CRYPTOPP_BOOL_X86
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
ASJ( jmp, 0, f)
#endif
ASL(2) // non-SSE2
AS2( mov esi, ecx)
AS2( lea edi, A(0))
AS2( mov ecx, 8)
AS1( rep movsd)
AS2( mov esi, K_END)
ASJ( jmp, 3, f)
#endif
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
ASL(0)
AS2( movdqa E(0), xmm1)
AS2( movdqa A(0), xmm0)
#endif
#if CRYPTOPP_BOOL_X86
ASL(3)
#endif
AS2( sub WORD_REG(si), 48*4)
SWAP_COPY(0) SWAP_COPY(1) SWAP_COPY(2) SWAP_COPY(3)
SWAP_COPY(4) SWAP_COPY(5) SWAP_COPY(6) SWAP_COPY(7)
#if CRYPTOPP_BOOL_X86
SWAP_COPY(8) SWAP_COPY(9) SWAP_COPY(10) SWAP_COPY(11)
SWAP_COPY(12) SWAP_COPY(13) SWAP_COPY(14) SWAP_COPY(15)
#endif
AS2( mov edi, E(0)) // E
AS2( mov eax, B(0)) // B
AS2( xor eax, C(0)) // B^C
AS2( mov ecx, A(0)) // A
ROUND(0, 0, eax, ecx, edi, edx)
ROUND(1, 0, ecx, eax, edx, edi)
ROUND(2, 0, eax, ecx, edi, edx)
ROUND(3, 0, ecx, eax, edx, edi)
ROUND(4, 0, eax, ecx, edi, edx)
ROUND(5, 0, ecx, eax, edx, edi)
ROUND(6, 0, eax, ecx, edi, edx)
ROUND(7, 0, ecx, eax, edx, edi)
ROUND(8, 0, eax, ecx, edi, edx)
ROUND(9, 0, ecx, eax, edx, edi)
ROUND(10, 0, eax, ecx, edi, edx)
ROUND(11, 0, ecx, eax, edx, edi)
ROUND(12, 0, eax, ecx, edi, edx)
ROUND(13, 0, ecx, eax, edx, edi)
ROUND(14, 0, eax, ecx, edi, edx)
ROUND(15, 0, ecx, eax, edx, edi)
ASL(1)
AS2(add WORD_REG(si), 4*16)
ROUND(0, 1, eax, ecx, edi, edx)
ROUND(1, 1, ecx, eax, edx, edi)
ROUND(2, 1, eax, ecx, edi, edx)
ROUND(3, 1, ecx, eax, edx, edi)
ROUND(4, 1, eax, ecx, edi, edx)
ROUND(5, 1, ecx, eax, edx, edi)
ROUND(6, 1, eax, ecx, edi, edx)
ROUND(7, 1, ecx, eax, edx, edi)
ROUND(8, 1, eax, ecx, edi, edx)
ROUND(9, 1, ecx, eax, edx, edi)
ROUND(10, 1, eax, ecx, edi, edx)
ROUND(11, 1, ecx, eax, edx, edi)
ROUND(12, 1, eax, ecx, edi, edx)
ROUND(13, 1, ecx, eax, edx, edi)
ROUND(14, 1, eax, ecx, edi, edx)
ROUND(15, 1, ecx, eax, edx, edi)
AS2( cmp WORD_REG(si), K_END)
ASJ( jb, 1, b)
AS2( mov WORD_REG(dx), DATA_SAVE)
AS2( add WORD_REG(dx), 64)
AS2( mov AS_REG_7, STATE_SAVE)
AS2( mov DATA_SAVE, WORD_REG(dx))
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
#if CRYPTOPP_BOOL_X86
AS2( test DWORD PTR K_END, 1)
ASJ( jz, 4, f)
#endif
AS2( movdqa xmm1, XMMWORD_PTR [AS_REG_7+1*16])
AS2( movdqa xmm0, XMMWORD_PTR [AS_REG_7+0*16])
AS2( paddd xmm1, E(0))
AS2( paddd xmm0, A(0))
AS2( movdqa [AS_REG_7+1*16], xmm1)
AS2( movdqa [AS_REG_7+0*16], xmm0)
AS2( cmp WORD_REG(dx), DATA_END)
ASJ( jb, 0, b)
#endif
#if CRYPTOPP_BOOL_X86
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
ASJ( jmp, 5, f)
ASL(4) // non-SSE2
#endif
AS2( add [AS_REG_7+0*4], ecx) // A
AS2( add [AS_REG_7+4*4], edi) // E
AS2( mov eax, B(0))
AS2( mov ebx, C(0))
AS2( mov ecx, D(0))
AS2( add [AS_REG_7+1*4], eax)
AS2( add [AS_REG_7+2*4], ebx)
AS2( add [AS_REG_7+3*4], ecx)
AS2( mov eax, F(0))
AS2( mov ebx, G(0))
AS2( mov ecx, H(0))
AS2( add [AS_REG_7+5*4], eax)
AS2( add [AS_REG_7+6*4], ebx)
AS2( add [AS_REG_7+7*4], ecx)
AS2( mov ecx, AS_REG_7d)
AS2( cmp WORD_REG(dx), DATA_END)
ASJ( jb, 2, b)
#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
ASL(5)
#endif
#endif
AS_POP_IF86(sp)
AS_POP_IF86(bp)
#if !defined(_MSC_VER) || (_MSC_VER < 1400)
AS_POP_IF86(bx)
#endif
#ifdef CRYPTOPP_GENERATE_X64_MASM
add rsp, LOCALS_SIZE+8
pop rbp
pop rbx
pop rdi
pop rsi
ret
X86_SHA256_HashBlocks ENDP
#endif
#ifdef __GNUC__
".att_syntax prefix;"
:
: "c" (state), "d" (data), "S" (SHA256_K+48), "D" (len)
#if CRYPTOPP_BOOL_X64
, "m" (workspace[0])
#endif
: "memory", "cc", "%eax"
#if CRYPTOPP_BOOL_X64
, "%rbx", "%r8", "%r10"
#endif
);
#endif
}
static inline bool HasSSE2(void) { return false; }
static void SHA256_Transform32(word32 *state, const word32 *data)
{
word32 W[16];
int i;
for (i = 0; i < 16; i++)
W[i] = ((word32 *)(data))[i];
X86_SHA256_HashBlocks(state, W, 16 * 4);
}
static void runhash32(void *state, const void *input, const void *init)
{
memcpy(state, init, 32);
SHA256_Transform32(state, input);
}
/* suspiciously similar to ScanHash* from bitcoin */
bool scanhash_asm32(const unsigned char *midstate, unsigned char *data,
unsigned char *hash1, unsigned char *hash,
unsigned long *hashes_done)
{
uint32_t *hash32 = (uint32_t *) hash;
uint32_t *nonce = (uint32_t *)(data + 12);
uint32_t n = 0;
unsigned long stat_ctr = 0;
while (1) {
n++;
*nonce = n;
runhash32(hash1, data, midstate);
runhash32(hash, hash1, sha256_init_state);
stat_ctr++;
if (hash32[7] == 0) {
char *hexstr;
hexstr = bin2hex(hash, 32);
fprintf(stderr,
"DBG: found zeroes in hash:\n%s\n",
hexstr);
free(hexstr);
*hashes_done = stat_ctr;
return true;
}
if ((n & 0xffffff) == 0) {
if (opt_debug)
fprintf(stderr, "DBG: end of nonce range\n");
*hashes_done = stat_ctr;
return false;
}
}
}
#endif // #if defined(WANT_CRYPTOPP_ASM32)