00001
00002
00003
00004
00005 #include "pch.h"
00006
00007 #ifndef CRYPTOPP_IMPORTS
00008 #ifndef CRYPTOPP_GENERATE_X64_MASM
00009
00010 #include "gcm.h"
00011 #include "cpu.h"
00012
00013 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
00014 #include <emmintrin.h>
00015 #endif
00016
00017 NAMESPACE_BEGIN(CryptoPP)
00018
00019 word16 GCM_Base::s_reductionTable[256];
00020 bool GCM_Base::s_reductionTableInitialized = false;
00021
00022 void GCM_Base::GCTR::IncrementCounterBy256()
00023 {
00024 IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
00025 }
00026
00027 #if 0
00028
00029 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
00030 {
00031 word64 Z0=0, Z1=0, V0, V1;
00032
00033 typedef BlockGetAndPut<word64, BigEndian> Block;
00034 Block::Get(a)(V0)(V1);
00035
00036 for (int i=0; i<16; i++)
00037 {
00038 for (int j=0x80; j!=0; j>>=1)
00039 {
00040 int x = b[i] & j;
00041 Z0 ^= x ? V0 : 0;
00042 Z1 ^= x ? V1 : 0;
00043 x = (int)V1 & 1;
00044 V1 = (V1>>1) | (V0<<63);
00045 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00046 }
00047 }
00048 Block::Put(NULL, c)(Z0)(Z1);
00049 }
00050 #endif
00051
00052 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00053 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
00054 {
00055 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
00056 *(__m128i *)a = _mm_xor_si128(*(__m128i *)b, *(__m128i *)c);
00057 #else
00058 asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
00059 #endif
00060 }
00061 #endif
00062
00063 inline static void Xor16(byte *a, const byte *b, const byte *c)
00064 {
00065 ((word64 *)a)[0] = ((word64 *)b)[0] ^ ((word64 *)c)[0];
00066 ((word64 *)a)[1] = ((word64 *)b)[1] ^ ((word64 *)c)[1];
00067 }
00068
00069 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs ¶ms)
00070 {
00071 BlockCipher &blockCipher = AccessBlockCipher();
00072 blockCipher.SetKey(userKey, keylength, params);
00073
00074 if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
00075 throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
00076
00077 int tableSize;
00078 if (params.GetIntValue(Name::TableSize(), tableSize))
00079 tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
00080 else
00081 tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
00082
00083 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
00084
00085 tableSize = 2*1024;
00086 #endif
00087
00088 m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
00089 byte *hashKey = HashKey();
00090 memset(hashKey, 0, REQUIRED_BLOCKSIZE);
00091 blockCipher.ProcessBlock(hashKey);
00092
00093 byte *table = MulTable();
00094 int i, j, k;
00095 word64 V0, V1;
00096
00097 typedef BlockGetAndPut<word64, BigEndian> Block;
00098 Block::Get(hashKey)(V0)(V1);
00099
00100 if (tableSize == 64*1024)
00101 {
00102 for (i=0; i<128; i++)
00103 {
00104 k = i%8;
00105 Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
00106
00107 int x = (int)V1 & 1;
00108 V1 = (V1>>1) | (V0<<63);
00109 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00110 }
00111
00112 for (i=0; i<16; i++)
00113 {
00114 memset(table+i*256*16, 0, 16);
00115 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00116 if (HasSSE2())
00117 for (j=2; j<=0x80; j*=2)
00118 for (k=1; k<j; k++)
00119 SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
00120 else
00121 #endif
00122 for (j=2; j<=0x80; j*=2)
00123 for (k=1; k<j; k++)
00124 Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
00125 }
00126 }
00127 else
00128 {
00129 if (!s_reductionTableInitialized)
00130 {
00131 s_reductionTable[0] = 0;
00132 word16 x = 0x01c2;
00133 s_reductionTable[1] = ByteReverse(x);
00134 for (int i=2; i<=0x80; i*=2)
00135 {
00136 x <<= 1;
00137 s_reductionTable[i] = ByteReverse(x);
00138 for (int j=1; j<i; j++)
00139 s_reductionTable[i+j] = s_reductionTable[i] ^ s_reductionTable[j];
00140 }
00141 s_reductionTableInitialized = true;
00142 }
00143
00144 for (i=0; i<128-24; i++)
00145 {
00146 k = i%32;
00147 if (k < 4)
00148 Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
00149 else if (k < 8)
00150 Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
00151
00152 int x = (int)V1 & 1;
00153 V1 = (V1>>1) | (V0<<63);
00154 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00155 }
00156
00157 for (i=0; i<4; i++)
00158 {
00159 memset(table+i*256, 0, 16);
00160 memset(table+1024+i*256, 0, 16);
00161 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00162 if (HasSSE2())
00163 for (j=2; j<=8; j*=2)
00164 for (k=1; k<j; k++)
00165 {
00166 SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
00167 SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
00168 }
00169 else
00170 #endif
00171 for (j=2; j<=8; j*=2)
00172 for (k=1; k<j; k++)
00173 {
00174 Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
00175 Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
00176 }
00177 }
00178 }
00179 }
00180
00181 void GCM_Base::Resync(const byte *iv, size_t len)
00182 {
00183 BlockCipher &cipher = AccessBlockCipher();
00184 byte *hashBuffer = HashBuffer();
00185
00186 if (len == 12)
00187 {
00188 memcpy(hashBuffer, iv, len);
00189 memset(hashBuffer+len, 0, 3);
00190 hashBuffer[len+3] = 1;
00191 }
00192 else
00193 {
00194 size_t origLen = len;
00195 memset(hashBuffer, 0, HASH_BLOCKSIZE);
00196
00197 if (len >= HASH_BLOCKSIZE)
00198 {
00199 len = GCM_Base::AuthenticateBlocks(iv, len);
00200 iv += (origLen - len);
00201 }
00202
00203 if (len > 0)
00204 {
00205 memcpy(m_buffer, iv, len);
00206 memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
00207 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00208 }
00209
00210 PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
00211 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00212 }
00213
00214 if (m_state >= State_IVSet)
00215 m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
00216 else
00217 m_ctr.SetCipherWithIV(cipher, hashBuffer);
00218
00219 m_ctr.Seek(HASH_BLOCKSIZE);
00220
00221 memset(hashBuffer, 0, HASH_BLOCKSIZE);
00222 }
00223
00224 unsigned int GCM_Base::OptimalDataAlignment() const
00225 {
00226 return HasSSE2() ? 16 : GetBlockCipher().OptimalDataAlignment();
00227 }
00228
00229 #pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
00230
00231 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00232
00233 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
00234 extern "C" {
00235 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
00236 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
00237 }
00238 #endif
00239
00240 #ifndef CRYPTOPP_GENERATE_X64_MASM
00241
00242 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
00243 {
00244 typedef BlockGetAndPut<word64, NativeByteOrder> Block;
00245 word64 *hashBuffer = (word64 *)HashBuffer();
00246
00247 switch (2*(m_buffer.size()>=64*1024)
00248 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
00249 + HasSSE2()
00250 #endif
00251 )
00252 {
00253 case 0:
00254 {
00255 byte *table = MulTable();
00256 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
00257
00258 do
00259 {
00260 word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
00261 Block::Get(data)(y0)(y1);
00262 x0 ^= y0;
00263 x1 ^= y1;
00264
00265 data += HASH_BLOCKSIZE;
00266 len -= HASH_BLOCKSIZE;
00267
00268 #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(table+(a*1024)+(b*256)+c+d*8)
00269
00270 #ifdef IS_LITTLE_ENDIAN
00271 #if CRYPTOPP_BOOL_SLOW_WORD64
00272 word32 z0 = (word32)x0;
00273 word32 z1 = (word32)(x0>>32);
00274 word32 z2 = (word32)x1;
00275 word32 z3 = (word32)(x1>>32);
00276 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
00277 #else
00278 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
00279 #endif
00280 #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
00281 #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
00282 #else
00283 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
00284 #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
00285 #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
00286 #endif
00287
00288 #define GF_MUL_32BY128(op, a, b, c) \
00289 a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
00290 a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
00291 b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
00292 b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
00293 c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
00294 c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
00295 d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
00296 d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
00297
00298 GF_MUL_32BY128(=, 0, 0, 0)
00299 GF_MUL_32BY128(^=, 0, 1, 1)
00300 GF_MUL_32BY128(^=, 1, 0, 2)
00301 GF_MUL_32BY128(^=, 1, 1, 3)
00302
00303 word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
00304 GF_SHIFT_8(d)
00305 c0 ^= d0; c1 ^= d1;
00306 r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
00307 GF_SHIFT_8(c)
00308 b0 ^= c0; b1 ^= c1;
00309 r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
00310 GF_SHIFT_8(b)
00311 a0 ^= b0; a1 ^= b1;
00312 a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
00313 x0 = a0; x1 = a1;
00314 }
00315 while (len >= HASH_BLOCKSIZE);
00316
00317 hashBuffer[0] = x0; hashBuffer[1] = x1;
00318 return len;
00319 }
00320
00321 case 2:
00322 {
00323 byte *table = MulTable();
00324 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
00325
00326 do
00327 {
00328 word64 y0, y1, a0, a1;
00329 Block::Get(data)(y0)(y1);
00330 x0 ^= y0;
00331 x1 ^= y1;
00332
00333 data += HASH_BLOCKSIZE;
00334 len -= HASH_BLOCKSIZE;
00335
00336 #undef READ_TABLE_WORD64_COMMON
00337 #undef READ_TABLE_WORD64
00338
00339 #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(table+(a)*256*16+(c)+(d)*8)
00340
00341 #ifdef IS_LITTLE_ENDIAN
00342 #if CRYPTOPP_BOOL_SLOW_WORD64
00343 word32 z0 = (word32)x0;
00344 word32 z1 = (word32)(x0>>32);
00345 word32 z2 = (word32)x1;
00346 word32 z3 = (word32)(x1>>32);
00347 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
00348 #else
00349 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
00350 #endif
00351 #else
00352 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
00353 #endif
00354
00355 #define GF_MUL_8BY128(op, b, c, d) \
00356 a0 op READ_TABLE_WORD64(b, c, d, 0);\
00357 a1 op READ_TABLE_WORD64(b, c, d, 1);\
00358
00359 GF_MUL_8BY128(=, 0, 0, 0)
00360 GF_MUL_8BY128(^=, 0, 0, 1)
00361 GF_MUL_8BY128(^=, 0, 0, 2)
00362 GF_MUL_8BY128(^=, 0, 0, 3)
00363 GF_MUL_8BY128(^=, 0, 1, 0)
00364 GF_MUL_8BY128(^=, 0, 1, 1)
00365 GF_MUL_8BY128(^=, 0, 1, 2)
00366 GF_MUL_8BY128(^=, 0, 1, 3)
00367 GF_MUL_8BY128(^=, 1, 2, 0)
00368 GF_MUL_8BY128(^=, 1, 2, 1)
00369 GF_MUL_8BY128(^=, 1, 2, 2)
00370 GF_MUL_8BY128(^=, 1, 2, 3)
00371 GF_MUL_8BY128(^=, 1, 3, 0)
00372 GF_MUL_8BY128(^=, 1, 3, 1)
00373 GF_MUL_8BY128(^=, 1, 3, 2)
00374 GF_MUL_8BY128(^=, 1, 3, 3)
00375
00376 x0 = a0; x1 = a1;
00377 }
00378 while (len >= HASH_BLOCKSIZE);
00379
00380 hashBuffer[0] = x0; hashBuffer[1] = x1;
00381 return len;
00382 }
00383 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00384
00385 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
00386 case 1:
00387 GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
00388 return len % 16;
00389 case 3:
00390 GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
00391 return len % 16;
00392 #endif
00393
00394 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00395 case 1:
00396 {
00397 #ifdef __GNUC__
00398 __asm__ __volatile__
00399 (
00400 ".intel_syntax noprefix;"
00401 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00402 ALIGN 8
00403 GCM_AuthenticateBlocks_2K PROC FRAME
00404 rex_push_reg rsi
00405 push_reg rdi
00406 push_reg rbx
00407 .endprolog
00408 mov rsi, r8
00409 mov r11, r9
00410 #else
00411 AS2( mov WORD_REG(cx), data )
00412 AS2( mov WORD_REG(dx), len )
00413 AS2( mov WORD_REG(si), hashBuffer )
00414 AS2( shr WORD_REG(dx), 4 )
00415 #endif
00416
00417 #if !defined(_MSC_VER) || (_MSC_VER < 1400)
00418 AS_PUSH_IF86( bx)
00419 #endif
00420 AS_PUSH_IF86( bp)
00421
00422 #ifdef __GNUC__
00423 AS2( mov AS_REG_7, WORD_REG(di))
00424 #elif CRYPTOPP_BOOL_X86
00425 AS2( lea AS_REG_7, s_reductionTable)
00426 #endif
00427
00428 AS2( movdqa xmm0, [WORD_REG(si)] )
00429
00430 #define MUL_TABLE_0 WORD_REG(si) + 32
00431 #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
00432 #define RED_TABLE AS_REG_7
00433
00434 ASL(0)
00435 AS2( movdqu xmm4, [WORD_REG(cx)] )
00436 AS2( pxor xmm0, xmm4 )
00437
00438 AS2( movd ebx, xmm0 )
00439 AS2( mov eax, AS_HEX(f0f0f0f0) )
00440 AS2( and eax, ebx )
00441 AS2( shl ebx, 4 )
00442 AS2( and ebx, AS_HEX(f0f0f0f0) )
00443 AS2( movzx edi, ah )
00444 AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00445 AS2( movzx edi, al )
00446 AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00447 AS2( shr eax, 16 )
00448 AS2( movzx edi, ah )
00449 AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00450 AS2( movzx edi, al )
00451 AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00452
00453 #define SSE2_MUL_32BITS(i) \
00454 AS2( psrldq xmm0, 4 )\
00455 AS2( movd eax, xmm0 )\
00456 AS2( and eax, AS_HEX(f0f0f0f0) )\
00457 AS2( movzx edi, bh )\
00458 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00459 AS2( movzx edi, bl )\
00460 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00461 AS2( shr ebx, 16 )\
00462 AS2( movzx edi, bh )\
00463 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00464 AS2( movzx edi, bl )\
00465 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00466 AS2( movd ebx, xmm0 )\
00467 AS2( shl ebx, 4 )\
00468 AS2( and ebx, AS_HEX(f0f0f0f0) )\
00469 AS2( movzx edi, ah )\
00470 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00471 AS2( movzx edi, al )\
00472 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00473 AS2( shr eax, 16 )\
00474 AS2( movzx edi, ah )\
00475 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00476 AS2( movzx edi, al )\
00477 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00478
00479 SSE2_MUL_32BITS(1)
00480 SSE2_MUL_32BITS(2)
00481 SSE2_MUL_32BITS(3)
00482
00483 AS2( movzx edi, bh )
00484 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00485 AS2( movzx edi, bl )
00486 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00487 AS2( shr ebx, 16 )
00488 AS2( movzx edi, bh )
00489 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00490 AS2( movzx edi, bl )
00491 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00492
00493 AS2( movdqa xmm0, xmm3 )
00494 AS2( pslldq xmm3, 1 )
00495 AS2( pxor xmm2, xmm3 )
00496 AS2( movdqa xmm1, xmm2 )
00497 AS2( pslldq xmm2, 1 )
00498 AS2( pxor xmm5, xmm2 )
00499
00500 AS2( psrldq xmm0, 15 )
00501 AS2( movd WORD_REG(di), xmm0 )
00502 AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00503 AS2( shl eax, 8 )
00504
00505 AS2( movdqa xmm0, xmm5 )
00506 AS2( pslldq xmm5, 1 )
00507 AS2( pxor xmm4, xmm5 )
00508
00509 AS2( psrldq xmm1, 15 )
00510 AS2( movd WORD_REG(di), xmm1 )
00511 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00512 AS2( shl eax, 8 )
00513
00514 AS2( psrldq xmm0, 15 )
00515 AS2( movd WORD_REG(di), xmm0 )
00516 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00517
00518 AS2( movd xmm0, eax )
00519 AS2( pxor xmm0, xmm4 )
00520
00521 AS2( add WORD_REG(cx), 16 )
00522 AS2( sub WORD_REG(dx), 1 )
00523 ASJ( jnz, 0, b )
00524 AS2( movdqa [WORD_REG(si)], xmm0 )
00525
00526 AS_POP_IF86( bp)
00527 #if !defined(_MSC_VER) || (_MSC_VER < 1400)
00528 AS_POP_IF86( bx)
00529 #endif
00530
00531 #ifdef __GNUC__
00532 ".att_syntax prefix;"
00533 :
00534 : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
00535 : "memory", "cc", "%eax"
00536 #if CRYPTOPP_BOOL_X64
00537 , "%ebx", "%r11"
00538 #endif
00539 );
00540 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00541 pop rbx
00542 pop rdi
00543 pop rsi
00544 ret
00545 GCM_AuthenticateBlocks_2K ENDP
00546 #endif
00547
00548 return len%16;
00549 }
00550 case 3:
00551 {
00552 #ifdef __GNUC__
00553 __asm__ __volatile__
00554 (
00555 ".intel_syntax noprefix;"
00556 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00557 ALIGN 8
00558 GCM_AuthenticateBlocks_64K PROC FRAME
00559 rex_push_reg rsi
00560 push_reg rdi
00561 .endprolog
00562 mov rsi, r8
00563 #else
00564 AS2( mov WORD_REG(cx), data )
00565 AS2( mov WORD_REG(dx), len )
00566 AS2( mov WORD_REG(si), hashBuffer )
00567 AS2( shr WORD_REG(dx), 4 )
00568 #endif
00569
00570 AS2( movdqa xmm0, [WORD_REG(si)] )
00571
00572 #undef MUL_TABLE
00573 #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
00574
00575 ASL(1)
00576 AS2( movdqu xmm1, [WORD_REG(cx)] )
00577 AS2( pxor xmm1, xmm0 )
00578 AS2( pxor xmm0, xmm0 )
00579
00580 #undef SSE2_MUL_32BITS
00581 #define SSE2_MUL_32BITS(i) \
00582 AS2( movd eax, xmm1 )\
00583 AS2( psrldq xmm1, 4 )\
00584 AS2( movzx edi, al )\
00585 AS2( add WORD_REG(di), WORD_REG(di) )\
00586 AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
00587 AS2( movzx edi, ah )\
00588 AS2( add WORD_REG(di), WORD_REG(di) )\
00589 AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
00590 AS2( shr eax, 16 )\
00591 AS2( movzx edi, al )\
00592 AS2( add WORD_REG(di), WORD_REG(di) )\
00593 AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
00594 AS2( movzx edi, ah )\
00595 AS2( add WORD_REG(di), WORD_REG(di) )\
00596 AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
00597
00598 SSE2_MUL_32BITS(0)
00599 SSE2_MUL_32BITS(1)
00600 SSE2_MUL_32BITS(2)
00601 SSE2_MUL_32BITS(3)
00602
00603 AS2( add WORD_REG(cx), 16 )
00604 AS2( sub WORD_REG(dx), 1 )
00605 ASJ( jnz, 1, b )
00606 AS2( movdqa [WORD_REG(si)], xmm0 )
00607
00608 #ifdef __GNUC__
00609 ".att_syntax prefix;"
00610 :
00611 : "c" (data), "d" (len/16), "S" (hashBuffer)
00612 : "memory", "cc", "%edi", "%eax"
00613 );
00614 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00615 pop rdi
00616 pop rsi
00617 ret
00618 GCM_AuthenticateBlocks_64K ENDP
00619 #endif
00620
00621 return len%16;
00622 }
00623 #endif
00624 #ifndef CRYPTOPP_GENERATE_X64_MASM
00625 }
00626
00627 return len%16;
00628 }
00629
00630 void GCM_Base::AuthenticateLastHeaderBlock()
00631 {
00632 if (m_bufferedDataLength > 0)
00633 {
00634 memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
00635 m_bufferedDataLength = 0;
00636 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00637 }
00638 }
00639
00640 void GCM_Base::AuthenticateLastConfidentialBlock()
00641 {
00642 GCM_Base::AuthenticateLastHeaderBlock();
00643 PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
00644 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00645 }
00646
00647 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
00648 {
00649 m_ctr.Seek(0);
00650 m_ctr.ProcessData(mac, HashBuffer(), macSize);
00651 }
00652
00653 NAMESPACE_END
00654
00655 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00656 #endif