Add BoringSSL Dispatch Test for aarch64 (aws#1093)

add armv8 dispatch tests checking for the following function hits:
* aes_hw_ctr32_encrypt_blocks
* aes_hw_encrypt
* aes_gcm_enc_kernel
* aes_hw_set_encrypt_key
* vpaes_encrypt
* vpaes_set_encrypt_key
* sha256_block_armv8
* aesv8_gcm_8x_enc_128
* sha512_block_armv8

Author: billbo-yang

crypto/fipsmodule/aes/asm/aesv8-armx.pl

@@ -99,6 +99,13 @@
 .Lenc_key:
 ___
 $code.=<<___	if ($flavour =~ /64/);
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key
+#endif
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 	AARCH64_VALID_CALL_TARGET
 	stp	x29,x30,[sp,#-16]!
@@ -343,6 +350,17 @@ ()
 .type	${prefix}_${dir}crypt,%function
 .align	5
 ${prefix}_${dir}crypt:
+___
+$code.=<<___	if ($flavour =~ /64/);
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#1] // kFlag_aes_hw_encrypt
+#endif
+___
+$code.=<<___;
 	AARCH64_VALID_CALL_TARGET
 	ldr	$rounds,[$key,#240]
 	vld1.32	{$rndkey0},[$key],#16
@@ -722,6 +740,13 @@ ()
 ${prefix}_ctr32_encrypt_blocks:
 ___
 $code.=<<___	if ($flavour =~ /64/);
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks
+#endif
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 	AARCH64_VALID_CALL_TARGET
 	stp		x29,x30,[sp,#-16]!

crypto/fipsmodule/aes/asm/vpaes-armv8.pl

@@ -264,6 +264,13 @@
 .type	vpaes_encrypt,%function
 .align	4
 vpaes_encrypt:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#4] // kFlag_vpaes_encrypt
+#endif
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29,x30,[sp,#-16]!
 	add	x29,sp,#0
@@ -1072,6 +1079,13 @@
 .type	vpaes_set_encrypt_key,%function
 .align	4
 vpaes_set_encrypt_key:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key
+#endif
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29,x30,[sp,#-16]!
 	add	x29,sp,#0

crypto/fipsmodule/cpucap/cpucap.c

@@ -85,8 +85,8 @@ HIDDEN uint32_t OPENSSL_armcap_P = 0;
 
 #if defined(BORINGSSL_DISPATCH_TEST)
 // This value must be explicitly initialized to zero. See similar comment above.
-HIDDEN uint8_t BORINGSSL_function_hit[8] = {0};
-#endif
+HIDDEN uint8_t BORINGSSL_function_hit[9] = {0};
+#endif // BORINGSSL_DISPATCH_TEST
 
 // This variable is used only for testing purposes to ensure that the library
 // constructor is executed and the capability variable is initialized.

crypto/fipsmodule/modes/asm/aesv8-gcm-armv8-unroll8.pl

@@ -262,6 +262,13 @@
 .type   aesv8_gcm_8x_enc_128,%function
 .align  4
 aesv8_gcm_8x_enc_128:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#7] // kFlag_aesv8_gcm_8x_enc_128
+#endif
 	AARCH64_VALID_CALL_TARGET
 	cbz	x1, .L128_enc_ret
 	stp	d8, d9, [sp, #-80]!

crypto/fipsmodule/modes/asm/aesv8-gcm-armv8.pl

@@ -288,6 +288,13 @@
 .type   aes_gcm_enc_kernel,%function
 .align  4
 aes_gcm_enc_kernel:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#2] // kFlag_aes_gcm_enc_kernel
+#endif
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29, x30, [sp, #-128]!
 	mov	x29, sp

crypto/fipsmodule/sha/asm/sha512-armv8.pl

@@ -362,6 +362,13 @@ sub BODY_00_xx {
 .align	6
 sha256_block_armv8:
 .Lv8_entry:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#6] // kFlag_sha256_hw
+#endif
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 	stp		x29,x30,[sp,#-16]!
 	add		x29,sp,#0
@@ -448,6 +455,13 @@ sub BODY_00_xx {
 .align	6
 sha512_block_armv8:
 .Lv8_entry:
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern        BORINGSSL_function_hit
+	adrp	x6,:pg_hi21:BORINGSSL_function_hit
+	add x6, x6, :lo12:BORINGSSL_function_hit
+	mov w7, #1
+	strb w7, [x6,#8] // kFlag_sha512_hw
+#endif
 	stp		x29,x30,[sp,#-16]!
 	add		x29,sp,#0
 

crypto/impl_dispatch_test.cc

@@ -36,13 +36,13 @@ class ImplDispatchTest : public ::testing::Test {
  public:
   void SetUp() override {
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
-    aesni_ = CRYPTO_is_AESNI_capable();
+    aes_hw_ = CRYPTO_is_AESNI_capable();
     avx_movbe_ = CRYPTO_is_AVX_capable() && CRYPTO_is_MOVBE_capable();
-    ssse3_ = CRYPTO_is_SSSE3_capable();
+    aes_vpaes_ = CRYPTO_is_SSSE3_capable();
     sha_ext_ = CRYPTO_is_SHAEXT_capable();
-    vaes_vpclmulqdq_ =
-        (OPENSSL_ia32cap_P[2] & 0xC0030000) &&         // AVX512{F+DQ+BW+VL}
-        (((OPENSSL_ia32cap_P[3] >> 9) & 0x3) == 0x3);  // VAES + VPCLMULQDQ
+    vaes_vpclmulqdq_ = CRYPTO_is_AVX512_capable() && 
+                        CRYPTO_is_VAES_capable() &&
+                        CRYPTO_is_VPCLMULQDQ_capable();
     is_x86_64_ =
 #if defined(OPENSSL_X86_64)
         true;
@@ -61,7 +61,14 @@ class ImplDispatchTest : public ::testing::Test {
 #else
         false;
 #endif // MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
-#endif  // X86 || X86_64
+#elif defined(OPENSSL_AARCH64)
+    aes_hw_ = CRYPTO_is_ARMv8_AES_capable();
+    aes_vpaes_ = CRYPTO_is_NEON_capable();
+    aes_gcm_pmull_ = CRYPTO_is_ARMv8_PMULL_capable();
+    aes_gcm_8x_ = CRYPTO_is_ARMv8_GCM_8x_capable();
+    sha_ext_ = OPENSSL_armcap_P & ARMV8_SHA256;
+    sha_512_ext_ = OPENSSL_armcap_P & ARMV8_SHA512;
+#endif
   }
 
  protected:
@@ -88,50 +95,70 @@ class ImplDispatchTest : public ::testing::Test {
     }
   }
 
+  bool aes_hw_ = false;
+  bool aes_vpaes_ = false;
+  bool sha_ext_ = false;
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
   bool vaes_vpclmulqdq_ = false;
-  bool aesni_ = false;
   bool avx_movbe_ = false;
-  bool ssse3_ = false;
-  bool sha_ext_ = false;
   bool is_x86_64_ = false;
   bool is_assembler_too_old = false;
   bool is_assembler_too_old_avx512 = false;
+#else // AARCH64
+  bool aes_gcm_pmull_ = false;
+  bool aes_gcm_8x_ = false;
+  bool sha_512_ext_ = false;
 #endif
+
 };
 
-#if !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64))
+#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86) || \
+    defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64))
 
 constexpr size_t kFlag_aes_hw_ctr32_encrypt_blocks = 0;
 constexpr size_t kFlag_aes_hw_encrypt = 1;
-constexpr size_t kFlag_aesni_gcm_encrypt = 2;
 constexpr size_t kFlag_aes_hw_set_encrypt_key = 3;
 constexpr size_t kFlag_vpaes_encrypt = 4;
 constexpr size_t kFlag_vpaes_set_encrypt_key = 5;
-constexpr size_t kFlag_sha256_shaext = 6;
+constexpr size_t kFlag_sha256_hw = 6;
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+constexpr size_t kFlag_aesni_gcm_encrypt = 2;
 constexpr size_t kFlag_aes_gcm_encrypt_avx512 = 7;
+#else // AARCH64
+constexpr size_t kFlag_aes_gcm_enc_kernel = 2;
+constexpr size_t kFlag_aesv8_gcm_8x_enc_128 = 7;
+constexpr size_t kFlag_sha512_hw = 8;
+#endif
 
 TEST_F(ImplDispatchTest, AEAD_AES_GCM) {
   AssertFunctionsHit(
       {
-          {kFlag_aes_hw_ctr32_encrypt_blocks, aesni_ &&
+          {kFlag_aes_hw_encrypt, aes_hw_},
+          {kFlag_aes_hw_set_encrypt_key, aes_hw_},
+          {kFlag_vpaes_encrypt, aes_vpaes_ && !aes_hw_},
+          {kFlag_vpaes_set_encrypt_key, aes_vpaes_ && !aes_hw_},
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+          {kFlag_aes_hw_ctr32_encrypt_blocks, aes_hw_ &&
            (is_assembler_too_old || !vaes_vpclmulqdq_)},
-          {kFlag_aes_hw_encrypt, aesni_},
-          {kFlag_aes_hw_set_encrypt_key, aesni_},
           {kFlag_aesni_gcm_encrypt,
-           is_x86_64_ && aesni_ && avx_movbe_ &&
+           is_x86_64_ && aes_hw_ && avx_movbe_ &&
            !is_assembler_too_old && !vaes_vpclmulqdq_},
-          {kFlag_vpaes_encrypt, ssse3_ && !aesni_},
-          {kFlag_vpaes_set_encrypt_key, ssse3_ && !aesni_},
           {kFlag_aes_gcm_encrypt_avx512,
-           is_x86_64_ && aesni_ &&
+           is_x86_64_ && aes_hw_ &&
            !is_assembler_too_old_avx512 &&
            vaes_vpclmulqdq_},
+#else // AARCH64
+          {kFlag_aes_hw_ctr32_encrypt_blocks, aes_hw_ &&
+           !aes_gcm_pmull_ && !aes_gcm_8x_},
+          {kFlag_aes_gcm_enc_kernel, aes_hw_ &&
+           aes_gcm_pmull_ && !aes_gcm_8x_},
+          {kFlag_aesv8_gcm_8x_enc_128, aes_hw_ &&
+           aes_gcm_pmull_ && aes_gcm_8x_}
+#endif
       },
       [] {
         const uint8_t kZeros[16] = {0};
-        const uint8_t kPlaintext[40] = {1, 2, 3, 4, 0};
+        const uint8_t kPlaintext[256] = {1, 2, 3, 4, 0};
         uint8_t ciphertext[sizeof(kPlaintext) + 16];
         size_t ciphertext_len;
         bssl::ScopedEVP_AEAD_CTX ctx;
@@ -148,8 +175,8 @@ TEST_F(ImplDispatchTest, AEAD_AES_GCM) {
 TEST_F(ImplDispatchTest, AES_set_encrypt_key) {
   AssertFunctionsHit(
       {
-          {kFlag_aes_hw_set_encrypt_key, aesni_},
-          {kFlag_vpaes_set_encrypt_key, ssse3_ && !aesni_},
+          {kFlag_aes_hw_set_encrypt_key, aes_hw_},
+          {kFlag_vpaes_set_encrypt_key, aes_vpaes_ && !aes_hw_},
       },
       [] {
         AES_KEY key;
@@ -165,8 +192,8 @@ TEST_F(ImplDispatchTest, AES_single_block) {
 
   AssertFunctionsHit(
       {
-          {kFlag_aes_hw_encrypt, aesni_},
-          {kFlag_vpaes_encrypt, ssse3_ && !aesni_},
+          {kFlag_aes_hw_encrypt, aes_hw_},
+          {kFlag_vpaes_encrypt, aes_vpaes_ && !aes_hw_},
       },
       [&key] {
         uint8_t in[AES_BLOCK_SIZE] = {0};
@@ -178,7 +205,7 @@ TEST_F(ImplDispatchTest, AES_single_block) {
 TEST_F(ImplDispatchTest, SHA256) {
   AssertFunctionsHit(
       {
-          {kFlag_sha256_shaext, sha_ext_},
+          {kFlag_sha256_hw, sha_ext_},
       },
       [] {
         const uint8_t in[32] = {0};
@@ -187,6 +214,20 @@ TEST_F(ImplDispatchTest, SHA256) {
       });
 }
 
-#endif  // X86 || X86_64
+#ifdef OPENSSL_AARCH64
+TEST_F(ImplDispatchTest, SHA512) {
+  AssertFunctionsHit(
+      {
+          {kFlag_sha512_hw, sha_512_ext_},
+      },
+      [] {
+        const uint8_t in[32] = {0};
+        uint8_t out[SHA512_DIGEST_LENGTH];
+        SHA512(in, 32, out);
+      });
+}
+#endif // OPENSSL_AARCH64
+
+#endif  // !OPENSSL_NO_ASM && (OPENSSL_X86 || OPENSSL_X86_64 || OPENSSL_AARCH64)
 
 #endif  // DISPATCH_TEST && !SHARED_LIBRARY

crypto/internal.h

@@ -1013,6 +1013,7 @@ OPENSSL_INLINE int boringssl_fips_break_test(const char *test) {
 
 // BORINGSSL_function_hit is an array of flags. The following functions will
 // set these flags if BORINGSSL_DISPATCH_TEST is defined.
+// On x86 and x86_64:
 //   0: aes_hw_ctr32_encrypt_blocks
 //   1: aes_hw_encrypt
 //   2: aesni_gcm_encrypt
@@ -1021,7 +1022,17 @@ OPENSSL_INLINE int boringssl_fips_break_test(const char *test) {
 //   5: vpaes_set_encrypt_key
 //   6: sha256_block_data_order_shaext
 //   7: aes_gcm_encrypt_avx512
-extern uint8_t BORINGSSL_function_hit[8];
+// On AARCH64:
+//   0: aes_hw_ctr32_encrypt_blocks
+//   1: aes_hw_encrypt
+//   2: aes_gcm_enc_kernel
+//   3: aes_hw_set_encrypt_key
+//   4: vpaes_encrypt
+//   5: vpaes_set_encrypt_key
+//   6: sha256_block_armv8
+//   7: aesv8_gcm_8x_enc_128
+//   8: sha512_block_armv8
+extern uint8_t BORINGSSL_function_hit[9];
 #endif  // BORINGSSL_DISPATCH_TEST
 
 #if !defined(AWSLC_FIPS) && !defined(BORINGSSL_SHARED_LIBRARY)

generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S

@@ -32,6 +32,13 @@ Lrcon:
 .align	5
 _aes_hw_set_encrypt_key:
 Lenc_key:
+#ifdef BORINGSSL_DISPATCH_TEST
+
+	adrp	x6,_BORINGSSL_function_hit@PAGE
+	add	x6, x6, _BORINGSSL_function_hit@PAGEOFF
+	mov	w7, #1
+	strb	w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key
+#endif
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 	AARCH64_VALID_CALL_TARGET
 	stp	x29,x30,[sp,#-16]!
@@ -244,6 +251,13 @@ Ldec_key_abort:
 
 .align	5
 _aes_hw_encrypt:
+#ifdef BORINGSSL_DISPATCH_TEST
+
+	adrp	x6,_BORINGSSL_function_hit@PAGE
+	add	x6, x6, _BORINGSSL_function_hit@PAGEOFF
+	mov	w7, #1
+	strb	w7, [x6,#1] // kFlag_aes_hw_encrypt
+#endif
 	AARCH64_VALID_CALL_TARGET
 	ldr	w3,[x2,#240]
 	ld1	{v0.4s},[x2],#16
@@ -275,6 +289,13 @@ Loop_enc:
 
 .align	5
 _aes_hw_decrypt:
+#ifdef BORINGSSL_DISPATCH_TEST
+
+	adrp	x6,_BORINGSSL_function_hit@PAGE
+	add	x6, x6, _BORINGSSL_function_hit@PAGEOFF
+	mov	w7, #1
+	strb	w7, [x6,#1] // kFlag_aes_hw_encrypt
+#endif
 	AARCH64_VALID_CALL_TARGET
 	ldr	w3,[x2,#240]
 	ld1	{v0.4s},[x2],#16
@@ -599,6 +620,13 @@ Lcbc_abort:
 
 .align	5
 _aes_hw_ctr32_encrypt_blocks:
+#ifdef BORINGSSL_DISPATCH_TEST
+
+	adrp	x6,_BORINGSSL_function_hit@PAGE
+	add	x6, x6, _BORINGSSL_function_hit@PAGEOFF
+	mov	w7, #1
+	strb	w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks
+#endif
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 	AARCH64_VALID_CALL_TARGET
 	stp	x29,x30,[sp,#-16]!