Add Runtime dispatch based on custom CPU capabilities function

.github/actions/config-variations/action.yml

@@ -124,6 +124,58 @@ runs:
         acvp: true
         opt: ${{ inputs.opt }}
         examples: false # Some examples use a custom config themselves
+    - name: "Custom native capability functions (static ON)"
+      if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'native-cap-ON') }}
+      uses: ./.github/actions/multi-functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        compile_mode: native
+        cflags: "-std=c11 -D_GNU_SOURCE -DMLD_CONFIG_FILE=\\\\\\\"../../test/custom_native_capability_config_1.h\\\\\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
+    - name: "Custom native capability functions (static OFF)"
+      if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'native-cap-OFF') }}
+      uses: ./.github/actions/multi-functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        compile_mode: native
+        cflags: "-std=c11 -D_GNU_SOURCE -DMLD_CONFIG_FILE=\\\\\\\"../../test/custom_native_capability_config_0.h\\\\\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
+    - name: "Custom native capability functions (ID_AA64PFR1_EL1 detection)"
+      if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'native-cap-ID_AA64PFR1_EL1')) && runner.os == 'Linux' && runner.arch == 'ARM64' }}
+      uses: ./.github/actions/multi-functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        compile_mode: native
+        cflags: "-std=c11 -march=armv8.4-a+sha3 -D_GNU_SOURCE -DMLD_CONFIG_FILE=\\\\\\\"../../test/custom_native_capability_config_ID_AA64PFR1_EL1.h\\\\\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
+    - name: "Custom native capability functions (CPUID AVX2 detection)"
+      if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'native-cap-CPUID_AVX2')) && runner.os == 'Linux' && runner.arch == 'X64' }}
+      uses: ./.github/actions/multi-functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        compile_mode: native
+        cflags: "-std=c11 -mavx2 -mbmi2 -mpopcnt -D_GNU_SOURCE -DMLD_CONFIG_FILE=\\\\\\\"../../test/custom_native_capability_config_CPUID_AVX2.h\\\\\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
     - name: "Serial FIPS202 (no batched Keccak)"
       if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'serial-fips202') }}
       uses: ./.github/actions/multi-functest

.github/workflows/ci.yml

@@ -513,32 +513,28 @@ jobs:
             ec2_volume_size: 20
             compile_mode: native
             opt: all
-            # TODO: This config do not exist for now, will be available after PR #607 ("Add Runtime Dispatch") is merged.
-            # config_variations: 'native-cap-CPUID_AVX2'
+            config_variations: 'native-cap-CPUID_AVX2'
           - name: Intel Xeon 4th gen (t3)
             ec2_instance_type: t3.small
             ec2_ami: ubuntu-latest (x86_64)
             ec2_volume_size: 20
             compile_mode: native
             opt: all
-            # TODO: This config do not exist for now, will be available after PR #607 ("Add Runtime Dispatch") is merged.
-            # config_variations: 'native-cap-CPUID_AVX2'
+            config_variations: 'native-cap-CPUID_AVX2'
           - name: Graviton2 (c6g.medium)
             ec2_instance_type: c6g.medium
             ec2_ami: ubuntu-latest (aarch64)
             ec2_volume_size: 20
             compile_mode: native
             opt: all
-            # TODO: This config do not exist for now, will be available after PR #607 ("Add Runtime Dispatch") is merged.
-            # config_variations: 'native-cap-ON native-cap-OFF native-cap-ID_AA64PFR1_EL1'
+            config_variations: 'native-cap-ON native-cap-OFF native-cap-ID_AA64PFR1_EL1'
           - name: Graviton3 (c7g.medium)
             ec2_instance_type: c7g.medium
             ec2_ami: ubuntu-latest (aarch64)
             ec2_volume_size: 20
             compile_mode: native
             opt: all
-            # TODO: This config do not exist for now, will be available after PR #607 ("Add Runtime Dispatch") is merged.
-            # config_variations: 'native-cap-ID_AA64PFR1_EL1'
+            config_variations: 'native-cap-ID_AA64PFR1_EL1'
     name: Platform tests  (${{ matrix.target.name }})
     permissions:
       contents: 'read'

BIBLIOGRAPHY.md

@@ -36,6 +36,10 @@ source code and documentation.
   - [test/break_pct_config.h](test/break_pct_config.h)
   - [test/custom_memcpy_config.h](test/custom_memcpy_config.h)
   - [test/custom_memset_config.h](test/custom_memset_config.h)
+  - [test/custom_native_capability_config_0.h](test/custom_native_capability_config_0.h)
+  - [test/custom_native_capability_config_1.h](test/custom_native_capability_config_1.h)
+  - [test/custom_native_capability_config_CPUID_AVX2.h](test/custom_native_capability_config_CPUID_AVX2.h)
+  - [test/custom_native_capability_config_ID_AA64PFR1_EL1.h](test/custom_native_capability_config_ID_AA64PFR1_EL1.h)
   - [test/custom_randombytes_config.h](test/custom_randombytes_config.h)
   - [test/custom_stdlib_config.h](test/custom_stdlib_config.h)
   - [test/custom_zeroize_config.h](test/custom_zeroize_config.h)
@@ -80,6 +84,10 @@ source code and documentation.
   - [test/break_pct_config.h](test/break_pct_config.h)
   - [test/custom_memcpy_config.h](test/custom_memcpy_config.h)
   - [test/custom_memset_config.h](test/custom_memset_config.h)
+  - [test/custom_native_capability_config_0.h](test/custom_native_capability_config_0.h)
+  - [test/custom_native_capability_config_1.h](test/custom_native_capability_config_1.h)
+  - [test/custom_native_capability_config_CPUID_AVX2.h](test/custom_native_capability_config_CPUID_AVX2.h)
+  - [test/custom_native_capability_config_ID_AA64PFR1_EL1.h](test/custom_native_capability_config_ID_AA64PFR1_EL1.h)
   - [test/custom_randombytes_config.h](test/custom_randombytes_config.h)
   - [test/custom_stdlib_config.h](test/custom_stdlib_config.h)
   - [test/custom_zeroize_config.h](test/custom_zeroize_config.h)

dev/aarch64_clean/meta.h

@@ -32,27 +32,31 @@
 
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/arith_native_aarch64.h"
 
-static MLD_INLINE void mld_ntt_native(int32_t data[MLDSA_N])
+static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N])
 {
   mld_ntt_asm(data, mld_aarch64_ntt_zetas_layer123456,
               mld_aarch64_ntt_zetas_layer78);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_intt_native(int32_t data[MLDSA_N])
+static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N])
 {
   mld_intt_asm(data, mld_aarch64_intt_zetas_layer78,
                mld_aarch64_intt_zetas_layer123456);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
 static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len,
                                              const uint8_t *buf,
                                              unsigned buflen)
 {
-  if (len != MLDSA_N || buflen % 24 != 0)
+  if (len != MLDSA_N ||
+      buflen % 24 != 0) /* NEON support is mandatory for AArch64 */
   {
-    return -1;
+    return MLD_NATIVE_FUNC_FALLBACK;
   }
 
   /* Safety: outlen is at most MLDSA_N, hence, this cast is safe. */
@@ -67,7 +71,7 @@ static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len,
   /* AArch64 implementation assumes specific buffer lengths */
   if (len != MLDSA_N || buflen != MLD_AARCH64_REJ_UNIFORM_ETA2_BUFLEN)
   {
-    return -1;
+    return MLD_NATIVE_FUNC_FALLBACK;
   }
   /* Constant time: Inputs and outputs to this function are secret.
    * It is safe to leak which coefficients are accepted/rejected.
@@ -91,7 +95,7 @@ static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len,
   /* AArch64 implementation assumes specific buffer lengths */
   if (len != MLDSA_N || buflen != MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN)
   {
-    return -1;
+    return MLD_NATIVE_FUNC_FALLBACK;
   }
   /* Constant time: Inputs and outputs to this function are secret.
    * It is safe to leak which coefficients are accepted/rejected.
@@ -107,81 +111,90 @@ static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len,
   return (int)outlen;
 }
 
-static MLD_INLINE void mld_poly_decompose_32_native(int32_t *a1, int32_t *a0,
-                                                    const int32_t *a)
+static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0,
+                                                   const int32_t *a)
 {
   mld_poly_decompose_32_asm(a1, a0, a);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_poly_decompose_88_native(int32_t *a1, int32_t *a0,
-                                                    const int32_t *a)
+static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0,
+                                                   const int32_t *a)
 {
   mld_poly_decompose_88_asm(a1, a0, a);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_poly_caddq_native(int32_t a[MLDSA_N])
+static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N])
 {
   mld_poly_caddq_asm(a);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_poly_use_hint_32_native(int32_t *b, const int32_t *a,
-                                                   const int32_t *h)
+static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *b, const int32_t *a,
+                                                  const int32_t *h)
 {
   mld_poly_use_hint_32_asm(b, a, h);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_poly_use_hint_88_native(int32_t *b, const int32_t *a,
-                                                   const int32_t *h)
+static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *b, const int32_t *a,
+                                                  const int32_t *h)
 {
   mld_poly_use_hint_88_asm(b, a, h);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
 static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B)
 {
   return mld_poly_chknorm_asm(a, B);
 }
 
-static MLD_INLINE void mld_polyz_unpack_17_native(int32_t *r,
-                                                  const uint8_t *buf)
+static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf)
 {
   mld_polyz_unpack_17_asm(r, buf, mld_polyz_unpack_17_indices);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_polyz_unpack_19_native(int32_t *r,
-                                                  const uint8_t *buf)
+static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *buf)
 {
   mld_polyz_unpack_19_asm(r, buf, mld_polyz_unpack_19_indices);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_poly_pointwise_montgomery_native(
+static MLD_INLINE int mld_poly_pointwise_montgomery_native(
     int32_t out[MLDSA_N], const int32_t in0[MLDSA_N],
     const int32_t in1[MLDSA_N])
 {
   mld_poly_pointwise_montgomery_asm(out, in0, in1);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_polyvecl_pointwise_acc_montgomery_l4_native(
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native(
     int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N],
     const int32_t v[4][MLDSA_N])
 {
   mld_polyvecl_pointwise_acc_montgomery_l4_asm(w, (const int32_t *)u,
                                                (const int32_t *)v);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_polyvecl_pointwise_acc_montgomery_l5_native(
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native(
     int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N],
     const int32_t v[5][MLDSA_N])
 {
   mld_polyvecl_pointwise_acc_montgomery_l5_asm(w, (const int32_t *)u,
                                                (const int32_t *)v);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
-static MLD_INLINE void mld_polyvecl_pointwise_acc_montgomery_l7_native(
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native(
     int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N],
     const int32_t v[7][MLDSA_N])
 {
   mld_polyvecl_pointwise_acc_montgomery_l7_asm(w, (const int32_t *)u,
                                                (const int32_t *)v);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 
 #endif /* !__ASSEMBLER__ */

dev/fips202/aarch64/x1_scalar.h

@@ -13,10 +13,12 @@
 #define MLD_FIPS202_AARCH64_NEED_X1_SCALAR
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/fips202_native_aarch64.h"
-static MLD_INLINE void mld_keccak_f1600_x1_native(uint64_t *state)
+static MLD_INLINE int mld_keccak_f1600_x1_native(uint64_t *state)
 {
   mld_keccak_f1600_x1_scalar_asm(state, mld_keccakf1600_round_constants);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 #endif /* !__ASSEMBLER__ */
 

dev/fips202/aarch64/x1_v84a.h

@@ -17,10 +17,17 @@
 #define MLD_FIPS202_AARCH64_NEED_X1_V84A
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/fips202_native_aarch64.h"
-static MLD_INLINE void mld_keccak_f1600_x1_native(uint64_t *state)
+static MLD_INLINE int mld_keccak_f1600_x1_native(uint64_t *state)
 {
+  if (!mld_sys_check_capability(MLD_SYS_CAP_SHA3))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+
   mld_keccak_f1600_x1_v84a_asm(state, mld_keccakf1600_round_constants);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 #endif /* !__ASSEMBLER__ */
 

dev/fips202/aarch64/x2_v84a.h

@@ -17,13 +17,20 @@
 #define MLD_FIPS202_AARCH64_NEED_X2_V84A
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/fips202_native_aarch64.h"
 
 
-static MLD_INLINE void mld_keccak_f1600_x4_native(uint64_t *state)
+static MLD_INLINE int mld_keccak_f1600_x4_native(uint64_t *state)
 {
+  if (!mld_sys_check_capability(MLD_SYS_CAP_SHA3))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+
   mld_keccak_f1600_x2_v84a_asm(state + 0 * 25, mld_keccakf1600_round_constants);
   mld_keccak_f1600_x2_v84a_asm(state + 2 * 25, mld_keccakf1600_round_constants);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 #endif /* !__ASSEMBLER__ */
 

dev/fips202/aarch64/x4_v8a_scalar.h

@@ -13,11 +13,17 @@
 #define MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/fips202_native_aarch64.h"
-static MLD_INLINE void mld_keccak_f1600_x4_native(uint64_t *state)
+static MLD_INLINE int mld_keccak_f1600_x4_native(uint64_t *state)
 {
+  if (!mld_sys_check_capability(MLD_SYS_CAP_SHA3))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
   mld_keccak_f1600_x4_scalar_v8a_hybrid_asm(state,
                                             mld_keccakf1600_round_constants);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 #endif /* !__ASSEMBLER__ */
 

dev/fips202/aarch64/x4_v8a_v84a_scalar.h

@@ -17,11 +17,18 @@
 #define MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID
 
 #if !defined(__ASSEMBLER__)
+#include "../api.h"
 #include "src/fips202_native_aarch64.h"
-static MLD_INLINE void mld_keccak_f1600_x4_native(uint64_t *state)
+static MLD_INLINE int mld_keccak_f1600_x4_native(uint64_t *state)
 {
+  if (!mld_sys_check_capability(MLD_SYS_CAP_SHA3))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+
   mld_keccak_f1600_x4_scalar_v8a_v84a_hybrid_asm(
       state, mld_keccakf1600_round_constants);
+  return MLD_NATIVE_FUNC_SUCCESS;
 }
 #endif /* !__ASSEMBLER__ */