arm64: Re-add arm crc32c hw acceleration

Use the .arch_extension directive so that a config options and
special cflags aren't required. Add a few tests for both the
software and hardware implementations

Author: AGSaidi

Makefile.am

@@ -4,7 +4,7 @@ noinst_PROGRAMS = memcached-debug sizes testapp timedrun
 
 BUILT_SOURCES=
 
-testapp_SOURCES = testapp.c util.c util.h stats_prefix.c stats_prefix.h jenkins_hash.c murmur3_hash.c hash.h cache.c
+testapp_SOURCES = testapp.c util.c util.h stats_prefix.c stats_prefix.h jenkins_hash.c murmur3_hash.c hash.h cache.c crc32c.c
 
 timedrun_SOURCES = timedrun.c
 

configure.ac

@@ -75,18 +75,17 @@ AS_IF([test "$ICC" = "yes" -o "$GCC" = "yes"],
     AS_IF(test "$CLANG" = "no",[CFLAGS="$CFLAGS -pthread"])
 ])
 
+dnl clang will error .arch_extension crc32 assembler directives to allow
+dnl assembling crc instructions without this
+AS_IF(test "$CLANG" = "yes",[CFLAGS="$CFLAGS -Wno-language-extension-token"])
+
 if test "$ICC" = "no"; then
    AC_PROG_CC_C99
 fi
 
 AM_PROG_CC_C_O
 AC_PROG_INSTALL
 
-dnl ARM crc32 optimization is disabled until we have hardware for an automated
-dnl regression test.
-dnl AC_ARG_ENABLE(arm_crc32,
-dnl  [AS_HELP_STRING([--enable-arm-crc32], [Enable ARMv8 CRC32 instructions])])
-
 AC_ARG_ENABLE(extstore,
   [AS_HELP_STRING([--disable-extstore], [Disable external storage (extstore)])])
 
@@ -206,10 +205,6 @@ if test "x$enable_tls" = "xyes"; then
     AC_DEFINE([TLS],1,[Set to nonzero if you want to enable TLS])
 fi
 
-if test "x$enable_arm_crc32" = "xyes"; then
-    AC_DEFINE([ARM_CRC32],1,[Set to nonzero if you want to enable ARMv8 crc32])
-fi
-
 if test "x$enable_asan" = "xyes"; then
     AC_DEFINE([ASAN],1,[Set to nonzero if you want to compile using ASAN])
 fi

crc32c.c

@@ -48,7 +48,6 @@ crc_func crc32c;
 /* CRC-32C (iSCSI) polynomial in reversed bit order. */
 #define POLY 0x82f63b78
 
-uint32_t crc32c_sw(uint32_t crc, void const *buf, size_t len);
 uint32_t crc32c_sw_little(uint32_t crc, void const *buf, size_t len);
 uint32_t crc32c_sw_big(uint32_t crc, void const *buf, size_t len);
 #ifdef __x86_64__
@@ -274,8 +273,70 @@ void crc32c_init(void) {
     }
 }
 
-#else /* !__x86_64__ */
+#elif defined(__aarch64__) && defined(__linux__)
+#include <sys/auxv.h>
 
+static inline uint32_t crc32cx(uint32_t crc, const uint64_t data)
+{
+        asm(".arch_extension crc\n"
+        "crc32cx %w0, %w0, %x1" : "+r" (crc) : "r" (data));
+        return crc;
+}
+
+static inline uint32_t crc32cb(uint32_t crc, const uint8_t data)
+{
+        asm(".arch_extension crc\n"
+            "crc32cb %w0, %w0, %w1" : "+r" (crc) : "r" (data));
+        return crc;
+}
+
+static uint32_t crc32c_hw(uint32_t crc, void const *buf, size_t len) {
+    crc = ~crc;
+    unsigned char const *next = buf;
+
+    while (((uintptr_t)next & 7) && len > 0) {
+        crc = crc32cb(crc, *(uint8_t *)next);
+        next++;
+        len--;
+    }
+
+    while (len >= 64) {
+        uint64_t *next8 = (uint64_t *)next;
+        crc = crc32cx(crc, next8[0]);
+        crc = crc32cx(crc, next8[1]);
+        crc = crc32cx(crc, next8[2]);
+        crc = crc32cx(crc, next8[3]);
+        crc = crc32cx(crc, next8[4]);
+        crc = crc32cx(crc, next8[5]);
+        crc = crc32cx(crc, next8[6]);
+        crc = crc32cx(crc, next8[7]);
+        next += 64;
+        len -= 64;
+    }
+
+    while (len >= 8) {
+        crc = crc32cx(crc, *(uint64_t *)next);
+        next += 8;
+        len -= 8;
+    }
+
+    while (len > 0) {
+        crc = crc32cb(crc, *(uint8_t *)next);
+        next++;
+        len--;
+    }
+
+    return ~crc;
+}
+
+void crc32c_init(void) {
+    uint64_t auxv = getauxval(AT_HWCAP);
+
+    crc32c = crc32c_sw;
+    if (auxv & HWCAP_CRC32)
+        crc32c = crc32c_hw;
+}
+#else /* !__x86_64__i && !__aarch64__ */
 void crc32c_init(void) {
     crc32c = crc32c_sw;
 }

crc32c.h

@@ -17,4 +17,7 @@ extern crc_func crc32c;
 
 void crc32c_init(void);
 
+// Expose a prototype for the crc32c software variant simply for testing purposes
+uint32_t crc32c_sw(uint32_t crc, void const *buf, size_t len);
+
 #endif    /* CRC32C_H */

memcached.spec.in

@@ -1,4 +1,3 @@
-%bcond_with arm_crc32
 %bcond_with extstore
 %bcond_with seccomp
 %bcond_with sasl
@@ -66,7 +65,6 @@ web applications by alleviating database load.
 
 %build
 %configure \
-  %{?with_arm_crc32:--enable-arm-crc32} \
   %{?with_extstore:--enable-extstore} \
   %{?with_seccomp:--enable-seccomp} \
   %{?with_sasl:--enable-sasl} \

testapp.c

@@ -20,6 +20,7 @@
 
 #include "config.h"
 #include "cache.h"
+#include "crc32c.h"
 #include "hash.h"
 #include "jenkins_hash.h"
 #include "stats_prefix.h"
@@ -898,6 +899,34 @@ static enum test_return test_issue_92(void) {
     return TEST_PASS;
 }
 
+static enum test_return test_crc32c(void) {
+    uint32_t crc_hw, crc_sw;
+
+    char buffer[256];
+    for (int x = 0; x < 256; x++)
+        buffer[x] = x;
+
+    /* Compare harware to software implementaiton */
+    crc_hw = crc32c(0, buffer, 256);
+    crc_sw = crc32c_sw(0, buffer, 256);
+    assert(crc_hw == 0x9c44184b);
+    assert(crc_sw == 0x9c44184b);
+
+    /* Test that passing a CRC in also works */
+    crc_hw = crc32c(crc_hw, buffer, 256);
+    crc_sw = crc32c_sw(crc_sw, buffer, 256);
+    assert(crc_hw == 0xae10ee5a);
+    assert(crc_sw == 0xae10ee5a);
+
+    /* Test odd offsets/sizes */
+    crc_hw = crc32c(crc_hw, buffer + 1, 256 - 2);
+    crc_sw = crc32c_sw(crc_sw, buffer + 1, 256 - 2);
+    assert(crc_hw == 0xed37b906);
+    assert(crc_sw == 0xed37b906);
+
+    return TEST_PASS;
+}
+
 static enum test_return test_issue_102(void) {
     char buffer[4096];
     memset(buffer, ' ', sizeof(buffer));
@@ -2297,6 +2326,7 @@ struct testcase testcases[] = {
     { "issue_44", test_issue_44 },
     { "vperror", test_vperror },
     { "issue_101", test_issue_101 },
+    { "crc32c", test_crc32c },
     /* The following tests all run towards the same server */
     { "start_server", start_memcached_server },
     { "issue_92", test_issue_92 },
@@ -2363,6 +2393,8 @@ int main(int argc, char **argv)
     hash = jenkins_hash;
     stats_prefix_init(':');
 
+    crc32c_init();
+
     for (num_cases = 0; testcases[num_cases].description; num_cases++) {
         /* Just counting */
     }