adds semantics for pmuludq and extends AVX/AVX2 pmuldq

Previously, we only supported SSE and SSE2 variants of pmuldq and
didn't support any of pmuludq (unsigned packed multiplication).

Author: ivg

plugins/x86/semantics/test.t

@@ -54,6 +54,38 @@ and the same for a memory operand
     YMM0 := high:128[YMM0].#1.#2
   }
 
+  $ mc 0x66,0x0f,0xf4,0x4f,0x0c
+  pmuludq 0xc(%rdi), %xmm1
+  {
+    #0 := mem[RDI + 0xC, el]:u128
+    #1 := pad:64[95:64[YMM1]] * pad:64[95:64[#0]]
+    #2 := pad:64[31:0[YMM1]] * pad:64[31:0[#0]]
+    YMM1 := high:128[YMM1].#1.#2
+  }
+  $ mc 0x66,0x0f,0xf4,0xcb
+  pmuludq %xmm3, %xmm1
+  {
+    #0 := pad:64[95:64[YMM1]] * pad:64[95:64[YMM3]]
+    #1 := pad:64[31:0[YMM1]] * pad:64[31:0[YMM3]]
+    YMM1 := high:128[YMM1].#0.#1
+  }
+  $ mc 0xc5,0xe1,0xf4,0xcc
+  vpmuludq %xmm4, %xmm3, %xmm1
+  {
+    #0 := pad:64[95:64[YMM1]] * pad:64[95:64[YMM4]]
+    #1 := pad:64[31:0[YMM1]] * pad:64[31:0[YMM4]]
+    YMM1 := 0.#0.#1
+  }
+  $ mc 0xc5,0xe5,0xf4,0xcc
+  vpmuludq %ymm4, %ymm3, %ymm1
+  {
+    #0 := pad:64[223:192[YMM1]] * pad:64[223:192[YMM4]]
+    #1 := pad:64[159:128[YMM1]] * pad:64[159:128[YMM4]]
+    #2 := pad:64[95:64[YMM1]] * pad:64[95:64[YMM4]]
+    #3 := pad:64[31:0[YMM1]] * pad:64[31:0[YMM4]]
+    YMM1 := #0.#1.#2.#3
+  }
+
 
 -----------------------------------------------------------
 #                     Testing xchgb                       #

plugins/x86/semantics/x86-64.lisp

@@ -63,19 +63,66 @@
 ;; pmuludq
 ;; Reference: Vol. 2B 4-370
 (defun PMULDQrr (dst _ src)
-  (pmul dst src))
+  (pmul set$ cast-signed dst src))
 
 (defun PMULDQrm (dst _ base _ _ off _)
   (let ((src (load-dword (+ base off))))
-    (pmul dst src)))
+    (pmul set$ cast-signed dst src)))
 
-(defun pmul (dst src)
+(defun PMULUDQrr (dst _ src)
+  (pmul set$ cast-unsigned dst src))
+
+(defun PMULUDQrm (dst _ base _ _ off _)
+  (let ((src (load-dword (+ base off))))
+    (pmul set$ cast-unsigned dst src)))
+
+(defun VPMULDQrr (dst _ src)
+  (pmul setv cast-signed dst src))
+
+(defun VPMULDQrm (dst _ base _ _ off _)
+  (let ((src (load-dword (+ base off))))
+    (pmul setv cast-signed dst src)))
+
+(defun VPMULUDQrr (dst _ src)
+  (pmul setv cast-unsigned dst src))
+
+(defun VPMULUDQrm (dst _ base _ _ off _)
+  (let ((src (load-dword (+ base off))))
+    (pmul setv cast-unsigned dst src)))
+
+(defun VPMULDQYrr (dst _ src)
+  (pmuly cast-signed dst src))
+
+(defun VPMULDQYrm (dst _ base _ _ off _)
+  (let ((src (load-dword (+ base off))))
+    (pmuly cast-signed dst src)))
+
+(defun VPMULUDQYrr (dst _ src)
+  (pmuly cast-unsigned dst src))
+
+(defun VPMULUDQYrm (dst _ base _ _ off _)
+  (let ((src (load-dword (+ base off))))
+    (pmuly cast-unsigned dst src)))
+
+(defmacro pmul (set cast dst src)
+  (declare (visibility :private))
+  (let ((hi (* (cast 64 (extract 95 64 dst))
+               (cast 64 (extract 95 64 src))))
+        (lo (* (cast 64 (extract 31  0 dst))
+               (cast 64 (extract 31  0 src)))))
+    (set dst (concat hi lo))))
+
+(defmacro pmuly (cast dst src)
   (declare (visibility :private))
-  (let ((hi (* (cast-signed 64 (extract 95 64 dst))
-               (cast-signed 64 (extract 95 64 src))))
-        (lo (* (cast-signed 64 (extract 31  0 dst))
-               (cast-signed 64 (extract 31  0 src)))))
-    (set$ dst (concat hi lo))))
+  (let ((w4 (* (cast 64 (extract 223 192 dst))
+               (cast 64 (extract 223 192 src))))
+        (w3 (* (cast 64 (extract 159 128 dst))
+               (cast 64 (extract 159 128 src))))
+        (w2 (* (cast 64 (extract 95 64 dst))
+               (cast 64 (extract 95 64 src))))
+        (w1 (* (cast 64 (extract 31  0 dst))
+               (cast 64 (extract 31  0 src)))))
+    (set$ dst (concat w4 w3 w2 w1))))
 
 
 ;; pack{u,s}sdw/pack{u,s}swb