Refactor implementation of *_bcst{_1,_2,_3} patterns.

Add new predicate bcst_mem_operand and corresponding constraint "Br"
to merge "$(pattern)_bcst{_1,_2,_3}" into "$(pattern)", also delete
those separate "*_bcst{_1,_2,_3}" patterns.

gcc/ChangeLog:

	PR target/87767
	* config/i386/constraints.md ("Br"): New special memory
	constraint.
	* config/i386/i386-expand.c (ix86_binary_operator_ok): Both
	source operand cannot be in memory or bcst_memory_operand.
	* config/i386/i386.c (ix86_print_operand): Print bcst_mem_operand.
	* config/i386/i386.h (VALID_BCST_MODE_P): New.
	* config/i386/predicates.md (bcst_mem_operand): New predicate
	for AVX512 embedding broadcast memory operand.
	(bcst_vector_operand): New predicate, vector_operand or
	bcst_mem_operand.
	* config/i386/sse.md
	(*<plusminus_insn><mode>3<mask_name><round_name>): Extend
	predicate and constraints to handle bcst_mem_operand.
	(*mul<mode>3<mask_name><round_name>): Ditto.
	(<sse>_div<mode>3<mask_name><round_name>): Ditto.
	(<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name><round_name>):
	Ditto.
	(<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name><round_name>):
	Ditto.
	(<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name><round_name>):
	Ditto.
	(<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name><round_name>):
	Ditto.
	(*<plusminus_insn><mode>3): Ditto.
	(avx512dq_mul<mode>3<mask_name>): Ditto.
	(*<sse4_1_avx2>_mul<mode>3<mask_name>): Ditto.
	(*andnot<mode>3): Ditto.
	(<mask_codefor><code><mode>3<mask_name>): Ditto.
	(*sub<mode>3<mask_name>_bcst): Removed.
	(*add<mode>3<mask_name>_bcst): Ditto.
	(*mul<mode>3<mask_name>_bcst): Ditto.
	(*<avx512>_div<mode>3<mask_name>_bcst): Ditto.
	(*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_1):
	Ditto.
	(*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_2):
	Ditto.
	(*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_3):
	Ditto.
	(*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_1):
	Ditto.
	(*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_2):
	Ditto.
	(*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_3):
	Ditto.
	(*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_1):
	Ditto.
	(*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_2):
	Ditto.
	(*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_3):
	Ditto.
	(*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_1):
	Ditto.
	(*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_2):
	Ditto.
	(*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_3):
	Ditto.
	(*sub<mode>3_bcst): Ditto.
	(*add<mode>3_bcst): Ditto.
	(*avx512dq_mul<mode>3<mask_name>_bcst): Ditto.
	(*avx512f_mul<mode>3<mask_name>_bcst): Ditto.
	(*andnot<mode>3_bcst): Ditto.
	(*<code><mode>3_bcst): Ditto.
	* config/i386/subst.md (bcst_round_constraint): New subst
	attribute.
	(bcst_round_nimm_predicate): Ditto.
	(bcst_mask_prefix3): Ditto.
	(bcst_mask_prefix4): Ditto.

diff --git a/gcc/config/i386/constraints.md b/gcc/config/i386/constraints.md
index af37f5cd537105d6b576341102365952146c90db..0b902d543b3ec59f7bd9862a51290037a229bb0d 100644 (file)
--- a/gcc/config/i386/constraints.md
+++ b/gcc/config/i386/constraints.md
@@ -183,6 +183,10 @@
   "@internal Memory operand without REX prefix."
   (match_operand 0 "norex_memory_operand"))
 
+(define_special_memory_constraint "Br"
+  "@internal bcst memory operand."
+  (match_operand 0 "bcst_mem_operand"))
+
 (define_constraint "Bs"
   "@internal Sibcall memory operand."
   (ior (and (not (match_test "TARGET_INDIRECT_BRANCH_REGISTER"))

diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
index d0d7a969637637651433e90e64457dd8702d11ca..3e8afe683dc57ef1ad568c0149d78d5e566189ee 100644 (file)
--- a/gcc/config/i386/i386-expand.c
+++ b/gcc/config/i386/i386-expand.c
@@ -1045,7 +1045,8 @@ ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
   rtx src2 = operands[2];
 
   /* Both source operands cannot be in memory.  */
-  if (MEM_P (src1) && MEM_P (src2))
+  if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
+      && (MEM_P (src2) || bcst_mem_operand (src2, mode)))
     return false;
 
   /* Canonicalize operand order for commutative operators.  */

diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 54c2cdaf060c683d4742eab22d495b2bdf7371bb..502d24057b54dffd549e25baabf96223fa7b2232 100644 (file)
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -13098,6 +13098,43 @@ ix86_print_operand (FILE *file, rtx x, int code)
       fputs (dstr, file);
     }
 
+  /* Print bcst_mem_operand.  */
+  else if (GET_CODE (x) == VEC_DUPLICATE)
+    {
+      machine_mode vmode = GET_MODE (x);
+      /* Must be bcst_memory_operand.  */
+      gcc_assert (bcst_mem_operand (x, vmode));
+
+      rtx mem = XEXP (x,0);
+      ix86_print_operand (file, mem, 0);
+
+      switch (vmode)
+       {
+       case E_V2DImode:
+       case E_V2DFmode:
+         fputs ("{1to2}", file);
+         break;
+       case E_V4SImode:
+       case E_V4SFmode:
+       case E_V4DImode:
+       case E_V4DFmode:
+         fputs ("{1to4}", file);
+         break;
+       case E_V8SImode:
+       case E_V8SFmode:
+       case E_V8DFmode:
+       case E_V8DImode:
+         fputs ("{1to8}", file);
+         break;
+       case E_V16SFmode:
+       case E_V16SImode:
+         fputs ("{1to16}", file);
+         break;
+       default:
+         gcc_unreachable ();
+       }
+    }
+
   else
     {
       /* We have patterns that allow zero sets of memory, for instance.

diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index e8bde37645a51bc1c572c29382ed1752d2d12b07..24207d042115bcac86e445fb0cc33681985ca70b 100644 (file)
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -1272,6 +1272,10 @@ extern const char *host_detect_local_cpu (int argc, const char **argv);
   (TARGET_FMA4 && ((MODE) == V4SFmode || (MODE) == V2DFmode \
                  || (MODE) == V8SFmode || (MODE) == V4DFmode))
 
+#define VALID_BCST_MODE_P(MODE)                        \
+  ((MODE) == SFmode || (MODE) == DFmode                \
+   || (MODE) == SImode || (MODE) == DImode)
+
 /* It is possible to write patterns to move flags; but until someone
    does it,  */
 #define AVOID_CCMODE_COPIES

diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index b03f9cd1c8cf8f1ed47e136e9082af1234634bd3..be57cdaf7680146f041a898ebc02a07730289d6e 100644 (file)
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -1081,6 +1081,19 @@
   (ior (match_operand 0 "register_operand")
        (match_operand 0 "vector_memory_operand")))
 
+(define_predicate "bcst_mem_operand"
+  (and (match_code "vec_duplicate")
+       (and (match_test "TARGET_AVX512F")
+           (ior (match_test "TARGET_AVX512VL")
+                (match_test "GET_MODE_SIZE (GET_MODE (op)) == 64")))
+       (match_test "VALID_BCST_MODE_P (GET_MODE_INNER (GET_MODE (op)))")
+       (match_test "memory_operand (XEXP (op, 0), GET_MODE (XEXP (op, 0)))")))
+
+; Return true when OP is bcst_mem_operand or vector_memory_operand.
+(define_predicate "bcst_vector_operand"
+  (ior (match_operand 0 "vector_operand")
+       (match_operand 0 "bcst_mem_operand")))
+
 ;; Return true when OP is either nonimmediate operand, or any
 ;; CONST_VECTOR.
 (define_predicate "nonimmediate_or_const_vector_operand"

diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 3689ab0e7b706edcbf4e5a14cc6e6b5dc3b9b27b..52635f6bc08bff39f9f94d2e851e9f654295dbfc 100644 (file)
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -1756,8 +1756,8 @@
 (define_insn "*<plusminus_insn><mode>3<mask_name><round_name>"
   [(set (match_operand:VF 0 "register_operand" "=x,v")
        (plusminus:VF
-         (match_operand:VF 1 "<round_nimm_predicate>" "<comm>0,v")
-         (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
+         (match_operand:VF 1 "<bcst_round_nimm_predicate>" "<comm>0,v")
+         (match_operand:VF 2 "<bcst_round_nimm_predicate>" "xBm,<bcst_round_constraint>")))]
   "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
    && <mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
@@ -1765,35 +1765,7 @@
    v<plusminus_mnemonic><ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
   [(set_attr "isa" "noavx,avx")
    (set_attr "type" "sseadd")
-   (set_attr "prefix" "<mask_prefix3>")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*sub<mode>3<mask_name>_bcst"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (minus:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "v")
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))]
-  "TARGET_AVX512F
-   && ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
-   && <mask_mode512bit_condition>"
-  "vsub<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<avx512bcst>}"
-  [(set_attr "prefix" "evex")
-   (set_attr "type" "sseadd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*add<mode>3<mask_name>_bcst"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (plus:VF_AVX512
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VF_AVX512 2 "register_operand" "v")))]
-  "TARGET_AVX512F
-   && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
-   && <mask_mode512bit_condition>"
-  "vadd<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
-  [(set_attr "prefix" "evex")
-   (set_attr "type" "sseadd")
+   (set_attr "prefix" "<bcst_mask_prefix3>")
    (set_attr "mode" "<MODE>")])
 
 ;; Standard scalar operation patterns which preserve the rest of the
@@ -1846,32 +1818,19 @@
 (define_insn "*mul<mode>3<mask_name><round_name>"
   [(set (match_operand:VF 0 "register_operand" "=x,v")
        (mult:VF
-         (match_operand:VF 1 "<round_nimm_predicate>" "%0,v")
-         (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
-  "TARGET_SSE
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))
+         (match_operand:VF 1 "<bcst_round_nimm_predicate>" "%0,v")
+         (match_operand:VF 2 "<bcst_round_nimm_predicate>" "xBm,<bcst_round_constraint>")))]
+  "TARGET_SSE && ix86_binary_operator_ok (MULT, <MODE>mode, operands)
    && <mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    mul<ssemodesuffix>\t{%2, %0|%0, %2}
    vmul<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
   [(set_attr "isa" "noavx,avx")
    (set_attr "type" "ssemul")
-   (set_attr "prefix" "<mask_prefix3>")
+   (set_attr "prefix" "<bcst_mask_prefix3>")
    (set_attr "btver2_decode" "direct,double")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*mul<mode>3<mask_name>_bcst"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (mult:VF_AVX512
-         (vec_duplicate:VF_AVX512
-            (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VF_AVX512 2 "register_operand" "v")))]
-  "TARGET_AVX512F && <mask_mode512bit_condition>"
-  "vmul<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
-  [(set_attr "prefix" "evex")
-   (set_attr "type" "ssemul")
-   (set_attr "mode" "<MODE>")])
-
 ;; Standard scalar operation patterns which preserve the rest of the
 ;; vector for combiner.
 (define_insn "*<sse>_vm<multdiv_mnemonic><mode>3"
@@ -1943,26 +1902,14 @@
   [(set (match_operand:VF 0 "register_operand" "=x,v")
        (div:VF
          (match_operand:VF 1 "register_operand" "0,v")
-         (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
+         (match_operand:VF 2 "<bcst_round_nimm_predicate>" "xBm,<bcst_round_constraint>")))]
   "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    div<ssemodesuffix>\t{%2, %0|%0, %2}
    vdiv<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
   [(set_attr "isa" "noavx,avx")
    (set_attr "type" "ssediv")
-   (set_attr "prefix" "<mask_prefix3>")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<avx512>_div<mode>3<mask_name>_bcst"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (div:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "v")
-         (vec_duplicate:VF_AVX512
-            (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))]
-  "TARGET_AVX512F && <mask_mode512bit_condition>"
-  "vdiv<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<avx512bcst>}"
-  [(set_attr "prefix" "evex")
-    (set_attr "type" "ssediv")
+   (set_attr "prefix" "<bcst_mask_prefix3>")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<sse>_rcp<mode>2"
@@ -4055,9 +4002,9 @@
 (define_insn "<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name><round_name>"
   [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
        (fma:VF_SF_AVX512VL
-         (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
-         (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
-         (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
+         (match_operand:VF_SF_AVX512VL 1 "<bcst_round_nimm_predicate>" "%0,0,v")
+         (match_operand:VF_SF_AVX512VL 2 "<bcst_round_nimm_predicate>" "<bcst_round_constraint>,v,<bcst_round_constraint>")
+         (match_operand:VF_SF_AVX512VL 3 "<bcst_round_nimm_predicate>" "v,<bcst_round_constraint>,0")))]
   "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    vfmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
@@ -4066,46 +4013,6 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_1"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (fma:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "%0")
-         (match_operand:VF_AVX512 2 "register_operand" "v")
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 3 "memory_operand" "m"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "vfmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_2"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m,m"))
-         (match_operand:VF_AVX512 2 "register_operand" "0,v")
-         (match_operand:VF_AVX512 3 "register_operand" "v,0")))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfmadd132<ssemodesuffix>\t{%1<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %1<avx512bcst>}
-   vfmadd231<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_3"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "0,v")
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m,m"))
-         (match_operand:VF_AVX512 3 "register_operand" "v,0")))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfmadd132<ssemodesuffix>\t{%2<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<avx512bcst>}
-   vfmadd231<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "<avx512>_fmadd_<mode>_mask<round_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
        (vec_merge:VF_AVX512VL
@@ -4171,10 +4078,10 @@
 (define_insn "<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name><round_name>"
   [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
        (fma:VF_SF_AVX512VL
-         (match_operand:VF_SF_AVX512VL   1 "<round_nimm_predicate>" "%0,0,v")
-         (match_operand:VF_SF_AVX512VL   2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
+         (match_operand:VF_SF_AVX512VL   1 "<bcst_round_nimm_predicate>" "%0,0,v")
+         (match_operand:VF_SF_AVX512VL   2 "<bcst_round_nimm_predicate>" "<bcst_round_constraint>,v,<bcst_round_constraint>")
          (neg:VF_SF_AVX512VL
-           (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
+           (match_operand:VF_SF_AVX512VL 3 "<bcst_round_nimm_predicate>" "v,<bcst_round_constraint>,0"))))]
   "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    vfmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
@@ -4183,49 +4090,6 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_1"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (fma:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "%0")
-         (match_operand:VF_AVX512 2 "register_operand" "v")
-         (neg:VF_AVX512
-           (vec_duplicate:VF_AVX512
-             (match_operand:<ssescalarmode> 3 "memory_operand" "m")))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "vfmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_2"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m,m"))
-         (match_operand:VF_AVX512 2 "register_operand" "0,v")
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 3 "register_operand" "v,0"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfmsub132<ssemodesuffix>\t{%1<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %1<avx512bcst>}
-   vfmsub231<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_3"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (match_operand:VF_AVX512 1 "register_operand" "0,v")
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m,m"))
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 3 "nonimmediate_operand" "v,0"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfmsub132<ssemodesuffix>\t{%2<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<avx512bcst>}
-   vfmsub231<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "<avx512>_fmsub_<mode>_mask<round_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
        (vec_merge:VF_AVX512VL
@@ -4294,9 +4158,9 @@
   [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
        (fma:VF_SF_AVX512VL
          (neg:VF_SF_AVX512VL
-           (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
-         (match_operand:VF_SF_AVX512VL   2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
-         (match_operand:VF_SF_AVX512VL   3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
+           (match_operand:VF_SF_AVX512VL 1 "<bcst_round_nimm_predicate>" "%0,0,v"))
+         (match_operand:VF_SF_AVX512VL   2 "<bcst_round_nimm_predicate>" "<bcst_round_constraint>,v,<bcst_round_constraint>")
+         (match_operand:VF_SF_AVX512VL   3 "<bcst_round_nimm_predicate>" "v,<bcst_round_constraint>,0")))]
   "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    vfnmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
@@ -4305,49 +4169,6 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_1"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 1 "register_operand" "%0"))
-         (match_operand:VF_AVX512 2 "register_operand" "v")
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 3 "memory_operand" "m"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "vfnmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_2"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (vec_duplicate:VF_AVX512
-             (match_operand:<ssescalarmode> 1 "memory_operand" "m,m")))
-         (match_operand:VF_AVX512 2 "register_operand" "0,v")
-         (match_operand:VF_AVX512 3 "register_operand" "v,0")))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfnmadd132<ssemodesuffix>\t{%1<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %1<avx512bcst>}
-   vfnmadd231<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_3"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 1 "register_operand" "0,v"))
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m,m"))
-         (match_operand:VF_AVX512 3 "register_operand" "v,0")))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfnmadd132<ssemodesuffix>\t{%2<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<avx512bcst>}
-   vfnmadd231<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "<avx512>_fnmadd_<mode>_mask<round_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
        (vec_merge:VF_AVX512VL
@@ -4417,10 +4238,10 @@
   [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
        (fma:VF_SF_AVX512VL
          (neg:VF_SF_AVX512VL
-           (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
-         (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
+           (match_operand:VF_SF_AVX512VL 1 "<bcst_round_nimm_predicate>" "%0,0,v"))
+         (match_operand:VF_SF_AVX512VL 2 "<bcst_round_nimm_predicate>" "<bcst_round_constraint>,v,<bcst_round_constraint>")
          (neg:VF_SF_AVX512VL
-           (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
+           (match_operand:VF_SF_AVX512VL 3 "<bcst_round_nimm_predicate>" "v,<bcst_round_constraint>,0"))))]
   "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
   "@
    vfnmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
@@ -4429,52 +4250,6 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_1"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 1 "register_operand" "%0"))
-         (match_operand:VF_AVX512 2 "register_operand" "v")
-         (neg:VF_AVX512
-           (vec_duplicate:VF_AVX512
-             (match_operand:<ssescalarmode> 3 "memory_operand" "m")))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "vfnmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_2"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (vec_duplicate:VF_AVX512
-             (match_operand:<ssescalarmode> 1 "memory_operand" "m,m")))
-         (match_operand:VF_AVX512 2 "register_operand" "0,v")
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 3 "register_operand" "v,0"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfnmsub132<ssemodesuffix>\t{%1<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %1<avx512bcst>}
-   vfnmsub231<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_3"
-  [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v")
-       (fma:VF_AVX512
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 1 "register_operand" "0,v"))
-         (vec_duplicate:VF_AVX512
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m,m"))
-         (neg:VF_AVX512
-           (match_operand:VF_AVX512 3 "register_operand" "v,0"))))]
-  "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
-  "@
-   vfnmsub132<ssemodesuffix>\t{%2<avx512bcst>, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<avx512bcst>}
-   vfnmsub231<ssemodesuffix>\t{%2<avx512bcst>, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<avx512bcst>}"
-  [(set_attr "type" "ssemuladd")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "<avx512>_fnmsub_<mode>_mask<round_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
        (vec_merge:VF_AVX512VL
@@ -11563,8 +11338,8 @@
 (define_insn "*<plusminus_insn><mode>3"
   [(set (match_operand:VI_AVX2 0 "register_operand" "=x,v")
        (plusminus:VI_AVX2
-         (match_operand:VI_AVX2 1 "vector_operand" "<comm>0,v")
-         (match_operand:VI_AVX2 2 "vector_operand" "xBm,vm")))]
+         (match_operand:VI_AVX2 1 "bcst_vector_operand" "<comm>0,v")
+         (match_operand:VI_AVX2 2 "bcst_vector_operand" "xBm,vmBr")))]
   "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
   "@
    p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
@@ -11572,31 +11347,7 @@
   [(set_attr "isa" "noavx,avx")
    (set_attr "type" "sseiadd")
    (set_attr "prefix_data16" "1,*")
-   (set_attr "prefix" "orig,vex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "*sub<mode>3_bcst"
-  [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
-       (minus:VI48_AVX512VL
-         (match_operand:VI48_AVX512VL 1 "register_operand" "v")
-         (vec_duplicate:VI48_AVX512VL
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))]
-  "TARGET_AVX512F && ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
-  "vpsub<ssemodesuffix>\t{%2<avx512bcst>, %1, %0|%0, %1, %2<avx512bcst>}"
-  [(set_attr "type" "sseiadd")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "*add<mode>3_bcst"
-  [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
-       (plus:VI48_AVX512VL
-         (vec_duplicate:VI48_AVX512VL
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VI48_AVX512VL 2 "register_operand" "v")))]
-  "TARGET_AVX512F && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
-  "vpadd<ssemodesuffix>\t{%1<avx512bcst>, %2, %0|%0, %2, %1<avx512bcst>}"
-  [(set_attr "type" "sseiadd")
-   (set_attr "prefix" "evex")
+   (set_attr "prefix" "orig,maybe_evex")
    (set_attr "mode" "<sseinsnmode>")])
 
 (define_insn "*<plusminus_insn><mode>3_mask"
@@ -12110,24 +11861,13 @@
    (set_attr "mode" "TI")])
 
 (define_insn "avx512dq_mul<mode>3<mask_name>"
-  [(set (match_operand:VI8 0 "register_operand" "=v")
-       (mult:VI8
-         (match_operand:VI8 1 "register_operand" "v")
-         (match_operand:VI8 2 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512DQ && <mask_mode512bit_condition>"
-  "vpmullq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
-  [(set_attr "type" "sseimul")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "*avx512dq_mul<mode>3<mask_name>_bcst"
   [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
        (mult:VI8_AVX512VL
-         (vec_duplicate:VI8_AVX512VL
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VI8_AVX512VL 2 "register_operand" "v")))]
-  "TARGET_AVX512DQ"
-  "vpmullq\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
+         (match_operand:VI8_AVX512VL 1 "bcst_vector_operand" "%v")
+         (match_operand:VI8_AVX512VL 2 "bcst_vector_operand" "vmBr")))]
+  "TARGET_AVX512DQ && <mask_mode512bit_condition>
+  && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
+  "vpmullq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sseimul")
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
@@ -12157,10 +11897,10 @@
 (define_insn "*<sse4_1_avx2>_mul<mode>3<mask_name>"
   [(set (match_operand:VI4_AVX512F 0 "register_operand" "=Yr,*x,v")
        (mult:VI4_AVX512F
-         (match_operand:VI4_AVX512F 1 "vector_operand" "%0,0,v")
-         (match_operand:VI4_AVX512F 2 "vector_operand" "YrBm,*xBm,vm")))]
-  "TARGET_SSE4_1 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
-   && <mask_mode512bit_condition>"
+         (match_operand:VI4_AVX512F 1 "bcst_vector_operand" "%0,0,v")
+         (match_operand:VI4_AVX512F 2 "bcst_vector_operand" "YrBm,*xBm,vmBr")))]
+  "TARGET_SSE4_1 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)
+  && <mask_mode512bit_condition>"
   "@
    pmulld\t{%2, %0|%0, %2}
    pmulld\t{%2, %0|%0, %2}
@@ -12168,22 +11908,10 @@
   [(set_attr "isa" "noavx,noavx,avx")
    (set_attr "type" "sseimul")
    (set_attr "prefix_extra" "1")
-   (set_attr "prefix" "<mask_prefix4>")
+   (set_attr "prefix" "<bcst_mask_prefix4>")
    (set_attr "btver2_decode" "vector,vector,vector")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "*avx512f_mul<mode>3<mask_name>_bcst"
-  [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
-       (mult:VI4_AVX512VL
-         (vec_duplicate:VI4_AVX512VL
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VI4_AVX512VL 2 "register_operand" "v")))]
-  "TARGET_AVX512F"
-   "vpmulld\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "sseimul")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
 (define_expand "mul<mode>3"
   [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand")
        (mult:VI8_AVX2_AVX512F
@@ -13210,7 +12938,7 @@
   [(set (match_operand:VI 0 "register_operand" "=x,x,v")
        (and:VI
          (not:VI (match_operand:VI 1 "register_operand" "0,x,v"))
-         (match_operand:VI 2 "vector_operand" "xBm,xm,vm")))]
+         (match_operand:VI 2 "bcst_vector_operand" "xBm,xm,vmBr")))]
   "TARGET_SSE"
 {
   char buf[64];
@@ -13309,19 +13037,6 @@
              ]
              (const_string "<sseinsnmode>")))])
 
-(define_insn "*andnot<mode>3_bcst"
-  [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
-       (and:VI48_AVX512VL
-         (not:VI48_AVX512VL
-            (match_operand:VI48_AVX512VL 1 "register_operand" "v"))
-         (vec_duplicate:VI48_AVX512VL
-           (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))]
-  "TARGET_AVX512F"
-  "vpandn<ssemodesuffix>\t{%2<avx512bcst>, %1, %0|%0, %1, %2<avx512bcst>}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
 (define_insn "*andnot<mode>3_mask"
   [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
        (vec_merge:VI48_AVX512VL
@@ -13351,10 +13066,10 @@
 (define_insn "<mask_codefor><code><mode>3<mask_name>"
   [(set (match_operand:VI48_AVX_AVX512F 0 "register_operand" "=x,x,v")
        (any_logic:VI48_AVX_AVX512F
-         (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,x,v")
-         (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
+         (match_operand:VI48_AVX_AVX512F 1 "bcst_vector_operand" "%0,x,v")
+         (match_operand:VI48_AVX_AVX512F 2 "bcst_vector_operand" "xBm,xm,vmBr")))]
   "TARGET_SSE && <mask_mode512bit_condition>
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+   && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 {
   char buf[64];
   const char *ops;
@@ -13540,18 +13255,6 @@
              ]
              (const_string "<sseinsnmode>")))])
 
-(define_insn "*<code><mode>3_bcst"
-  [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
-       (any_logic:VI48_AVX512VL
-         (vec_duplicate:VI48_AVX512VL
-           (match_operand:<ssescalarmode> 1 "memory_operand" "m"))
-         (match_operand:VI48_AVX512VL 2 "register_operand" "v")))]
-  "TARGET_AVX512F && <mask_avx512vl_condition>"
-  "vp<logic><ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
-  [(set_attr "type" "sseiadd")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
 (define_mode_iterator VI1248_AVX512VLBW
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX512VL && TARGET_AVX512BW")
    (V16QI "TARGET_AVX512VL && TARGET_AVX512BW")

diff --git a/gcc/config/i386/subst.md b/gcc/config/i386/subst.md
index 58ea9dc83e27c74a2f2e84670b1c3be6bd78c6bf..e037a9649aa22650822e12c1c75b46233d6fd369 100644 (file)
--- a/gcc/config/i386/subst.md
+++ b/gcc/config/i386/subst.md
@@ -60,7 +60,9 @@
 (define_subst_attr "mask_prefix" "mask" "vex" "evex")
 (define_subst_attr "mask_prefix2" "mask" "maybe_vex" "evex")
 (define_subst_attr "mask_prefix3" "mask" "orig,vex" "evex,evex")
+(define_subst_attr "bcst_mask_prefix3" "mask" "orig,maybe_evex" "evex,evex")
 (define_subst_attr "mask_prefix4" "mask" "orig,orig,vex" "evex,evex,evex")
+(define_subst_attr "bcst_mask_prefix4" "mask" "orig,orig,maybe_evex" "evex,evex,evex")
 (define_subst_attr "mask_expand_op3" "mask" "3" "5")
 
 (define_subst "mask"
@@ -130,9 +132,11 @@
 (define_subst_attr "round_mask_op4" "round" "" "<round_mask_operand4>")
 (define_subst_attr "round_sd_mask_op4" "round" "" "<round_sd_mask_operand4>")
 (define_subst_attr "round_constraint" "round" "vm" "v")
+(define_subst_attr "bcst_round_constraint" "round" "vmBr" "v")
 (define_subst_attr "round_constraint2" "round" "m" "v")
 (define_subst_attr "round_constraint3" "round" "rm" "r")
 (define_subst_attr "round_nimm_predicate" "round" "vector_operand" "register_operand")
+(define_subst_attr "bcst_round_nimm_predicate" "round" "bcst_vector_operand" "register_operand")
 (define_subst_attr "round_nimm_scalar_predicate" "round" "nonimmediate_operand" "register_operand")
 (define_subst_attr "round_prefix" "round" "vex" "evex")
 (define_subst_attr "round_mode512bit_condition" "round" "1" "(<MODE>mode == V16SFmode