gcc/config/aarch64/aarch64-simd-builtins.def

   1 /* Machine description for AArch64 architecture.
   2    Copyright (C) 2012-2021 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 /* In the list below, the BUILTIN_<ITERATOR> macros expand to create
  22    builtins for each of the modes described by <ITERATOR>.  When adding
  23    new builtins to this list, a helpful idiom to follow is to add
  24    a line for each pattern in the md file.  Thus, ADDP, which has one
  25    pattern defined for the VD_BHSI iterator, and one for DImode, has two
  26    entries below.
  27
  28    Parameter 1 is the 'type' of the intrinsic.  This is used to
  29    describe the type modifiers (for example; unsigned) applied to
  30    each of the parameters to the intrinsic function.
  31
  32    Parameter 2 is the name of the intrinsic.  This is appended
  33    to `__builtin_aarch64_<name><mode>` to give the intrinsic name
  34    as exported to the front-ends.
  35
  36    Parameter 3 describes how to map from the name to the CODE_FOR_
  37    macro holding the RTL pattern for the intrinsic.  This mapping is:
  38    0 - CODE_FOR_aarch64_<name><mode>
  39    1-9 - CODE_FOR_<name><mode><1-9>
  40    10 - CODE_FOR_<name><mode>.
  41
  42    Parameter 4 is the 'flag' of the intrinsic.  This is used to
  43    help describe the attributes (for example, pure) for the intrinsic
  44    function.  */
  45
  46   BUILTIN_VDC (COMBINE, combine, 0, ALL)
  47   VAR1 (COMBINEP, combine, 0, ALL, di)
  48   BUILTIN_VB (BINOP, pmul, 0, NONE)
  49   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0, FP)
  50   BUILTIN_VHSDF_DF (UNOP, sqrt, 2, FP)
  51   BUILTIN_VD_BHSI (BINOP, addp, 0, NONE)
  52   VAR1 (UNOP, addp, 0, NONE, di)
  53   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2, ALL)
  54   BUILTIN_VDQ_BHSI (UNOP, clz, 2, ALL)
  55   BUILTIN_VS (UNOP, ctz, 2, ALL)
  56   BUILTIN_VB (UNOP, popcount, 2, ALL)
  57
  58   /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
  59   BUILTIN_VSDQ_I (BINOP, sqshl, 0, NONE)
  60   BUILTIN_VSDQ_I (BINOP_UUS, uqshl, 0, NONE)
  61   BUILTIN_VSDQ_I (BINOP, sqrshl, 0, NONE)
  62   BUILTIN_VSDQ_I (BINOP_UUS, uqrshl, 0, NONE)
  63   /* Implemented by aarch64_<su_optab><optab><mode>.  */
  64   BUILTIN_VSDQ_I (BINOP, sqadd, 0, NONE)
  65   BUILTIN_VSDQ_I (BINOPU, uqadd, 0, NONE)
  66   BUILTIN_VSDQ_I (BINOP, sqsub, 0, NONE)
  67   BUILTIN_VSDQ_I (BINOPU, uqsub, 0, NONE)
  68   /* Implemented by aarch64_<sur>qadd<mode>.  */
  69   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0, NONE)
  70   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0, NONE)
  71
  72   /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
  73   BUILTIN_VDC (GETREG, get_dregoi, 0, AUTO_FP)
  74   BUILTIN_VDC (GETREG, get_dregci, 0, AUTO_FP)
  75   BUILTIN_VDC (GETREG, get_dregxi, 0, AUTO_FP)
  76   VAR1 (GETREGP, get_dregoi, 0, AUTO_FP, di)
  77   VAR1 (GETREGP, get_dregci, 0, AUTO_FP, di)
  78   VAR1 (GETREGP, get_dregxi, 0, AUTO_FP, di)
  79   /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
  80   BUILTIN_VQ (GETREG, get_qregoi, 0, AUTO_FP)
  81   BUILTIN_VQ (GETREG, get_qregci, 0, AUTO_FP)
  82   BUILTIN_VQ (GETREG, get_qregxi, 0, AUTO_FP)
  83   VAR1 (GETREGP, get_qregoi, 0, AUTO_FP, v2di)
  84   VAR1 (GETREGP, get_qregci, 0, AUTO_FP, v2di)
  85   VAR1 (GETREGP, get_qregxi, 0, AUTO_FP, v2di)
  86   /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
  87   BUILTIN_VQ (SETREG, set_qregoi, 0, AUTO_FP)
  88   BUILTIN_VQ (SETREG, set_qregci, 0, AUTO_FP)
  89   BUILTIN_VQ (SETREG, set_qregxi, 0, AUTO_FP)
  90   VAR1 (SETREGP, set_qregoi, 0, AUTO_FP, v2di)
  91   VAR1 (SETREGP, set_qregci, 0, AUTO_FP, v2di)
  92   VAR1 (SETREGP, set_qregxi, 0, AUTO_FP, v2di)
  93   /* Implemented by aarch64_ld1x2<VQ:mode>. */
  94   BUILTIN_VQ (LOADSTRUCT, ld1x2, 0, ALL)
  95   /* Implemented by aarch64_ld1x2<VDC:mode>. */
  96   BUILTIN_VDC (LOADSTRUCT, ld1x2, 0, ALL)
  97   /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
  98   BUILTIN_VDC (LOADSTRUCT, ld2, 0, ALL)
  99   BUILTIN_VDC (LOADSTRUCT, ld3, 0, ALL)
 100   BUILTIN_VDC (LOADSTRUCT, ld4, 0, ALL)
 101   /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
 102   BUILTIN_VQ (LOADSTRUCT, ld2, 0, ALL)
 103   BUILTIN_VQ (LOADSTRUCT, ld3, 0, ALL)
 104   BUILTIN_VQ (LOADSTRUCT, ld4, 0, ALL)
 105   /* Implemented by aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>.  */
 106   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0, ALL)
 107   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0, ALL)
 108   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0, ALL)
 109   /* Implemented by aarch64_ld<VSTRUCT:nregs>_lane<VQ:mode>.  */
 110   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0, ALL)
 111   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0, ALL)
 112   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0, ALL)
 113   /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
 114   BUILTIN_VDC (STORESTRUCT, st2, 0, STORE)
 115   BUILTIN_VDC (STORESTRUCT, st3, 0, STORE)
 116   BUILTIN_VDC (STORESTRUCT, st4, 0, STORE)
 117   /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
 118   BUILTIN_VQ (STORESTRUCT, st2, 0, STORE)
 119   BUILTIN_VQ (STORESTRUCT, st3, 0, STORE)
 120   BUILTIN_VQ (STORESTRUCT, st4, 0, STORE)
 121
 122   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0, ALL)
 123   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0, ALL)
 124   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0, ALL)
 125
 126   BUILTIN_VQW (BINOP, saddl2, 0, NONE)
 127   BUILTIN_VQW (BINOP, uaddl2, 0, NONE)
 128   BUILTIN_VQW (BINOP, ssubl2, 0, NONE)
 129   BUILTIN_VQW (BINOP, usubl2, 0, NONE)
 130   BUILTIN_VQW (BINOP, saddw2, 0, NONE)
 131   BUILTIN_VQW (BINOP, uaddw2, 0, NONE)
 132   BUILTIN_VQW (BINOP, ssubw2, 0, NONE)
 133   BUILTIN_VQW (BINOP, usubw2, 0, NONE)
 134   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
 135   BUILTIN_VD_BHSI (BINOP, saddl, 0, NONE)
 136   BUILTIN_VD_BHSI (BINOP, uaddl, 0, NONE)
 137   BUILTIN_VD_BHSI (BINOP, ssubl, 0, NONE)
 138   BUILTIN_VD_BHSI (BINOP, usubl, 0, NONE)
 139   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
 140   BUILTIN_VD_BHSI (BINOP, saddw, 0, NONE)
 141   BUILTIN_VD_BHSI (BINOP, uaddw, 0, NONE)
 142   BUILTIN_VD_BHSI (BINOP, ssubw, 0, NONE)
 143   BUILTIN_VD_BHSI (BINOP, usubw, 0, NONE)
 144   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
 145   BUILTIN_VDQ_BHSI (BINOP, shadd, 0, NONE)
 146   BUILTIN_VDQ_BHSI (BINOP, shsub, 0, NONE)
 147   BUILTIN_VDQ_BHSI (BINOP, uhadd, 0, NONE)
 148   BUILTIN_VDQ_BHSI (BINOP, uhsub, 0, NONE)
 149   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0, NONE)
 150   BUILTIN_VDQ_BHSI (BINOP, urhadd, 0, NONE)
 151
 152   /* Implemented by aarch64_<su>abd<mode>.  */
 153   BUILTIN_VDQ_BHSI (BINOP, sabd, 0, NONE)
 154   BUILTIN_VDQ_BHSI (BINOPU, uabd, 0, NONE)
 155
 156   /* Implemented by aarch64_<su>aba<mode>.  */
 157   BUILTIN_VDQ_BHSI (TERNOP, saba, 0, NONE)
 158   BUILTIN_VDQ_BHSI (TERNOPU, uaba, 0, NONE)
 159
 160   BUILTIN_VDQV_S (BINOP, sadalp, 0, NONE)
 161   BUILTIN_VDQV_S (BINOPU, uadalp, 0, NONE)
 162
 163   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
 164   BUILTIN_VQN (BINOP, addhn, 0, NONE)
 165   BUILTIN_VQN (BINOP, subhn, 0, NONE)
 166   BUILTIN_VQN (BINOP, raddhn, 0, NONE)
 167   BUILTIN_VQN (BINOP, rsubhn, 0, NONE)
 168   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
 169   BUILTIN_VQN (TERNOP, addhn2, 0, NONE)
 170   BUILTIN_VQN (TERNOP, subhn2, 0, NONE)
 171   BUILTIN_VQN (TERNOP, raddhn2, 0, NONE)
 172   BUILTIN_VQN (TERNOP, rsubhn2, 0, NONE)
 173
 174   /* Implemented by aarch64_<us>xtl<mode>.  */
 175   BUILTIN_VQN (UNOP, sxtl, 0, NONE)
 176   BUILTIN_VQN (UNOPU, uxtl, 0, NONE)
 177
 178   /* Implemented by aarch64_xtn<mode>.  */
 179   BUILTIN_VQN (UNOP, xtn, 0, NONE)
 180
 181   /* Implemented by aarch64_<su>mlsl<mode>.  */
 182   BUILTIN_VD_BHSI (TERNOP, smlsl, 0, NONE)
 183   BUILTIN_VD_BHSI (TERNOPU, umlsl, 0, NONE)
 184
 185   /* Implemented by aarch64_<su>mlsl_hi<mode>.  */
 186   BUILTIN_VQW (TERNOP, smlsl_hi, 0, NONE)
 187   BUILTIN_VQW (TERNOPU, umlsl_hi, 0, NONE)
 188
 189   BUILTIN_VSQN_HSDI (UNOPUS, sqmovun, 0, NONE)
 190   /* Implemented by aarch64_<sur>qmovn<mode>.  */
 191   BUILTIN_VSQN_HSDI (UNOP, sqmovn, 0, NONE)
 192   BUILTIN_VSQN_HSDI (UNOP, uqmovn, 0, NONE)
 193
 194   /* Implemented by aarch64_<su>qxtn2<mode>.  */
 195   BUILTIN_VQN (BINOP, sqxtn2, 0, NONE)
 196   BUILTIN_VQN (BINOPU, uqxtn2, 0, NONE)
 197
 198   /* Implemented by aarch64_s<optab><mode>.  */
 199   BUILTIN_VSDQ_I (UNOP, sqabs, 0, NONE)
 200   BUILTIN_VSDQ_I (UNOP, sqneg, 0, NONE)
 201
 202   /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
 203   BUILTIN_VSD_HSI (TERNOP, sqdmlal, 0, NONE)
 204   BUILTIN_VSD_HSI (TERNOP, sqdmlsl, 0, NONE)
 205   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_lane<mode>.  */
 206   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_lane, 0, NONE)
 207   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_lane, 0, NONE)
 208   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_laneq<mode>.  */
 209   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_laneq, 0, NONE)
 210   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_laneq, 0, NONE)
 211   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
 212   BUILTIN_VD_HSI (TERNOP, sqdmlal_n, 0, NONE)
 213   BUILTIN_VD_HSI (TERNOP, sqdmlsl_n, 0, NONE)
 214
 215   BUILTIN_VQ_HSI (TERNOP, sqdmlal2, 0, NONE)
 216   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2, 0, NONE)
 217   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_lane, 0, NONE)
 218   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_lane, 0, NONE)
 219   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_laneq, 0, NONE)
 220   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_laneq, 0, NONE)
 221   BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n, 0, NONE)
 222   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n, 0, NONE)
 223
 224   BUILTIN_VD_BHSI (BINOP, intrinsic_vec_smult_lo_, 0, NONE)
 225   BUILTIN_VD_BHSI (BINOPU, intrinsic_vec_umult_lo_, 0, NONE)
 226
 227   BUILTIN_VQW (BINOP, vec_widen_smult_hi_, 10, NONE)
 228   BUILTIN_VQW (BINOPU, vec_widen_umult_hi_, 10, NONE)
 229
 230   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_lane_, 0, ALL)
 231   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_lane_, 0, ALL)
 232   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_laneq_, 0, ALL)
 233   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_laneq_, 0, ALL)
 234   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_lane_, 0, ALL)
 235   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_lane_, 0, ALL)
 236   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_laneq_, 0, ALL)
 237   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_laneq_, 0, ALL)
 238
 239   BUILTIN_VSD_HSI (BINOP, sqdmull, 0, NONE)
 240   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_lane, 0, NONE)
 241   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_laneq, 0, NONE)
 242   BUILTIN_VD_HSI (BINOP, sqdmull_n, 0, NONE)
 243   BUILTIN_VQ_HSI (BINOP, sqdmull2, 0, NONE)
 244   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_lane, 0, NONE)
 245   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_laneq, 0, NONE)
 246   BUILTIN_VQ_HSI (BINOP, sqdmull2_n, 0, NONE)
 247   /* Implemented by aarch64_sq<r>dmulh<mode>.  */
 248   BUILTIN_VSDQ_HSI (BINOP, sqdmulh, 0, NONE)
 249   BUILTIN_VSDQ_HSI (BINOP, sqrdmulh, 0, NONE)
 250   /* Implemented by aarch64_sq<r>dmulh_lane<q><mode>.  */
 251   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_lane, 0, NONE)
 252   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_laneq, 0, NONE)
 253   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_lane, 0, NONE)
 254   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_laneq, 0, NONE)
 255
 256   BUILTIN_VSDQ_I_DI (BINOP, ashl, 3, ALL)
 257   /* Implemented by aarch64_<sur>shl<mode>.  */
 258   BUILTIN_VSDQ_I_DI (BINOP, sshl, 0, ALL)
 259   BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0, ALL)
 260   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0, ALL)
 261   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0, ALL)
 262
 263   /* Implemented by aarch64_<sur><dotprod>{_lane}{q}<dot_mode>.  */
 264   BUILTIN_VB (TERNOP, sdot, 0, NONE)
 265   BUILTIN_VB (TERNOPU, udot, 0, NONE)
 266   BUILTIN_VB (TERNOP_SSUS, usdot, 0, NONE)
 267   BUILTIN_VB (QUADOP_LANE, sdot_lane, 0, ALL)
 268   BUILTIN_VB (QUADOPU_LANE, udot_lane, 0, ALL)
 269   BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0, ALL)
 270   BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0, ALL)
 271   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_lane, 0, ALL)
 272   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_laneq, 0, ALL)
 273   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_lane, 0, ALL)
 274   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_laneq, 0, ALL)
 275
 276   /* Implemented by aarch64_fcadd<rot><mode>.   */
 277   BUILTIN_VHSDF (BINOP, fcadd90, 0, FP)
 278   BUILTIN_VHSDF (BINOP, fcadd270, 0, FP)
 279
 280   /* Implemented by aarch64_fcmla{_lane}{q}<rot><mode>.   */
 281   BUILTIN_VHSDF (TERNOP, fcmla0, 0, FP)
 282   BUILTIN_VHSDF (TERNOP, fcmla90, 0, FP)
 283   BUILTIN_VHSDF (TERNOP, fcmla180, 0, FP)
 284   BUILTIN_VHSDF (TERNOP, fcmla270, 0, FP)
 285   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane0, 0, ALL)
 286   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane90, 0, ALL)
 287   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane180, 0, ALL)
 288   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane270, 0, ALL)
 289
 290   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane0, 0, ALL)
 291   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane90, 0, ALL)
 292   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane180, 0, ALL)
 293   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane270, 0, ALL)
 294
 295   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, ALL)
 296   VAR1 (SHIFTIMM, ashr_simd, 0, ALL, di)
 297   BUILTIN_VDQ_I (SHIFTIMM, lshr, 3, ALL)
 298   VAR1 (USHIFTIMM, lshr_simd, 0, ALL, di)
 299   /* Implemented by aarch64_<sur>shr_n<mode>.  */
 300   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0, ALL)
 301   BUILTIN_VSDQ_I_DI (USHIFTIMM, urshr_n, 0, ALL)
 302   /* Implemented by aarch64_<sur>sra_n<mode>.  */
 303   BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n, 0, ALL)
 304   BUILTIN_VSDQ_I_DI (USHIFTACC, usra_n, 0, ALL)
 305   BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n, 0, ALL)
 306   BUILTIN_VSDQ_I_DI (USHIFTACC, ursra_n, 0, ALL)
 307   /* Implemented by aarch64_<sur>shll_n<mode>.  */
 308   BUILTIN_VD_BHSI (SHIFTIMM, sshll_n, 0, ALL)
 309   BUILTIN_VD_BHSI (USHIFTIMM, ushll_n, 0, ALL)
 310   /* Implemented by aarch64_<sur>shll2_n<mode>.  */
 311   BUILTIN_VQW (SHIFTIMM, sshll2_n, 0, ALL)
 312   BUILTIN_VQW (SHIFTIMM, ushll2_n, 0, ALL)
 313   /* Implemented by aarch64_<sur>q<r>shr<u>n_n<mode>.  */
 314   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrun_n, 0, NONE)
 315   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrun_n, 0, NONE)
 316   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrn_n, 0, NONE)
 317   BUILTIN_VSQN_HSDI (USHIFTIMM, uqshrn_n, 0, NONE)
 318   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrn_n, 0, NONE)
 319   BUILTIN_VSQN_HSDI (USHIFTIMM, uqrshrn_n, 0, NONE)
 320   /* Implemented by aarch64_<sur>q<r>shr<u>n2_n<mode>.  */
 321   BUILTIN_VQN (SHIFT2IMM_UUSS, sqshrun2_n, 0, NONE)
 322   BUILTIN_VQN (SHIFT2IMM_UUSS, sqrshrun2_n, 0, NONE)
 323   BUILTIN_VQN (SHIFT2IMM, sqshrn2_n, 0, NONE)
 324   BUILTIN_VQN (USHIFT2IMM, uqshrn2_n, 0, NONE)
 325   BUILTIN_VQN (SHIFT2IMM, sqrshrn2_n, 0, NONE)
 326   BUILTIN_VQN (USHIFT2IMM, uqrshrn2_n, 0, NONE)
 327   /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
 328   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n, 0, ALL)
 329   BUILTIN_VSDQ_I_DI (USHIFTACC, usri_n, 0, ALL)
 330   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n, 0, ALL)
 331   VAR2 (SHIFTINSERTP, ssli_n, 0, ALL, di, v2di)
 332   BUILTIN_VSDQ_I_DI (USHIFTACC, usli_n, 0, ALL)
 333   /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
 334   BUILTIN_VSDQ_I (SHIFTIMM_USS, sqshlu_n, 0, NONE)
 335   BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n, 0, NONE)
 336   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0, NONE)
 337
 338   /* Implemented by aarch64_xtn2<mode>.  */
 339   BUILTIN_VQN (UNOP, xtn2, 0, NONE)
 340
 341   /* Implemented by aarch64_reduc_plus_<mode>.  */
 342   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, NONE)
 343
 344   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
 345   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10, NONE)
 346   BUILTIN_VDQIF_F16 (UNOP, reduc_smin_scal_, 10, NONE)
 347   BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, NONE)
 348   BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, NONE)
 349   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10, NONE)
 350   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10, NONE)
 351
 352   /* Implemented by <maxmin_uns><mode>3.
 353      smax variants map to fmaxnm,
 354      smax_nan variants map to fmax.  */
 355   BUILTIN_VDQ_BHSI (BINOP, smax, 3, NONE)
 356   BUILTIN_VDQ_BHSI (BINOP, smin, 3, NONE)
 357   BUILTIN_VDQ_BHSI (BINOP, umax, 3, NONE)
 358   BUILTIN_VDQ_BHSI (BINOP, umin, 3, NONE)
 359   BUILTIN_VHSDF_DF (BINOP, smax_nan, 3, NONE)
 360   BUILTIN_VHSDF_DF (BINOP, smin_nan, 3, NONE)
 361
 362   /* Implemented by <maxmin_uns><mode>3.  */
 363   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3, FP)
 364   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3, FP)
 365
 366   /* Implemented by aarch64_<maxmin_uns>p<mode>.  */
 367   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0, NONE)
 368   BUILTIN_VDQ_BHSI (BINOP, sminp, 0, NONE)
 369   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0, NONE)
 370   BUILTIN_VDQ_BHSI (BINOP, uminp, 0, NONE)
 371   BUILTIN_VHSDF (BINOP, smaxp, 0, NONE)
 372   BUILTIN_VHSDF (BINOP, sminp, 0, NONE)
 373   BUILTIN_VHSDF (BINOP, smax_nanp, 0, NONE)
 374   BUILTIN_VHSDF (BINOP, smin_nanp, 0, NONE)
 375
 376   /* Implemented by <frint_pattern><mode>2.  */
 377   BUILTIN_VHSDF (UNOP, btrunc, 2, FP)
 378   BUILTIN_VHSDF (UNOP, ceil, 2, FP)
 379   BUILTIN_VHSDF (UNOP, floor, 2, FP)
 380   BUILTIN_VHSDF (UNOP, nearbyint, 2, FP)
 381   BUILTIN_VHSDF (UNOP, rint, 2, FP)
 382   BUILTIN_VHSDF (UNOP, round, 2, FP)
 383   BUILTIN_VHSDF_HSDF (UNOP, frintn, 2, FP)
 384
 385   VAR1 (UNOP, btrunc, 2, FP, hf)
 386   VAR1 (UNOP, ceil, 2, FP, hf)
 387   VAR1 (UNOP, floor, 2, FP, hf)
 388   VAR1 (UNOP, nearbyint, 2, FP, hf)
 389   VAR1 (UNOP, rint, 2, FP, hf)
 390   VAR1 (UNOP, round, 2, FP, hf)
 391
 392   /* Implemented by l<fcvt_pattern><su_optab><VQDF:mode><vcvt_target>2.  */
 393   VAR1 (UNOP, lbtruncv4hf, 2, FP, v4hi)
 394   VAR1 (UNOP, lbtruncv8hf, 2, FP, v8hi)
 395   VAR1 (UNOP, lbtruncv2sf, 2, FP, v2si)
 396   VAR1 (UNOP, lbtruncv4sf, 2, FP, v4si)
 397   VAR1 (UNOP, lbtruncv2df, 2, FP, v2di)
 398
 399   VAR1 (UNOPUS, lbtruncuv4hf, 2, FP, v4hi)
 400   VAR1 (UNOPUS, lbtruncuv8hf, 2, FP, v8hi)
 401   VAR1 (UNOPUS, lbtruncuv2sf, 2, FP, v2si)
 402   VAR1 (UNOPUS, lbtruncuv4sf, 2, FP, v4si)
 403   VAR1 (UNOPUS, lbtruncuv2df, 2, FP, v2di)
 404
 405   VAR1 (UNOP, lroundv4hf, 2, FP, v4hi)
 406   VAR1 (UNOP, lroundv8hf, 2, FP, v8hi)
 407   VAR1 (UNOP, lroundv2sf, 2, FP, v2si)
 408   VAR1 (UNOP, lroundv4sf, 2, FP, v4si)
 409   VAR1 (UNOP, lroundv2df, 2, FP, v2di)
 410   /* Implemented by l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2.  */
 411   BUILTIN_GPI_I16 (UNOP, lroundhf, 2, FP)
 412   VAR1 (UNOP, lroundsf, 2, FP, si)
 413   VAR1 (UNOP, lrounddf, 2, FP, di)
 414
 415   VAR1 (UNOPUS, lrounduv4hf, 2, FP, v4hi)
 416   VAR1 (UNOPUS, lrounduv8hf, 2, FP, v8hi)
 417   VAR1 (UNOPUS, lrounduv2sf, 2, FP, v2si)
 418   VAR1 (UNOPUS, lrounduv4sf, 2, FP, v4si)
 419   VAR1 (UNOPUS, lrounduv2df, 2, FP, v2di)
 420   BUILTIN_GPI_I16 (UNOPUS, lrounduhf, 2, FP)
 421   VAR1 (UNOPUS, lroundusf, 2, FP, si)
 422   VAR1 (UNOPUS, lroundudf, 2, FP, di)
 423
 424   VAR1 (UNOP, lceilv4hf, 2, FP, v4hi)
 425   VAR1 (UNOP, lceilv8hf, 2, FP, v8hi)
 426   VAR1 (UNOP, lceilv2sf, 2, FP, v2si)
 427   VAR1 (UNOP, lceilv4sf, 2, FP, v4si)
 428   VAR1 (UNOP, lceilv2df, 2, FP, v2di)
 429   BUILTIN_GPI_I16 (UNOP, lceilhf, 2, FP)
 430
 431   VAR1 (UNOPUS, lceiluv4hf, 2, FP, v4hi)
 432   VAR1 (UNOPUS, lceiluv8hf, 2, FP, v8hi)
 433   VAR1 (UNOPUS, lceiluv2sf, 2, FP, v2si)
 434   VAR1 (UNOPUS, lceiluv4sf, 2, FP, v4si)
 435   VAR1 (UNOPUS, lceiluv2df, 2, FP, v2di)
 436   BUILTIN_GPI_I16 (UNOPUS, lceiluhf, 2, FP)
 437   VAR1 (UNOPUS, lceilusf, 2, FP, si)
 438   VAR1 (UNOPUS, lceiludf, 2, FP, di)
 439
 440   VAR1 (UNOP, lfloorv4hf, 2, FP, v4hi)
 441   VAR1 (UNOP, lfloorv8hf, 2, FP, v8hi)
 442   VAR1 (UNOP, lfloorv2sf, 2, FP, v2si)
 443   VAR1 (UNOP, lfloorv4sf, 2, FP, v4si)
 444   VAR1 (UNOP, lfloorv2df, 2, FP, v2di)
 445   BUILTIN_GPI_I16 (UNOP, lfloorhf, 2, FP)
 446
 447   VAR1 (UNOPUS, lflooruv4hf, 2, FP, v4hi)
 448   VAR1 (UNOPUS, lflooruv8hf, 2, FP, v8hi)
 449   VAR1 (UNOPUS, lflooruv2sf, 2, FP, v2si)
 450   VAR1 (UNOPUS, lflooruv4sf, 2, FP, v4si)
 451   VAR1 (UNOPUS, lflooruv2df, 2, FP, v2di)
 452   BUILTIN_GPI_I16 (UNOPUS, lflooruhf, 2, FP)
 453   VAR1 (UNOPUS, lfloorusf, 2, FP, si)
 454   VAR1 (UNOPUS, lfloorudf, 2, FP, di)
 455
 456   VAR1 (UNOP, lfrintnv4hf, 2, FP, v4hi)
 457   VAR1 (UNOP, lfrintnv8hf, 2, FP, v8hi)
 458   VAR1 (UNOP, lfrintnv2sf, 2, FP, v2si)
 459   VAR1 (UNOP, lfrintnv4sf, 2, FP, v4si)
 460   VAR1 (UNOP, lfrintnv2df, 2, FP, v2di)
 461   BUILTIN_GPI_I16 (UNOP, lfrintnhf, 2, FP)
 462   VAR1 (UNOP, lfrintnsf, 2, FP, si)
 463   VAR1 (UNOP, lfrintndf, 2, FP, di)
 464
 465   VAR1 (UNOPUS, lfrintnuv4hf, 2, FP, v4hi)
 466   VAR1 (UNOPUS, lfrintnuv8hf, 2, FP, v8hi)
 467   VAR1 (UNOPUS, lfrintnuv2sf, 2, FP, v2si)
 468   VAR1 (UNOPUS, lfrintnuv4sf, 2, FP, v4si)
 469   VAR1 (UNOPUS, lfrintnuv2df, 2, FP, v2di)
 470   BUILTIN_GPI_I16 (UNOPUS, lfrintnuhf, 2, FP)
 471   VAR1 (UNOPUS, lfrintnusf, 2, FP, si)
 472   VAR1 (UNOPUS, lfrintnudf, 2, FP, di)
 473
 474   /* Implemented by <optab><fcvt_target><VDQF:mode>2.  */
 475   VAR1 (UNOP, floatv4hi, 2, FP, v4hf)
 476   VAR1 (UNOP, floatv8hi, 2, FP, v8hf)
 477   VAR1 (UNOP, floatv2si, 2, FP, v2sf)
 478   VAR1 (UNOP, floatv4si, 2, FP, v4sf)
 479   VAR1 (UNOP, floatv2di, 2, FP, v2df)
 480
 481   VAR1 (UNOP, floatunsv4hi, 2, FP, v4hf)
 482   VAR1 (UNOP, floatunsv8hi, 2, FP, v8hf)
 483   VAR1 (UNOP, floatunsv2si, 2, FP, v2sf)
 484   VAR1 (UNOP, floatunsv4si, 2, FP, v4sf)
 485   VAR1 (UNOP, floatunsv2di, 2, FP, v2df)
 486
 487   VAR5 (UNOPU, bswap, 2, ALL, v4hi, v8hi, v2si, v4si, v2di)
 488
 489   BUILTIN_VB (UNOP, rbit, 0, ALL)
 490
 491   /* Implemented by
 492      aarch64_<PERMUTE:perm_insn><mode>.  */
 493   BUILTIN_VALL (BINOP, zip1, 0, ALL)
 494   BUILTIN_VALL (BINOP, zip2, 0, ALL)
 495   BUILTIN_VALL (BINOP, uzp1, 0, ALL)
 496   BUILTIN_VALL (BINOP, uzp2, 0, ALL)
 497   BUILTIN_VALL (BINOP, trn1, 0, ALL)
 498   BUILTIN_VALL (BINOP, trn2, 0, ALL)
 499
 500   BUILTIN_GPF_F16 (UNOP, frecpe, 0, FP)
 501   BUILTIN_GPF_F16 (UNOP, frecpx, 0, FP)
 502
 503   BUILTIN_VDQ_SI (UNOP, urecpe, 0, NONE)
 504
 505   BUILTIN_VHSDF (UNOP, frecpe, 0, FP)
 506   BUILTIN_VHSDF_HSDF (BINOP, frecps, 0, FP)
 507
 508   /* Implemented by a mixture of abs2 patterns.  Note the DImode builtin is
 509      only ever used for the int64x1_t intrinsic, there is no scalar version.  */
 510   BUILTIN_VSDQ_I_DI (UNOP, abs, 0, AUTO_FP)
 511   BUILTIN_VHSDF (UNOP, abs, 2, AUTO_FP)
 512   VAR1 (UNOP, abs, 2, AUTO_FP, hf)
 513
 514   BUILTIN_VQ_HSF (UNOP, vec_unpacks_hi_, 10, FP)
 515   VAR1 (BINOP, float_truncate_hi_, 0, FP, v4sf)
 516   VAR1 (BINOP, float_truncate_hi_, 0, FP, v8hf)
 517
 518   VAR1 (UNOP, float_extend_lo_, 0, FP, v2df)
 519   VAR1 (UNOP, float_extend_lo_,  0, FP, v4sf)
 520   BUILTIN_VDF (UNOP, float_truncate_lo_, 0, FP)
 521
 522   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
 523   BUILTIN_VALL_F16 (LOAD1, ld1, 0, ALL)
 524   VAR1(STORE1P, ld1, 0, ALL, v2di)
 525
 526   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
 527   BUILTIN_VALL_F16 (STORE1, st1, 0, STORE)
 528   VAR1 (STORE1P, st1, 0, STORE, v2di)
 529
 530   /* Implemented by aarch64_ld1x3<VALLDIF:mode>.  */
 531   BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0, ALL)
 532
 533   /* Implemented by aarch64_ld1x4<VALLDIF:mode>.  */
 534   BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0, ALL)
 535
 536   /* Implemented by aarch64_st1x2<VALLDIF:mode>.  */
 537   BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0, STORE)
 538
 539   /* Implemented by aarch64_st1x3<VALLDIF:mode>.  */
 540   BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0, STORE)
 541
 542   /* Implemented by aarch64_st1x4<VALLDIF:mode>.  */
 543   BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0, STORE)
 544
 545   /* Implemented by fma<mode>4.  */
 546   BUILTIN_VHSDF (TERNOP, fma, 4, FP)
 547   VAR1 (TERNOP, fma, 4, FP, hf)
 548   /* Implemented by fnma<mode>4.  */
 549   BUILTIN_VHSDF (TERNOP, fnma, 4, FP)
 550   VAR1 (TERNOP, fnma, 4, FP, hf)
 551
 552   /* Implemented by aarch64_simd_bsl<mode>.  */
 553   BUILTIN_VDQQH (BSL_P, simd_bsl, 0, ALL)
 554   VAR2 (BSL_P, simd_bsl,0, ALL, di, v2di)
 555   BUILTIN_VSDQ_I_DI (BSL_U, simd_bsl, 0, ALL)
 556   BUILTIN_VALLDIF (BSL_S, simd_bsl, 0, ALL)
 557
 558   /* Implemented by aarch64_crypto_aes<op><mode>.  */
 559   VAR1 (BINOPU, crypto_aese, 0, NONE, v16qi)
 560   VAR1 (BINOPU, crypto_aesd, 0, NONE, v16qi)
 561   VAR1 (UNOPU, crypto_aesmc, 0, NONE, v16qi)
 562   VAR1 (UNOPU, crypto_aesimc, 0, NONE, v16qi)
 563
 564   /* Implemented by aarch64_crypto_sha1<op><mode>.  */
 565   VAR1 (UNOPU, crypto_sha1h, 0, NONE, si)
 566   VAR1 (BINOPU, crypto_sha1su1, 0, NONE, v4si)
 567   VAR1 (TERNOPU, crypto_sha1c, 0, NONE, v4si)
 568   VAR1 (TERNOPU, crypto_sha1m, 0, NONE, v4si)
 569   VAR1 (TERNOPU, crypto_sha1p, 0, NONE, v4si)
 570   VAR1 (TERNOPU, crypto_sha1su0, 0, NONE, v4si)
 571
 572   /* Implemented by aarch64_crypto_sha256<op><mode>.  */
 573   VAR1 (TERNOPU, crypto_sha256h, 0, NONE, v4si)
 574   VAR1 (TERNOPU, crypto_sha256h2, 0, NONE, v4si)
 575   VAR1 (BINOPU, crypto_sha256su0, 0, NONE, v4si)
 576   VAR1 (TERNOPU, crypto_sha256su1, 0, NONE, v4si)
 577
 578   /* Implemented by aarch64_crypto_pmull<mode>.  */
 579   VAR1 (BINOPP, crypto_pmull, 0, NONE, di)
 580   VAR1 (BINOPP, crypto_pmull, 0, NONE, v2di)
 581
 582   /* Implemented by aarch64_tbl3<mode>.  */
 583   VAR1 (BINOP, tbl3, 0, NONE, v8qi)
 584   VAR1 (BINOP, tbl3, 0, NONE, v16qi)
 585
 586   /* Implemented by aarch64_qtbl3<mode>.  */
 587   VAR1 (BINOP, qtbl3, 0, NONE, v8qi)
 588   VAR1 (BINOP, qtbl3, 0, NONE, v16qi)
 589
 590   /* Implemented by aarch64_qtbl4<mode>.  */
 591   VAR1 (BINOP, qtbl4, 0, NONE, v8qi)
 592   VAR1 (BINOP, qtbl4, 0, NONE, v16qi)
 593
 594   /* Implemented by aarch64_tbx4<mode>.  */
 595   VAR1 (TERNOP, tbx4, 0, NONE, v8qi)
 596   VAR1 (TERNOP, tbx4, 0, NONE, v16qi)
 597
 598   /* Implemented by aarch64_qtbx3<mode>.  */
 599   VAR1 (TERNOP, qtbx3, 0, NONE, v8qi)
 600   VAR1 (TERNOP, qtbx3, 0, NONE, v16qi)
 601
 602   /* Implemented by aarch64_qtbx4<mode>.  */
 603   VAR1 (TERNOP, qtbx4, 0, NONE, v8qi)
 604   VAR1 (TERNOP, qtbx4, 0, NONE, v16qi)
 605
 606   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 607
 608   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>.  */
 609   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlah, 0, NONE)
 610   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlsh, 0, NONE)
 611
 612   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>.  */
 613   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_lane, 0, NONE)
 614   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_lane, 0, NONE)
 615
 616   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>.  */
 617   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_laneq, 0, NONE)
 618   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_laneq, 0, NONE)
 619
 620   /* Implemented by <FCVT_F2FIXED/FIXED2F:fcvt_fixed_insn><*><*>3.  */
 621   BUILTIN_VSDQ_HSDI (SHIFTIMM, scvtf, 3, ALL)
 622   BUILTIN_VSDQ_HSDI (FCVTIMM_SUS, ucvtf, 3, ALL)
 623   BUILTIN_VHSDF_HSDF (SHIFTIMM, fcvtzs, 3, ALL)
 624   BUILTIN_VHSDF_HSDF (SHIFTIMM_USS, fcvtzu, 3, ALL)
 625   VAR1 (SHIFTIMM, scvtfsi, 3, ALL, hf)
 626   VAR1 (SHIFTIMM, scvtfdi, 3, ALL, hf)
 627   VAR1 (FCVTIMM_SUS, ucvtfsi, 3, ALL, hf)
 628   VAR1 (FCVTIMM_SUS, ucvtfdi, 3, ALL, hf)
 629   BUILTIN_GPI (SHIFTIMM, fcvtzshf, 3, ALL)
 630   BUILTIN_GPI (SHIFTIMM_USS, fcvtzuhf, 3, ALL)
 631
 632   /* Implemented by aarch64_rsqrte<mode>.  */
 633   BUILTIN_VHSDF_HSDF (UNOP, rsqrte, 0, FP)
 634
 635   /* Implemented by aarch64_rsqrts<mode>.  */
 636   BUILTIN_VHSDF_HSDF (BINOP, rsqrts, 0, FP)
 637
 638   /* Implemented by fabd<mode>3.  */
 639   BUILTIN_VHSDF_HSDF (BINOP, fabd, 3, FP)
 640
 641   /* Implemented by aarch64_faddp<mode>.  */
 642   BUILTIN_VHSDF (BINOP, faddp, 0, FP)
 643
 644   /* Implemented by aarch64_cm<optab><mode>.  */
 645   BUILTIN_VHSDF_HSDF (BINOP_USS, cmeq, 0, FP)
 646   BUILTIN_VHSDF_HSDF (BINOP_USS, cmge, 0, FP)
 647   BUILTIN_VHSDF_HSDF (BINOP_USS, cmgt, 0, FP)
 648   BUILTIN_VHSDF_HSDF (BINOP_USS, cmle, 0, FP)
 649   BUILTIN_VHSDF_HSDF (BINOP_USS, cmlt, 0, FP)
 650
 651   /* Implemented by neg<mode>2.  */
 652   BUILTIN_VHSDF_HSDF (UNOP, neg, 2, ALL)
 653
 654   /* Implemented by aarch64_fac<optab><mode>.  */
 655   BUILTIN_VHSDF_HSDF (BINOP_USS, faclt, 0, FP)
 656   BUILTIN_VHSDF_HSDF (BINOP_USS, facle, 0, FP)
 657   BUILTIN_VHSDF_HSDF (BINOP_USS, facgt, 0, FP)
 658   BUILTIN_VHSDF_HSDF (BINOP_USS, facge, 0, FP)
 659
 660   /* Implemented by sqrt<mode>2.  */
 661   VAR1 (UNOP, sqrt, 2, FP, hf)
 662
 663   /* Implemented by <optab><mode>hf2.  */
 664   VAR1 (UNOP, floatdi, 2, FP, hf)
 665   VAR1 (UNOP, floatsi, 2, FP, hf)
 666   VAR1 (UNOP, floathi, 2, FP, hf)
 667   VAR1 (UNOPUS, floatunsdi, 2, FP, hf)
 668   VAR1 (UNOPUS, floatunssi, 2, FP, hf)
 669   VAR1 (UNOPUS, floatunshi, 2, FP, hf)
 670   BUILTIN_GPI_I16 (UNOP, fix_trunchf, 2, FP)
 671   BUILTIN_GPI (UNOP, fix_truncsf, 2, FP)
 672   BUILTIN_GPI (UNOP, fix_truncdf, 2, FP)
 673   BUILTIN_GPI_I16 (UNOPUS, fixuns_trunchf, 2, FP)
 674   BUILTIN_GPI (UNOPUS, fixuns_truncsf, 2, FP)
 675   BUILTIN_GPI (UNOPUS, fixuns_truncdf, 2, FP)
 676
 677   /* Implemented by aarch64_sm3ss1qv4si.  */
 678   VAR1 (TERNOPU, sm3ss1q, 0, NONE, v4si)
 679   /* Implemented by aarch64_sm3tt<sm3tt_op>qv4si.  */
 680   VAR1 (QUADOPUI, sm3tt1aq, 0, NONE, v4si)
 681   VAR1 (QUADOPUI, sm3tt1bq, 0, NONE, v4si)
 682   VAR1 (QUADOPUI, sm3tt2aq, 0, NONE, v4si)
 683   VAR1 (QUADOPUI, sm3tt2bq, 0, NONE, v4si)
 684   /* Implemented by aarch64_sm3partw<sm3part_op>qv4si.  */
 685   VAR1 (TERNOPU, sm3partw1q, 0, NONE, v4si)
 686   VAR1 (TERNOPU, sm3partw2q, 0, NONE, v4si)
 687   /* Implemented by aarch64_sm4eqv4si.  */
 688   VAR1 (BINOPU, sm4eq, 0, NONE, v4si)
 689   /* Implemented by aarch64_sm4ekeyqv4si.  */
 690   VAR1 (BINOPU, sm4ekeyq, 0, NONE, v4si)
 691   /* Implemented by aarch64_crypto_sha512hqv2di.  */
 692   VAR1 (TERNOPU, crypto_sha512hq, 0, NONE, v2di)
 693   /* Implemented by aarch64_sha512h2qv2di.  */
 694   VAR1 (TERNOPU, crypto_sha512h2q, 0, NONE, v2di)
 695   /* Implemented by aarch64_crypto_sha512su0qv2di.  */
 696   VAR1 (BINOPU, crypto_sha512su0q, 0, NONE, v2di)
 697   /* Implemented by aarch64_crypto_sha512su1qv2di.  */
 698   VAR1 (TERNOPU, crypto_sha512su1q, 0, NONE, v2di)
 699   /* Implemented by eor3q<mode>4.  */
 700   BUILTIN_VQ_I (TERNOPU, eor3q, 4, ALL)
 701   BUILTIN_VQ_I (TERNOP, eor3q, 4, ALL)
 702   /* Implemented by aarch64_rax1qv2di.  */
 703   VAR1 (BINOPU, rax1q, 0, ALL, v2di)
 704   /* Implemented by aarch64_xarqv2di.  */
 705   VAR1 (TERNOPUI, xarq, 0, ALL, v2di)
 706   /* Implemented by bcaxq<mode>4.  */
 707   BUILTIN_VQ_I (TERNOPU, bcaxq, 4, ALL)
 708   BUILTIN_VQ_I (TERNOP, bcaxq, 4, ALL)
 709
 710   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_low<mode>.  */
 711   VAR1 (TERNOP, fmlal_low, 0, FP, v2sf)
 712   VAR1 (TERNOP, fmlsl_low, 0, FP, v2sf)
 713   VAR1 (TERNOP, fmlalq_low, 0, FP, v4sf)
 714   VAR1 (TERNOP, fmlslq_low, 0, FP, v4sf)
 715   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_high<mode>.  */
 716   VAR1 (TERNOP, fmlal_high, 0, FP, v2sf)
 717   VAR1 (TERNOP, fmlsl_high, 0, FP, v2sf)
 718   VAR1 (TERNOP, fmlalq_high, 0, FP, v4sf)
 719   VAR1 (TERNOP, fmlslq_high, 0, FP, v4sf)
 720   /* Implemented by aarch64_fml<f16mac1>l_lane_lowv2sf.  */
 721   VAR1 (QUADOP_LANE, fmlal_lane_low, 0, ALL, v2sf)
 722   VAR1 (QUADOP_LANE, fmlsl_lane_low, 0, ALL, v2sf)
 723   /* Implemented by aarch64_fml<f16mac1>l_laneq_lowv2sf.  */
 724   VAR1 (QUADOP_LANE, fmlal_laneq_low, 0, ALL, v2sf)
 725   VAR1 (QUADOP_LANE, fmlsl_laneq_low, 0, ALL, v2sf)
 726   /* Implemented by aarch64_fml<f16mac1>lq_lane_lowv4sf.  */
 727   VAR1 (QUADOP_LANE, fmlalq_lane_low, 0, ALL, v4sf)
 728   VAR1 (QUADOP_LANE, fmlslq_lane_low, 0, ALL, v4sf)
 729   /* Implemented by aarch64_fml<f16mac1>lq_laneq_lowv4sf.  */
 730   VAR1 (QUADOP_LANE, fmlalq_laneq_low, 0, ALL, v4sf)
 731   VAR1 (QUADOP_LANE, fmlslq_laneq_low, 0, ALL, v4sf)
 732   /* Implemented by aarch64_fml<f16mac1>l_lane_highv2sf.  */
 733   VAR1 (QUADOP_LANE, fmlal_lane_high, 0, ALL, v2sf)
 734   VAR1 (QUADOP_LANE, fmlsl_lane_high, 0, ALL, v2sf)
 735   /* Implemented by aarch64_fml<f16mac1>l_laneq_highv2sf.  */
 736   VAR1 (QUADOP_LANE, fmlal_laneq_high, 0, ALL, v2sf)
 737   VAR1 (QUADOP_LANE, fmlsl_laneq_high, 0, ALL, v2sf)
 738   /* Implemented by aarch64_fml<f16mac1>lq_lane_highv4sf.  */
 739   VAR1 (QUADOP_LANE, fmlalq_lane_high, 0, ALL, v4sf)
 740   VAR1 (QUADOP_LANE, fmlslq_lane_high, 0, ALL, v4sf)
 741   /* Implemented by aarch64_fml<f16mac1>lq_laneq_highv4sf.  */
 742   VAR1 (QUADOP_LANE, fmlalq_laneq_high, 0, ALL, v4sf)
 743   VAR1 (QUADOP_LANE, fmlslq_laneq_high, 0, ALL, v4sf)
 744
 745   /* Implemented by aarch64_<frintnzs_op><mode>.  */
 746   BUILTIN_VSFDF (UNOP, frint32z, 0, FP)
 747   BUILTIN_VSFDF (UNOP, frint32x, 0, FP)
 748   BUILTIN_VSFDF (UNOP, frint64z, 0, FP)
 749   BUILTIN_VSFDF (UNOP, frint64x, 0, FP)
 750
 751   /* Implemented by aarch64_bfdot{_lane}{q}<mode>.  */
 752   VAR2 (TERNOP, bfdot, 0, AUTO_FP, v2sf, v4sf)
 753   VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, ALL, v2sf, v4sf)
 754   VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, ALL, v2sf, v4sf)
 755
 756   /* Implemented by aarch64_bfmmlaqv4sf  */
 757   VAR1 (TERNOP, bfmmlaq, 0, AUTO_FP, v4sf)
 758
 759   /* Implemented by aarch64_bfmlal<bt>{_lane{q}}v4sf  */
 760   VAR1 (TERNOP, bfmlalb, 0, FP, v4sf)
 761   VAR1 (TERNOP, bfmlalt, 0, FP, v4sf)
 762   VAR1 (QUADOP_LANE, bfmlalb_lane, 0, ALL, v4sf)
 763   VAR1 (QUADOP_LANE, bfmlalt_lane, 0, ALL, v4sf)
 764   VAR1 (QUADOP_LANE, bfmlalb_lane_q, 0, ALL, v4sf)
 765   VAR1 (QUADOP_LANE, bfmlalt_lane_q, 0, ALL, v4sf)
 766
 767   /* Implemented by aarch64_vget_lo/hi_halfv8bf.  */
 768   VAR1 (UNOP, vget_lo_half, 0, AUTO_FP, v8bf)
 769   VAR1 (UNOP, vget_hi_half, 0, AUTO_FP, v8bf)
 770
 771   /* Implemented by aarch64_simd_<sur>mmlav16qi.  */
 772   VAR1 (TERNOP, simd_smmla, 0, NONE, v16qi)
 773   VAR1 (TERNOPU, simd_ummla, 0, NONE, v16qi)
 774   VAR1 (TERNOP_SSUS, simd_usmmla, 0, NONE, v16qi)
 775
 776   /* Implemented by aarch64_bfcvtn{q}{2}<mode>  */
 777   VAR1 (UNOP, bfcvtn, 0, FP, v4bf)
 778   VAR1 (UNOP, bfcvtn_q, 0, FP, v8bf)
 779   VAR1 (BINOP, bfcvtn2, 0, FP, v8bf)
 780   VAR1 (UNOP, bfcvt, 0, FP, bf)
 781
 782   /* Implemented by aarch64_{v}bfcvt{_high}<mode>.  */
 783   VAR2 (UNOP, vbfcvt, 0, AUTO_FP, v4bf, v8bf)
 784   VAR1 (UNOP, vbfcvt_high, 0, AUTO_FP, v8bf)
 785   VAR1 (UNOP, bfcvt, 0, AUTO_FP, sf)