elf/dl-hwcaps.c

   1 /* Hardware capability support for run-time dynamic loader.
   2    Copyright (C) 2012-2022 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <elf.h>
  21 #include <errno.h>
  22 #include <libintl.h>
  23 #include <unistd.h>
  24 #include <ldsodefs.h>
  25 #include <fcntl.h>
  26 #include <sysdep.h>
  27 #include <not-errno.h>
  28
  29 #include <dl-procinfo.h>
  30 #include <dl-hwcaps.h>
  31
  32 /* This is the result of counting the substrings in a colon-separated
  33    hwcaps string.  */
  34 struct hwcaps_counts
  35 {
  36   /* Number of substrings.  */
  37   size_t count;
  38
  39   /* Sum of the individual substring lengths (without separators or
  40      null terminators).  */
  41   size_t total_length;
  42
  43   /* Maximum length of an individual substring.  */
  44   size_t maximum_length;
  45 };
  46
  47 /* Update *COUNTS according to the contents of HWCAPS.  Skip over
  48    entries whose bit is not set in MASK.  */
  49 static void
  50 update_hwcaps_counts (struct hwcaps_counts *counts, const char *hwcaps,
  51                       uint32_t bitmask, const char *mask)
  52 {
  53   struct dl_hwcaps_split_masked sp;
  54   _dl_hwcaps_split_masked_init (&sp, hwcaps, bitmask, mask);
  55   while (_dl_hwcaps_split_masked (&sp))
  56     {
  57       ++counts->count;
  58       counts->total_length += sp.split.length;
  59       if (sp.split.length > counts->maximum_length)
  60         counts->maximum_length = sp.split.length;
  61     }
  62 }
  63
  64 /* State for copy_hwcaps.  Must be initialized to point to
  65    the storage areas for the array and the strings themselves.  */
  66 struct copy_hwcaps
  67 {
  68   struct r_strlenpair *next_pair;
  69   char *next_string;
  70 };
  71
  72 /* Copy HWCAPS into the string pairs and strings, advancing *TARGET.
  73    Skip over entries whose bit is not set in MASK.  */
  74 static void
  75 copy_hwcaps (struct copy_hwcaps *target, const char *hwcaps,
  76              uint32_t bitmask, const char *mask)
  77 {
  78   struct dl_hwcaps_split_masked sp;
  79   _dl_hwcaps_split_masked_init (&sp, hwcaps, bitmask, mask);
  80   while (_dl_hwcaps_split_masked (&sp))
  81     {
  82       target->next_pair->str = target->next_string;
  83       char *slash = __mempcpy (__mempcpy (target->next_string,
  84                                           GLIBC_HWCAPS_PREFIX,
  85                                           strlen (GLIBC_HWCAPS_PREFIX)),
  86                                sp.split.segment, sp.split.length);
  87       *slash = '/';
  88       target->next_pair->len
  89         = strlen (GLIBC_HWCAPS_PREFIX) + sp.split.length + 1;
  90       ++target->next_pair;
  91       target->next_string = slash + 1;
  92     }
  93 }
  94
  95 struct dl_hwcaps_priority *_dl_hwcaps_priorities;
  96 uint32_t _dl_hwcaps_priorities_length;
  97
  98 /* Allocate _dl_hwcaps_priorities and fill it with data.  */
  99 static void
 100 compute_priorities (size_t total_count, const char *prepend,
 101                     uint32_t bitmask, const char *mask)
 102 {
 103   _dl_hwcaps_priorities = malloc (total_count
 104                                   * sizeof (*_dl_hwcaps_priorities));
 105   if (_dl_hwcaps_priorities == NULL)
 106     _dl_signal_error (ENOMEM, NULL, NULL,
 107                       N_("cannot create HWCAP priorities"));
 108   _dl_hwcaps_priorities_length = total_count;
 109
 110   /* First the prepended subdirectories.  */
 111   size_t i = 0;
 112   {
 113     struct dl_hwcaps_split sp;
 114     _dl_hwcaps_split_init (&sp, prepend);
 115     while (_dl_hwcaps_split (&sp))
 116       {
 117         _dl_hwcaps_priorities[i].name = sp.segment;
 118         _dl_hwcaps_priorities[i].name_length = sp.length;
 119         _dl_hwcaps_priorities[i].priority = i + 1;
 120         ++i;
 121       }
 122   }
 123
 124   /* Then the built-in subdirectories that are actually active.  */
 125   {
 126     struct dl_hwcaps_split_masked sp;
 127     _dl_hwcaps_split_masked_init (&sp, _dl_hwcaps_subdirs, bitmask, mask);
 128     while (_dl_hwcaps_split_masked (&sp))
 129       {
 130         _dl_hwcaps_priorities[i].name = sp.split.segment;
 131         _dl_hwcaps_priorities[i].name_length = sp.split.length;
 132         _dl_hwcaps_priorities[i].priority = i + 1;
 133         ++i;
 134       }
 135   }
 136   assert (i == total_count);
 137 }
 138
 139 /* Sort the _dl_hwcaps_priorities array by name.  */
 140 static void
 141 sort_priorities_by_name (void)
 142 {
 143   /* Insertion sort.  There is no need to link qsort into the dynamic
 144      loader for such a short array.  */
 145   for (size_t i = 1; i < _dl_hwcaps_priorities_length; ++i)
 146     for (size_t j = i; j > 0; --j)
 147       {
 148         struct dl_hwcaps_priority *previous = _dl_hwcaps_priorities + j - 1;
 149         struct dl_hwcaps_priority *current = _dl_hwcaps_priorities + j;
 150
 151         /* Bail out if current is greater or equal to the previous
 152            value.  */
 153         uint32_t to_compare;
 154         if (current->name_length < previous->name_length)
 155           to_compare = current->name_length;
 156         else
 157           to_compare = previous->name_length;
 158         int cmp = memcmp (current->name, previous->name, to_compare);
 159         if (cmp > 0
 160             || (cmp == 0 && current->name_length >= previous->name_length))
 161           break;
 162
 163         /* Swap *previous and *current.  */
 164         struct dl_hwcaps_priority tmp = *previous;
 165         *previous = *current;
 166         *current = tmp;
 167       }
 168 }
 169
 170 /* Return an array of useful/necessary hardware capability names.  */
 171 const struct r_strlenpair *
 172 _dl_important_hwcaps (const char *glibc_hwcaps_prepend,
 173                       const char *glibc_hwcaps_mask,
 174                       size_t *sz, size_t *max_capstrlen)
 175 {
 176   uint64_t hwcap_mask = GET_HWCAP_MASK();
 177   /* Determine how many important bits are set.  */
 178   uint64_t masked = GLRO(dl_hwcap) & hwcap_mask;
 179   size_t cnt = GLRO (dl_platform) != NULL;
 180   size_t n, m;
 181   struct r_strlenpair *result;
 182   struct r_strlenpair *rp;
 183   char *cp;
 184
 185   /* glibc-hwcaps subdirectories.  These are exempted from the power
 186      set construction below.  */
 187   uint32_t hwcaps_subdirs_active = _dl_hwcaps_subdirs_active ();
 188   struct hwcaps_counts hwcaps_counts =  { 0, };
 189   update_hwcaps_counts (&hwcaps_counts, glibc_hwcaps_prepend, -1, NULL);
 190   update_hwcaps_counts (&hwcaps_counts, _dl_hwcaps_subdirs,
 191                         hwcaps_subdirs_active, glibc_hwcaps_mask);
 192   compute_priorities (hwcaps_counts.count, glibc_hwcaps_prepend,
 193                       hwcaps_subdirs_active, glibc_hwcaps_mask);
 194   sort_priorities_by_name ();
 195
 196   /* Each hwcaps subdirectory has a GLIBC_HWCAPS_PREFIX string prefix
 197      and a "/" suffix once stored in the result.  */
 198   hwcaps_counts.maximum_length += strlen (GLIBC_HWCAPS_PREFIX) + 1;
 199   size_t hwcaps_sz = (hwcaps_counts.count * (strlen (GLIBC_HWCAPS_PREFIX) + 1)
 200                   + hwcaps_counts.total_length);
 201
 202   /* Count the number of bits set in the masked value.  */
 203   for (n = 0; (~((1ULL << n) - 1) & masked) != 0; ++n)
 204     if ((masked & (1ULL << n)) != 0)
 205       ++cnt;
 206
 207   /* For TLS enabled builds always add 'tls'.  */
 208   ++cnt;
 209
 210 #ifdef NEED_LD_SO_NOHWCAP
 211   if (__access_noerrno ("/etc/ld.so.nohwcap", F_OK) == 0)
 212     {
 213       /* If hwcap is disabled, we only have the base directory to search.  */
 214       result = (struct r_strlenpair *) malloc (sizeof (*result));
 215       if (result == NULL)
 216         _dl_signal_error (ENOMEM, NULL, NULL,
 217                          N_("cannot create capability list"));
 218
 219       result[0].str = (char *) result;  /* Does not really matter.  */
 220       result[0].len = 0;
 221
 222       *sz = 1;
 223       return result;
 224     }
 225 #endif
 226
 227   /* Create temporary data structure to generate result table.  */
 228   struct r_strlenpair temp[cnt];
 229   m = 0;
 230   for (n = 0; masked != 0; ++n)
 231     if ((masked & (1ULL << n)) != 0)
 232       {
 233         temp[m].str = _dl_hwcap_string (n);
 234         temp[m].len = strlen (temp[m].str);
 235         masked ^= 1ULL << n;
 236         ++m;
 237       }
 238   if (GLRO (dl_platform) != NULL)
 239     {
 240       temp[m].str = GLRO (dl_platform);
 241       temp[m].len = GLRO (dl_platformlen);
 242       ++m;
 243     }
 244
 245   temp[m].str = "tls";
 246   temp[m].len = 3;
 247   ++m;
 248
 249   assert (m == cnt);
 250
 251   /* Determine the total size of all strings together.  */
 252   size_t total;
 253   if (cnt == 1)
 254     total = temp[0].len + 1;
 255   else
 256     {
 257       total = temp[0].len + temp[cnt - 1].len + 2;
 258       if (cnt > 2)
 259         {
 260           total <<= 1;
 261           for (n = 1; n + 1 < cnt; ++n)
 262             total += temp[n].len + 1;
 263           if (cnt > 3
 264               && (cnt >= sizeof (size_t) * 8
 265                   || total + (sizeof (*result) << 3)
 266                      >= (1UL << (sizeof (size_t) * 8 - cnt + 3))))
 267             _dl_signal_error (ENOMEM, NULL, NULL,
 268                               N_("cannot create capability list"));
 269
 270           total <<= cnt - 3;
 271         }
 272     }
 273
 274   *sz = hwcaps_counts.count + (1 << cnt);
 275
 276   /* This is the overall result, including both glibc-hwcaps
 277      subdirectories and the legacy hwcaps subdirectories using the
 278      power set construction.  */
 279   total += hwcaps_sz;
 280   struct r_strlenpair *overall_result
 281     = malloc (*sz * sizeof (*result) + total);
 282   if (overall_result == NULL)
 283     _dl_signal_error (ENOMEM, NULL, NULL,
 284                       N_("cannot create capability list"));
 285
 286   /* Fill in the glibc-hwcaps subdirectories.  */
 287   {
 288     struct copy_hwcaps target;
 289     target.next_pair = overall_result;
 290     target.next_string = (char *) (overall_result + *sz);
 291     copy_hwcaps (&target, glibc_hwcaps_prepend, -1, NULL);
 292     copy_hwcaps (&target, _dl_hwcaps_subdirs,
 293                  hwcaps_subdirs_active, glibc_hwcaps_mask);
 294     /* Set up the write target for the power set construction.  */
 295     result = target.next_pair;
 296     cp = target.next_string;
 297   }
 298
 299
 300   /* Power set construction begins here.  We use a very compressed way
 301      to store the various combinations of capability names.  */
 302
 303   if (cnt == 1)
 304     {
 305       result[0].str = cp;
 306       result[0].len = temp[0].len + 1;
 307       result[1].str = cp;
 308       result[1].len = 0;
 309       cp = __mempcpy (cp, temp[0].str, temp[0].len);
 310       *cp = '/';
 311       if (result[0].len > hwcaps_counts.maximum_length)
 312         *max_capstrlen = result[0].len;
 313       else
 314         *max_capstrlen = hwcaps_counts.maximum_length;
 315
 316       return overall_result;
 317     }
 318
 319   /* Fill in the information.  This follows the following scheme
 320      (indices from TEMP for four strings):
 321         entry #0: 0, 1, 2, 3    binary: 1111
 322               #1: 0, 1, 3               1101
 323               #2: 0, 2, 3               1011
 324               #3: 0, 3                  1001
 325      This allows the representation of all possible combinations of
 326      capability names in the string.  First generate the strings.  */
 327   result[1].str = result[0].str = cp;
 328 #define add(idx) \
 329       cp = __mempcpy (__mempcpy (cp, temp[idx].str, temp[idx].len), "/", 1);
 330   if (cnt == 2)
 331     {
 332       add (1);
 333       add (0);
 334     }
 335   else
 336     {
 337       n = 1 << (cnt - 1);
 338       do
 339         {
 340           n -= 2;
 341
 342           /* We always add the last string.  */
 343           add (cnt - 1);
 344
 345           /* Add the strings which have the bit set in N.  */
 346           for (m = cnt - 2; m > 0; --m)
 347             if ((n & (1 << m)) != 0)
 348               add (m);
 349
 350           /* Always add the first string.  */
 351           add (0);
 352         }
 353       while (n != 0);
 354     }
 355 #undef add
 356
 357   /* Now we are ready to install the string pointers and length.  */
 358   for (n = 0; n < (1UL << cnt); ++n)
 359     result[n].len = 0;
 360   n = cnt;
 361   do
 362     {
 363       size_t mask = 1 << --n;
 364
 365       rp = result;
 366       for (m = 1 << cnt; m > 0; ++rp)
 367         if ((--m & mask) != 0)
 368           rp->len += temp[n].len + 1;
 369     }
 370   while (n != 0);
 371
 372   /* The first half of the strings all include the first string.  */
 373   n = (1 << cnt) - 2;
 374   rp = &result[2];
 375   while (n != (1UL << (cnt - 1)))
 376     {
 377       if ((--n & 1) != 0)
 378         rp[0].str = rp[-2].str + rp[-2].len;
 379       else
 380         rp[0].str = rp[-1].str;
 381       ++rp;
 382     }
 383
 384   /* The second half starts right after the first part of the string of
 385      the corresponding entry in the first half.  */
 386   do
 387     {
 388       rp[0].str = rp[-(1 << (cnt - 1))].str + temp[cnt - 1].len + 1;
 389       ++rp;
 390     }
 391   while (--n != 0);
 392
 393   /* The maximum string length.  */
 394   if (result[0].len > hwcaps_counts.maximum_length)
 395     *max_capstrlen = result[0].len;
 396   else
 397     *max_capstrlen = hwcaps_counts.maximum_length;
 398
 399   return overall_result;
 400 }