fbtl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2013 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <not-cancel.h>
  24 #include "pthreadP.h"
  25 #include <lowlevellock.h>
  26 #include <stap-probe.h>
  27
  28
  29 #ifndef LLL_MUTEX_LOCK
  30 # define LLL_MUTEX_LOCK(mutex) \
  31   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  32 # define LLL_MUTEX_TRYLOCK(mutex) \
  33   lll_trylock ((mutex)->__data.__lock)
  34 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  35   lll_robust_lock ((mutex)->__data.__lock, id, \
  36                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  37 #endif
  38
  39
  40 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  41      __attribute_noinline__;
  42
  43
  44 int
  45 __pthread_mutex_lock (pthread_mutex_t *mutex)
  46 {
  47   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  48
  49   unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
  50
  51   LIBC_PROBE (mutex_entry, 1, mutex);
  52
  53   if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
  54     return __pthread_mutex_lock_full (mutex);
  55
  56   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  57
  58   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  59       == PTHREAD_MUTEX_TIMED_NP)
  60     {
  61     simple:
  62       /* Normal mutex.  */
  63       LLL_MUTEX_LOCK (mutex);
  64       assert (mutex->__data.__owner == 0);
  65     }
  66   else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  67     {
  68       /* Recursive mutex.  */
  69
  70       /* Check whether we already hold the mutex.  */
  71       if (mutex->__data.__owner == id)
  72         {
  73           /* Just bump the counter.  */
  74           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
  75             /* Overflow of the counter.  */
  76             return EAGAIN;
  77
  78           ++mutex->__data.__count;
  79
  80           return 0;
  81         }
  82
  83       /* We have to get the mutex.  */
  84       LLL_MUTEX_LOCK (mutex);
  85
  86       assert (mutex->__data.__owner == 0);
  87       mutex->__data.__count = 1;
  88     }
  89   else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  90     {
  91       if (! __is_smp)
  92         goto simple;
  93
  94       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
  95         {
  96           int cnt = 0;
  97           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  98                              mutex->__data.__spins * 2 + 10);
  99           do
 100             {
 101               if (cnt++ >= max_cnt)
 102                 {
 103                   LLL_MUTEX_LOCK (mutex);
 104                   break;
 105                 }
 106
 107 #ifdef BUSY_WAIT_NOP
 108               BUSY_WAIT_NOP;
 109 #endif
 110             }
 111           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 112
 113           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 114         }
 115       assert (mutex->__data.__owner == 0);
 116     }
 117   else
 118     {
 119       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
 120       /* Check whether we already hold the mutex.  */
 121       if (__glibc_unlikely (mutex->__data.__owner == id))
 122         return EDEADLK;
 123       goto simple;
 124     }
 125
 126   /* Record the ownership.  */
 127   mutex->__data.__owner = id;
 128 #ifndef NO_INCR
 129   ++mutex->__data.__nusers;
 130 #endif
 131
 132   LIBC_PROBE (mutex_acquired, 1, mutex);
 133
 134   return 0;
 135 }
 136
 137 static int
 138 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 139 {
 140 #if 1
 141   return EINVAL;
 142 #else
 143   int oldval;
 144   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 145
 146   switch (PTHREAD_MUTEX_TYPE (mutex))
 147     {
 148     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 149     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 150     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 151     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 152       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 153                      &mutex->__data.__list.__next);
 154
 155       oldval = mutex->__data.__lock;
 156       do
 157         {
 158         again:
 159           if ((oldval & FUTEX_OWNER_DIED) != 0)
 160             {
 161               /* The previous owner died.  Try locking the mutex.  */
 162               int newval = id;
 163 #ifdef NO_INCR
 164               newval |= FUTEX_WAITERS;
 165 #else
 166               newval |= (oldval & FUTEX_WAITERS);
 167 #endif
 168
 169               newval
 170                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 171                                                        newval, oldval);
 172
 173               if (newval != oldval)
 174                 {
 175                   oldval = newval;
 176                   goto again;
 177                 }
 178
 179               /* We got the mutex.  */
 180               mutex->__data.__count = 1;
 181               /* But it is inconsistent unless marked otherwise.  */
 182               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 183
 184               ENQUEUE_MUTEX (mutex);
 185               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 186
 187               /* Note that we deliberately exit here.  If we fall
 188                  through to the end of the function __nusers would be
 189                  incremented which is not correct because the old
 190                  owner has to be discounted.  If we are not supposed
 191                  to increment __nusers we actually have to decrement
 192                  it here.  */
 193 #ifdef NO_INCR
 194               --mutex->__data.__nusers;
 195 #endif
 196
 197               return EOWNERDEAD;
 198             }
 199
 200           /* Check whether we already hold the mutex.  */
 201           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 202             {
 203               int kind = PTHREAD_MUTEX_TYPE (mutex);
 204               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 205                 {
 206                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 207                                  NULL);
 208                   return EDEADLK;
 209                 }
 210
 211               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 212                 {
 213                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 214                                  NULL);
 215
 216                   /* Just bump the counter.  */
 217                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 218                     /* Overflow of the counter.  */
 219                     return EAGAIN;
 220
 221                   ++mutex->__data.__count;
 222
 223                   return 0;
 224                 }
 225             }
 226
 227           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 228
 229           if (__builtin_expect (mutex->__data.__owner
 230                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 231             {
 232               /* This mutex is now not recoverable.  */
 233               mutex->__data.__count = 0;
 234               lll_unlock (mutex->__data.__lock,
 235                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 236               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 237               return ENOTRECOVERABLE;
 238             }
 239         }
 240       while ((oldval & FUTEX_OWNER_DIED) != 0);
 241
 242       mutex->__data.__count = 1;
 243       ENQUEUE_MUTEX (mutex);
 244       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 245       break;
 246
 247     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 248     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 249     case PTHREAD_MUTEX_PI_NORMAL_NP:
 250     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 251     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 252     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 253     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 254     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 255       {
 256         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 257         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 258
 259         if (robust)
 260           /* Note: robust PI futexes are signaled by setting bit 0.  */
 261           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 262                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 263                                    | 1));
 264
 265         oldval = mutex->__data.__lock;
 266
 267         /* Check whether we already hold the mutex.  */
 268         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 269           {
 270             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 271               {
 272                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 273                 return EDEADLK;
 274               }
 275
 276             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 277               {
 278                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 279
 280                 /* Just bump the counter.  */
 281                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 282                   /* Overflow of the counter.  */
 283                   return EAGAIN;
 284
 285                 ++mutex->__data.__count;
 286
 287                 return 0;
 288               }
 289           }
 290
 291         int newval = id;
 292 #ifdef NO_INCR
 293         newval |= FUTEX_WAITERS;
 294 #endif
 295         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 296                                                       newval, 0);
 297
 298         if (oldval != 0)
 299           {
 300             /* The mutex is locked.  The kernel will now take care of
 301                everything.  */
 302             int private = (robust
 303                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 304                            : PTHREAD_MUTEX_PSHARED (mutex));
 305             INTERNAL_SYSCALL_DECL (__err);
 306             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 307                                       __lll_private_flag (FUTEX_LOCK_PI,
 308                                                           private), 1, 0);
 309
 310             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 311                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 312                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 313               {
 314                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 315                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 316                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 317                 /* ESRCH can happen only for non-robust PI mutexes where
 318                    the owner of the lock died.  */
 319                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 320
 321                 /* Delay the thread indefinitely.  */
 322                 while (1)
 323                   pause_not_cancel ();
 324               }
 325
 326             oldval = mutex->__data.__lock;
 327
 328             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 329           }
 330
 331         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 332           {
 333             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 334
 335             /* We got the mutex.  */
 336             mutex->__data.__count = 1;
 337             /* But it is inconsistent unless marked otherwise.  */
 338             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 339
 340             ENQUEUE_MUTEX_PI (mutex);
 341             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 342
 343             /* Note that we deliberately exit here.  If we fall
 344                through to the end of the function __nusers would be
 345                incremented which is not correct because the old owner
 346                has to be discounted.  If we are not supposed to
 347                increment __nusers we actually have to decrement it here.  */
 348 #ifdef NO_INCR
 349             --mutex->__data.__nusers;
 350 #endif
 351
 352             return EOWNERDEAD;
 353           }
 354
 355         if (robust
 356             && __builtin_expect (mutex->__data.__owner
 357                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 358           {
 359             /* This mutex is now not recoverable.  */
 360             mutex->__data.__count = 0;
 361
 362             INTERNAL_SYSCALL_DECL (__err);
 363             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 364                               __lll_private_flag (FUTEX_UNLOCK_PI,
 365                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 366                               0, 0);
 367
 368             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 369             return ENOTRECOVERABLE;
 370           }
 371
 372         mutex->__data.__count = 1;
 373         if (robust)
 374           {
 375             ENQUEUE_MUTEX_PI (mutex);
 376             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 377           }
 378       }
 379       break;
 380
 381     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 382     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 383     case PTHREAD_MUTEX_PP_NORMAL_NP:
 384     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 385       {
 386         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 387
 388         oldval = mutex->__data.__lock;
 389
 390         /* Check whether we already hold the mutex.  */
 391         if (mutex->__data.__owner == id)
 392           {
 393             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 394               return EDEADLK;
 395
 396             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 397               {
 398                 /* Just bump the counter.  */
 399                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 400                   /* Overflow of the counter.  */
 401                   return EAGAIN;
 402
 403                 ++mutex->__data.__count;
 404
 405                 return 0;
 406               }
 407           }
 408
 409         int oldprio = -1, ceilval;
 410         do
 411           {
 412             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 413                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 414
 415             if (__pthread_current_priority () > ceiling)
 416               {
 417                 if (oldprio != -1)
 418                   __pthread_tpp_change_priority (oldprio, -1);
 419                 return EINVAL;
 420               }
 421
 422             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 423             if (retval)
 424               return retval;
 425
 426             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 427             oldprio = ceiling;
 428
 429             oldval
 430               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 431 #ifdef NO_INCR
 432                                                      ceilval | 2,
 433 #else
 434                                                      ceilval | 1,
 435 #endif
 436                                                      ceilval);
 437
 438             if (oldval == ceilval)
 439               break;
 440
 441             do
 442               {
 443                 oldval
 444                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 445                                                          ceilval | 2,
 446                                                          ceilval | 1);
 447
 448                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 449                   break;
 450
 451                 if (oldval != ceilval)
 452                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 453                                   PTHREAD_MUTEX_PSHARED (mutex));
 454               }
 455             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 456                                                         ceilval | 2, ceilval)
 457                    != ceilval);
 458           }
 459         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 460
 461         assert (mutex->__data.__owner == 0);
 462         mutex->__data.__count = 1;
 463       }
 464       break;
 465
 466     default:
 467       /* Correct code cannot set any other type.  */
 468       return EINVAL;
 469     }
 470
 471   /* Record the ownership.  */
 472   mutex->__data.__owner = id;
 473 #ifndef NO_INCR
 474   ++mutex->__data.__nusers;
 475 #endif
 476
 477   LIBC_PROBE (mutex_acquired, 1, mutex);
 478
 479   return 0;
 480 #endif
 481 }
 482 #ifndef __pthread_mutex_lock
 483 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 484 hidden_def (__pthread_mutex_lock)
 485 #endif
 486
 487
 488 #ifdef NO_INCR
 489 void
 490 internal_function
 491 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
 492 {
 493   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 494   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 495   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 496
 497   /* Record the ownership.  */
 498   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 499   mutex->__data.__owner = id;
 500
 501   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 502     ++mutex->__data.__count;
 503 }
 504 #endif