diff options
| -rw-r--r-- | absl/synchronization/internal/kernel_timeout.h | 2 | ||||
| -rw-r--r-- | absl/synchronization/internal/waiter.cc | 73 | ||||
| -rw-r--r-- | absl/synchronization/internal/waiter.h | 9 | ||||
| -rw-r--r-- | absl/time/time.h | 4 |
4 files changed, 55 insertions, 33 deletions
diff --git a/absl/synchronization/internal/kernel_timeout.h b/absl/synchronization/internal/kernel_timeout.h index a83c427bf499..0d132d98f886 100644 --- a/absl/synchronization/internal/kernel_timeout.h +++ b/absl/synchronization/internal/kernel_timeout.h @@ -39,6 +39,7 @@ namespace absl { namespace synchronization_internal { +class Futex; class Waiter; class KernelTimeout { @@ -139,6 +140,7 @@ class KernelTimeout { } #endif + friend class Futex; friend class Waiter; }; diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc index cd16c7887c08..db1b54f6d039 100644 --- a/absl/synchronization/internal/waiter.cc +++ b/absl/synchronization/internal/waiter.cc @@ -39,10 +39,12 @@ #include <atomic> #include <cassert> +#include <cstdint> #include "absl/base/internal/malloc_extension.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/thread_identity.h" +#include "absl/base/optimization.h" #include "absl/synchronization/internal/kernel_timeout.h" namespace absl { @@ -82,6 +84,42 @@ static void MaybeBecomeIdle() { #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF #endif #endif +class Futex { + public: + static int WaitUntil(std::atomic<int32_t> *v, int32_t val, + KernelTimeout t) { + int err = 0; + if (t.has_timeout()) { + // https://locklessinc.com/articles/futex_cheat_sheet/ + // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. + struct timespec abs_timeout = t.MakeAbsTimespec(); + // Atomically check that the futex value is still 0, and if it + // is, sleep until abs_timeout or until woken by FUTEX_WAKE. + err = syscall( + SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, + &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); + } else { + // Atomically check that the futex value is still 0, and if it + // is, sleep until woken by FUTEX_WAKE. + err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr); + } + if (err != 0) { + err = -errno; + } + return err; + } + + static int Wake(std::atomic<int32_t> *v, int32_t count) { + int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); + if (ABSL_PREDICT_FALSE(err < 0)) { + err = -errno; + } + return err; + } +}; void Waiter::Init() { futex_.store(0, std::memory_order_relaxed); @@ -91,7 +129,7 @@ bool Waiter::Wait(KernelTimeout t) { // Loop until we can atomically decrement futex from a positive // value, waiting on a futex while we believe it is zero. while (true) { - int x = futex_.load(std::memory_order_relaxed); + int32_t x = futex_.load(std::memory_order_relaxed); if (x != 0) { if (!futex_.compare_exchange_weak(x, x - 1, std::memory_order_acquire, @@ -101,30 +139,14 @@ bool Waiter::Wait(KernelTimeout t) { return true; // Consumed a wakeup, we are done. } - int err = 0; - if (t.has_timeout()) { - // https://locklessinc.com/articles/futex_cheat_sheet/ - // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. - struct timespec abs_timeout = t.MakeAbsTimespec(); - // Atomically check that the futex value is still 0, and if it - // is, sleep until abs_timeout or until woken by FUTEX_WAKE. - err = syscall( - SYS_futex, reinterpret_cast<int *>(&futex_), - FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, 0, - &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); - } else { - // Atomically check that the futex value is still 0, and if it - // is, sleep until woken by FUTEX_WAKE. - err = syscall(SYS_futex, reinterpret_cast<int *>(&futex_), - FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, nullptr); - } + const int err = Futex::WaitUntil(&futex_, 0, t); if (err != 0) { - if (errno == EINTR || errno == EWOULDBLOCK) { + if (err == -EINTR || err == -EWOULDBLOCK) { // Do nothing, the loop will retry. - } else if (errno == ETIMEDOUT) { - return false; // Timeout. + } else if (err == -ETIMEDOUT) { + return false; } else { - ABSL_RAW_LOG(FATAL, "Futex operation failed with errno %d\n", errno); + ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); } } @@ -141,10 +163,9 @@ void Waiter::Post() { void Waiter::Poke() { // Wake one thread waiting on the futex. - int err = syscall(SYS_futex, reinterpret_cast<int *>(&futex_), - FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1); - if (err < 0) { - ABSL_RAW_LOG(FATAL, "FUTEX_WAKE failed with errno %d\n", errno); + const int err = Futex::Wake(&futex_, 1); + if (ABSL_PREDICT_FALSE(err < 0)) { + ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); } } diff --git a/absl/synchronization/internal/waiter.h b/absl/synchronization/internal/waiter.h index 025ace42e261..23166f4bfa37 100644 --- a/absl/synchronization/internal/waiter.h +++ b/absl/synchronization/internal/waiter.h @@ -29,6 +29,7 @@ #endif #include <atomic> +#include <cstdint> #include "absl/base/internal/thread_identity.h" #include "absl/synchronization/internal/kernel_timeout.h" @@ -99,10 +100,10 @@ class Waiter { private: #if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX - // Futexes are defined by specification to be ints. - // Thus std::atomic<int> must be just an int with lockfree methods. - std::atomic<int> futex_; - static_assert(sizeof(int) == sizeof(futex_), "Wrong size for futex"); + // Futexes are defined by specification to be 32-bits. + // Thus std::atomic<int32_t> must be just an int32_t with lockfree methods. + std::atomic<int32_t> futex_; + static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex"); #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR pthread_mutex_t mu_; diff --git a/absl/time/time.h b/absl/time/time.h index 58f25e14ef94..77366b58a165 100644 --- a/absl/time/time.h +++ b/absl/time/time.h @@ -506,9 +506,7 @@ std::string UnparseFlag(Duration d); // // `absl::Time` assumes there are 60 seconds in a minute, which means the // underlying time scales must be "smeared" to eliminate leap seconds. -// POSIX, for example, legislates that a `time_t` value of `536457599` shall -// correspond to "1986-12-31 23:59:59 +0000". -// +// See https://developers.google.com/time/smear. // // Even though `absl::Time` supports a wide range of timestamps, exercise // caution when using values in the distant past. `absl::Time` uses the |