9 files changed, 390 insertions, 134 deletions

diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel
index f52e9d41644a..53e7988457e9 100644
--- a/absl/synchronization/BUILD.bazel
+++ b/absl/synchronization/BUILD.bazel
@@ -15,7 +15,7 @@
 #
 
 load(
-    "//absl:copts.bzl",
+    "//absl:copts/configure_copts.bzl",
     "ABSL_DEFAULT_COPTS",
     "ABSL_TEST_COPTS",
 )
@@ -170,18 +170,31 @@ cc_test(
     ],
 )
 
-cc_test(
-    name = "mutex_benchmark",
+cc_library(
+    name = "mutex_benchmark_common",
+    testonly = 1,
     srcs = ["mutex_benchmark.cc"],
-    copts = ABSL_TEST_COPTS,
-    tags = ["benchmark"],
-    visibility = ["//visibility:private"],
+    copts = ABSL_DEFAULT_COPTS,
+    visibility = [
+        "//absl/synchronization:__pkg__",
+    ],
     deps = [
         ":synchronization",
         ":thread_pool",
         "//absl/base",
         "@com_github_google_benchmark//:benchmark_main",
     ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "mutex_benchmark",
+    testonly = 1,
+    copts = ABSL_DEFAULT_COPTS,
+    visibility = ["//visibility:private"],
+    deps = [
+        ":mutex_benchmark_common",
+    ],
 )
 
 cc_test(
diff --git a/absl/synchronization/CMakeLists.txt b/absl/synchronization/CMakeLists.txt
index de0d7b7d4500..cb77b685647f 100644
--- a/absl/synchronization/CMakeLists.txt
+++ b/absl/synchronization/CMakeLists.txt
@@ -14,142 +14,182 @@
 # limitations under the License.
 #
 
-list(APPEND SYNCHRONIZATION_PUBLIC_HEADERS
-  "barrier.h"
-  "blocking_counter.h"
-  "mutex.h"
-  "notification.h"
+absl_cc_library(
+  NAME
+    graphcycles_internal
+  HDRS
+    "internal/graphcycles.h"
+  SRCS
+    "internal/graphcycles.cc"
+  COPTS
+    ${ABSL_DEFAULT_COPTS}
+  DEPS
+    absl::base
+    absl::base_internal
+    absl::core_headers
+    absl::malloc_internal
 )
 
-
-list(APPEND SYNCHRONIZATION_INTERNAL_HEADERS
-  "internal/create_thread_identity.h"
-  "internal/graphcycles.h"
-  "internal/kernel_timeout.h"
-  "internal/per_thread_sem.h"
-  "internal/thread_pool.h"
-  "internal/waiter.h"
-)
-
-
-
-# synchronization library
-list(APPEND SYNCHRONIZATION_SRC
-  "barrier.cc"
-  "blocking_counter.cc"
-  "internal/create_thread_identity.cc"
-  "internal/per_thread_sem.cc"
-  "internal/waiter.cc"
-  "internal/graphcycles.cc"
-  "notification.cc"
-  "mutex.cc"
-)
-
-set(SYNCHRONIZATION_PUBLIC_LIBRARIES absl::base absl::stacktrace absl::symbolize absl::time)
-
-absl_library(
-  TARGET
-    absl_synchronization
-  SOURCES
-    ${SYNCHRONIZATION_SRC}
-  PUBLIC_LIBRARIES
-    ${SYNCHRONIZATION_PUBLIC_LIBRARIES}
-  EXPORT_NAME
+absl_cc_library(
+  NAME
     synchronization
+  HDRS
+    "barrier.h"
+    "blocking_counter.h"
+    "internal/create_thread_identity.h"
+    "internal/kernel_timeout.h"
+    "internal/mutex_nonprod.inc"
+    "internal/per_thread_sem.h"
+    "internal/waiter.h"
+    "mutex.h"
+    "notification.h"
+  SRCS
+    "barrier.cc"
+    "blocking_counter.cc"
+    "internal/create_thread_identity.cc"
+    "internal/per_thread_sem.cc"
+    "internal/waiter.cc"
+    "notification.cc"
+    "mutex.cc"
+  COPTS
+    ${ABSL_DEFAULT_COPTS}
+  DEPS
+    absl::graphcycles_internal
+    absl::base
+    absl::base_internal
+    absl::config
+    absl::core_headers
+    absl::dynamic_annotations
+    absl::malloc_internal
+    absl::stacktrace
+    absl::symbolize
+    absl::time
+  PUBLIC
 )
 
-
-#
-## TESTS
-#
-
-
-# test barrier_test
-set(BARRIER_TEST_SRC "barrier_test.cc")
-set(BARRIER_TEST_PUBLIC_LIBRARIES absl::synchronization)
-
-absl_test(
-  TARGET
+absl_cc_test(
+  NAME
     barrier_test
-  SOURCES
-    ${BARRIER_TEST_SRC}
-  PUBLIC_LIBRARIES
-    ${BARRIER_TEST_PUBLIC_LIBRARIES}
+  SRCS
+    "barrier_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::time
+    gmock_main
 )
 
-
-# test blocking_counter_test
-set(BLOCKING_COUNTER_TEST_SRC "blocking_counter_test.cc")
-set(BLOCKING_COUNTER_TEST_PUBLIC_LIBRARIES absl::synchronization)
-
-absl_test(
-  TARGET
+absl_cc_test(
+  NAME
     blocking_counter_test
-  SOURCES
-    ${BLOCKING_COUNTER_TEST_SRC}
-  PUBLIC_LIBRARIES
-    ${BLOCKING_COUNTER_TEST_PUBLIC_LIBRARIES}
+  SRCS
+    "blocking_counter_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::time
+    gmock_main
 )
 
-
-# test graphcycles_test
-set(GRAPHCYCLES_TEST_SRC "internal/graphcycles_test.cc")
-set(GRAPHCYCLES_TEST_PUBLIC_LIBRARIES absl::synchronization)
-
-absl_test(
-  TARGET
+absl_cc_test(
+  NAME
     graphcycles_test
-  SOURCES
-    ${GRAPHCYCLES_TEST_SRC}
-  PUBLIC_LIBRARIES
-    ${GRAPHCYCLES_TEST_PUBLIC_LIBRARIES}
+  SRCS
+    "internal/graphcycles_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::graphcycles_internal
+    absl::base
+    absl::core_headers
+    gmock_main
 )
 
+absl_cc_library(
+  NAME
+    thread_pool
+  HDRS
+    "internal/thread_pool.h"
+  DEPS
+    absl::synchronization
+    absl::core_headers
+  TESTONLY
+)
 
-# test mutex_test
-set(MUTEX_TEST_SRC "mutex_test.cc")
-set(MUTEX_TEST_PUBLIC_LIBRARIES absl::synchronization)
-
-absl_test(
-  TARGET
+absl_cc_test(
+  NAME
     mutex_test
-  SOURCES
-    ${MUTEX_TEST_SRC}
-  PUBLIC_LIBRARIES
-    ${MUTEX_TEST_PUBLIC_LIBRARIES}
+  SRCS
+    "mutex_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::thread_pool
+    absl::base
+    absl::core_headers
+    absl::memory
+    absl::time
+    gmock_main
 )
 
-
-# test notification_test
-set(NOTIFICATION_TEST_SRC "notification_test.cc")
-set(NOTIFICATION_TEST_PUBLIC_LIBRARIES absl::synchronization)
-
-absl_test(
-  TARGET
+absl_cc_test(
+  NAME
     notification_test
-  SOURCES
-    ${NOTIFICATION_TEST_SRC}
-  PUBLIC_LIBRARIES
-    ${NOTIFICATION_TEST_PUBLIC_LIBRARIES}
+  SRCS
+    "notification_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::time
+    gmock_main
 )
 
-
-# test per_thread_sem_test_common
-set(PER_THREAD_SEM_TEST_COMMON_SRC "internal/per_thread_sem_test.cc")
-set(PER_THREAD_SEM_TEST_COMMON_PUBLIC_LIBRARIES absl::synchronization absl::strings)
-
-absl_test(
-  TARGET
+absl_cc_library(
+  NAME
     per_thread_sem_test_common
-  SOURCES
-    ${PER_THREAD_SEM_TEST_COMMON_SRC}
-  PUBLIC_LIBRARIES
-    ${PER_THREAD_SEM_TEST_COMMON_PUBLIC_LIBRARIES}
+  SRCS
+    "internal/per_thread_sem_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::base
+    absl::strings
+    absl::time
+    gmock
+  TESTONLY
 )
 
+absl_cc_test(
+  NAME
+    per_thread_sem_test
+  SRCS
+    "internal/per_thread_sem_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::per_thread_sem_test_common
+    absl::synchronization
+    absl::base
+    absl::strings
+    absl::time
+    gmock_main
+)
 
-
-
-
-
-
+absl_cc_test(
+  NAME
+    lifetime_test
+  SRCS
+    "lifetime_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::synchronization
+    absl::base
+    absl::core_headers
+    Threads::Threads
+)
diff --git a/absl/synchronization/internal/kernel_timeout.h b/absl/synchronization/internal/kernel_timeout.h
index 76e7983ae06c..9e1eed75d3aa 100644
--- a/absl/synchronization/internal/kernel_timeout.h
+++ b/absl/synchronization/internal/kernel_timeout.h
@@ -100,8 +100,8 @@ class KernelTimeout {
     if (n < 0) n = 0;
 
     struct timespec abstime;
-    int64_t seconds = std::min(n / kNanosPerSecond,
-                             int64_t{(std::numeric_limits<time_t>::max)()});
+    int64_t seconds = (std::min)(n / kNanosPerSecond,
+                               int64_t{(std::numeric_limits<time_t>::max)()});
     abstime.tv_sec = static_cast<time_t>(seconds);
     abstime.tv_nsec =
         static_cast<decltype(abstime.tv_nsec)>(n % kNanosPerSecond);
diff --git a/absl/synchronization/internal/mutex_nonprod.inc b/absl/synchronization/internal/mutex_nonprod.inc
index 0aab3d1314e6..b8d5af79ad38 100644
--- a/absl/synchronization/internal/mutex_nonprod.inc
+++ b/absl/synchronization/internal/mutex_nonprod.inc
@@ -214,6 +214,9 @@ class SynchronizationStorage {
   // stack) should use this constructor.
   explicit SynchronizationStorage(base_internal::LinkerInitialized) {}
 
+  constexpr explicit SynchronizationStorage(absl::ConstInitType)
+      : is_dynamic_(false), once_(), space_{{0}} {}
+
   SynchronizationStorage(SynchronizationStorage&) = delete;
   SynchronizationStorage& operator=(SynchronizationStorage&) = delete;
 
diff --git a/absl/synchronization/lifetime_test.cc b/absl/synchronization/lifetime_test.cc
index b7360c29016b..8b168e21a316 100644
--- a/absl/synchronization/lifetime_test.cc
+++ b/absl/synchronization/lifetime_test.cc
@@ -17,6 +17,7 @@
 #include <type_traits>
 
 #include "absl/base/attributes.h"
+#include "absl/base/const_init.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/thread_annotations.h"
 #include "absl/synchronization/mutex.h"
@@ -95,6 +96,10 @@ void TestLocals() {
   RunTests(&mutex, &condvar);
 }
 
+// Normal kConstInit usage
+ABSL_CONST_INIT absl::Mutex const_init_mutex(absl::kConstInit);
+void TestConstInitGlobal() { RunTests(&const_init_mutex, nullptr); }
+
 // Global variables during start and termination
 //
 // In a translation unit, static storage duration variables are initialized in
@@ -117,10 +122,53 @@ class OnDestruction {
   Function fn_;
 };
 
+// kConstInit
+// Test early usage.  (Declaration comes first; definitions must appear after
+// the test runner.)
+extern absl::Mutex early_const_init_mutex;
+// (Normally I'd write this +[], to make the cast-to-function-pointer explicit,
+// but in some MSVC setups we support, lambdas provide conversion operators to
+// different flavors of function pointers, making this trick ambiguous.)
+OnConstruction test_early_const_init([] {
+  RunTests(&early_const_init_mutex, nullptr);
+});
+// This definition appears before test_early_const_init, but it should be
+// initialized first (due to constant initialization).  Test that the object
+// actually works when constructed this way.
+ABSL_CONST_INIT absl::Mutex early_const_init_mutex(absl::kConstInit);
+
+// Furthermore, test that the const-init c'tor doesn't stomp over the state of
+// a Mutex.  Really, this is a test that the platform under test correctly
+// supports C++11 constant initialization.  (The constant-initialization
+// constructors of globals "happen at link time"; memory is pre-initialized,
+// before the constructors of either grab_lock or check_still_locked are run.)
+extern absl::Mutex const_init_sanity_mutex;
+OnConstruction grab_lock([]() NO_THREAD_SAFETY_ANALYSIS {
+  const_init_sanity_mutex.Lock();
+});
+ABSL_CONST_INIT absl::Mutex const_init_sanity_mutex(absl::kConstInit);
+OnConstruction check_still_locked([]() NO_THREAD_SAFETY_ANALYSIS {
+  const_init_sanity_mutex.AssertHeld();
+  const_init_sanity_mutex.Unlock();
+});
+
+// Test shutdown usage.  (Declarations come first; definitions must appear after
+// the test runner.)
+extern absl::Mutex late_const_init_mutex;
+// OnDestruction is being used here as a global variable, even though it has a
+// non-trivial destructor.  This is against the style guide.  We're violating
+// that rule here to check that the exception we allow for kConstInit is safe.
+// NOLINTNEXTLINE
+OnDestruction test_late_const_init([] {
+  RunTests(&late_const_init_mutex, nullptr);
+});
+ABSL_CONST_INIT absl::Mutex late_const_init_mutex(absl::kConstInit);
+
 }  // namespace
 
 int main() {
   TestLocals();
+  TestConstInitGlobal();
   // Explicitly call exit(0) here, to make it clear that we intend for the
   // above global object destructors to run.
   std::exit(0);
diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc
index 812197981214..f1b42db15f99 100644
--- a/absl/synchronization/mutex.cc
+++ b/absl/synchronization/mutex.cc
@@ -1079,7 +1079,7 @@ void Mutex::TryRemove(PerThreadSynch *s) {
 // if the wait extends past the absolute time specified, even if "s" is still
 // on the mutex queue.  In this case, remove "s" from the queue and return
 // true, otherwise return false.
-void Mutex::Block(PerThreadSynch *s) {
+ABSL_XRAY_LOG_ARGS(1) void Mutex::Block(PerThreadSynch *s) {
   while (s->state.load(std::memory_order_acquire) == PerThreadSynch::kQueued) {
     if (!DecrementSynchSem(this, s, s->waitp->timeout)) {
       // After a timeout, we go into a spin loop until we remove ourselves
diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h
index aeef3c95978c..4b65e92cb3d0 100644
--- a/absl/synchronization/mutex.h
+++ b/absl/synchronization/mutex.h
@@ -61,6 +61,7 @@
 #include <cstdint>
 #include <string>
 
+#include "absl/base/const_init.h"
 #include "absl/base/internal/identity.h"
 #include "absl/base/internal/low_level_alloc.h"
 #include "absl/base/internal/thread_identity.h"
@@ -136,7 +137,26 @@ struct SynchWaitParams;
 
 class LOCKABLE Mutex {
  public:
+  // Creates a `Mutex` that is not held by anyone. This constructor is
+  // typically used for Mutexes allocated on the heap or the stack.
+  //
+  // To create `Mutex` instances with static storage duration
+  // (e.g. a namespace-scoped or global variable), see
+  // `Mutex::Mutex(absl::kConstInit)` below instead.
   Mutex();
+
+  // Creates a mutex with static storage duration.  A global variable
+  // constructed this way avoids the lifetime issues that can occur on program
+  // startup and shutdown.  (See absl/base/const_init.h.)
+  //
+  // For Mutexes allocated on the heap and stack, instead use the default
+  // constructor, which can interact more fully with the thread sanitizer.
+  //
+  // Example usage:
+  //   namespace foo {
+  //   ABSL_CONST_INIT Mutex mu(absl::kConstInit);
+  //   }
+  explicit constexpr Mutex(absl::ConstInitType);
   ~Mutex();
 
   // Mutex::Lock()
@@ -879,10 +899,12 @@ class SCOPED_LOCKABLE ReleasableMutexLock {
 };
 
 #ifdef ABSL_INTERNAL_USE_NONPROD_MUTEX
+inline constexpr Mutex::Mutex(absl::ConstInitType) : impl_(absl::kConstInit) {}
 #else
 inline Mutex::Mutex() : mu_(0) {
   ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
 }
+inline constexpr Mutex::Mutex(absl::ConstInitType) : mu_(0) {}
 
 inline CondVar::CondVar() : cv_(0) {}
 #endif
diff --git a/absl/synchronization/mutex_benchmark.cc b/absl/synchronization/mutex_benchmark.cc
index 1e019e001ae4..2652bb974e99 100644
--- a/absl/synchronization/mutex_benchmark.cc
+++ b/absl/synchronization/mutex_benchmark.cc
@@ -12,16 +12,154 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
+#include <mutex>  // NOLINT(build/c++11)
 #include <vector>
 
-#include "benchmark/benchmark.h"
-#include "absl/base/internal/sysinfo.h"
+#include "absl/base/internal/cycleclock.h"
+#include "absl/base/internal/spinlock.h"
 #include "absl/synchronization/blocking_counter.h"
 #include "absl/synchronization/internal/thread_pool.h"
 #include "absl/synchronization/mutex.h"
+#include "benchmark/benchmark.h"
 
 namespace {
 
+void BM_Mutex(benchmark::State& state) {
+  static absl::Mutex* mu = new absl::Mutex;
+  for (auto _ : state) {
+    absl::MutexLock lock(mu);
+  }
+}
+BENCHMARK(BM_Mutex)->UseRealTime()->Threads(1)->ThreadPerCpu();
+
+static void DelayNs(int64_t ns, int* data) {
+  int64_t end = absl::base_internal::CycleClock::Now() +
+                ns * absl::base_internal::CycleClock::Frequency() / 1e9;
+  while (absl::base_internal::CycleClock::Now() < end) {
+    ++(*data);
+    benchmark::DoNotOptimize(*data);
+  }
+}
+
+template <typename MutexType>
+class RaiiLocker {
+ public:
+  explicit RaiiLocker(MutexType* mu) : mu_(mu) { mu_->Lock(); }
+  ~RaiiLocker() { mu_->Unlock(); }
+ private:
+  MutexType* mu_;
+};
+
+template <>
+class RaiiLocker<std::mutex> {
+ public:
+  explicit RaiiLocker(std::mutex* mu) : mu_(mu) { mu_->lock(); }
+  ~RaiiLocker() { mu_->unlock(); }
+ private:
+  std::mutex* mu_;
+};
+
+template <typename MutexType>
+void BM_Contended(benchmark::State& state) {
+  struct Shared {
+    MutexType mu;
+    int data = 0;
+  };
+  static auto* shared = new Shared;
+  int local = 0;
+  for (auto _ : state) {
+    // Here we model both local work outside of the critical section as well as
+    // some work inside of the critical section. The idea is to capture some
+    // more or less realisitic contention levels.
+    // If contention is too low, the benchmark won't measure anything useful.
+    // If contention is unrealistically high, the benchmark will favor
+    // bad mutex implementations that block and otherwise distract threads
+    // from the mutex and shared state for as much as possible.
+    // To achieve this amount of local work is multiplied by number of threads
+    // to keep ratio between local work and critical section approximately
+    // equal regardless of number of threads.
+    DelayNs(100 * state.threads, &local);
+    RaiiLocker<MutexType> locker(&shared->mu);
+    DelayNs(state.range(0), &shared->data);
+  }
+}
+
+BENCHMARK_TEMPLATE(BM_Contended, absl::Mutex)
+    ->UseRealTime()
+    // ThreadPerCpu poorly handles non-power-of-two CPU counts.
+    ->Threads(1)
+    ->Threads(2)
+    ->Threads(4)
+    ->Threads(6)
+    ->Threads(8)
+    ->Threads(12)
+    ->Threads(16)
+    ->Threads(24)
+    ->Threads(32)
+    ->Threads(48)
+    ->Threads(64)
+    ->Threads(96)
+    ->Threads(128)
+    ->Threads(192)
+    ->Threads(256)
+    // Some empirically chosen amounts of work in critical section.
+    // 1 is low contention, 200 is high contention and few values in between.
+    ->Arg(1)
+    ->Arg(20)
+    ->Arg(50)
+    ->Arg(200);
+
+BENCHMARK_TEMPLATE(BM_Contended, absl::base_internal::SpinLock)
+    ->UseRealTime()
+    // ThreadPerCpu poorly handles non-power-of-two CPU counts.
+    ->Threads(1)
+    ->Threads(2)
+    ->Threads(4)
+    ->Threads(6)
+    ->Threads(8)
+    ->Threads(12)
+    ->Threads(16)
+    ->Threads(24)
+    ->Threads(32)
+    ->Threads(48)
+    ->Threads(64)
+    ->Threads(96)
+    ->Threads(128)
+    ->Threads(192)
+    ->Threads(256)
+    // Some empirically chosen amounts of work in critical section.
+    // 1 is low contention, 200 is high contention and few values in between.
+    ->Arg(1)
+    ->Arg(20)
+    ->Arg(50)
+    ->Arg(200);
+
+BENCHMARK_TEMPLATE(BM_Contended, std::mutex)
+    ->UseRealTime()
+    // ThreadPerCpu poorly handles non-power-of-two CPU counts.
+    ->Threads(1)
+    ->Threads(2)
+    ->Threads(4)
+    ->Threads(6)
+    ->Threads(8)
+    ->Threads(12)
+    ->Threads(16)
+    ->Threads(24)
+    ->Threads(32)
+    ->Threads(48)
+    ->Threads(64)
+    ->Threads(96)
+    ->Threads(128)
+    ->Threads(192)
+    ->Threads(256)
+    // Some empirically chosen amounts of work in critical section.
+    // 1 is low contention, 200 is high contention and few values in between.
+    ->Arg(1)
+    ->Arg(20)
+    ->Arg(50)
+    ->Arg(200);
+
 // Measure the overhead of conditions on mutex release (when they must be
 // evaluated).  Mutex has (some) support for equivalence classes allowing
 // Conditions with the same function/argument to potentially not be multiply
@@ -82,13 +220,4 @@ constexpr int kMaxConditionWaiters = 1024;
 #endif
 BENCHMARK(BM_ConditionWaiters)->RangePair(0, 2, 1, kMaxConditionWaiters);
 
-void BM_ContendedMutex(benchmark::State& state) {
-  static absl::Mutex* mu = new absl::Mutex;
-  for (auto _ : state) {
-    absl::MutexLock lock(mu);
-  }
-}
-BENCHMARK(BM_ContendedMutex)->Threads(1);
-BENCHMARK(BM_ContendedMutex)->ThreadPerCpu();
-
 }  // namespace
diff --git a/absl/synchronization/notification.h b/absl/synchronization/notification.h
index 107932f2909e..f95f4d142a00 100644
--- a/absl/synchronization/notification.h
+++ b/absl/synchronization/notification.h
@@ -52,6 +52,7 @@
 
 #include <atomic>
 
+#include "absl/base/macros.h"
 #include "absl/synchronization/mutex.h"
 #include "absl/time/time.h"