diff options
Diffstat (limited to 'absl/flags')
| -rw-r--r-- | absl/flags/config.h | 11 | ||||
| -rw-r--r-- | absl/flags/flag.cc | 22 | ||||
| -rw-r--r-- | absl/flags/flag.h | 45 | ||||
| -rw-r--r-- | absl/flags/internal/commandlineflag.h | 32 | ||||
| -rw-r--r-- | absl/flags/internal/flag.cc | 15 | ||||
| -rw-r--r-- | absl/flags/internal/flag.h | 95 | ||||
| -rw-r--r-- | absl/flags/parse.cc | 4 |
7 files changed, 160 insertions, 64 deletions
diff --git a/absl/flags/config.h b/absl/flags/config.h index a9fd97ad0f37..fbe349611f54 100644 --- a/absl/flags/config.h +++ b/absl/flags/config.h @@ -45,4 +45,15 @@ #define ABSL_FLAGS_STRIP_HELP ABSL_FLAGS_STRIP_NAMES #endif +// ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD macro is used for using atomics with +// double words, e.g. absl::Duration. +// For reasons in bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878, modern +// versions of GCC do not support cmpxchg16b instruction in standard atomics. +#ifdef ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD +#error "ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD should not be defined." +#elif defined(__clang__) && defined(__x86_64__) && \ + defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) +#define ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD 1 +#endif + #endif // ABSL_FLAGS_CONFIG_H_ diff --git a/absl/flags/flag.cc b/absl/flags/flag.cc index 7faa7ade0d0d..491a66bcf2da 100644 --- a/absl/flags/flag.cc +++ b/absl/flags/flag.cc @@ -20,28 +20,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// We want to validate the type mismatch between type definition and -// declaration. The lock-free implementation does not allow us to do it, -// so in debug builds we always use the slower implementation, which always -// validates the type. -#ifndef NDEBUG -#define ABSL_FLAGS_ATOMIC_GET(T) \ - T GetFlag(const absl::Flag<T>& flag) { return flag.Get(); } -#else -#define ABSL_FLAGS_ATOMIC_GET(T) \ - T GetFlag(const absl::Flag<T>& flag) { \ - T result; \ - if (flag.AtomicGet(&result)) { \ - return result; \ - } \ - return flag.Get(); \ - } -#endif - -ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_ATOMIC_GET) - -#undef ABSL_FLAGS_ATOMIC_GET - // This global nutex protects on-demand construction of flag objects in MSVC // builds. #if defined(_MSC_VER) && !defined(__clang__) diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 326fb8ee6894..62e73f8418d1 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h @@ -29,6 +29,8 @@ #ifndef ABSL_FLAGS_FLAG_H_ #define ABSL_FLAGS_FLAG_H_ +#include <type_traits> + #include "absl/base/attributes.h" #include "absl/base/casts.h" #include "absl/flags/config.h" @@ -181,23 +183,42 @@ class Flag { // // // FLAGS_firstname is a Flag of type `std::string` // std::string first_name = absl::GetFlag(FLAGS_firstname); -template <typename T> +template <typename T, + typename std::enable_if< + !flags_internal::IsAtomicFlagTypeTrait<T>::value, int>::type = 0> ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag) { -#define ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE(BIT) \ - static_assert( \ - !std::is_same<T, BIT>::value, \ - "Do not specify explicit template parameters to absl::GetFlag"); - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE) -#undef ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE - return flag.Get(); } +// We want to validate the type mismatch between type definition and +// declaration. The lock-free implementation does not allow us to do it, +// so in debug builds we always use the slower implementation, which always +// validates the type. +#ifndef NDEBUG +template <typename T, + typename std::enable_if< + flags_internal::IsAtomicFlagTypeTrait<T>::value, int>::type = 0> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag) { + return flag.Get(); +} +#else // Overload for `GetFlag()` for types that support lock-free reads. -#define ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT(T) \ - ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag); -ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT) -#undef ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT +template <typename T, + typename std::enable_if< + flags_internal::IsAtomicFlagTypeTrait<T>::value, int>::type = 0> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag) { + // T might not be default constructible. + union U { + T value; + U() {} + }; + U result; + if (flag.AtomicGet(&result.value)) { + return result.value; + } + return flag.Get(); +} +#endif // SetFlag() // diff --git a/absl/flags/internal/commandlineflag.h b/absl/flags/internal/commandlineflag.h index a0c18e80b6c7..1862306d97f4 100644 --- a/absl/flags/internal/commandlineflag.h +++ b/absl/flags/internal/commandlineflag.h @@ -259,22 +259,22 @@ class CommandLineFlag { virtual void Read(void* dst) const = 0; }; -// This macro is the "source of truth" for the list of supported flag types we -// expect to perform lock free operations on. Specifically it generates code, -// a one argument macro operating on a type, supplied as a macro argument, for -// each type in the list. -#define ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(A) \ - A(bool) \ - A(short) \ - A(unsigned short) \ - A(int) \ - A(unsigned int) \ - A(long) \ - A(unsigned long) \ - A(long long) \ - A(unsigned long long) \ - A(double) \ - A(float) +// This macro is the "source of truth" for the list of supported flag built-in +// types. +#define ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ + A(bool) \ + A(short) \ + A(unsigned short) \ + A(int) \ + A(unsigned int) \ + A(long) \ + A(unsigned long) \ + A(long long) \ + A(unsigned long long) \ + A(double) \ + A(float) \ + A(std::string) \ + A(std::vector<std::string>) } // namespace flags_internal ABSL_NAMESPACE_END diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index bb9a98f34404..6979dc464c78 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -16,6 +16,7 @@ #include "absl/flags/internal/flag.h" #include "absl/base/optimization.h" +#include "absl/flags/config.h" #include "absl/flags/usage_config.h" #include "absl/synchronization/mutex.h" @@ -35,9 +36,7 @@ namespace { bool ShouldValidateFlagValue(FlagOpFn flag_type_id) { #define DONT_VALIDATE(T) \ if (flag_type_id == &flags_internal::FlagOps<T>) return false; - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(DONT_VALIDATE) - DONT_VALIDATE(std::string) - DONT_VALIDATE(std::vector<std::string>) + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(DONT_VALIDATE) #undef DONT_VALIDATE return true; @@ -85,7 +84,6 @@ void FlagImpl::Init() { cur_ = MakeInitValue().release(); StoreAtomic(); inited_.store(true, std::memory_order_release); - InvokeCallback(); } } @@ -264,8 +262,15 @@ void FlagImpl::StoreAtomic() { if (data_size <= sizeof(int64_t)) { int64_t t = 0; std::memcpy(&t, cur_, data_size); - atomic_.store(t, std::memory_order_release); + atomics_.small_atomic.store(t, std::memory_order_release); } +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + else if (data_size <= sizeof(FlagsInternalTwoWordsType)) { + FlagsInternalTwoWordsType t{0, 0}; + std::memcpy(&t, cur_, data_size); + atomics_.big_atomic.store(t, std::memory_order_release); + } +#endif } void FlagImpl::Write(const void* src, const flags_internal::FlagOpFn src_op) { diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 7d5271c40de6..a5edfd17d5cc 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -20,6 +20,7 @@ #include <cstring> #include "absl/base/thread_annotations.h" +#include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/flags/internal/registry.h" #include "absl/memory/memory.h" @@ -30,7 +31,61 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace flags_internal { -constexpr int64_t AtomicInit() { return 0xababababababababll; } +// The minimum atomic size we believe to generate lock free code, i.e. all +// trivially copyable types not bigger this size generate lock free code. +static constexpr int kMinLockFreeAtomicSize = 8; + +// The same as kMinLockFreeAtomicSize but maximum atomic size. As double words +// might use two registers, we want to dispatch the logic for them. +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) +static constexpr int kMaxLockFreeAtomicSize = 16; +#else +static constexpr int kMaxLockFreeAtomicSize = 8; +#endif + +// We can use atomic in cases when it fits in the register, trivially copyable +// in order to make memcpy operations. +template <typename T> +struct IsAtomicFlagTypeTrait { + static constexpr bool value = + (sizeof(T) <= kMaxLockFreeAtomicSize && + type_traits_internal::is_trivially_copyable<T>::value); +}; + +// Clang does not always produce cmpxchg16b instruction when alignment of a 16 +// bytes type is not 16. +struct alignas(16) FlagsInternalTwoWordsType { + int64_t first; + int64_t second; +}; + +constexpr bool operator==(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return that.first == other.first && that.second == other.second; +} +constexpr bool operator!=(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return !(that == other); +} + +constexpr int64_t SmallAtomicInit() { return 0xababababababababll; } + +template <typename T, typename S = void> +struct BestAtomicType { + using type = int64_t; + static constexpr int64_t AtomicInit() { return SmallAtomicInit(); } +}; + +template <typename T> +struct BestAtomicType< + T, typename std::enable_if<(kMinLockFreeAtomicSize < sizeof(T) && + sizeof(T) <= kMaxLockFreeAtomicSize), + void>::type> { + using type = FlagsInternalTwoWordsType; + static constexpr FlagsInternalTwoWordsType AtomicInit() { + return {SmallAtomicInit(), SmallAtomicInit()}; + } +}; template <typename T> class Flag; @@ -182,14 +237,15 @@ class FlagImpl { // it replaces `dst` with the new value. bool TryParse(void** dst, absl::string_view value, std::string* err) const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + template <typename T> bool AtomicGet(T* v) const { - const int64_t r = atomic_.load(std::memory_order_acquire); - if (r != flags_internal::AtomicInit()) { - std::memcpy(v, &r, sizeof(T)); + using U = flags_internal::BestAtomicType<T>; + const typename U::type r = atomics_.template load<T>(); + if (r != U::AtomicInit()) { + std::memcpy(static_cast<void*>(v), &r, sizeof(T)); return true; } - return false; } @@ -271,7 +327,34 @@ class FlagImpl { int64_t counter_ ABSL_GUARDED_BY(*DataGuard()) = 0; // For some types, a copy of the current value is kept in an atomically // accessible field. - std::atomic<int64_t> atomic_{flags_internal::AtomicInit()}; + union Atomics { + // Using small atomic for small types. + std::atomic<int64_t> small_atomic; + template <typename T, + typename K = typename std::enable_if< + (sizeof(T) <= kMinLockFreeAtomicSize), void>::type> + int64_t load() const { + return small_atomic.load(std::memory_order_acquire); + } + +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + // Using big atomics for big types. + std::atomic<FlagsInternalTwoWordsType> big_atomic; + template <typename T, typename K = typename std::enable_if< + (kMinLockFreeAtomicSize < sizeof(T) && + sizeof(T) <= kMaxLockFreeAtomicSize), + void>::type> + FlagsInternalTwoWordsType load() const { + return big_atomic.load(std::memory_order_acquire); + } + constexpr Atomics() + : big_atomic{FlagsInternalTwoWordsType{SmallAtomicInit(), + SmallAtomicInit()}} {} +#else + constexpr Atomics() : small_atomic{SmallAtomicInit()} {} +#endif + }; + Atomics atomics_{}; struct CallbackData { FlagCallback func; diff --git a/absl/flags/parse.cc b/absl/flags/parse.cc index 2f07725f5614..a288ace843fe 100644 --- a/absl/flags/parse.cc +++ b/absl/flags/parse.cc @@ -280,9 +280,7 @@ void CheckDefaultValuesParsingRoundtrip() { #define IGNORE_TYPE(T) \ if (flag->IsOfType<T>()) return; - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(IGNORE_TYPE) - IGNORE_TYPE(std::string) - IGNORE_TYPE(std::vector<std::string>) + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(IGNORE_TYPE) #undef IGNORE_TYPE flag->CheckDefaultValueParsingRoundtrip(); |