Sign-extend timestamp deltas in SpinLock::DecodeWaitCycles. This prevents small negative CycleClock deltas between the sleeping waiter's core and the unlocking holder's core from underflowing the 29-bit unsigned timestamp in the lock word to ~2^36 cycles. Interpret the 29-bit difference as signed so negative deltas decode as <= 0 and are ignored by SlowUnlock, and increase kProfileTimestampShift from 7 to 8 to preserve the ~2^36-cycle positive range. PiperOrigin-RevId: 1016328794
diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 57966d2..bc79c1e 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc
@@ -85,10 +85,10 @@ // counter wrap very quickly on high-frequency processors we divide to reduce // the granularity to 2^kProfileTimestampShift sized units. Because only 29 // bits are stored in the 32-bit lock word (and uncontended lock() does not read -// CycleClock), SlowUnlock only knows the start time modulo 2^36 CycleClock -// ticks (=~ 34 seconds at a 2 GHz CycleClock) and cannot distinguish a wrapped -// hold to saturate. -static constexpr int kProfileTimestampShift = 7; +// CycleClock), SlowUnlock only knows the start time modulo 2^37 CycleClock +// ticks (=~ +/-34 seconds as a signed offset at a 2 GHz CycleClock) and cannot +// distinguish a wrapped hold to saturate. +static constexpr int kProfileTimestampShift = 8; // We currently reserve the lower 3 bits. static constexpr int kLockwordReservedShift = 3; @@ -239,13 +239,16 @@ } int64_t SpinLock::DecodeWaitCycles(uint32_t lock_value, int64_t now) { - // Modular subtraction in the width of the stored timestamp. Encode `now` - // the same way as `lock_value` so a timestamp truncated to zero does not - // underflow against EncodeWaitStart's rounding to one. - const uint32_t scaled_wait_time = - (EncodeWaitStart(now) - (lock_value & kWaitTimeMask)) >> + // Signed modular subtraction in the width of the stored timestamp so that a + // negative clock delta (e.g. from cross-core TSC skew) decodes as <= 0 + // rather than wrapping to ~2^37 cycles. Encode `now` the same way as + // `lock_value` so a timestamp truncated to zero does not underflow against + // EncodeWaitStart's rounding to one. + const int32_t scaled_wait_time = + static_cast<int32_t>(EncodeWaitStart(now) - + (lock_value & kWaitTimeMask)) >> kLockwordReservedShift; - return static_cast<int64_t>(scaled_wait_time) << kProfileTimestampShift; + return int64_t{scaled_wait_time} * (int64_t{1} << kProfileTimestampShift); } } // namespace base_internal
diff --git a/absl/base/spinlock_profiling_test.cc b/absl/base/spinlock_profiling_test.cc index 8ffc604..c1f9622 100644 --- a/absl/base/spinlock_profiling_test.cc +++ b/absl/base/spinlock_profiling_test.cc
@@ -37,6 +37,13 @@ return (l.lockword_.load(std::memory_order_relaxed) & SpinLock::kWaitTimeMask) != 0; } + + static void LockAndUnlockWithWaitStart(SpinLock& l, int64_t wait_start_time) { + l.lock(); + l.lockword_.fetch_or(SpinLock::EncodeWaitStart(wait_start_time), + std::memory_order_relaxed); + l.unlock(); + } }; namespace { @@ -131,6 +138,16 @@ EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); } +TEST_F(SpinLockProfiling, NegativeClockSkewIsNotReported) { + SpinLock spinlock(SCHEDULE_KERNEL_ONLY); + profiled_lock = &spinlock; + SpinLockProfilingTestPeer::LockAndUnlockWithWaitStart( + spinlock, CycleClock::Now() + (int64_t{1} << 20)); + profiled_lock = nullptr; + + EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); +} + } // namespace } // namespace base_internal ABSL_NAMESPACE_END
diff --git a/absl/base/spinlock_test_common.cc b/absl/base/spinlock_test_common.cc index a29a7f5..1ffa037 100644 --- a/absl/base/spinlock_test_common.cc +++ b/absl/base/spinlock_test_common.cc
@@ -143,14 +143,16 @@ TEST(SpinLock, WaitStartEncoding) { // These are implementation details not exported by SpinLock. - const int kProfileTimestampShift = 7; + const int kProfileTimestampShift = 8; const int kLockwordReservedShift = 3; // The encoded timestamp wraps around at 2^kWrapShift cycles, and the lower - // kProfileTimestampShift bits are dropped. + // kProfileTimestampShift bits are dropped. Deltas are interpreted as signed, + // so positive waits up to 2^(kWrapShift - 1) - step decode as positive and + // negative clock skews decode as negative. const int kWrapShift = 32 - kLockwordReservedShift + kProfileTimestampShift; const int64_t step = (1 << kProfileTimestampShift); - const int64_t kMaxCycles = (int64_t{1} << kWrapShift) - step; + const int64_t kMaxCycles = (int64_t{1} << (kWrapShift - 1)) - step; // These bits should be zero after encoding. const uint32_t kLockwordReservedMask = (1 << kLockwordReservedShift) - 1; @@ -195,13 +197,20 @@ EXPECT_EQ(0, SpinLockTest::DecodeWaitCycles(lock_value, start_time)); EXPECT_EQ(step, SpinLockTest::DecodeWaitCycles(lock_value, start_time + step)); + EXPECT_EQ(-step, + SpinLockTest::DecodeWaitCycles(lock_value, start_time - step)); EXPECT_EQ(kMaxCycles, SpinLockTest::DecodeWaitCycles( lock_value, start_time + kMaxCycles)); + // Past the signed half-range, deltas are treated as negative clock skew. + EXPECT_EQ(-kMaxCycles - step, + SpinLockTest::DecodeWaitCycles(lock_value, + start_time + kMaxCycles + step)); // Once the clock wraps all the way around, the elapsed time wraps to zero. - EXPECT_EQ(0, SpinLockTest::DecodeWaitCycles(lock_value, - start_time + kMaxCycles + step)); - EXPECT_EQ(step, SpinLockTest::DecodeWaitCycles( - lock_value, start_time + kMaxCycles + 2 * step)); + EXPECT_EQ(0, SpinLockTest::DecodeWaitCycles( + lock_value, start_time + (int64_t{1} << kWrapShift))); + EXPECT_EQ(step, + SpinLockTest::DecodeWaitCycles( + lock_value, start_time + (int64_t{1} << kWrapShift) + step)); // A timestamp whose truncation is zero must not encode as "no waiter", and // decoding in the same tick must return zero rather than underflowing.