From 3980b1cfb0b8e9a7d47b215139e7cba1741454a3 Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Tue, 17 Nov 2026 21:48:56 -0800 Subject: [PATCH] Throttle heap profile compressibility analysis by the time since the previous heap profile, so that frequent heap profiling does not repeatedly read sampled pages and reset their kernel page aging. PiperOrigin-RevId: 1024327583 --- tcmalloc/internal/BUILD | 1 + tcmalloc/internal/profile_builder.cc | 50 +++++++++- tcmalloc/internal/profile_builder.h | 39 +++++++- tcmalloc/internal/profile_builder_test.cc | 116 ++++++++++++++++++++++ 4 files changed, 202 insertions(+), 4 deletions(-) diff --git a/tcmalloc/internal/BUILD b/tcmalloc/internal/BUILD index bceb382e2..4b2630e1c 100644 --- a/tcmalloc/internal/BUILD +++ b/tcmalloc/internal/BUILD @@ -1238,6 +1238,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/hash", "@com_google_absl//absl/random", + "@com_google_absl//absl/random:distributions", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/tcmalloc/internal/profile_builder.cc b/tcmalloc/internal/profile_builder.cc index 7cd49253f..d95deaa29 100644 --- a/tcmalloc/internal/profile_builder.cc +++ b/tcmalloc/internal/profile_builder.cc @@ -23,10 +23,12 @@ #include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" #include "absl/hash/hash.h" +#include "absl/random/distributions.h" #include "absl/random/random.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" +#include "absl/time/clock.h" #include "absl/time/time.h" #include "absl/types/span.h" #include "tcmalloc/internal/compressibility.h" @@ -232,7 +234,8 @@ struct SampleMergeResult { SampleMergeResult MergeProfileSamplesAndMaybeGetResidencyInfo( const tcmalloc::Profile& profile, PageFlagsBase* pageflags, - Residency* residency, bool collecting_compressibility) { + Residency* residency, bool collecting_compressibility, + double compressibility_fraction) { SampleMergeResult result; SampleMergedMap& map = result.samples; std::optional compression_analyzer; @@ -339,6 +342,13 @@ SampleMergeResult MergeProfileSamplesAndMaybeGetResidencyInfo( break; } const CompressibilityCandidate& cand = data.candidates[round]; + // Reading the span sets its pages' accessed bits and resets their + // kstaled aging; see CompressibilityThrottle. Unselected candidates + // are neither read nor counted, as if they had not been sampled. + if (compressibility_fraction < 1.0 && + !absl::Bernoulli(rng, compressibility_fraction)) { + continue; + } absl::Span sample_mem( reinterpret_cast(cand.span_start_address), cand.size); absl::StatusOr res = @@ -1034,9 +1044,34 @@ std::unique_ptr ProfileBuilder::Finalize() && { return std::move(profile_); } +double CompressibilityThrottle::Fraction(absl::Time now, + absl::Duration period) { + const int64_t now_nanos = absl::ToUnixNanos(now); + const int64_t last_nanos = + last_conversion_nanos_.exchange(now_nanos, std::memory_order_relaxed); + if (period <= absl::ZeroDuration() || last_nanos == 0) { + return 1.0; + } + const absl::Duration elapsed = absl::Nanoseconds(now_nanos - last_nanos); + if (elapsed <= absl::ZeroDuration()) { + return 0.0; + } + if (elapsed >= period) { + return 1.0; + } + return absl::FDivDuration(elapsed, period); +} + absl::StatusOr> MakeProfileProto( const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags, Residency* residency) { + return MakeProfileProto(profile, pageflags, residency, + /*compressibility_fraction=*/1.0); +} + +absl::StatusOr> MakeProfileProto( + const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags, + Residency* residency, double compressibility_fraction) { if (profile.Type() == ProfileType::kDoNotUse) { if (MallocExtension::GetProfileSamplingInterval() == 0) { return absl::NotFoundError("Profile sampling is disabled"); @@ -1164,7 +1199,8 @@ absl::StatusOr> MakeProfileProto( // compressible rather than as having no data. const auto [samples, exporting_compressibility] = MergeProfileSamplesAndMaybeGetResidencyInfo(profile, pageflags, residency, - collecting_compressibility); + collecting_compressibility, + compressibility_fraction); if (exporting_compressibility) { perftools::profiles::ValueType* sample_type = nullptr; @@ -1254,12 +1290,20 @@ absl::StatusOr> MakeProfileProto( PageFlags* p = nullptr; Residency* r = nullptr; + double compressibility_fraction = 1.0; if (profile.Type() == ProfileType::kHeap) { p = &pageflags.emplace(); r = &residency.emplace(); + + absl::Duration period = absl::ZeroDuration(); + bool collecting_compressibility = false; + if (collecting_compressibility) { + ABSL_CONST_INIT static CompressibilityThrottle throttle; + compressibility_fraction = throttle.Fraction(absl::Now(), period); + } } - return MakeProfileProto(profile, p, r); + return MakeProfileProto(profile, p, r, compressibility_fraction); } } // namespace tcmalloc_internal diff --git a/tcmalloc/internal/profile_builder.h b/tcmalloc/internal/profile_builder.h index 331a2c9f7..ba55f072b 100644 --- a/tcmalloc/internal/profile_builder.h +++ b/tcmalloc/internal/profile_builder.h @@ -20,6 +20,7 @@ #include #endif // defined(__linux__) +#include #include #include #include @@ -30,6 +31,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/time/time.h" #include "absl/types/span.h" #include "tcmalloc/malloc_extension.h" @@ -90,11 +92,46 @@ class PageFlagsBase; class PageFlags; class Residency; -// Exposed to facilitate testing. +// Exposed to facilitate testing. Analyzes every compressibility candidate. [[nodiscard]] absl::StatusOr> MakeProfileProto(const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags, Residency* residency); +// Exposed to facilitate testing. Analyzes each compressibility candidate with +// probability `compressibility_fraction` (see CompressibilityThrottle). +[[nodiscard]] absl::StatusOr> +MakeProfileProto(const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags, + Residency* residency, double compressibility_fraction); + +// Bounds how often heap profile conversions read sampled spans for +// compressibility analysis, independently of how often heap profiles are +// requested. +// +// Compressibility analysis copies the contents of sampled resident spans, which +// sets their pages' accessed bits. kstaled then treats those pages as recently +// used and `stale_space` reports them as not stale for the next one or two scan +// periods. A process that converts heap profiles more often than the stale scan +// period would therefore hide the staleness of its sampled pages from every +// other heap profile. To keep the average rate at which a sampled span is read +// at about once per `period` regardless of the number of callers, each +// conversion analyzes each candidate with probability +// min(1, elapsed / period), where `elapsed` is the time since the previous +// conversion that collected compressibility in this process. +class CompressibilityThrottle { + public: + constexpr CompressibilityThrottle() = default; + + // Returns the fraction of compressibility candidates to analyze for a + // conversion at `now`, and records `now` as the latest conversion. Returns 1 + // for the first conversion, when `period` is not positive, or when at least + // `period` has elapsed; 0 if `now` is not after the previous conversion. + double Fraction(absl::Time now, absl::Duration period); + + private: + // absl::ToUnixNanos() of the previous conversion; 0 if there was none. + std::atomic last_conversion_nanos_{0}; +}; + } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/internal/profile_builder_test.cc b/tcmalloc/internal/profile_builder_test.cc index c02666caa..05e2de38e 100644 --- a/tcmalloc/internal/profile_builder_test.cc +++ b/tcmalloc/internal/profile_builder_test.cc @@ -1714,6 +1714,122 @@ TEST(ProfileConverterTest, SmallSizeClassGroupAnalyzesHundredsOfSamples) { EXPECT_EQ(values["analyzed_compressibility_space"], kTotalBacked); } +TEST(CompressibilityThrottleTest, FractionTracksTimeSincePreviousConversion) { + CompressibilityThrottle throttle; + const absl::Duration period = absl::Minutes(30); + const absl::Time start = absl::FromUnixSeconds(1'700'000'000); + + // The first conversion analyzes everything. + EXPECT_EQ(throttle.Fraction(start, period), 1.0); + // 3 minutes later: a tenth of the candidates. + EXPECT_DOUBLE_EQ(throttle.Fraction(start + absl::Minutes(3), period), 0.1); + // Measured from the previous conversion, not from the first one. + EXPECT_DOUBLE_EQ(throttle.Fraction(start + absl::Minutes(18), period), 0.5); + // At least a period later: everything again. + EXPECT_EQ(throttle.Fraction(start + absl::Minutes(48), period), 1.0); + // A conversion at the same time, or with the clock moving backwards, + // analyzes nothing. + EXPECT_EQ(throttle.Fraction(start + absl::Minutes(48), period), 0.0); + EXPECT_EQ(throttle.Fraction(start + absl::Minutes(40), period), 0.0); +} + +TEST(CompressibilityThrottleTest, NonPositivePeriodAnalyzesEverything) { + CompressibilityThrottle throttle; + const absl::Time start = absl::FromUnixSeconds(1'700'000'000); + EXPECT_EQ(throttle.Fraction(start, absl::ZeroDuration()), 1.0); + EXPECT_EQ(throttle.Fraction(start + absl::Seconds(1), absl::ZeroDuration()), + 1.0); + EXPECT_EQ(throttle.Fraction(start + absl::Seconds(2), -absl::Seconds(1)), + 1.0); +} + +// Builds a kHeap profile of `num_spans` resident 1 KiB spans in one callstack +// group, half of them all zeroes, and converts it with the given fraction. +absl::flat_hash_map ConvertWithCompressibilityFraction( + size_t num_spans, double fraction, std::vector>& bufs, + bool& has_compressibility_columns) { + constexpr size_t kSpanSize = 1024; + bufs.assign(num_spans, std::vector(kSpanSize)); + StubPageFlags pageflags; + StubResidency residency; + std::vector samples; + samples.reserve(num_spans); + for (size_t i = 0; i < num_spans; ++i) { + std::memset(bufs[i].data(), (i % 2 == 0) ? 0 : 0x42, kSpanSize); + Profile::Sample sample = {}; + sample.sum = kSpanSize; + sample.count = 1; + sample.requested_size = kSpanSize; + sample.allocated_size = kSpanSize; + sample.span_start_address = bufs[i].data(); + sample.depth = 1; + sample.stack[0] = reinterpret_cast(&RealPath); + sample.type = AllocationType::Malloc; + samples.push_back(sample); + + Residency::Info info; + info.bytes_resident = kSpanSize; + info.page_is_resident.SetBit(0); + info.page_is_resident.SetBit(1); + residency.SetInfo(bufs[i].data(), info); + pageflags.set_bytes_stale(bufs[i].data(), kSpanSize); + } + + auto fake_profile = std::make_unique(); + fake_profile->SetType(ProfileType::kHeap); + fake_profile->SetDuration(absl::Milliseconds(100)); + fake_profile->SetSamples(std::move(samples)); + Profile profile = ProfileAccessor::MakeProfile(std::move(fake_profile)); + + auto converted_or = + MakeProfileProto(profile, &pageflags, &residency, fraction); + CHECK_OK(converted_or.status()); + const auto& converted = **converted_or; + CHECK_EQ(converted.sample_size(), 1); + absl::flat_hash_map values; + for (int i = 0; i < converted.sample_type_size(); ++i) { + values[converted.string_table(converted.sample_type(i).type())] = + converted.sample(0).value(i); + } + has_compressibility_columns = + values.contains("analyzed_compressibility_space"); + return values; +} + +TEST(ProfileConverterTest, ZeroCompressibilityFractionAnalyzesNothing) { + std::vector> bufs; + bool has_compressibility_columns = true; + auto values = ConvertWithCompressibilityFraction( + /*num_spans=*/100, /*fraction=*/0.0, bufs, has_compressibility_columns); + // Residency and pageflags columns are unaffected. + EXPECT_EQ(values["resident_space"], 100 * 1024); + EXPECT_EQ(values["stale_space"], 100 * 1024); + // No span was analyzed, so the compressibility columns are omitted rather + // than reported as zero. + EXPECT_FALSE(has_compressibility_columns); +} + +TEST(ProfileConverterTest, CompressibilityFractionSubsamplesCandidates) { + constexpr size_t kNumSpans = 400; + constexpr int64_t kTotalBacked = kNumSpans * 1024; + std::vector> bufs; + bool has_compressibility_columns = false; + auto values = ConvertWithCompressibilityFraction( + kNumSpans, /*fraction=*/0.25, bufs, has_compressibility_columns); + + return; + + EXPECT_EQ(values["resident_space"], kTotalBacked); + EXPECT_EQ(values["stale_space"], kTotalBacked); + ASSERT_TRUE(has_compressibility_columns); + // About a quarter of the 400 spans is analyzed (binomial: mean 100, standard + // deviation 8.7), and zero_space is still extrapolated to the whole group. + EXPECT_GE(values["analyzed_compressibility_space"], kTotalBacked / 10); + EXPECT_LE(values["analyzed_compressibility_space"], kTotalBacked * 4 / 10); + EXPECT_GE(values["zero_space"], kTotalBacked * 3 / 10); + EXPECT_LE(values["zero_space"], kTotalBacked * 7 / 10); +} + #if defined(__linux__) TEST(ProfileConverterTest, UnanalyzedGroupUsesConservativeIncompressibleFallback) {