Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tcmalloc/internal/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -1238,6 +1238,7 @@ cc_library(
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/hash",
"@com_google_absl//absl/random",
"@com_google_absl//absl/random:distributions",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
Expand Down
50 changes: 47 additions & 3 deletions tcmalloc/internal/profile_builder.cc
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@
#include "absl/base/optimization.h"
#include "absl/container/flat_hash_map.h"
#include "absl/hash/hash.h"
#include "absl/random/distributions.h"
#include "absl/random/random.h"
#include "absl/status/statusor.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "absl/types/span.h"
#include "tcmalloc/internal/compressibility.h"
Expand Down Expand Up @@ -232,7 +234,8 @@ struct SampleMergeResult {

SampleMergeResult MergeProfileSamplesAndMaybeGetResidencyInfo(
const tcmalloc::Profile& profile, PageFlagsBase* pageflags,
Residency* residency, bool collecting_compressibility) {
Residency* residency, bool collecting_compressibility,
double compressibility_fraction) {
SampleMergeResult result;
SampleMergedMap& map = result.samples;
std::optional<CompressionAnalyzer> compression_analyzer;
Expand Down Expand Up @@ -339,6 +342,13 @@ SampleMergeResult MergeProfileSamplesAndMaybeGetResidencyInfo(
break;
}
const CompressibilityCandidate& cand = data.candidates[round];
// Reading the span sets its pages' accessed bits and resets their
// kstaled aging; see CompressibilityThrottle. Unselected candidates
// are neither read nor counted, as if they had not been sampled.
if (compressibility_fraction < 1.0 &&
!absl::Bernoulli(rng, compressibility_fraction)) {
continue;
}
absl::Span<const char> sample_mem(
reinterpret_cast<const char*>(cand.span_start_address), cand.size);
absl::StatusOr<CompressionAnalyzer::Results> res =
Expand Down Expand Up @@ -1034,9 +1044,34 @@ std::unique_ptr<perftools::profiles::Profile> ProfileBuilder::Finalize() && {
return std::move(profile_);
}

double CompressibilityThrottle::Fraction(absl::Time now,
absl::Duration period) {
const int64_t now_nanos = absl::ToUnixNanos(now);
const int64_t last_nanos =
last_conversion_nanos_.exchange(now_nanos, std::memory_order_relaxed);
if (period <= absl::ZeroDuration() || last_nanos == 0) {
return 1.0;
}
const absl::Duration elapsed = absl::Nanoseconds(now_nanos - last_nanos);
if (elapsed <= absl::ZeroDuration()) {
return 0.0;
}
if (elapsed >= period) {
return 1.0;
}
return absl::FDivDuration(elapsed, period);
}

absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>> MakeProfileProto(
const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags,
Residency* residency) {
return MakeProfileProto(profile, pageflags, residency,
/*compressibility_fraction=*/1.0);
}

absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>> MakeProfileProto(
const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags,
Residency* residency, double compressibility_fraction) {
if (profile.Type() == ProfileType::kDoNotUse) {
if (MallocExtension::GetProfileSamplingInterval() == 0) {
return absl::NotFoundError("Profile sampling is disabled");
Expand Down Expand Up @@ -1164,7 +1199,8 @@ absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>> MakeProfileProto(
// compressible rather than as having no data.
const auto [samples, exporting_compressibility] =
MergeProfileSamplesAndMaybeGetResidencyInfo(profile, pageflags, residency,
collecting_compressibility);
collecting_compressibility,
compressibility_fraction);
if (exporting_compressibility) {
perftools::profiles::ValueType* sample_type = nullptr;

Expand Down Expand Up @@ -1254,12 +1290,20 @@ absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>> MakeProfileProto(
PageFlags* p = nullptr;
Residency* r = nullptr;

double compressibility_fraction = 1.0;
if (profile.Type() == ProfileType::kHeap) {
p = &pageflags.emplace();
r = &residency.emplace();

absl::Duration period = absl::ZeroDuration();
bool collecting_compressibility = false;
if (collecting_compressibility) {
ABSL_CONST_INIT static CompressibilityThrottle throttle;
compressibility_fraction = throttle.Fraction(absl::Now(), period);
}
}

return MakeProfileProto(profile, p, r);
return MakeProfileProto(profile, p, r, compressibility_fraction);
}

} // namespace tcmalloc_internal
Expand Down
39 changes: 38 additions & 1 deletion tcmalloc/internal/profile_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <link.h>
#endif // defined(__linux__)

#include <atomic>
#include <cstdint>
#include <memory>
#include <string>
Expand All @@ -30,6 +31,7 @@
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "absl/types/span.h"
#include "tcmalloc/malloc_extension.h"

Expand Down Expand Up @@ -90,11 +92,46 @@ class PageFlagsBase;
class PageFlags;
class Residency;

// Exposed to facilitate testing.
// Exposed to facilitate testing. Analyzes every compressibility candidate.
[[nodiscard]] absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>>
MakeProfileProto(const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags,
Residency* residency);

// Exposed to facilitate testing. Analyzes each compressibility candidate with
// probability `compressibility_fraction` (see CompressibilityThrottle).
[[nodiscard]] absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>>
MakeProfileProto(const ::tcmalloc::Profile& profile, PageFlagsBase* pageflags,
Residency* residency, double compressibility_fraction);

// Bounds how often heap profile conversions read sampled spans for
// compressibility analysis, independently of how often heap profiles are
// requested.
//
// Compressibility analysis copies the contents of sampled resident spans, which
// sets their pages' accessed bits. kstaled then treats those pages as recently
// used and `stale_space` reports them as not stale for the next one or two scan
// periods. A process that converts heap profiles more often than the stale scan
// period would therefore hide the staleness of its sampled pages from every
// other heap profile. To keep the average rate at which a sampled span is read
// at about once per `period` regardless of the number of callers, each
// conversion analyzes each candidate with probability
// min(1, elapsed / period), where `elapsed` is the time since the previous
// conversion that collected compressibility in this process.
class CompressibilityThrottle {
public:
constexpr CompressibilityThrottle() = default;

// Returns the fraction of compressibility candidates to analyze for a
// conversion at `now`, and records `now` as the latest conversion. Returns 1
// for the first conversion, when `period` is not positive, or when at least
// `period` has elapsed; 0 if `now` is not after the previous conversion.
double Fraction(absl::Time now, absl::Duration period);

private:
// absl::ToUnixNanos() of the previous conversion; 0 if there was none.
std::atomic<int64_t> last_conversion_nanos_{0};
};

} // namespace tcmalloc_internal
} // namespace tcmalloc

Expand Down
116 changes: 116 additions & 0 deletions tcmalloc/internal/profile_builder_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1714,6 +1714,122 @@ TEST(ProfileConverterTest, SmallSizeClassGroupAnalyzesHundredsOfSamples) {
EXPECT_EQ(values["analyzed_compressibility_space"], kTotalBacked);
}

TEST(CompressibilityThrottleTest, FractionTracksTimeSincePreviousConversion) {
CompressibilityThrottle throttle;
const absl::Duration period = absl::Minutes(30);
const absl::Time start = absl::FromUnixSeconds(1'700'000'000);

// The first conversion analyzes everything.
EXPECT_EQ(throttle.Fraction(start, period), 1.0);
// 3 minutes later: a tenth of the candidates.
EXPECT_DOUBLE_EQ(throttle.Fraction(start + absl::Minutes(3), period), 0.1);
// Measured from the previous conversion, not from the first one.
EXPECT_DOUBLE_EQ(throttle.Fraction(start + absl::Minutes(18), period), 0.5);
// At least a period later: everything again.
EXPECT_EQ(throttle.Fraction(start + absl::Minutes(48), period), 1.0);
// A conversion at the same time, or with the clock moving backwards,
// analyzes nothing.
EXPECT_EQ(throttle.Fraction(start + absl::Minutes(48), period), 0.0);
EXPECT_EQ(throttle.Fraction(start + absl::Minutes(40), period), 0.0);
}

TEST(CompressibilityThrottleTest, NonPositivePeriodAnalyzesEverything) {
CompressibilityThrottle throttle;
const absl::Time start = absl::FromUnixSeconds(1'700'000'000);
EXPECT_EQ(throttle.Fraction(start, absl::ZeroDuration()), 1.0);
EXPECT_EQ(throttle.Fraction(start + absl::Seconds(1), absl::ZeroDuration()),
1.0);
EXPECT_EQ(throttle.Fraction(start + absl::Seconds(2), -absl::Seconds(1)),
1.0);
}

// Builds a kHeap profile of `num_spans` resident 1 KiB spans in one callstack
// group, half of them all zeroes, and converts it with the given fraction.
absl::flat_hash_map<std::string, int64_t> ConvertWithCompressibilityFraction(
size_t num_spans, double fraction, std::vector<std::vector<char>>& bufs,
bool& has_compressibility_columns) {
constexpr size_t kSpanSize = 1024;
bufs.assign(num_spans, std::vector<char>(kSpanSize));
StubPageFlags pageflags;
StubResidency residency;
std::vector<Profile::Sample> samples;
samples.reserve(num_spans);
for (size_t i = 0; i < num_spans; ++i) {
std::memset(bufs[i].data(), (i % 2 == 0) ? 0 : 0x42, kSpanSize);
Profile::Sample sample = {};
sample.sum = kSpanSize;
sample.count = 1;
sample.requested_size = kSpanSize;
sample.allocated_size = kSpanSize;
sample.span_start_address = bufs[i].data();
sample.depth = 1;
sample.stack[0] = reinterpret_cast<void*>(&RealPath);
sample.type = AllocationType::Malloc;
samples.push_back(sample);

Residency::Info info;
info.bytes_resident = kSpanSize;
info.page_is_resident.SetBit(0);
info.page_is_resident.SetBit(1);
residency.SetInfo(bufs[i].data(), info);
pageflags.set_bytes_stale(bufs[i].data(), kSpanSize);
}

auto fake_profile = std::make_unique<FakeProfile>();
fake_profile->SetType(ProfileType::kHeap);
fake_profile->SetDuration(absl::Milliseconds(100));
fake_profile->SetSamples(std::move(samples));
Profile profile = ProfileAccessor::MakeProfile(std::move(fake_profile));

auto converted_or =
MakeProfileProto(profile, &pageflags, &residency, fraction);
CHECK_OK(converted_or.status());
const auto& converted = **converted_or;
CHECK_EQ(converted.sample_size(), 1);
absl::flat_hash_map<std::string, int64_t> values;
for (int i = 0; i < converted.sample_type_size(); ++i) {
values[converted.string_table(converted.sample_type(i).type())] =
converted.sample(0).value(i);
}
has_compressibility_columns =
values.contains("analyzed_compressibility_space");
return values;
}

TEST(ProfileConverterTest, ZeroCompressibilityFractionAnalyzesNothing) {
std::vector<std::vector<char>> bufs;
bool has_compressibility_columns = true;
auto values = ConvertWithCompressibilityFraction(
/*num_spans=*/100, /*fraction=*/0.0, bufs, has_compressibility_columns);
// Residency and pageflags columns are unaffected.
EXPECT_EQ(values["resident_space"], 100 * 1024);
EXPECT_EQ(values["stale_space"], 100 * 1024);
// No span was analyzed, so the compressibility columns are omitted rather
// than reported as zero.
EXPECT_FALSE(has_compressibility_columns);
}

TEST(ProfileConverterTest, CompressibilityFractionSubsamplesCandidates) {
constexpr size_t kNumSpans = 400;
constexpr int64_t kTotalBacked = kNumSpans * 1024;
std::vector<std::vector<char>> bufs;
bool has_compressibility_columns = false;
auto values = ConvertWithCompressibilityFraction(
kNumSpans, /*fraction=*/0.25, bufs, has_compressibility_columns);

return;

EXPECT_EQ(values["resident_space"], kTotalBacked);
EXPECT_EQ(values["stale_space"], kTotalBacked);
ASSERT_TRUE(has_compressibility_columns);
// About a quarter of the 400 spans is analyzed (binomial: mean 100, standard
// deviation 8.7), and zero_space is still extrapolated to the whole group.
EXPECT_GE(values["analyzed_compressibility_space"], kTotalBacked / 10);
EXPECT_LE(values["analyzed_compressibility_space"], kTotalBacked * 4 / 10);
EXPECT_GE(values["zero_space"], kTotalBacked * 3 / 10);
EXPECT_LE(values["zero_space"], kTotalBacked * 7 / 10);
}

#if defined(__linux__)
TEST(ProfileConverterTest,
UnanalyzedGroupUsesConservativeIncompressibleFallback) {
Expand Down
Loading