Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/stats.md
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,7 @@ PARAMETER tcmalloc_min_free_bytes_to_break_sparse_hugepage 0
PARAMETER tcmalloc_track_broken_hugepages 1
PARAMETER tcmalloc_preserve_donated_on_collapse 0
PARAMETER tcmalloc_collapse_released_hugepages 0
PARAMETER tcmalloc_subrelease_order_by_allocation_time 0
PARAMETER tcmalloc_madvise_sampled_allocations 1
PARAMETER tcmalloc_use_wider_slabs 1
PARAMETER heap_partitioning 0
Expand Down
2 changes: 2 additions & 0 deletions tcmalloc/experiment_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ enum class Experiment : int {
TCMALLOC_SONIC_PRESERVE_DONATED_ON_COLLAPSE, // TODO: b/572530574 - Complete experiment.
TCMALLOC_SONIC_SHARDED_TRANSFER_CACHE, // TODO: b/538230058 - Complete experiment.
TCMALLOC_SONIC_SPAN_BITMAP_ABLATION, // TODO: b/575736447 - Complete experiment.
TCMALLOC_SONIC_SUBRELEASE_ORDER_BY_ALLOCATION_TIME, // TODO: b/578767065 - Complete experiment.
TCMALLOC_SONIC_TRACK_BROKEN_HUGEPAGES, // TODO: b/572530574 - Complete experiment.
TEST_ONLY_MM_VCPU, // TODO: b/245776120 - Complete experiment.
TEST_ONLY_TCMALLOC_POW2_SIZECLASS,
Expand Down Expand Up @@ -74,6 +75,7 @@ inline constexpr ExperimentConfig experiments[] = {
{Experiment::TCMALLOC_SONIC_PRESERVE_DONATED_ON_COLLAPSE, "TCMALLOC_SONIC_PRESERVE_DONATED_ON_COLLAPSE", /*brittle=*/false, /*force_disable=*/false, /*rollout_lower_bound=*/0, /*rollout_upper_bound=*/0.01},
{Experiment::TCMALLOC_SONIC_SHARDED_TRANSFER_CACHE, "TCMALLOC_SONIC_SHARDED_TRANSFER_CACHE", /*brittle=*/false, /*force_disable=*/false, /*rollout_lower_bound=*/0, /*rollout_upper_bound=*/0.01},
{Experiment::TCMALLOC_SONIC_SPAN_BITMAP_ABLATION, "TCMALLOC_SONIC_SPAN_BITMAP_ABLATION", /*brittle=*/false, /*force_disable=*/false, /*rollout_lower_bound=*/0, /*rollout_upper_bound=*/0.1},
{Experiment::TCMALLOC_SONIC_SUBRELEASE_ORDER_BY_ALLOCATION_TIME, "TCMALLOC_SONIC_SUBRELEASE_ORDER_BY_ALLOCATION_TIME"},
{Experiment::TCMALLOC_SONIC_TRACK_BROKEN_HUGEPAGES, "TCMALLOC_SONIC_TRACK_BROKEN_HUGEPAGES", /*brittle=*/false, /*force_disable=*/false, /*rollout_lower_bound=*/0, /*rollout_upper_bound=*/0.01},
{Experiment::TEST_ONLY_MM_VCPU, "TEST_ONLY_MM_VCPU"},
{Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS, "TEST_ONLY_TCMALLOC_POW2_SIZECLASS", /*brittle=*/true},
Expand Down
6 changes: 6 additions & 0 deletions tcmalloc/experiment_variants.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ experiment_variants = [
"deps": ["//tcmalloc:common_8k_pages"],
"env": {"BORG_EXPERIMENTS": "TCMALLOC_SONIC_SPAN_BITMAP_ABLATION"},
},
{
"name": "tcmalloc_subrelease_order_by_allocation_time",
"malloc": "//tcmalloc",
"deps": ["//tcmalloc:common_8k_pages"],
"env": {"BORG_EXPERIMENTS": "TCMALLOC_SONIC_SUBRELEASE_ORDER_BY_ALLOCATION_TIME"},
},
{
"name": "tcmalloc_track_broken_hugepages",
"malloc": "//tcmalloc",
Expand Down
8 changes: 8 additions & 0 deletions tcmalloc/global_stats.cc
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,11 @@ void DumpStats(Printer& out, int level) {
Parameters::filler_append_on_free() == FillerAppendOnFree::kEnabled
? 1
: 0);
out.printf("PARAMETER tcmalloc_subrelease_order_by_allocation_time %d\n",
Parameters::subrelease_order_by_allocation_time() ==
SubreleaseOrderByAllocationTime::kEnabled
? 1
: 0);
out.printf("PARAMETER tcmalloc_madvise_sampled_allocations %d\n",
Parameters::madvise_sampled_allocations() ==
MadviseSampledAllocations::kEnabled);
Expand Down Expand Up @@ -962,6 +967,9 @@ void DumpStatsInPbtxt(Printer& out, int level) {
region.PrintBool(
"tcmalloc_filler_append_on_free",
Parameters::filler_append_on_free() == FillerAppendOnFree::kEnabled);
region.PrintBool("tcmalloc_subrelease_order_by_allocation_time",
Parameters::subrelease_order_by_allocation_time() ==
SubreleaseOrderByAllocationTime::kEnabled);
region.PrintI64("profile_sampling_interval",
Parameters::profile_sampling_interval());
region.PrintRaw("percpu_vcpu_type",
Expand Down
12 changes: 7 additions & 5 deletions tcmalloc/huge_page_aware_allocator.h
Original file line number Diff line number Diff line change
Expand Up @@ -986,7 +986,8 @@ inline Length HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPages(
SkipSubreleaseIntervals{.short_interval = short_interval,
.long_interval = long_interval},
forwarder_.release_partial_alloc_pages(),
/*hit_limit=*/false, min_free_pages_to_break(/*hit_limit=*/false));
/*hit_limit=*/false, min_free_pages_to_break(/*hit_limit=*/false),
forwarder_.subrelease_order_by_allocation_time());
DrainFreedTrackers();
}
}
Expand Down Expand Up @@ -1199,10 +1200,11 @@ HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPagesBreakingHugepages(
return released;
}

released += filler_.ReleasePages(n - released, SkipSubreleaseIntervals{},
ReleasePartialAllocPages::kDisabled,
/*hit_limit=*/true,
min_free_pages_to_break(/*hit_limit=*/true));
released += filler_.ReleasePages(
n - released, SkipSubreleaseIntervals{},
ReleasePartialAllocPages::kDisabled,
/*hit_limit=*/true, min_free_pages_to_break(/*hit_limit=*/true),
forwarder_.subrelease_order_by_allocation_time());
DrainFreedTrackers();

info_.RecordRelease(n, released, reason);
Expand Down
24 changes: 23 additions & 1 deletion tcmalloc/huge_page_aware_allocator_fuzz.cc
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,19 @@ struct SetMinFreePagesToBreakSparseHugepage {
}
};

struct SetSubreleaseOrderByAllocationTime {
bool value;

void Perform(State& state) const;

template <typename Sink>
friend void AbslStringify(Sink& sink,
const SetSubreleaseOrderByAllocationTime& s) {
absl::Format(&sink, "SetSubreleaseOrderByAllocationTime{.value=%v}",
s.value);
}
};

struct Instruction;

template <typename Sink>
Expand All @@ -671,7 +684,8 @@ using ParamOp = std::variant<
SetReleaseMaxFillerPages, SetPreferBackedFillerPages, SetFillerAppendOnFree,
SetEnableReleaseStalePages, SetCollapseReleasedHugepages,
SetMadvNoHugepageHugeRegions, UpdateBitmaps, SetUsageLimitPressure,
SetMinFreePagesToBreakDenseHugepage, SetMinFreePagesToBreakSparseHugepage>;
SetMinFreePagesToBreakDenseHugepage, SetMinFreePagesToBreakSparseHugepage,
SetSubreleaseOrderByAllocationTime>;

template <typename Sink>
void AbslStringify(Sink& sink, const ParamOp& p) {
Expand Down Expand Up @@ -1376,6 +1390,12 @@ void SetMinFreePagesToBreakSparseHugepage::Perform(State& state) const {
std::clamp(Length(pages), Length(1), kPagesPerHugePage));
}

void SetSubreleaseOrderByAllocationTime::Perform(State& state) const {
state.allocator.forwarder().subrelease_order_by_allocation_time.set(
value ? SubreleaseOrderByAllocationTime::kEnabled
: SubreleaseOrderByAllocationTime::kDisabled);
}

void UpdateBitmaps::Perform(State& state) const {
if (hugepage_backed_set) {
state.is_hugepage_backed = hugepage_backed_val;
Expand Down Expand Up @@ -2195,6 +2215,8 @@ TEST(HugePageAwareAllocatorTest, PrinterTest) {
"SetMinFreePagesToBreakDenseHugepage{.pages=8}");
EXPECT_EQ(absl::StrCat(SetMinFreePagesToBreakSparseHugepage{.pages = 16}),
"SetMinFreePagesToBreakSparseHugepage{.pages=16}");
EXPECT_EQ(absl::StrCat(SetSubreleaseOrderByAllocationTime{.value = true}),
"SetSubreleaseOrderByAllocationTime{.value=true}");
}

// Regression tests for the three lock-drop interleavings while collapsing a
Expand Down
90 changes: 90 additions & 0 deletions tcmalloc/huge_page_aware_allocator_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3176,6 +3176,96 @@ TEST_P(HugePageAwareAllocatorTest, MinFreePagesToBreakHugepage) {
}
}

TEST_P(HugePageAwareAllocatorTest, SubreleaseOrderByAllocationTime) {
constexpr SpanAllocInfo kSparseInfo = {
.objects_per_span = 1,
.density = AccessDensityPrediction::kSparse,
};
constexpr Length kBand = BytesToLengthCeil(kHugePageSize / 16);

allocator_->forwarder().hpaa_subrelease.set(true);
allocator_->forwarder().filler_skip_subrelease_short_interval.set(
absl::ZeroDuration());
allocator_->forwarder().filler_skip_subrelease_long_interval.set(
absl::ZeroDuration());
allocator_->forwarder().release_max_filler_pages.set(false);
allocator_->forwarder().min_free_pages_to_break_sparse_hugepage.set(
Length(1));

struct TrackedPage {
AllocationState keep;
AllocationState fill1;
AllocationState fill2;
HugePage hp;
Length free_pages;
};
// Populate four sparse hugepages in band 1 from oldest (most used pages) to
// newest (fewest used pages, placed at the front of the filler freelist).
std::array<TrackedPage, 4> pages;
for (int i = 0; i < 4; ++i) {
if (i > 0) {
huge_page_allocator_internal::FakeHugePageStaticForwarder::AdvanceClock(
absl::Seconds(120));
}
const Length used = kBand >= Length(4) ? kBand + Length(3 - i) : Length(1);
const Length rem = kPagesPerHugePage - used;
pages[i].keep = New(used, kSparseInfo);
pages[i].fill1 = New(rem / 2, kSparseInfo);
pages[i].fill2 = New(rem - rem / 2, kSparseInfo);
pages[i].hp = HugePageContaining(pages[i].keep.r.p);
pages[i].free_pages = rem;
ASSERT_EQ(HugePageContaining(pages[i].fill1.r.p), pages[i].hp);
ASSERT_EQ(HugePageContaining(pages[i].fill2.r.p), pages[i].hp);
}
for (TrackedPage& page : pages) {
DeleteWithInfo(page.fill1, kSparseInfo);
DeleteWithInfo(page.fill2, kSparseInfo);
}

// Flip back and forth between modes with live trackers in flight:
// kEnabled -> oldest pages[0] (background release)
// kDisabled -> emptiest/MRU pages[3] (background release)
// kEnabled -> oldest remaining pages[1] (hard-limit release)
// kDisabled -> remaining pages[2] (background release)
allocator_->forwarder().subrelease_order_by_allocation_time.set(
SubreleaseOrderByAllocationTime::kEnabled);
EXPECT_EQ(
ReleasePages(Length(1), PageReleaseReason::kProcessBackgroundActions),
pages[0].free_pages);
EXPECT_FALSE(GetHugepageState(pages[0].hp).unbroken);
EXPECT_TRUE(GetHugepageState(pages[3].hp).unbroken);

allocator_->forwarder().subrelease_order_by_allocation_time.set(
SubreleaseOrderByAllocationTime::kDisabled);
EXPECT_EQ(
ReleasePages(Length(1), PageReleaseReason::kProcessBackgroundActions),
pages[3].free_pages);
EXPECT_FALSE(GetHugepageState(pages[3].hp).unbroken);
EXPECT_TRUE(GetHugepageState(pages[1].hp).unbroken);

allocator_->forwarder().subrelease_order_by_allocation_time.set(
SubreleaseOrderByAllocationTime::kEnabled);
{
PageHeapSpinLockHolder l;
EXPECT_EQ(allocator_->ReleaseAtLeastNPagesBreakingHugepages(
Length(1), PageReleaseReason::kSoftLimitExceeded),
pages[1].free_pages);
}
EXPECT_FALSE(GetHugepageState(pages[1].hp).unbroken);
EXPECT_TRUE(GetHugepageState(pages[2].hp).unbroken);

allocator_->forwarder().subrelease_order_by_allocation_time.set(
SubreleaseOrderByAllocationTime::kDisabled);
EXPECT_EQ(
ReleasePages(Length(1), PageReleaseReason::kProcessBackgroundActions),
pages[2].free_pages);
EXPECT_FALSE(GetHugepageState(pages[2].hp).unbroken);

for (TrackedPage& page : pages) {
DeleteWithInfo(page.keep, kSparseInfo);
}
}

TEST(HugePageAwareAllocatorDeathTest, DoubleFreeReportsError) {
FakeHugePageAwareAllocator allocator({.tag = MemoryTag::kNormal});
constexpr SpanAllocInfo kSpanInfo = {
Expand Down
Loading
Loading