inline: FixedBlockPage ++
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatedSizeTracker.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatedSizeTracker.cpp
index fe29ea6..49c789c 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatedSizeTracker.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatedSizeTracker.cpp
@@ -10,7 +10,7 @@
using namespace kotlin;
-NO_INLINE void alloc::AllocatedSizeTracker::Page::onPageOverflow(std::size_t allocatedBytes) noexcept {
+void alloc::AllocatedSizeTracker::Page::onPageOverflow(std::size_t allocatedBytes) noexcept {
RuntimeAssert(allocatedBytes >= allocatedBytesLastRecorded_,
"A page can't overflow with less allocated bytes (%zu) than there were after the last sweep (%zu)",
allocatedBytes, allocatedBytesLastRecorded_);
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp
index c46f4f5..43c7fd6 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp
@@ -17,7 +17,7 @@
CustomAllocDebug("Cell@%p::Cell(%u)", this, size);
}
-uint8_t* Cell::TryAllocate(uint32_t cellsNeeded) noexcept {
+ALWAYS_INLINE uint8_t* Cell::TryAllocate(uint32_t cellsNeeded) noexcept {
CustomAllocDebug("Cell@%p{ allocated = %d, size = %u }::TryAllocate(%u)", this, isAllocated_, size_, cellsNeeded);
if (isAllocated_ || cellsNeeded > size_) {
CustomAllocDebug("Failed to allocate in Cell");
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp
index c580c92..ba869aa 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp
@@ -128,7 +128,7 @@
}
}
-ALWAYS_INLINE uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept {
+uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept {
// TODO no slow path here?
CustomAllocDebug("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount);
// FIXME WTF is TryAllocate?
@@ -159,16 +159,19 @@
CustomAllocDebug("CustomAllocator::AllocateInFixedBlockPage(%u)", cellCount);
FixedBlockPage* page = fixedBlockPages_[cellCount];
if (page) {
- uint8_t* block = page->TryAllocate();
+ uint8_t* block = page->TryAllocate(cellCount);
if (block) return block;
}
- return AllocateInFixedBlockPageSlowPath(cellCount);
+ return AllocateInFixedBlockPageSlowPath(page, cellCount);
}
-NO_INLINE uint8_t* CustomAllocator::AllocateInFixedBlockPageSlowPath(uint32_t cellCount) noexcept {
- CustomAllocDebug("Failed to allocate in current FixedBlockPage");
+NO_INLINE uint8_t* CustomAllocator::AllocateInFixedBlockPageSlowPath(FixedBlockPage* overflownPage, uint32_t cellCount) noexcept {
+ CustomAllocDebug("Failed to allocate in current FixedBlockPage(%p)", overflownPage);
+ if (overflownPage != nullptr) {
+ overflownPage->OnPageOverflow();
+ }
while (auto* page = heap_.GetFixedBlockPage(cellCount, finalizerQueue_)) {
- uint8_t* block = page->TryAllocate();
+ uint8_t* block = page->TryAllocate(cellCount);
if (block) {
fixedBlockPages_[cellCount] = page;
return block;
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp
index f15091b..2ae26bc 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp
@@ -52,7 +52,7 @@
uint8_t* AllocateInNextFitPageSlowPath(uint32_t cellCount) noexcept;
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount) noexcept;
- uint8_t* AllocateInFixedBlockPageSlowPath(uint32_t cellCount) noexcept;
+ uint8_t* AllocateInFixedBlockPageSlowPath(FixedBlockPage* overflownPage, uint32_t cellCount) noexcept;
Heap& heap_;
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp
index be39553..aeaf630 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp
@@ -33,14 +33,16 @@
end_ = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize;
}
-ALWAYS_INLINE uint8_t* FixedBlockPage::TryAllocate() noexcept {
+ALWAYS_INLINE uint8_t* FixedBlockPage::TryAllocate(uint32_t blockSize) noexcept {
+ RuntimeAssert(blockSize == blockSize_, "Trying to allocate block of size %d in the FixedBlockPage with block size %d", blockSize, blockSize_);
uint32_t next = nextFree_.first;
if (next < nextFree_.last) {
- nextFree_.first += blockSize_;
+ nextFree_.first += blockSize;
return cells_[next].data;
}
- if (next >= end_) {
- allocatedSizeTracker_.onPageOverflow(end_ * sizeof(FixedBlockCell));
+ // FIXME generalize
+ auto end = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize;
+ if (next >= end) {
return nullptr;
}
nextFree_ = cells_[next].nextFree;
@@ -48,6 +50,11 @@
return cells_[next].data;
}
+void FixedBlockPage::OnPageOverflow() noexcept {
+ RuntimeAssert(nextFree_.first >= end_, "Page must overflow"); // FIXME only == ?
+ allocatedSizeTracker_.onPageOverflow(end_ * sizeof(FixedBlockCell));
+}
+
bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
FixedCellRange nextFree = nextFree_; // Accessing the previous free list structure.
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
index 464d76d..3987513 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
@@ -41,7 +41,9 @@
void Destroy() noexcept;
// Tries to allocate in current page, returns null if no free block in page
- uint8_t* TryAllocate() noexcept;
+ uint8_t* TryAllocate(uint32_t blockSize) noexcept;
+
+ void OnPageOverflow() noexcept;
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp
index 0f23388..d99c397 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp
@@ -27,7 +27,7 @@
}
uint8_t* alloc(FixedBlockPage* page, size_t blockSize) {
- uint8_t* ptr = page->TryAllocate();
+ uint8_t* ptr = page->TryAllocate(blockSize);
if (ptr) {
EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0);
reinterpret_cast<uint64_t*>(ptr)[1] = reinterpret_cast<uint64_t>(&fakeType);
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp
index 975bd21..59a6a70 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp
@@ -47,7 +47,7 @@
kotlin::alloc::FinalizerQueue finalizerQueue;
for (int blocks = MIN; blocks < MAX; ++blocks) {
pages[blocks] = heap.GetFixedBlockPage(blocks, finalizerQueue);
- uint8_t* obj = pages[blocks]->TryAllocate();
+ uint8_t* obj = pages[blocks]->TryAllocate(blocks);
size_t size = installType(obj, &fakeTypes[blocks]);
EXPECT_EQ(size, static_cast<size_t>(blocks * 8));
mark(obj); // to make the page survive a sweep
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp
index 5040dbd..ee19ac7 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp
@@ -30,7 +30,7 @@
cells_[1] = Cell(NEXT_FIT_PAGE_CELL_COUNT - 1);
}
-uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept {
+ALWAYS_INLINE uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept {
CustomAllocDebug("NextFitPage@%p::TryAllocate(%u)", this, blockSize);
// +1 accounts for header, since cell->size also includes header cell
uint32_t cellsNeeded = blockSize + 1;
@@ -79,7 +79,7 @@
return aliveBytes > 0;
}
-void NextFitPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept {
+NO_INLINE void NextFitPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept {
CustomAllocDebug("NextFitPage@%p::UpdateCurBlock(%u)", this, cellsNeeded);
if (curBlock_ == cells_) curBlock_ = cells_ + 1; // only used as a starting point
Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT;
