[K/N] custom-alloc: Encode consecutive free blocks ^KT-55364

Co-authored-by: Troels Lund <troels@google.com>

Merge-request: KOTLIN-MR-662
Merged-by: Alexander Shabalin <alexander.shabalin@jetbrains.com>

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt

@@ -54,6 +54,8 @@ object BinaryOptions : BinaryOptionRegistry() {
     val compileBitcodeWithXcodeLlvm by booleanOption()
 
     val objcDisposeOnMain by booleanOption()
+
+    val disableMmap by booleanOption()
 }
 
 open class BinaryOption<T : Any>(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -123,6 +123,21 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         realGc
     }
     val runtimeAssertsMode: RuntimeAssertsMode get() = configuration.get(BinaryOptions.runtimeAssertionsMode) ?: RuntimeAssertsMode.IGNORE
+    private val defaultDisableMmap get() = target.family == Family.MINGW
+    val disableMmap: Boolean by lazy {
+        when (configuration.get(BinaryOptions.disableMmap)) {
+            null -> defaultDisableMmap
+            true -> true
+            false -> {
+                if (target.family == Family.MINGW) {
+                    configuration.report(CompilerMessageSeverity.STRONG_WARNING, "MinGW target does not support mmap/munmap")
+                    true
+                } else {
+                    false
+                }
+            }
+        }
+    }
     val workerExceptionHandling: WorkerExceptionHandling get() = configuration.get(KonanConfigKeys.WORKER_EXCEPTION_HANDLING) ?: when (memoryModel) {
             MemoryModel.EXPERIMENTAL -> WorkerExceptionHandling.USE_HOOK
             else -> WorkerExceptionHandling.LEGACY
@@ -434,6 +449,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
             append("-gc-scheduler=${gcSchedulerType.name}")
         if (runtimeAssertsMode != RuntimeAssertsMode.IGNORE)
             append("-runtime_asserts=${runtimeAssertsMode.name}")
+        if (disableMmap != defaultDisableMmap)
+            append("-disable_mmap=${disableMmap}")
     }
 
     private val userCacheFlavorString = buildString {

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2958,6 +2958,7 @@ internal fun NativeGenerationState.generateRuntimeConstantsModule() : LLVMModule
 
     setRuntimeConstGlobal("Kotlin_needDebugInfo", llvm.constInt32(if (shouldContainDebugInfo()) 1 else 0))
     setRuntimeConstGlobal("Kotlin_runtimeAssertsMode", llvm.constInt32(config.runtimeAssertsMode.value))
+    setRuntimeConstGlobal("Kotlin_disableMmap", llvm.constInt32(if (config.disableMmap) 1 else 0))
     val runtimeLogs = config.runtimeLogs?.let {
         static.cStringLiteral(it)
     } ?: NullPointer(llvm.int8Type)

kotlin-native/runtime/src/custom_alloc/README.md

@@ -200,41 +200,56 @@ the OS.
 ```cpp
 class FixedBlockPage {
 public:
-    FixedBlockPage(uint32_t blockSize);
-    uint8_t* TryAllocate();
+    FixedBlockPage(uint32_t blockSize) noexcept;
+
+    uint8_t* TryAllocate() noexcept;
+
     bool Sweep() noexcept;
 
 private:
-    FixedBlockPage* next_; // used by AtomicStack
+    FixedBlockPage* next_;
+    FixedCellRange nextFree_;
     uint32_t blockSize_;
-    FixedBlockCell* nextFree_;
+    uint32_t end_;
     FixedBlockCell cells_[];
 };
+};
 ```
 
 All sufficiently small allocations (currently arbitrary <1KiB) are directed to
 a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations
-are expected to be in this page type. A `FixedBlockPage` has a singly-linked
-free-list of all free blocks. Allocating always happens in the first free block
-in the page.
+are expected to be in this page type. A `FixedBlockPage` consists of a number
+of equally sized blocks, where each allocation will take up exactly one such
+block.
 
 ```cpp
 struct FixedBlockCell {
     union {
         uint8_t data[];
-        FixedBlockCell* nextFree;
+        FixedCellRange nextFree;
     }
 };
+
+struct alignas(8) FixedCellRange {
+    uint32_t first;
+    uint32_t last;
+};
 ```
 
-The important point is that all links in the list point forward in the page, so
-all blocks between two consecutive links are implicitly allocated. Sweeping a
+Consecutive unallocated cells are represented by a `FixedCellRange`, with
+`.first` and `.last` being the inclusive end points of the range of unallocated
+cells. The `FixedBlockCell` at the the `.last` index contains a
+`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange`
+of unallocated ranges thus form a linked list.
+
+The important point is that all links in this list point forward in the page, so
+all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a
 `FixedBlockPage` consists of walking the free-list forward, and sweeping all
 blocks in between the links, maintaining the free list when blocks are freed.
 
 Each small page takes up the same amount of space, independent of block size,
 so larger block size implies fewer blocks per page.  This size is arbitrarily
-chosen to be 64 KiB, but this might change.
+chosen to be 256 KiB, but this might change.
 
 ## [NextFitPage](cpp/NextFitPage.hpp)
 

kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp

@@ -36,6 +36,7 @@ uint8_t* Cell::TryAllocate(uint32_t cellsNeeded) noexcept {
 void Cell::Deallocate() noexcept {
     CustomAllocDebug("Cell@%p{ allocated = %d, size = %u }::Deallocate()", this, isAllocated_, size_);
     RuntimeAssert(isAllocated_, "Cell is not currently allocated");
+    memset(data_, 0, (size_ - 1) * sizeof(Cell));
     isAllocated_ = false;
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp

@@ -9,6 +9,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cinttypes>
+#include <cstring>
 #include <new>
 
 #include "ConcurrentMarkAndSweep.hpp"
@@ -17,6 +18,7 @@
 #include "ExtraObjectData.hpp"
 #include "ExtraObjectPage.hpp"
 #include "GCScheduler.hpp"
+#include "KAssert.h"
 #include "SingleObjectPage.hpp"
 #include "NextFitPage.hpp"
 #include "Memory.h"
@@ -124,6 +126,7 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept {
 }
 
 uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
+    RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes");
     gcScheduler_.OnSafePointAllocation(size);
     CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size);
     uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell);
@@ -135,7 +138,7 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
     } else {
         ptr = AllocateInNextFitPage(cellCount);
     }
-    memset(ptr, 0, size);
+    RuntimeAssert(ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0, "CustomAllocator::Allocate: memory not zero!");
     return ptr;
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp

@@ -28,51 +28,62 @@ void FixedBlockPage::Destroy() noexcept {
 
 FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) {
     CustomAllocInfo("FixedBlockPage(%p)::FixedBlockPage(%u)", this, blockSize);
-    nextFree_ = cells_;
-    FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_);
-    for (FixedBlockCell* cell = cells_; cell < end; cell = cell->nextFree) {
-        cell->nextFree = cell + blockSize;
-    }
+    nextFree_.first = 0;
+    nextFree_.last = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize;
+    end_ = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize;
 }
 
 uint8_t* FixedBlockPage::TryAllocate() noexcept {
-    FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_);
-    FixedBlockCell* freeBlock = nextFree_;
-    if (freeBlock >= end) {
-        return nullptr;
+    uint32_t next = nextFree_.first;
+    if (next < nextFree_.last) {
+        nextFree_.first += blockSize_;
+        return cells_[next].data;
     }
-    nextFree_ = freeBlock->nextFree;
-    CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock->data);
-    return freeBlock->data;
+    if (next >= end_) return nullptr;
+    nextFree_ = cells_[next].nextFree;
+    memset(&cells_[next], 0, sizeof(cells_[next]));
+    return cells_[next].data;
 }
 
 bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
     CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
-    // `end` is after the last legal allocation of a block, but does not
-    // necessarily match an actual block starting point.
-    FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_);
-    bool alive = false;
-    FixedBlockCell** nextFree = &nextFree_;
-    for (FixedBlockCell* cell = cells_; cell < end; cell += blockSize_) {
-        // If the current cell is free, move on.
-        if (cell == *nextFree) {
-            nextFree = &cell->nextFree;
+    FixedCellRange nextFree = nextFree_; // Accessing the previous free list structure.
+    FixedCellRange* prevRange = &nextFree_; // Creating the new free list structure.
+    uint32_t prevLive = -blockSize_;
+    for (uint32_t cell = 0 ; cell < end_ ; cell += blockSize_) {
+        // Go through the occupied cells.
+        for (; cell < nextFree.first ; cell += blockSize_) {
+            if (!SweepObject(cells_[cell].data, finalizerQueue, sweepHandle)) {
+                // We should null this cell out, but we will do so in batch later.
+                continue;
+            }
+            if (prevLive + blockSize_ < cell) {
+                // We found an alive cell that ended a run of swept cells or a known unoccupied range.
+                uint32_t prevCell = cell - blockSize_;
+                // Nulling in batch.
+                memset(&cells_[prevLive + blockSize_], 0, (prevCell - prevLive) * sizeof(FixedBlockCell));
+                // Updating the free list structure.
+                prevRange->first = prevLive + blockSize_;
+                prevRange->last = prevCell;
+                // And the next unoccupied range will be stored in the last unoccupied cell.
+                prevRange = &cells_[prevCell].nextFree;
+            }
+            prevLive = cell;
+        }
+        // `cell` now points to a known unoccupied range.
+        if (nextFree.last < end_) {
+            cell = nextFree.last;
+            nextFree = cells_[cell].nextFree;
             continue;
         }
-        // If the current cell was marked, it's alive, and the whole page is alive.
-        if (SweepObject(cell->data, finalizerQueue, sweepHandle)) {
-            alive = true;
-            sweepHandle.addKeptObject();
-            continue;
-        }
-        CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell);
-        // Free the current block and insert it into the free list.
-        cell->nextFree = *nextFree;
-        *nextFree = cell;
-        nextFree = &cell->nextFree;
-        sweepHandle.addSweptObject();
+        prevRange->first = prevLive + blockSize_;
+        memset(&cells_[prevLive + blockSize_], 0, (cell - prevLive - blockSize_) * sizeof(FixedBlockCell));
+        prevRange->last = end_;
+        // And we're done.
+        break;
     }
-    return alive;
+    // The page is alive iff a range stored in the page header covers the entire page.
+    return nextFree_.first > 0 || nextFree_.last < end_;
 }
 
 } // namespace kotlin::alloc

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp

@@ -15,11 +15,16 @@
 
 namespace kotlin::alloc {
 
+struct alignas(8) FixedCellRange {
+    uint32_t first;
+    uint32_t last;
+};
+
 struct alignas(8) FixedBlockCell {
     // The FixedBlockCell either contains data or a pointer to the next free cell
     union {
         uint8_t data[];
-        FixedBlockCell* nextFree;
+        FixedCellRange nextFree;
     };
 };
 
@@ -39,14 +44,15 @@ public:
     bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
 
 private:
-    friend class AtomicStack<FixedBlockPage>;
-
     explicit FixedBlockPage(uint32_t blockSize) noexcept;
 
+    friend class AtomicStack<FixedBlockPage>;
+
     // Used for linking pages together in `pages` queue or in `unswept` queue.
     FixedBlockPage* next_;
+    FixedCellRange nextFree_;
     uint32_t blockSize_;
-    FixedBlockCell* nextFree_;
+    uint32_t end_;
     FixedBlockCell cells_[];
 };
 

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp

@@ -29,7 +29,7 @@ void mark(void* obj) {
 uint8_t* alloc(FixedBlockPage* page, size_t blockSize) {
     uint8_t* ptr = page->TryAllocate();
     if (ptr) {
-        memset(ptr, 0, 8 * blockSize);
+        EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0);
         reinterpret_cast<uint64_t*>(ptr)[1] = reinterpret_cast<uint64_t>(&fakeType);
     }
     return ptr;

kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp

@@ -6,9 +6,15 @@
 #include "GCApi.hpp"
 
 #include <atomic>
+#include <cstdint>
 #include <limits>
 
+#ifndef KONAN_WINDOWS
+#include <sys/mman.h>
+#endif
+
 #include "ConcurrentMarkAndSweep.hpp"
+#include "CompilerConstants.hpp"
 #include "CustomLogging.hpp"
 #include "ExtraObjectData.hpp"
 #include "ExtraObjectPage.hpp"
@@ -63,17 +69,42 @@ bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExt
 }
 
 void* SafeAlloc(uint64_t size) noexcept {
-    void* memory;
-    if (size > std::numeric_limits<size_t>::max() || !(memory = std_support::malloc(size))) {
+    if (size > std::numeric_limits<size_t>::max()) {
         konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes. Aborting.\n", size);
         konan::abort();
     }
+    void* memory;
+    bool error;
+    if (compiler::disableMmap()) {
+        memory = calloc(size, 1);
+        error = memory == nullptr;
+    } else {
+#if KONAN_WINDOWS
+        RuntimeFail("mmap is not available on mingw");
+#else
+        memory = mmap(nullptr, size, PROT_WRITE | PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+        error = memory == MAP_FAILED;
+#endif
+    }
+    if (error) {
+        konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes: %s. Aborting.\n", size, strerror(errno));
+        konan::abort();
+    }
     allocatedBytesCounter.fetch_add(static_cast<size_t>(size), std::memory_order_relaxed);
     return memory;
 }
 
 void Free(void* ptr, size_t size) noexcept {
-    std_support::free(ptr);
+    if (compiler::disableMmap()) {
+        free(ptr);
+    } else {
+#if KONAN_WINDOWS
+        RuntimeFail("mmap is not available on mingw");
+#else
+        auto result = munmap(ptr, size);
+        RuntimeAssert(result == 0, "Failed to munmap: %s", strerror(errno));
+#endif
+    }
     allocatedBytesCounter.fetch_sub(static_cast<size_t>(size), std::memory_order_relaxed);
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp

@@ -40,6 +40,7 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::
 bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& sweepScope) noexcept;
 
 void* SafeAlloc(uint64_t size) noexcept;
+
 void Free(void* ptr, size_t size) noexcept;
 
 size_t GetAllocatedBytes() noexcept;

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp

@@ -57,8 +57,12 @@ bool NextFitPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueu
     Cell* maxBlock = cells_; // size 0 block
     for (Cell* block = cells_ + 1; block != end; block = block->Next()) {
         if (block->isAllocated_) continue;
-        while (block->Next() != end && !block->Next()->isAllocated_) {
-            block->size_ += block->Next()->size_;
+        for (auto* next = block->Next(); next != end; next = block->Next()) {
+            if (next->isAllocated_) {
+                break;
+            }
+            block->size_ += next->size_;
+            memset(next, 0, sizeof(*next));
         }
         if (block->size_ > maxBlock->size_) maxBlock = block;
     }

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp

@@ -35,7 +35,7 @@ public:
     bool CheckInvariants() noexcept;
 
 private:
-    NextFitPage(uint32_t cellCount) noexcept;
+    explicit NextFitPage(uint32_t cellCount) noexcept;
 
     // Looks for a block big enough to hold cellsNeeded. If none big enough is
     // found, update to the largest one.

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp

@@ -33,7 +33,7 @@ uint8_t* alloc(NextFitPage* page, uint32_t blockSize) {
         return nullptr;
     }
     if (ptr == nullptr) return nullptr;
-    memset(ptr, 0, 8 * blockSize);
+    EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0);
     reinterpret_cast<uint64_t*>(ptr)[1] = reinterpret_cast<uint64_t>(&fakeType);
     if (!page->CheckInvariants()) {
         ADD_FAILURE();

kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp

@@ -44,6 +44,10 @@ public:
 
     T* GetPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept {
         T* page;
+        if ((page = ready_.Pop())) {
+            used_.Push(page);
+            return page;
+        }
         if ((page = unswept_.Pop())) {
             // If there're unswept_ pages, the GC is in progress.
             GCSweepScope sweepHandle = T::currentGCSweepScope();
@@ -51,10 +55,6 @@ public:
                 return page;
             }
         }
-        if ((page = ready_.Pop())) {
-            used_.Push(page);
-            return page;
-        }
         if ((page = empty_.Pop())) {
             used_.Push(page);
             return page;

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp

@@ -18,11 +18,11 @@ SingleObjectPage* SingleObjectPage::Create(uint64_t cellCount) noexcept {
     CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount);
     RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage");
     uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t);
-    auto* page = new (SafeAlloc(size)) SingleObjectPage();
-    page->size_ = size;
-    return page;
+    return new (SafeAlloc(size)) SingleObjectPage(size);
 }
 
+SingleObjectPage::SingleObjectPage(size_t size) noexcept : size_(size) {}
+
 void SingleObjectPage::Destroy() noexcept {
     Free(this, size_);
 }

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp

@@ -33,6 +33,9 @@ public:
 
 private:
     friend class AtomicStack<SingleObjectPage>;
+
+    explicit SingleObjectPage(size_t size) noexcept;
+
     SingleObjectPage* next_;
     bool isAllocated_ = false;
     size_t size_;

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp

@@ -26,9 +26,9 @@ void mark(void* obj) {
 
 SingleObjectPage* alloc(uint64_t blockSize) {
     SingleObjectPage* page = SingleObjectPage::Create(blockSize);
-    uint64_t* ptr = reinterpret_cast<uint64_t*>(page->TryAllocate());
-    memset(ptr, 0, 8 * blockSize);
-    ptr[1] = reinterpret_cast<uint64_t>(&fakeType);
+    uint8_t* ptr = page->TryAllocate();
+    EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0);
+    reinterpret_cast<uint64_t*>(ptr)[1] = reinterpret_cast<uint64_t>(&fakeType);
     return page;
 }
 

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -35,6 +35,7 @@ using string_view = std::experimental::string_view;
  */
 extern "C" const int32_t Kotlin_needDebugInfo;
 extern "C" const int32_t Kotlin_runtimeAssertsMode;
+extern "C" const int32_t Kotlin_disableMmap;
 extern "C" const char* const Kotlin_runtimeLogs;
 extern "C" const int32_t Kotlin_gcSchedulerType;
 extern "C" const int32_t Kotlin_freezingEnabled;
@@ -90,6 +91,10 @@ ALWAYS_INLINE inline bool runtimeAssertsEnabled() noexcept {
     return runtimeAssertsMode() != RuntimeAssertsMode::kIgnore;
 }
 
+ALWAYS_INLINE inline bool disableMmap() noexcept {
+    return Kotlin_disableMmap != 0;
+}
+
 ALWAYS_INLINE inline std::string_view runtimeLogs() noexcept {
     return Kotlin_runtimeLogs == nullptr ? std::string_view() : std::string_view(Kotlin_runtimeLogs);
 }

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -68,6 +68,11 @@ extern "C" {
 
 extern const int32_t Kotlin_needDebugInfo = 1;
 extern const int32_t Kotlin_runtimeAssertsMode = static_cast<int32_t>(kotlin::compiler::RuntimeAssertsMode::kPanic);
+#if KONAN_WINDOWS
+extern const int32_t Kotlin_disableMmap = 1;
+#else
+extern const int32_t Kotlin_disableMmap = 0;
+#endif
 extern const char* const Kotlin_runtimeLogs = nullptr;
 extern const int32_t Kotlin_gcSchedulerType = static_cast<int32_t>(kotlin::compiler::GCSchedulerType::kDisabled);
 extern const int32_t Kotlin_freezingChecksEnabled = 1;