diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt index 68175e8692e..3af01236fff 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt @@ -54,6 +54,8 @@ object BinaryOptions : BinaryOptionRegistry() { val compileBitcodeWithXcodeLlvm by booleanOption() val objcDisposeOnMain by booleanOption() + + val disableMmap by booleanOption() } open class BinaryOption( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index edd29e1d919..78748eba72b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -123,6 +123,21 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration realGc } val runtimeAssertsMode: RuntimeAssertsMode get() = configuration.get(BinaryOptions.runtimeAssertionsMode) ?: RuntimeAssertsMode.IGNORE + private val defaultDisableMmap get() = target.family == Family.MINGW + val disableMmap: Boolean by lazy { + when (configuration.get(BinaryOptions.disableMmap)) { + null -> defaultDisableMmap + true -> true + false -> { + if (target.family == Family.MINGW) { + configuration.report(CompilerMessageSeverity.STRONG_WARNING, "MinGW target does not support mmap/munmap") + true + } else { + false + } + } + } + } val workerExceptionHandling: WorkerExceptionHandling get() = configuration.get(KonanConfigKeys.WORKER_EXCEPTION_HANDLING) ?: when (memoryModel) { MemoryModel.EXPERIMENTAL -> WorkerExceptionHandling.USE_HOOK else -> WorkerExceptionHandling.LEGACY @@ -434,6 +449,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration append("-gc-scheduler=${gcSchedulerType.name}") if (runtimeAssertsMode != RuntimeAssertsMode.IGNORE) append("-runtime_asserts=${runtimeAssertsMode.name}") + if (disableMmap != defaultDisableMmap) + append("-disable_mmap=${disableMmap}") } private val userCacheFlavorString = buildString { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 8ec77f6bc95..a4a6f2ba900 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -2958,6 +2958,7 @@ internal fun NativeGenerationState.generateRuntimeConstantsModule() : LLVMModule setRuntimeConstGlobal("Kotlin_needDebugInfo", llvm.constInt32(if (shouldContainDebugInfo()) 1 else 0)) setRuntimeConstGlobal("Kotlin_runtimeAssertsMode", llvm.constInt32(config.runtimeAssertsMode.value)) + setRuntimeConstGlobal("Kotlin_disableMmap", llvm.constInt32(if (config.disableMmap) 1 else 0)) val runtimeLogs = config.runtimeLogs?.let { static.cStringLiteral(it) } ?: NullPointer(llvm.int8Type) diff --git a/kotlin-native/runtime/src/custom_alloc/README.md b/kotlin-native/runtime/src/custom_alloc/README.md index 0b0e323eeb3..6d20c03cc49 100644 --- a/kotlin-native/runtime/src/custom_alloc/README.md +++ b/kotlin-native/runtime/src/custom_alloc/README.md @@ -200,41 +200,56 @@ the OS. ```cpp class FixedBlockPage { public: - FixedBlockPage(uint32_t blockSize); - uint8_t* TryAllocate(); + FixedBlockPage(uint32_t blockSize) noexcept; + + uint8_t* TryAllocate() noexcept; + bool Sweep() noexcept; private: - FixedBlockPage* next_; // used by AtomicStack + FixedBlockPage* next_; + FixedCellRange nextFree_; uint32_t blockSize_; - FixedBlockCell* nextFree_; + uint32_t end_; FixedBlockCell cells_[]; }; +}; ``` All sufficiently small allocations (currently arbitrary <1KiB) are directed to a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations -are expected to be in this page type. A `FixedBlockPage` has a singly-linked -free-list of all free blocks. Allocating always happens in the first free block -in the page. +are expected to be in this page type. A `FixedBlockPage` consists of a number +of equally sized blocks, where each allocation will take up exactly one such +block. ```cpp struct FixedBlockCell { union { uint8_t data[]; - FixedBlockCell* nextFree; + FixedCellRange nextFree; } }; + +struct alignas(8) FixedCellRange { + uint32_t first; + uint32_t last; +}; ``` -The important point is that all links in the list point forward in the page, so -all blocks between two consecutive links are implicitly allocated. Sweeping a +Consecutive unallocated cells are represented by a `FixedCellRange`, with +`.first` and `.last` being the inclusive end points of the range of unallocated +cells. The `FixedBlockCell` at the the `.last` index contains a +`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange` +of unallocated ranges thus form a linked list. + +The important point is that all links in this list point forward in the page, so +all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a `FixedBlockPage` consists of walking the free-list forward, and sweeping all blocks in between the links, maintaining the free list when blocks are freed. Each small page takes up the same amount of space, independent of block size, so larger block size implies fewer blocks per page. This size is arbitrarily -chosen to be 64 KiB, but this might change. +chosen to be 256 KiB, but this might change. ## [NextFitPage](cpp/NextFitPage.hpp) diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp index 7a8add04afe..c46f4f5b71b 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp @@ -36,6 +36,7 @@ uint8_t* Cell::TryAllocate(uint32_t cellsNeeded) noexcept { void Cell::Deallocate() noexcept { CustomAllocDebug("Cell@%p{ allocated = %d, size = %u }::Deallocate()", this, isAllocated_, size_); RuntimeAssert(isAllocated_, "Cell is not currently allocated"); + memset(data_, 0, (size_ - 1) * sizeof(Cell)); isAllocated_ = false; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 319910fcc69..dff6abbf00d 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "ConcurrentMarkAndSweep.hpp" @@ -17,6 +18,7 @@ #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" #include "GCScheduler.hpp" +#include "KAssert.h" #include "SingleObjectPage.hpp" #include "NextFitPage.hpp" #include "Memory.h" @@ -124,6 +126,7 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept { } uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { + RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes"); gcScheduler_.OnSafePointAllocation(size); CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size); uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell); @@ -135,7 +138,7 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { } else { ptr = AllocateInNextFitPage(cellCount); } - memset(ptr, 0, size); + RuntimeAssert(ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0, "CustomAllocator::Allocate: memory not zero!"); return ptr; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp index 1d52a2c743f..d8356e10304 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -28,51 +28,62 @@ void FixedBlockPage::Destroy() noexcept { FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) { CustomAllocInfo("FixedBlockPage(%p)::FixedBlockPage(%u)", this, blockSize); - nextFree_ = cells_; - FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); - for (FixedBlockCell* cell = cells_; cell < end; cell = cell->nextFree) { - cell->nextFree = cell + blockSize; - } + nextFree_.first = 0; + nextFree_.last = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize; + end_ = FIXED_BLOCK_PAGE_CELL_COUNT / blockSize * blockSize; } uint8_t* FixedBlockPage::TryAllocate() noexcept { - FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); - FixedBlockCell* freeBlock = nextFree_; - if (freeBlock >= end) { - return nullptr; + uint32_t next = nextFree_.first; + if (next < nextFree_.last) { + nextFree_.first += blockSize_; + return cells_[next].data; } - nextFree_ = freeBlock->nextFree; - CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock->data); - return freeBlock->data; + if (next >= end_) return nullptr; + nextFree_ = cells_[next].nextFree; + memset(&cells_[next], 0, sizeof(cells_[next])); + return cells_[next].data; } bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this); - // `end` is after the last legal allocation of a block, but does not - // necessarily match an actual block starting point. - FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); - bool alive = false; - FixedBlockCell** nextFree = &nextFree_; - for (FixedBlockCell* cell = cells_; cell < end; cell += blockSize_) { - // If the current cell is free, move on. - if (cell == *nextFree) { - nextFree = &cell->nextFree; + FixedCellRange nextFree = nextFree_; // Accessing the previous free list structure. + FixedCellRange* prevRange = &nextFree_; // Creating the new free list structure. + uint32_t prevLive = -blockSize_; + for (uint32_t cell = 0 ; cell < end_ ; cell += blockSize_) { + // Go through the occupied cells. + for (; cell < nextFree.first ; cell += blockSize_) { + if (!SweepObject(cells_[cell].data, finalizerQueue, sweepHandle)) { + // We should null this cell out, but we will do so in batch later. + continue; + } + if (prevLive + blockSize_ < cell) { + // We found an alive cell that ended a run of swept cells or a known unoccupied range. + uint32_t prevCell = cell - blockSize_; + // Nulling in batch. + memset(&cells_[prevLive + blockSize_], 0, (prevCell - prevLive) * sizeof(FixedBlockCell)); + // Updating the free list structure. + prevRange->first = prevLive + blockSize_; + prevRange->last = prevCell; + // And the next unoccupied range will be stored in the last unoccupied cell. + prevRange = &cells_[prevCell].nextFree; + } + prevLive = cell; + } + // `cell` now points to a known unoccupied range. + if (nextFree.last < end_) { + cell = nextFree.last; + nextFree = cells_[cell].nextFree; continue; } - // If the current cell was marked, it's alive, and the whole page is alive. - if (SweepObject(cell->data, finalizerQueue, sweepHandle)) { - alive = true; - sweepHandle.addKeptObject(); - continue; - } - CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell); - // Free the current block and insert it into the free list. - cell->nextFree = *nextFree; - *nextFree = cell; - nextFree = &cell->nextFree; - sweepHandle.addSweptObject(); + prevRange->first = prevLive + blockSize_; + memset(&cells_[prevLive + blockSize_], 0, (cell - prevLive - blockSize_) * sizeof(FixedBlockCell)); + prevRange->last = end_; + // And we're done. + break; } - return alive; + // The page is alive iff a range stored in the page header covers the entire page. + return nextFree_.first > 0 || nextFree_.last < end_; } } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp index cfd629f2070..d2c9ac54e3e 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp @@ -15,11 +15,16 @@ namespace kotlin::alloc { +struct alignas(8) FixedCellRange { + uint32_t first; + uint32_t last; +}; + struct alignas(8) FixedBlockCell { // The FixedBlockCell either contains data or a pointer to the next free cell union { uint8_t data[]; - FixedBlockCell* nextFree; + FixedCellRange nextFree; }; }; @@ -39,14 +44,15 @@ public: bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; private: - friend class AtomicStack; - explicit FixedBlockPage(uint32_t blockSize) noexcept; + friend class AtomicStack; + // Used for linking pages together in `pages` queue or in `unswept` queue. FixedBlockPage* next_; + FixedCellRange nextFree_; uint32_t blockSize_; - FixedBlockCell* nextFree_; + uint32_t end_; FixedBlockCell cells_[]; }; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp index 70ee0d91be9..ef483f00a67 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp @@ -29,7 +29,7 @@ void mark(void* obj) { uint8_t* alloc(FixedBlockPage* page, size_t blockSize) { uint8_t* ptr = page->TryAllocate(); if (ptr) { - memset(ptr, 0, 8 * blockSize); + EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0); reinterpret_cast(ptr)[1] = reinterpret_cast(&fakeType); } return ptr; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 4dcf0be196b..40ac83e34ee 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -6,9 +6,15 @@ #include "GCApi.hpp" #include +#include #include +#ifndef KONAN_WINDOWS +#include +#endif + #include "ConcurrentMarkAndSweep.hpp" +#include "CompilerConstants.hpp" #include "CustomLogging.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" @@ -63,17 +69,42 @@ bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExt } void* SafeAlloc(uint64_t size) noexcept { - void* memory; - if (size > std::numeric_limits::max() || !(memory = std_support::malloc(size))) { + if (size > std::numeric_limits::max()) { konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes. Aborting.\n", size); konan::abort(); } + void* memory; + bool error; + if (compiler::disableMmap()) { + memory = calloc(size, 1); + error = memory == nullptr; + } else { +#if KONAN_WINDOWS + RuntimeFail("mmap is not available on mingw"); +#else + memory = mmap(nullptr, size, PROT_WRITE | PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); + error = memory == MAP_FAILED; +#endif + } + if (error) { + konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes: %s. Aborting.\n", size, strerror(errno)); + konan::abort(); + } allocatedBytesCounter.fetch_add(static_cast(size), std::memory_order_relaxed); return memory; } void Free(void* ptr, size_t size) noexcept { - std_support::free(ptr); + if (compiler::disableMmap()) { + free(ptr); + } else { +#if KONAN_WINDOWS + RuntimeFail("mmap is not available on mingw"); +#else + auto result = munmap(ptr, size); + RuntimeAssert(result == 0, "Failed to munmap: %s", strerror(errno)); +#endif + } allocatedBytesCounter.fetch_sub(static_cast(size), std::memory_order_relaxed); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index f920e7f51be..1897ca5b815 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -40,6 +40,7 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle:: bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& sweepScope) noexcept; void* SafeAlloc(uint64_t size) noexcept; + void Free(void* ptr, size_t size) noexcept; size_t GetAllocatedBytes() noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index 12b13e40078..0b075c953e8 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -57,8 +57,12 @@ bool NextFitPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueu Cell* maxBlock = cells_; // size 0 block for (Cell* block = cells_ + 1; block != end; block = block->Next()) { if (block->isAllocated_) continue; - while (block->Next() != end && !block->Next()->isAllocated_) { - block->size_ += block->Next()->size_; + for (auto* next = block->Next(); next != end; next = block->Next()) { + if (next->isAllocated_) { + break; + } + block->size_ += next->size_; + memset(next, 0, sizeof(*next)); } if (block->size_ > maxBlock->size_) maxBlock = block; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp index 9f1277d7836..cd32d33ea3f 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp @@ -35,7 +35,7 @@ public: bool CheckInvariants() noexcept; private: - NextFitPage(uint32_t cellCount) noexcept; + explicit NextFitPage(uint32_t cellCount) noexcept; // Looks for a block big enough to hold cellsNeeded. If none big enough is // found, update to the largest one. diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp index aba608e1512..88c3dfe0652 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp @@ -33,7 +33,7 @@ uint8_t* alloc(NextFitPage* page, uint32_t blockSize) { return nullptr; } if (ptr == nullptr) return nullptr; - memset(ptr, 0, 8 * blockSize); + EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0); reinterpret_cast(ptr)[1] = reinterpret_cast(&fakeType); if (!page->CheckInvariants()) { ADD_FAILURE(); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp index 31176f8b588..104fa49020d 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp @@ -44,6 +44,10 @@ public: T* GetPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept { T* page; + if ((page = ready_.Pop())) { + used_.Push(page); + return page; + } if ((page = unswept_.Pop())) { // If there're unswept_ pages, the GC is in progress. GCSweepScope sweepHandle = T::currentGCSweepScope(); @@ -51,10 +55,6 @@ public: return page; } } - if ((page = ready_.Pop())) { - used_.Push(page); - return page; - } if ((page = empty_.Pop())) { used_.Push(page); return page; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp index 1c6f60a60cd..e38d2d42a33 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -18,11 +18,11 @@ SingleObjectPage* SingleObjectPage::Create(uint64_t cellCount) noexcept { CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount); RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage"); uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t); - auto* page = new (SafeAlloc(size)) SingleObjectPage(); - page->size_ = size; - return page; + return new (SafeAlloc(size)) SingleObjectPage(size); } +SingleObjectPage::SingleObjectPage(size_t size) noexcept : size_(size) {} + void SingleObjectPage::Destroy() noexcept { Free(this, size_); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index a0eef0a81e5..435013a5f36 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -33,6 +33,9 @@ public: private: friend class AtomicStack; + + explicit SingleObjectPage(size_t size) noexcept; + SingleObjectPage* next_; bool isAllocated_ = false; size_t size_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp index 5f137851827..0ec0601d031 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp @@ -26,9 +26,9 @@ void mark(void* obj) { SingleObjectPage* alloc(uint64_t blockSize) { SingleObjectPage* page = SingleObjectPage::Create(blockSize); - uint64_t* ptr = reinterpret_cast(page->TryAllocate()); - memset(ptr, 0, 8 * blockSize); - ptr[1] = reinterpret_cast(&fakeType); + uint8_t* ptr = page->TryAllocate(); + EXPECT_TRUE(ptr[0] == 0 && memcmp(ptr, ptr + 1, blockSize * 8 - 1) == 0); + reinterpret_cast(ptr)[1] = reinterpret_cast(&fakeType); return page; } diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp index 7cbf31f0a34..1dfaa4008aa 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp @@ -35,6 +35,7 @@ using string_view = std::experimental::string_view; */ extern "C" const int32_t Kotlin_needDebugInfo; extern "C" const int32_t Kotlin_runtimeAssertsMode; +extern "C" const int32_t Kotlin_disableMmap; extern "C" const char* const Kotlin_runtimeLogs; extern "C" const int32_t Kotlin_gcSchedulerType; extern "C" const int32_t Kotlin_freezingEnabled; @@ -90,6 +91,10 @@ ALWAYS_INLINE inline bool runtimeAssertsEnabled() noexcept { return runtimeAssertsMode() != RuntimeAssertsMode::kIgnore; } +ALWAYS_INLINE inline bool disableMmap() noexcept { + return Kotlin_disableMmap != 0; +} + ALWAYS_INLINE inline std::string_view runtimeLogs() noexcept { return Kotlin_runtimeLogs == nullptr ? std::string_view() : std::string_view(Kotlin_runtimeLogs); } diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index de869abd725..9fb718b7c52 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -68,6 +68,11 @@ extern "C" { extern const int32_t Kotlin_needDebugInfo = 1; extern const int32_t Kotlin_runtimeAssertsMode = static_cast(kotlin::compiler::RuntimeAssertsMode::kPanic); +#if KONAN_WINDOWS +extern const int32_t Kotlin_disableMmap = 1; +#else +extern const int32_t Kotlin_disableMmap = 0; +#endif extern const char* const Kotlin_runtimeLogs = nullptr; extern const int32_t Kotlin_gcSchedulerType = static_cast(kotlin::compiler::GCSchedulerType::kDisabled); extern const int32_t Kotlin_freezingChecksEnabled = 1;