diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index a0bb391793d..a268488147c 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -2192,6 +2192,37 @@ void updateHeapRef(ObjHeader** location, const ObjHeader* object) { } } +template +void updateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) { + // In case of coping inside same array number of decrements and increments of RC can be decreased. + auto countIndex = [=](int i) { return (fromIndex < toIndex) ? count - 1 - i : i; }; + int rewrittenElementsNumber = std::abs(fromIndex - toIndex); + // Release rewritten elements. + for (int i = 0; i < rewrittenElementsNumber; i++) { + int index = countIndex(i); + ObjHeader* old = *ArrayAddressOfElementAt(array, toIndex + index); + *const_cast(ArrayAddressOfElementAt(array, toIndex + index)) = + *ArrayAddressOfElementAt(array, fromIndex + index); + if (old != nullptr) { + releaseHeapRef(old); + } + } + for (int i = rewrittenElementsNumber; i < count - rewrittenElementsNumber; i++) { + int index = countIndex(i); + *const_cast(ArrayAddressOfElementAt(array, toIndex + index)) = + *ArrayAddressOfElementAt(array, fromIndex + index); + } + for (int i = count - rewrittenElementsNumber; i < count; i++) { + int index = countIndex(i); + ObjHeader* object = *ArrayAddressOfElementAt(array, fromIndex + index); + // Add extra heap ref for copied elements. + if (object != nullptr) { + addHeapRef(object); + } + *const_cast(ArrayAddressOfElementAt(array, toIndex + index)) = object; + } +} + template void updateStackRef(ObjHeader** location, const ObjHeader* object) { UPDATE_REF_EVENT(memoryState, *location, object, location, 1) @@ -3367,6 +3398,15 @@ RUNTIME_NOTHROW void UpdateHeapRefRelaxed(ObjHeader** location, const ObjHeader* updateHeapRef(location, object); } +RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayStrict(const ArrayHeader* array, int fromIndex, int toIndex, + int count) { + updateHeapRefsInsideOneArray(array, fromIndex, toIndex, count); +} +RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayRelaxed(const ArrayHeader* array, int fromIndex, int toIndex, + int count) { + updateHeapRefsInsideOneArray(array, fromIndex, toIndex, count); +} + RUNTIME_NOTHROW void UpdateReturnRefStrict(ObjHeader** returnSlot, const ObjHeader* value) { updateReturnRef(returnSlot, value); } diff --git a/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp b/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp index 01deeccecd0..75f523ca6f0 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp +++ b/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp @@ -322,6 +322,8 @@ MODEL_VARIANTS(void, UpdateStackRef, ObjHeader** location, const ObjHeader* obje MODEL_VARIANTS(void, UpdateHeapRef, ObjHeader** location, const ObjHeader* object); MODEL_VARIANTS(void, UpdateHeapRefIfNull, ObjHeader** location, const ObjHeader* object); MODEL_VARIANTS(void, UpdateReturnRef, ObjHeader** returnSlot, const ObjHeader* object); +MODEL_VARIANTS(void, UpdateHeapRefsInsideOneArray, const ArrayHeader* array, int fromIndex, int toIndex, + int count); MODEL_VARIANTS(void, EnterFrame, ObjHeader** start, int parameters, int count); MODEL_VARIANTS(void, LeaveFrame, ObjHeader** start, int parameters, int count); diff --git a/kotlin-native/runtime/src/main/cpp/Arrays.cpp b/kotlin-native/runtime/src/main/cpp/Arrays.cpp index 164a9fd5e88..398a3709559 100644 --- a/kotlin-native/runtime/src/main/cpp/Arrays.cpp +++ b/kotlin-native/runtime/src/main/cpp/Arrays.cpp @@ -133,15 +133,20 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex, ThrowArrayIndexOutOfBoundsException(); } mutabilityCheck(destination); - if (fromIndex >= toIndex) { - for (int index = 0; index < count; index++) { - UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), - *ArrayAddressOfElementAt(array, fromIndex + index)); - } + if (CurrentMemoryModel != MemoryModel::kExperimental && array == destinationArray && + std::abs(fromIndex - toIndex) < count) { + UpdateHeapRefsInsideOneArray(array, fromIndex, toIndex, count); } else { - for (int index = count - 1; index >= 0; index--) { - UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), - *ArrayAddressOfElementAt(array, fromIndex + index)); + if (fromIndex >= toIndex) { + for (int index = 0; index < count; index++) { + UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + *ArrayAddressOfElementAt(array, fromIndex + index)); + } + } else { + for (int index = count - 1; index >= 0; index--) { + UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + *ArrayAddressOfElementAt(array, fromIndex + index)); + } } } } diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 6bf213010ca..1f5bbc5afe6 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -210,6 +210,8 @@ void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW; void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates heap/static data location. void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Updates heap/static data in one array. +void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW; // Updates location if it is null, atomically. void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates reference in return slot. diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 67e8ce2f928..37994c8e530 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -189,6 +189,11 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** lo mm::CompareAndSwapHeapRef(location, nullptr, const_cast(object), &result); } +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, + int toIndex, int count) { + RuntimeFail("Only for legacy MM"); +} + extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { mm::SetStackRef(returnSlot, const_cast(object)); } diff --git a/kotlin-native/runtime/src/relaxed/cpp/MemoryImpl.cpp b/kotlin-native/runtime/src/relaxed/cpp/MemoryImpl.cpp index 458d2177ae1..e12cd3c5eb7 100644 --- a/kotlin-native/runtime/src/relaxed/cpp/MemoryImpl.cpp +++ b/kotlin-native/runtime/src/relaxed/cpp/MemoryImpl.cpp @@ -67,4 +67,8 @@ RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* objec UpdateStackRefRelaxed(location, object); } +RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) { + UpdateHeapRefsInsideOneArrayRelaxed(array, fromIndex, toIndex, count); +} + } // extern "C" diff --git a/kotlin-native/runtime/src/strict/cpp/MemoryImpl.cpp b/kotlin-native/runtime/src/strict/cpp/MemoryImpl.cpp index 8c0b65891d3..113576c81e1 100644 --- a/kotlin-native/runtime/src/strict/cpp/MemoryImpl.cpp +++ b/kotlin-native/runtime/src/strict/cpp/MemoryImpl.cpp @@ -67,4 +67,8 @@ RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* objec UpdateStackRefStrict(location, object); } +RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) { + UpdateHeapRefsInsideOneArrayStrict(array, fromIndex, toIndex, count); +} + } // extern "C"