[LEGACY MM] Escape making extra increments and decrements during coping to the same array (#4731)

(cherry picked from commit 8b601d8e2c6bc772386ebd88d303a032a6f54237)

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -2192,6 +2192,37 @@ void updateHeapRef(ObjHeader** location, const ObjHeader* object) {
   }
 }
 
+template <bool Strict>
+void updateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) {
+  // In case of coping inside same array number of decrements and increments of RC can be decreased.
+  auto countIndex = [=](int i) { return (fromIndex < toIndex) ? count - 1 - i : i; };
+  int rewrittenElementsNumber = std::abs(fromIndex - toIndex);
+  // Release rewritten elements.
+  for (int i = 0; i < rewrittenElementsNumber; i++) {
+    int index = countIndex(i);
+    ObjHeader* old = *ArrayAddressOfElementAt(array, toIndex + index);
+    *const_cast<const ObjHeader**>(ArrayAddressOfElementAt(array, toIndex + index)) =
+      *ArrayAddressOfElementAt(array, fromIndex + index);
+    if (old != nullptr) {
+      releaseHeapRef<Strict>(old);
+    }
+  }
+  for (int i = rewrittenElementsNumber; i < count - rewrittenElementsNumber; i++) {
+    int index = countIndex(i);
+    *const_cast<const ObjHeader**>(ArrayAddressOfElementAt(array, toIndex + index)) =
+         *ArrayAddressOfElementAt(array, fromIndex + index);
+  }
+  for (int i = count - rewrittenElementsNumber; i < count; i++) {
+    int index = countIndex(i);
+    ObjHeader* object = *ArrayAddressOfElementAt(array, fromIndex + index);
+    // Add extra heap ref for copied elements.
+    if (object != nullptr) {
+      addHeapRef(object);
+    }
+    *const_cast<const ObjHeader**>(ArrayAddressOfElementAt(array, toIndex + index)) = object;
+  }
+}
+
 template <bool Strict>
 void updateStackRef(ObjHeader** location, const ObjHeader* object) {
   UPDATE_REF_EVENT(memoryState, *location, object, location, 1)
@@ -3367,6 +3398,15 @@ RUNTIME_NOTHROW void UpdateHeapRefRelaxed(ObjHeader** location, const ObjHeader*
   updateHeapRef<false>(location, object);
 }
 
+RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayStrict(const ArrayHeader* array, int fromIndex, int toIndex,
+                                                        int count) {
+  updateHeapRefsInsideOneArray<true>(array, fromIndex, toIndex, count);
+}
+RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayRelaxed(const ArrayHeader* array, int fromIndex, int toIndex,
+                                                         int count) {
+  updateHeapRefsInsideOneArray<false>(array, fromIndex, toIndex, count);
+}
+
 RUNTIME_NOTHROW void UpdateReturnRefStrict(ObjHeader** returnSlot, const ObjHeader* value) {
   updateReturnRef<true>(returnSlot, value);
 }

kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp

@@ -322,6 +322,8 @@ MODEL_VARIANTS(void, UpdateStackRef, ObjHeader** location, const ObjHeader* obje
 MODEL_VARIANTS(void, UpdateHeapRef, ObjHeader** location, const ObjHeader* object);
 MODEL_VARIANTS(void, UpdateHeapRefIfNull, ObjHeader** location, const ObjHeader* object);
 MODEL_VARIANTS(void, UpdateReturnRef, ObjHeader** returnSlot, const ObjHeader* object);
+MODEL_VARIANTS(void, UpdateHeapRefsInsideOneArray, const ArrayHeader* array, int fromIndex, int toIndex,
+               int count);
 MODEL_VARIANTS(void, EnterFrame, ObjHeader** start, int parameters, int count);
 MODEL_VARIANTS(void, LeaveFrame, ObjHeader** start, int parameters, int count);
 

kotlin-native/runtime/src/main/cpp/Arrays.cpp

@@ -133,15 +133,20 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex,
     ThrowArrayIndexOutOfBoundsException();
   }
   mutabilityCheck(destination);
-  if (fromIndex >= toIndex) {
-    for (int index = 0; index < count; index++) {
-      UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
-                      *ArrayAddressOfElementAt(array, fromIndex + index));
-    }
+  if (CurrentMemoryModel != MemoryModel::kExperimental && array == destinationArray &&
+      std::abs(fromIndex - toIndex) < count) {
+    UpdateHeapRefsInsideOneArray(array, fromIndex, toIndex, count);
   } else {
-    for (int index = count - 1; index >= 0; index--) {
-      UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
-                      *ArrayAddressOfElementAt(array, fromIndex + index));
+    if (fromIndex >= toIndex) {
+      for (int index = 0; index < count; index++) {
+        UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
+                        *ArrayAddressOfElementAt(array, fromIndex + index));
+      }
+    } else {
+      for (int index = count - 1; index >= 0; index--) {
+        UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
+                        *ArrayAddressOfElementAt(array, fromIndex + index));
+      }
     }
   }
 }

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -210,6 +210,8 @@ void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW;
 void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates heap/static data location.
 void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
+// Updates heap/static data in one array.
+void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW;
 // Updates location if it is null, atomically.
 void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates reference in return slot.

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -189,6 +189,11 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** lo
     mm::CompareAndSwapHeapRef(location, nullptr, const_cast<ObjHeader*>(object), &result);
 }
 
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex,
+                                                                           int toIndex, int count) {
+    RuntimeFail("Only for legacy MM");
+}
+
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
     mm::SetStackRef(returnSlot, const_cast<ObjHeader*>(object));
 }

kotlin-native/runtime/src/relaxed/cpp/MemoryImpl.cpp

@@ -67,4 +67,8 @@ RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* objec
     UpdateStackRefRelaxed(location, object);
 }
 
+RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) {
+  UpdateHeapRefsInsideOneArrayRelaxed(array, fromIndex, toIndex, count);
+}
+
 }  // extern "C"

kotlin-native/runtime/src/strict/cpp/MemoryImpl.cpp

@@ -67,4 +67,8 @@ RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* objec
     UpdateStackRefStrict(location, object);
 }
 
+RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) {
+  UpdateHeapRefsInsideOneArrayStrict(array, fromIndex, toIndex, count);
+}
+
 }  // extern "C"