Frame-local allocations. (#193)

2017-01-24 17:16:54 +03:00
parent afeb1b2d55
commit 66293cb89f
11 changed files with 400 additions and 206 deletions
@@ -21,11 +21,16 @@
 #define TRACE_GC_PHASES 0

 ContainerHeader ObjHeader::theStaticObjectsContainer = {
-  CONTAINER_TAG_NOCOUNT | CONTAINER_TAG_INCREMENT
+  CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT
 };

 namespace {

+// Granularity of arena container chunks.
+constexpr container_size_t kContainerAlignment = 1024;
+// Single object alignment.
+constexpr container_size_t kObjectAlignment = 8;
+
 #if USE_GC
 // Collection threshold default (collect after having so many elements in the
 // release candidates set).
@@ -63,11 +68,30 @@ struct MemoryState {

 MemoryState* memoryState = nullptr;

-#if USE_GC
-bool isPermanent(const ContainerHeader* header) {
-  return (header->ref_count_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NOCOUNT;
+// TODO: use those allocators for STL containers as well.
+template <typename T>
+inline T* allocMemory(container_size_t size) {
+  return reinterpret_cast<T*>(calloc(1, size));
 }

+inline void freeMemory(void* memory) {
+  free(memory);
+}
+
+inline bool isFreeable(const ContainerHeader* header) {
+  return (header->refCount_ & CONTAINER_TAG_MASK) < CONTAINER_TAG_PERMANENT;
+}
+
+inline bool isPermanent(const ContainerHeader* header) {
+  return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
+}
+
+inline container_size_t alignUp(container_size_t size, int alignment) {
+  return (size + alignment - 1) & ~(alignment - 1);
+}
+
+#if USE_GC
+
 // Must be vector or map 'container -> number', to keep reference counters correct.
 ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
  ContainerHeaderList result;
@@ -98,8 +122,8 @@ ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
 void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) {
  fprintf(stderr, "%s: %p (%08x): %d refs %s\n",
          prefix,
-          header, header->ref_count_, header->ref_count_ >> CONTAINER_TAG_SHIFT,
-          (header->ref_count_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
+          header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT,
+          (header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
  seen->insert(header);
  auto children = collectMutableReferred(header);
  for (auto child : children) {
@@ -117,22 +141,22 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
 }

 void phase1(ContainerHeader* header) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0)
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0)
    return;
-  header->ref_count_ |= CONTAINER_TAG_SEEN;
+  header->refCount_ |= CONTAINER_TAG_SEEN;
  auto containers = collectMutableReferred(header);
  for (auto container : containers) {
-      container->ref_count_ -= CONTAINER_TAG_INCREMENT;
+      container->refCount_ -= CONTAINER_TAG_INCREMENT;
      phase1(container);
  }
 }

 void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) == 0)
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0)
    return;
-  if ((header->ref_count_ >> CONTAINER_TAG_SHIFT) != 0)
+  if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0)
    rootset->insert(header);
-  header->ref_count_ &= ~CONTAINER_TAG_SEEN;
+  header->refCount_ &= ~CONTAINER_TAG_SEEN;
  auto containers = collectMutableReferred(header);
  for (auto container : containers) {
    phase2(container, rootset);
@@ -140,32 +164,32 @@ void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) {
 }

 void phase3(ContainerHeader* header) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0) {
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) {
    return;
  }
-  header->ref_count_ |= CONTAINER_TAG_SEEN;
+  header->refCount_ |= CONTAINER_TAG_SEEN;
  auto containers = collectMutableReferred(header);
  for (auto container : containers) {
-    container->ref_count_ += CONTAINER_TAG_INCREMENT;
+    container->refCount_ += CONTAINER_TAG_INCREMENT;
    phase3(container);
  }
 }

 void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) {
-  auto ref_count = header->ref_count_ >> CONTAINER_TAG_SHIFT;
-  bool seen = (ref_count > 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) == 0) ||
-      (ref_count == 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) != 0);
+  auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT;
+  bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) ||
+      (refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0);
  if (seen) return;

  // Add to toRemove set.
-  if (ref_count == 0)
+  if (refCount == 0)
    toRemove->insert(header);

  // Update seen bit.
-  if (ref_count == 0)
-    header->ref_count_ |= CONTAINER_TAG_SEEN;
+  if (refCount == 0)
+    header->refCount_ |= CONTAINER_TAG_SEEN;
  else
-    header->ref_count_ &= ~CONTAINER_TAG_SEEN;
+    header->refCount_ &= ~CONTAINER_TAG_SEEN;
  auto containers = collectMutableReferred(header);
  for (auto container : containers) {
    phase4(container, toRemove);
@@ -174,91 +198,109 @@ void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) {

 #endif // USE_GC

+// We use first slot as place to store frame-local arena container.
+ArenaContainer* initedArena(ObjHeader** auxSlot) {
+  ObjHeader* slotValue = *auxSlot;
+  if (slotValue) return reinterpret_cast<ArenaContainer*>(slotValue);
+  ArenaContainer* arena = allocMemory<ArenaContainer>(sizeof(ArenaContainer));
+  arena->Init();
+  *auxSlot = reinterpret_cast<ObjHeader*>(arena);
+  return arena;
+}
+
 }  // namespace

 ContainerHeader* AllocContainer(size_t size) {
-  ContainerHeader* result = reinterpret_cast<ContainerHeader*>(calloc(1, size));
+  ContainerHeader* result = allocMemory<ContainerHeader>(size);
 #if TRACE_MEMORY
-  fprintf(stderr, ">>> alloc %d -> %p\n", (int)size, result);
-   memoryState->containers->insert(result);
+  fprintf(stderr, ">>> alloc %d -> %p\n", static_cast<int>(size), result);
+  memoryState->containers->insert(result);
 #endif
  // TODO: atomic increment in concurrent case.
  memoryState->allocCount++;
  return result;
 }

+// TODO: shall we do padding for alignment?
+uint32_t ObjectSize(const ObjHeader* obj) {
+  const TypeInfo* type_info = obj->type_info();
+  if (type_info->instanceSize_ < 0) {
+    // An array.
+    return ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader);
+  } else {
+    return type_info->instanceSize_ + sizeof(ObjHeader);
+  }
+}
+
 void FreeContainer(ContainerHeader* header) {
  RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
 #if TRACE_MEMORY
-  fprintf(stderr, "<<< free %p\n", header);
-  memoryState->containers->erase(header);
+  if (isFreeable(header)) {
+    fprintf(stderr, "<<< free %p\n", header);
+    memoryState->containers->erase(header);
+  }
 #endif
-  header->ref_count_ = CONTAINER_TAG_INVALID;
+
 #if USE_GC
-  if (memoryState->toFree)
+  if (memoryState->toFree && isFreeable(header))
    memoryState->toFree->erase(header);
 #endif
  // Now let's clean all object's fields in this container.
-  // TODO: this is gross hack, relying on the fact that we now only alloc
-  // ArenaContainer and ObjectContainer, which both have single element.
  ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
-  const TypeInfo* typeInfo = obj->type_info();

-  // We use *local* versions as no other threads could see dead objects.
-  for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
-    ObjHeader** location = reinterpret_cast<ObjHeader**>(
-        reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
-    UpdateLocalRef(location, nullptr);
-  }
-  // Object arrays are *special*.
-  if (typeInfo == theArrayTypeInfo) {
-    ArrayHeader* array = obj->array();
-    ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_);
+  for (int index = 0; index < header->objectCount_; index++) {
+    const TypeInfo* typeInfo = obj->type_info();
+
+    // We use *local* versions as no other threads could see dead objects.
+    for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
+      ObjHeader** location = reinterpret_cast<ObjHeader**>(
+          reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
+      UpdateLocalRef(location, nullptr);
+    }
+    // Object arrays are *special*.
+    if (typeInfo == theArrayTypeInfo) {
+      ArrayHeader* array = obj->array();
+      ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_);
+    }
+    obj = reinterpret_cast<ObjHeader*>(reinterpret_cast<uintptr_t>(obj) + ObjectSize(obj));
  }

  // And release underlying memory.
-  // TODO: atomic decrement in concurrent case.
+  if (isFreeable(header)) {
+    // TODO: atomic decrement in concurrent case.
 #if CONCURRENT
-  #error "Atomic update of allocCount"
+    #error "Atomic update of allocCount"
 #endif
-  memoryState->allocCount--;
-  free(header);
+    memoryState->allocCount--;
+    freeMemory(header);
+  }
 }

 #if USE_GC
 void FreeContainerNoRef(ContainerHeader* header) {
-  RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
+  RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed");
 #if TRACE_MEMORY
  fprintf(stderr, "<<< free %p\n", header);
  memoryState->containers->erase(header);
 #endif
-  header->ref_count_ = CONTAINER_TAG_INVALID;
 #if USE_GC
  if (memoryState->toFree)
    memoryState->toFree->erase(header);
 #endif
  memoryState->allocCount--;
-  free(header);
+  freeMemory(header);
 }
 #endif

-ArenaContainer::ArenaContainer(uint32_t size) {
-    ArenaContainerHeader* header =
-        static_cast<ArenaContainerHeader*>(AllocContainer(size + sizeof(ArenaContainerHeader)));
-    header_ = header;
-    // header->ref_count_ is zero initialized by AllocContainer().
-    header->current_ =
-        reinterpret_cast<uint8_t*>(header_) + sizeof(ArenaContainerHeader);
-    header->end_ = header->current_ + size;
-}
-
 void ObjectContainer::Init(const TypeInfo* type_info) {
  RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
  uint32_t alloc_size =
      sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_;
  header_ = AllocContainer(alloc_size);
  if (header_) {
-     // header->ref_count_ is zero initialized by AllocContainer().
+    // One object in this container.
+    header_->objectCount_ = 1;
+     // header->refCount_ is zero initialized by AllocContainer().
    SetMeta(GetPlace(), type_info);
 #if TRACE_MEMORY
    fprintf(stderr, "object at %p\n", GetPlace());
@@ -274,7 +316,9 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
  header_ = AllocContainer(alloc_size);
  RuntimeAssert(header_ != nullptr, "Cannot alloc memory");
  if (header_) {
-    // header->ref_count_ is zero initialized by AllocContainer().
+    // One object in this container.
+    header_->objectCount_ = 1;
+    // header->refCount_ is zero initialized by AllocContainer().
    GetPlace()->count_ = elements;
    SetMeta(GetPlace()->obj(), type_info);
 #if TRACE_MEMORY
@@ -283,25 +327,73 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
  }
 }

-ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
-  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
-  uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
-  ObjHeader* result = reinterpret_cast<ObjHeader*>(Place(size));
-  if (!result) {
-      return nullptr;
+void ArenaContainer::Init() {
+  allocContainer(1024);
+}
+
+void ArenaContainer::Deinit() {
+  auto chunk = currentChunk_;
+  while (chunk != nullptr) {
+    auto toRemove = chunk;
+    // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set.
+    FreeContainer(chunk->asHeader());
+    chunk = chunk->next;
+    freeMemory(toRemove);
  }
-  SetMeta(result, type_info);
+}
+
+bool ArenaContainer::allocContainer(container_size_t minSize) {
+  auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk);
+  size = alignUp(size, kContainerAlignment);
+  ContainerChunk* result = allocMemory<ContainerChunk>(size);
+  RuntimeAssert(result != nullptr, "Cannot alloc memory");
+  if (result == nullptr) return false;
+  result->next = currentChunk_;
+  result->asHeader()->refCount_ = (CONTAINER_TAG_STACK | CONTAINER_TAG_INCREMENT);
+  currentChunk_ = result;
+  current_ = reinterpret_cast<uint8_t*>(result->asHeader() + 1);
+  end_ = reinterpret_cast<uint8_t*>(result) + size;
+  return true;
+}
+
+void* ArenaContainer::place(container_size_t size) {
+  size = alignUp(size, kObjectAlignment);
+  // Fast path.
+  if (current_ + size < end_) {
+    void* result = current_;
+    current_ += size;
+    return result;
+  }
+  if (!allocContainer(size)) {
+    return nullptr;
+  }
+  void* result = current_;
+  current_ += size;
+  RuntimeAssert(current_ <= end_, "Must not overflow");
  return result;
 }

-ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, int count) {
+ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
+  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
+  uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
+  ObjHeader* result = reinterpret_cast<ObjHeader*>(place(size));
+  if (!result) {
+      return nullptr;
+  }
+  currentChunk_->asHeader()->objectCount_++;
+  setMeta(result, type_info);
+  return result;
+}
+
+ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t count) {
  RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
-  uint32_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count;
-  ArrayHeader* result = reinterpret_cast<ArrayHeader*>(Place(size));
+  container_size_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count;
+  ArrayHeader* result = reinterpret_cast<ArrayHeader*>(place(size));
  if (!result) {
    return nullptr;
  }
-  SetMeta(result->obj(), type_info);
+  currentChunk_->asHeader()->objectCount_++;
+  setMeta(result->obj(), type_info);
  result->count_ = count;
  return result;
 }
@@ -384,6 +476,8 @@ void DeinitMemory() {

 #if USE_GC
  GarbageCollect();
+  delete memoryState->toFree;
+  memoryState->toFree = nullptr;
 #endif // USE_GC

  if (memoryState->allocCount > 0) {
@@ -399,20 +493,28 @@ void DeinitMemory() {
  memoryState = nullptr;
 }

-// Now we ignore all placement hints and always allocate heap space for new object.
-OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint) {
+ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) {
+  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
+  return initedArena(auxSlot)->PlaceObject(type_info);
+}
+
+OBJ_GETTER(AllocInstance, const TypeInfo* type_info) {
  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
  RETURN_OBJ(ObjectContainer(type_info).GetPlace());
 }

-OBJ_GETTER(AllocArrayInstance,
-           const TypeInfo* type_info, PlacementHint hint, uint32_t elements) {
+ObjHeader* ArenaAllocArrayInstance(
+    const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) {
+  RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
+  return initedArena(auxSlot)->PlaceArray(type_info, elements)->obj();
+}
+
+OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) {
  RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
  RETURN_OBJ(ArrayContainer(type_info, elements).GetPlace()->obj());
 }

-OBJ_GETTER(AllocStringInstance,
-  PlacementHint hint, const char* data, uint32_t length) {
+OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) {
  ArrayHeader* array = ArrayContainer(theStringTypeInfo, length).GetPlace();
  memcpy(
      ByteArrayAddressOfElementAt(array, 0),
@@ -422,8 +524,7 @@ OBJ_GETTER(AllocStringInstance,
 }

 OBJ_GETTER(InitInstance,
-    ObjHeader** location, const TypeInfo* type_info, PlacementHint hint,
-    void (*ctor)(ObjHeader*)) {
+    ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) {
  ObjHeader* sentinel = reinterpret_cast<ObjHeader*>(1);
  ObjHeader* value;
  // Wait until other initializers.
@@ -438,7 +539,7 @@ OBJ_GETTER(InitInstance,
    RETURN_OBJ(value);
  }

-  AllocInstance(type_info, hint, OBJ_RESULT);
+  AllocInstance(type_info, OBJ_RESULT);
  ObjHeader* object = *OBJ_RESULT;
  UpdateGlobalRef(location, object);
  try {
@@ -523,6 +624,15 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) {
 #endif
 }

+void LeaveFrame(ObjHeader** start, int count) {
+  ReleaseLocalRefs(start + 1, count - 1);
+  if (*start != nullptr) {
+    auto arena = initedArena(start);
+    arena->Deinit();
+    freeMemory(arena);
+  }
+}
+
 void ReleaseLocalRefs(ObjHeader** start, int count) {
 #if TRACE_MEMORY
  fprintf(stderr, "ReleaseLocalRefs %p .. %p\n", start, start + count);
@@ -623,7 +733,7 @@ void GarbageCollect() {
  memoryState->toFree->clear();

  for (auto header : toRemove) {
-    RuntimeAssert((header->ref_count_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen");
+    RuntimeAssert((header->refCount_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen");
    FreeContainerNoRef(header);
  }