From 0fdf58cbd83ec1204e1df6f383abaac61eeef6e4 Mon Sep 17 00:00:00 2001 From: Alexey Andreev Date: Tue, 30 Jul 2019 19:12:55 +0300 Subject: [PATCH] C: implement heap defragmentation in GC --- .../org/teavm/classlib/java/lang/TSystem.java | 2 +- core/src/main/java/org/teavm/runtime/GC.java | 424 ++++++++++++++++-- .../java/org/teavm/runtime/Relocation.java | 25 ++ .../org/teavm/runtime/RelocationBlock.java | 25 ++ .../main/java/org/teavm/interop/Address.java | 4 + 5 files changed, 444 insertions(+), 36 deletions(-) create mode 100644 core/src/main/java/org/teavm/runtime/Relocation.java create mode 100644 core/src/main/java/org/teavm/runtime/RelocationBlock.java diff --git a/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java b/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java index 2cce62f14..cd621e8e7 100644 --- a/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java +++ b/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java @@ -248,7 +248,7 @@ public final class TSystem extends TObject { } private static void gcLowLevel() { - GC.collectGarbage(0); + GC.collectGarbage(); } public static void runFinalization() { diff --git a/core/src/main/java/org/teavm/runtime/GC.java b/core/src/main/java/org/teavm/runtime/GC.java index bc186ead5..80622d3b4 100644 --- a/core/src/main/java/org/teavm/runtime/GC.java +++ b/core/src/main/java/org/teavm/runtime/GC.java @@ -34,6 +34,8 @@ public final class GC { static int freeMemory = (int) availableBytes(); static RuntimeReference firstWeakReference; + static RelocationBlock lastRelocationTarget; + static native Address gcStorageAddress(); static native int gcStorageSize(); @@ -92,7 +94,7 @@ public final class GC { if (getAvailableChunkIfPossible(size)) { return; } - collectGarbage(size); + collectGarbage(); if (!getAvailableChunkIfPossible(size)) { ExceptionHandling.printStack(); outOfMemory(); @@ -121,12 +123,13 @@ public final class GC { return true; } - public static boolean collectGarbage(int size) { + public static void collectGarbage() { mark(); processReferences(); sweep(); + defragment(); + sortFreeChunks(); updateFreeMemory(); - return true; } private static void mark() { @@ -286,12 +289,26 @@ public final class GC { long heapSize = availableBytes(); long reclaimedSpace = 0; long maxFreeChunk = 0; - int currentRegionIndex = 0; int regionsCount = (int) ((heapSize - 1) / regionSize()) + 1; - Address currentRegionEnd = object.toAddress().add(regionSize()); + Address currentRegionEnd = null; Address limit = heapAddress().add(heapSize); loop: while (object.toAddress().isLessThan(limit)) { + if (!object.toAddress().isLessThan(currentRegionEnd)) { + int currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); + Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + while (currentRegion.start == 0) { + if (++currentRegionIndex == regionsCount) { + object = limit.toStructure(); + break loop; + } + currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + } + Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize()); + object = newRegionStart.add(currentRegion.start - 1).toStructure(); + currentRegionEnd = newRegionStart.add(regionSize()); + } + int tag = object.classReference; boolean free; if (tag == 0) { @@ -308,22 +325,6 @@ public final class GC { if (lastFreeSpace == null) { lastFreeSpace = object; } - - if (!object.toAddress().isLessThan(currentRegionEnd)) { - currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); - Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); - while (currentRegion.start == 0) { - if (++currentRegionIndex == regionsCount) { - object = limit.toStructure(); - break loop; - } - currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); - } - Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize()); - object = newRegionStart.add(currentRegion.start - 1).toStructure(); - currentRegionEnd = newRegionStart.add(regionSize()); - continue; - } } else { if (lastFreeSpace != null) { lastFreeSpace.classReference = 0; @@ -357,6 +358,355 @@ public final class GC { } currentChunkPointer = gcStorageAddress().toStructure(); + } + + private static void defragment() { + markStackRoots(); + calculateRelocationTargets(); + updatePointersFromStaticRoots(); + updatePointersFromObjects(); + restoreObjectHeaders(); + relocateObjects(); + putNewFreeChunks(); + } + + private static void markStackRoots() { + Address relocationThreshold = currentChunkPointer.value.toAddress(); + + for (Address stackRoots = ShadowStack.getStackTop(); stackRoots != null; + stackRoots = ShadowStack.getNextStackFrame(stackRoots)) { + int count = ShadowStack.getStackRootCount(stackRoots); + Address stackRootsPtr = ShadowStack.getStackRootPointer(stackRoots); + while (count-- > 0) { + RuntimeObject obj = stackRootsPtr.getAddress().toStructure(); + if (!obj.toAddress().isLessThan(relocationThreshold)) { + obj.classReference |= RuntimeObject.GC_MARKED; + stackRootsPtr = stackRootsPtr.add(Address.sizeOf()); + } + } + } + } + + private static void calculateRelocationTargets() { + Address start = heapAddress(); + long heapSize = availableBytes(); + Address limit = start.add(heapSize); + + FreeChunkHolder freeChunkPointer = currentChunkPointer; + int freeChunks = GC.freeChunks; + FreeChunk freeChunk = currentChunkPointer.value; + FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure(); + + RelocationBlock relocationTarget = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks) + .toAddress().toStructure(); + relocationTarget.start = freeChunk.toAddress(); + relocationTarget.current = relocationTarget.start; + relocationTarget.end = limit; + RelocationBlock lastRelocationTarget = relocationTarget; + + Address relocations = Structure.add(FreeChunk.class, freeChunk, 1).toAddress(); + Address relocationsLimit = freeChunk.toAddress().add(freeChunk.size); + + boolean lastWasLocked = false; + objects: while (object.toAddress().isLessThan(limit)) { + int size = objectSize(object); + if (object.classReference != 0) { + if ((object.classReference & RuntimeObject.GC_MARKED) != 0 + || !relocationTarget.start.isLessThan(object.toAddress())) { + if (!lastWasLocked) { + lastRelocationTarget.end = object.toAddress(); + lastRelocationTarget = Structure.add(RelocationBlock.class, lastRelocationTarget, 1); + lastRelocationTarget.end = limit; + lastWasLocked = true; + } + lastRelocationTarget.start = object.toAddress().add(size); + lastRelocationTarget.current = lastRelocationTarget.start; + object.classReference &= ~RuntimeObject.GC_MARKED; + } else { + lastRelocationTarget.end = object.toAddress().add(size); + lastWasLocked = false; + + Address nextRelocationTarget; + while (true) { + nextRelocationTarget = relocationTarget.current.add(size); + if (nextRelocationTarget.isLessThan(relocationTarget.end)) { + break; + } + + relocationTarget = Structure.add(RelocationBlock.class, relocationTarget, 1); + if (lastRelocationTarget.toAddress().isLessThan(relocationTarget.toAddress())) { + break objects; + } + } + + while (!relocations.add(Structure.sizeOf(Relocation.class)).isLessThan(relocationsLimit)) { + if (--freeChunks == 0) { + break objects; + } + freeChunkPointer = Structure.add(FreeChunkHolder.class, freeChunkPointer, 1); + freeChunk = freeChunkPointer.value; + relocations = Structure.add(FreeChunk.class, freeChunk, 1).toAddress(); + relocationsLimit = freeChunk.toAddress().add(freeChunk.size); + } + + Relocation relocation = relocations.toStructure(); + relocation.classBackup = object.classReference; + relocation.sizeBackup = object.size; + relocation.newAddress = relocationTarget.current; + relocations = relocations.add(Structure.sizeOf(Relocation.class)); + + long targetAddress = relocation.toAddress().toLong(); + object.classReference = (int) (targetAddress >>> 33) | RuntimeObject.GC_MARKED; + object.size = (int) (targetAddress >> 1); + relocationTarget.current = nextRelocationTarget; + } + } else { + lastWasLocked = false; + lastRelocationTarget.end = object.toAddress().add(object.size); + } + object = object.toAddress().add(size).toStructure(); + } + + while (object.toAddress().isLessThan(limit)) { + int size = objectSize(object); + if (object.classReference != 0) { + object.classReference &= ~RuntimeObject.GC_MARKED; + } else { + lastRelocationTarget = Structure.add(RelocationBlock.class, lastRelocationTarget, 1); + lastRelocationTarget.start = object.toAddress(); + lastRelocationTarget.current = lastRelocationTarget.start; + lastRelocationTarget.end = lastRelocationTarget.start.add(size); + } + object = object.toAddress().add(size).toStructure(); + } + + GC.lastRelocationTarget = lastRelocationTarget; + } + + private static void updatePointersFromStaticRoots() { + Address staticRoots = Mutator.getStaticGCRoots(); + int staticCount = staticRoots.getInt(); + staticRoots = staticRoots.add(Address.sizeOf()); + while (staticCount-- > 0) { + Address staticRoot = staticRoots.getAddress(); + staticRoot.putAddress(updatePointer(staticRoot.getAddress())); + staticRoots = staticRoots.add(Address.sizeOf()); + } + } + + private static void updatePointersFromObjects() { + Address start = heapAddress(); + long heapSize = availableBytes(); + Address limit = start.add(heapSize); + + FreeChunk object = heapAddress().toStructure(); + + while (object.toAddress().isLessThan(limit)) { + int classRef = object.classReference; + int size; + if (classRef != 0) { + Relocation relocation = getRelocation(object.toAddress()); + if (relocation != null) { + classRef = relocation.classBackup; + } + RuntimeClass cls = RuntimeClass.unpack(classRef); + RuntimeObject realObject = object.toAddress().toStructure(); + updatePointers(cls, realObject); + size = objectSize(realObject, cls); + } else { + size = object.size; + } + + object = object.toAddress().add(size).toStructure(); + } + } + + private static void updatePointers(RuntimeClass cls, RuntimeObject object) { + if (cls.itemType == null) { + updatePointersInObject(cls, object); + } else { + updatePointersInArray(cls, (RuntimeArray) object); + } + } + + private static void updatePointersInObject(RuntimeClass cls, RuntimeObject object) { + while (cls != null) { + int type = (cls.flags >> RuntimeClass.VM_TYPE_SHIFT) & RuntimeClass.VM_TYPE_MASK; + switch (type) { + case RuntimeClass.VM_TYPE_WEAKREFERENCE: + updatePointersInWeakReference((RuntimeReference) object); + break; + + case RuntimeClass.VM_TYPE_REFERENCEQUEUE: + updatePointersInReferenceQueue((RuntimeReferenceQueue) object); + break; + + default: + updatePointersInFields(cls, object); + break; + } + cls = cls.parent; + } + } + + private static void updatePointersInWeakReference(RuntimeReference object) { + object.queue = updatePointer(object.queue.toAddress()).toStructure(); + object.next = updatePointer(object.next.toAddress()).toStructure(); + object.object = updatePointer(object.object.toAddress()).toStructure(); + } + + private static void updatePointersInReferenceQueue(RuntimeReferenceQueue object) { + object.first = updatePointer(object.first.toAddress()).toStructure(); + object.last = updatePointer(object.last.toAddress()).toStructure(); + } + + private static void updatePointersInFields(RuntimeClass cls, RuntimeObject object) { + Address layout = cls.layout; + if (layout != null) { + short fieldCount = layout.getShort(); + while (fieldCount-- > 0) { + layout = layout.add(2); + int fieldOffset = layout.getShort(); + Address referenceHolder = object.toAddress().add(fieldOffset); + referenceHolder.putAddress(updatePointer(referenceHolder.getAddress())); + } + } + } + + private static void updatePointersInArray(RuntimeClass cls, RuntimeArray array) { + if ((cls.itemType.flags & RuntimeClass.PRIMITIVE) != 0) { + return; + } + Address base = Address.align(array.toAddress().add(RuntimeArray.class, 1), Address.sizeOf()); + int size = array.size; + for (int i = 0; i < size; ++i) { + base.putAddress(updatePointer(base.getAddress())); + base = base.add(Address.sizeOf()); + } + } + + private static Address updatePointer(Address address) { + if (address == null) { + return null; + } + Relocation relocation = getRelocation(address); + return relocation != null ? relocation.newAddress : address; + } + + private static Relocation getRelocation(Address address) { + if (address.isLessThan(heapAddress()) || !address.isLessThan(heapAddress().add(availableBytes()))) { + return null; + } + FreeChunk obj = address.toStructure(); + if ((obj.classReference & RuntimeObject.GC_MARKED) == 0) { + return null; + } + long result = (((long) obj.classReference & 0xFFFFFFFFL) << 33) | (((long) obj.size & 0xFFFFFFFFL) << 1); + return Address.fromLong(result).toStructure(); + } + + private static void restoreObjectHeaders() { + Address start = heapAddress(); + long heapSize = availableBytes(); + Address limit = start.add(heapSize); + + FreeChunk freeChunk = currentChunkPointer.value; + FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure(); + + while (object.toAddress().isLessThan(limit)) { + Relocation relocation = getRelocation(object.toAddress()); + if (relocation != null) { + object.classReference = relocation.classBackup | RuntimeObject.GC_MARKED; + object.size = relocation.sizeBackup; + } + int size = objectSize(object); + object = object.toAddress().add(size).toStructure(); + } + } + + private static void relocateObjects() { + Address start = heapAddress(); + long heapSize = availableBytes(); + Address limit = start.add(heapSize); + + int freeChunks = GC.freeChunks; + FreeChunk freeChunk = currentChunkPointer.value; + FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure(); + + RelocationBlock relocationTarget = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks) + .toAddress().toStructure(); + Address currentTarget = relocationTarget.start; + + Address blockTarget = null; + Address blockSource = null; + int blockSize = 0; + + objects: while (object.toAddress().isLessThan(limit)) { + int size = objectSize(object); + if ((object.classReference & RuntimeObject.GC_MARKED) != 0) { + object.classReference &= ~RuntimeObject.GC_MARKED; + + Address nextRelocationTarget; + while (true) { + nextRelocationTarget = currentTarget.add(size); + if (nextRelocationTarget.isLessThan(relocationTarget.end)) { + break; + } + + if (blockSize != 0) { + Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); + blockSource = null; + blockSize = 0; + } + + relocationTarget = Structure.add(RelocationBlock.class, relocationTarget, 1); + if (lastRelocationTarget.toAddress().isLessThan(relocationTarget.toAddress())) { + break objects; + } + currentTarget = relocationTarget.start; + } + + if (blockSource == null) { + blockSource = object.toAddress(); + blockTarget = currentTarget; + } + currentTarget = nextRelocationTarget; + blockSize += size; + } else if (blockSource != null) { + Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); + blockSource = null; + blockSize = 0; + } + + object = object.toAddress().add(size).toStructure(); + } + + if (blockSource != null) { + Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); + } + } + + private static void putNewFreeChunks() { + FreeChunkHolder freeChunkPointer = currentChunkPointer; + RelocationBlock relocationBlock = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks) + .toAddress().toStructure(); + freeChunks = 0; + while (!lastRelocationTarget.toAddress().isLessThan(relocationBlock.toAddress())) { + if (!relocationBlock.current.isLessThan(relocationBlock.end)) { + continue; + } + FreeChunk freeChunk = relocationBlock.current.toStructure(); + freeChunk.size = (int) (relocationBlock.end.toLong() - relocationBlock.current.toLong()); + freeChunk.classReference = 0; + freeChunkPointer.value = freeChunk; + freeChunkPointer = Structure.add(FreeChunkHolder.class, freeChunkPointer, 1); + freeChunks++; + relocationBlock = Structure.add(RelocationBlock.class, relocationBlock, 1); + } + } + + private static void sortFreeChunks() { + currentChunkPointer = gcStorageAddress().toStructure(); sortFreeChunks(0, freeChunks - 1); currentChunk = currentChunkPointer.value; currentChunkLimit = currentChunk.toAddress().add(currentChunk.size); @@ -434,23 +784,27 @@ public final class GC { if (object.classReference == 0) { return object.size; } else { - RuntimeClass cls = RuntimeClass.getClass(object.toAddress().toStructure()); - if (cls.itemType == null) { - return cls.size; - } else { - int itemSize = (cls.itemType.flags & RuntimeClass.PRIMITIVE) == 0 - ? Address.sizeOf() - : cls.itemType.size; - RuntimeArray array = object.toAddress().toStructure(); - Address address = Address.fromInt(Structure.sizeOf(RuntimeArray.class)); - address = Address.align(address, itemSize); - address = address.add(itemSize * array.size); - address = Address.align(address, Address.sizeOf()); - return address.toInt(); - } + RuntimeObject realObject = object.toAddress().toStructure(); + RuntimeClass cls = RuntimeClass.getClass(realObject); + return objectSize(realObject, cls); } } + private static int objectSize(RuntimeObject object, RuntimeClass cls) { + if (cls.itemType == null) { + return cls.size; + } + int itemSize = (cls.itemType.flags & RuntimeClass.PRIMITIVE) == 0 + ? Address.sizeOf() + : cls.itemType.size; + RuntimeArray array = object.toAddress().toStructure(); + Address address = Address.fromInt(Structure.sizeOf(RuntimeArray.class)); + address = Address.align(address, itemSize); + address = address.add(itemSize * array.size); + address = Address.align(address, Address.sizeOf()); + return address.toInt(); + } + private static boolean isMarked(RuntimeObject object) { return (object.classReference & RuntimeObject.GC_MARKED) != 0; } diff --git a/core/src/main/java/org/teavm/runtime/Relocation.java b/core/src/main/java/org/teavm/runtime/Relocation.java new file mode 100644 index 000000000..7bf869444 --- /dev/null +++ b/core/src/main/java/org/teavm/runtime/Relocation.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 konsoletyper. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.runtime; + +import org.teavm.interop.Address; +import org.teavm.interop.Structure; + +class Relocation extends Structure { + int classBackup; + int sizeBackup; + Address newAddress; +} diff --git a/core/src/main/java/org/teavm/runtime/RelocationBlock.java b/core/src/main/java/org/teavm/runtime/RelocationBlock.java new file mode 100644 index 000000000..85c8610bc --- /dev/null +++ b/core/src/main/java/org/teavm/runtime/RelocationBlock.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 konsoletyper. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.runtime; + +import org.teavm.interop.Address; +import org.teavm.interop.Structure; + +class RelocationBlock extends Structure { + Address start; + Address current; + Address end; +} diff --git a/interop/core/src/main/java/org/teavm/interop/Address.java b/interop/core/src/main/java/org/teavm/interop/Address.java index 382565ac5..75ff7a0bb 100644 --- a/interop/core/src/main/java/org/teavm/interop/Address.java +++ b/interop/core/src/main/java/org/teavm/interop/Address.java @@ -89,4 +89,8 @@ public final class Address { public static native int sizeOf(); public native Address add(Class type, int offset); + + public long diff(Address that) { + return toLong() - that.toLong(); + } }