C: implement heap defragmentation in GC

classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java

@@ -248,7 +248,7 @@ public final class TSystem extends TObject {
     }
 
     private static void gcLowLevel() {
-        GC.collectGarbage(0);
+        GC.collectGarbage();
     }
 
     public static void runFinalization() {

core/src/main/java/org/teavm/runtime/GC.java

@@ -34,6 +34,8 @@ public final class GC {
     static int freeMemory = (int) availableBytes();
     static RuntimeReference firstWeakReference;
 
+    static RelocationBlock lastRelocationTarget;
+
     static native Address gcStorageAddress();
 
     static native int gcStorageSize();
@@ -92,7 +94,7 @@ public final class GC {
         if (getAvailableChunkIfPossible(size)) {
             return;
         }
-        collectGarbage(size);
+        collectGarbage();
         if (!getAvailableChunkIfPossible(size)) {
             ExceptionHandling.printStack();
             outOfMemory();
@@ -121,12 +123,13 @@ public final class GC {
         return true;
     }
 
-    public static boolean collectGarbage(int size) {
+    public static void collectGarbage() {
         mark();
         processReferences();
         sweep();
+        defragment();
+        sortFreeChunks();
         updateFreeMemory();
-        return true;
     }
 
     private static void mark() {
@@ -286,12 +289,26 @@ public final class GC {
         long heapSize = availableBytes();
         long reclaimedSpace = 0;
         long maxFreeChunk = 0;
-        int currentRegionIndex = 0;
         int regionsCount = (int) ((heapSize - 1) / regionSize()) + 1;
-        Address currentRegionEnd = object.toAddress().add(regionSize());
+        Address currentRegionEnd = null;
         Address limit = heapAddress().add(heapSize);
 
         loop: while (object.toAddress().isLessThan(limit)) {
+            if (!object.toAddress().isLessThan(currentRegionEnd)) {
+                int currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize());
+                Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex);
+                while (currentRegion.start == 0) {
+                    if (++currentRegionIndex == regionsCount) {
+                        object = limit.toStructure();
+                        break loop;
+                    }
+                    currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex);
+                }
+                Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize());
+                object = newRegionStart.add(currentRegion.start - 1).toStructure();
+                currentRegionEnd = newRegionStart.add(regionSize());
+            }
+
             int tag = object.classReference;
             boolean free;
             if (tag == 0) {
@@ -308,22 +325,6 @@ public final class GC {
                 if (lastFreeSpace == null) {
                     lastFreeSpace = object;
                 }
-
-                if (!object.toAddress().isLessThan(currentRegionEnd)) {
-                    currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize());
-                    Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex);
-                    while (currentRegion.start == 0) {
-                        if (++currentRegionIndex == regionsCount) {
-                            object = limit.toStructure();
-                            break loop;
-                        }
-                        currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex);
-                    }
-                    Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize());
-                    object = newRegionStart.add(currentRegion.start - 1).toStructure();
-                    currentRegionEnd = newRegionStart.add(regionSize());
-                    continue;
-                }
             } else {
                 if (lastFreeSpace != null) {
                     lastFreeSpace.classReference = 0;
@@ -357,6 +358,355 @@ public final class GC {
         }
 
         currentChunkPointer = gcStorageAddress().toStructure();
+    }
+
+    private static void defragment() {
+        markStackRoots();
+        calculateRelocationTargets();
+        updatePointersFromStaticRoots();
+        updatePointersFromObjects();
+        restoreObjectHeaders();
+        relocateObjects();
+        putNewFreeChunks();
+    }
+
+    private static void markStackRoots() {
+        Address relocationThreshold = currentChunkPointer.value.toAddress();
+
+        for (Address stackRoots = ShadowStack.getStackTop(); stackRoots != null;
+             stackRoots = ShadowStack.getNextStackFrame(stackRoots)) {
+            int count = ShadowStack.getStackRootCount(stackRoots);
+            Address stackRootsPtr = ShadowStack.getStackRootPointer(stackRoots);
+            while (count-- > 0) {
+                RuntimeObject obj = stackRootsPtr.getAddress().toStructure();
+                if (!obj.toAddress().isLessThan(relocationThreshold)) {
+                    obj.classReference |= RuntimeObject.GC_MARKED;
+                    stackRootsPtr = stackRootsPtr.add(Address.sizeOf());
+                }
+            }
+        }
+    }
+
+    private static void calculateRelocationTargets() {
+        Address start = heapAddress();
+        long heapSize = availableBytes();
+        Address limit = start.add(heapSize);
+
+        FreeChunkHolder freeChunkPointer = currentChunkPointer;
+        int freeChunks = GC.freeChunks;
+        FreeChunk freeChunk = currentChunkPointer.value;
+        FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure();
+
+        RelocationBlock relocationTarget = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks)
+                .toAddress().toStructure();
+        relocationTarget.start = freeChunk.toAddress();
+        relocationTarget.current = relocationTarget.start;
+        relocationTarget.end = limit;
+        RelocationBlock lastRelocationTarget = relocationTarget;
+
+        Address relocations = Structure.add(FreeChunk.class, freeChunk, 1).toAddress();
+        Address relocationsLimit = freeChunk.toAddress().add(freeChunk.size);
+
+        boolean lastWasLocked = false;
+        objects: while (object.toAddress().isLessThan(limit)) {
+            int size = objectSize(object);
+            if (object.classReference != 0) {
+                if ((object.classReference & RuntimeObject.GC_MARKED) != 0
+                        || !relocationTarget.start.isLessThan(object.toAddress())) {
+                    if (!lastWasLocked) {
+                        lastRelocationTarget.end = object.toAddress();
+                        lastRelocationTarget = Structure.add(RelocationBlock.class, lastRelocationTarget, 1);
+                        lastRelocationTarget.end = limit;
+                        lastWasLocked = true;
+                    }
+                    lastRelocationTarget.start = object.toAddress().add(size);
+                    lastRelocationTarget.current = lastRelocationTarget.start;
+                    object.classReference &= ~RuntimeObject.GC_MARKED;
+                } else {
+                    lastRelocationTarget.end = object.toAddress().add(size);
+                    lastWasLocked = false;
+
+                    Address nextRelocationTarget;
+                    while (true) {
+                        nextRelocationTarget = relocationTarget.current.add(size);
+                        if (nextRelocationTarget.isLessThan(relocationTarget.end)) {
+                            break;
+                        }
+
+                        relocationTarget = Structure.add(RelocationBlock.class, relocationTarget, 1);
+                        if (lastRelocationTarget.toAddress().isLessThan(relocationTarget.toAddress())) {
+                            break objects;
+                        }
+                    }
+
+                    while (!relocations.add(Structure.sizeOf(Relocation.class)).isLessThan(relocationsLimit)) {
+                        if (--freeChunks == 0) {
+                            break objects;
+                        }
+                        freeChunkPointer = Structure.add(FreeChunkHolder.class, freeChunkPointer, 1);
+                        freeChunk = freeChunkPointer.value;
+                        relocations = Structure.add(FreeChunk.class, freeChunk, 1).toAddress();
+                        relocationsLimit = freeChunk.toAddress().add(freeChunk.size);
+                    }
+
+                    Relocation relocation = relocations.toStructure();
+                    relocation.classBackup = object.classReference;
+                    relocation.sizeBackup = object.size;
+                    relocation.newAddress = relocationTarget.current;
+                    relocations = relocations.add(Structure.sizeOf(Relocation.class));
+
+                    long targetAddress = relocation.toAddress().toLong();
+                    object.classReference = (int) (targetAddress >>> 33) | RuntimeObject.GC_MARKED;
+                    object.size = (int) (targetAddress >> 1);
+                    relocationTarget.current = nextRelocationTarget;
+                }
+            } else {
+                lastWasLocked = false;
+                lastRelocationTarget.end = object.toAddress().add(object.size);
+            }
+            object = object.toAddress().add(size).toStructure();
+        }
+
+        while (object.toAddress().isLessThan(limit)) {
+            int size = objectSize(object);
+            if (object.classReference != 0) {
+                object.classReference &= ~RuntimeObject.GC_MARKED;
+            } else {
+                lastRelocationTarget = Structure.add(RelocationBlock.class, lastRelocationTarget, 1);
+                lastRelocationTarget.start = object.toAddress();
+                lastRelocationTarget.current = lastRelocationTarget.start;
+                lastRelocationTarget.end = lastRelocationTarget.start.add(size);
+            }
+            object = object.toAddress().add(size).toStructure();
+        }
+
+        GC.lastRelocationTarget = lastRelocationTarget;
+    }
+
+    private static void updatePointersFromStaticRoots() {
+        Address staticRoots = Mutator.getStaticGCRoots();
+        int staticCount = staticRoots.getInt();
+        staticRoots = staticRoots.add(Address.sizeOf());
+        while (staticCount-- > 0) {
+            Address staticRoot = staticRoots.getAddress();
+            staticRoot.putAddress(updatePointer(staticRoot.getAddress()));
+            staticRoots = staticRoots.add(Address.sizeOf());
+        }
+    }
+
+    private static void updatePointersFromObjects() {
+        Address start = heapAddress();
+        long heapSize = availableBytes();
+        Address limit = start.add(heapSize);
+
+        FreeChunk object = heapAddress().toStructure();
+
+        while (object.toAddress().isLessThan(limit)) {
+            int classRef = object.classReference;
+            int size;
+            if (classRef != 0) {
+                Relocation relocation = getRelocation(object.toAddress());
+                if (relocation != null) {
+                    classRef = relocation.classBackup;
+                }
+                RuntimeClass cls = RuntimeClass.unpack(classRef);
+                RuntimeObject realObject = object.toAddress().toStructure();
+                updatePointers(cls, realObject);
+                size = objectSize(realObject, cls);
+            } else {
+                size = object.size;
+            }
+
+            object = object.toAddress().add(size).toStructure();
+        }
+    }
+
+    private static void updatePointers(RuntimeClass cls, RuntimeObject object) {
+        if (cls.itemType == null) {
+            updatePointersInObject(cls, object);
+        } else {
+            updatePointersInArray(cls, (RuntimeArray) object);
+        }
+    }
+
+    private static void updatePointersInObject(RuntimeClass cls, RuntimeObject object) {
+        while (cls != null) {
+            int type = (cls.flags >> RuntimeClass.VM_TYPE_SHIFT) & RuntimeClass.VM_TYPE_MASK;
+            switch (type) {
+                case RuntimeClass.VM_TYPE_WEAKREFERENCE:
+                    updatePointersInWeakReference((RuntimeReference) object);
+                    break;
+
+                case RuntimeClass.VM_TYPE_REFERENCEQUEUE:
+                    updatePointersInReferenceQueue((RuntimeReferenceQueue) object);
+                    break;
+
+                default:
+                    updatePointersInFields(cls, object);
+                    break;
+            }
+            cls = cls.parent;
+        }
+    }
+
+    private static void updatePointersInWeakReference(RuntimeReference object) {
+        object.queue = updatePointer(object.queue.toAddress()).toStructure();
+        object.next = updatePointer(object.next.toAddress()).toStructure();
+        object.object = updatePointer(object.object.toAddress()).toStructure();
+    }
+
+    private static void updatePointersInReferenceQueue(RuntimeReferenceQueue object) {
+        object.first = updatePointer(object.first.toAddress()).toStructure();
+        object.last = updatePointer(object.last.toAddress()).toStructure();
+    }
+
+    private static void updatePointersInFields(RuntimeClass cls, RuntimeObject object) {
+        Address layout = cls.layout;
+        if (layout != null) {
+            short fieldCount = layout.getShort();
+            while (fieldCount-- > 0) {
+                layout = layout.add(2);
+                int fieldOffset = layout.getShort();
+                Address referenceHolder = object.toAddress().add(fieldOffset);
+                referenceHolder.putAddress(updatePointer(referenceHolder.getAddress()));
+            }
+        }
+    }
+
+    private static void updatePointersInArray(RuntimeClass cls, RuntimeArray array) {
+        if ((cls.itemType.flags & RuntimeClass.PRIMITIVE) != 0) {
+            return;
+        }
+        Address base = Address.align(array.toAddress().add(RuntimeArray.class, 1), Address.sizeOf());
+        int size = array.size;
+        for (int i = 0; i < size; ++i) {
+            base.putAddress(updatePointer(base.getAddress()));
+            base = base.add(Address.sizeOf());
+        }
+    }
+
+    private static Address updatePointer(Address address) {
+        if (address == null) {
+            return null;
+        }
+        Relocation relocation = getRelocation(address);
+        return relocation != null ? relocation.newAddress : address;
+    }
+
+    private static Relocation getRelocation(Address address) {
+        if (address.isLessThan(heapAddress()) || !address.isLessThan(heapAddress().add(availableBytes()))) {
+            return null;
+        }
+        FreeChunk obj = address.toStructure();
+        if ((obj.classReference & RuntimeObject.GC_MARKED) == 0) {
+            return null;
+        }
+        long result = (((long) obj.classReference & 0xFFFFFFFFL) << 33) | (((long) obj.size & 0xFFFFFFFFL) << 1);
+        return Address.fromLong(result).toStructure();
+    }
+
+    private static void restoreObjectHeaders() {
+        Address start = heapAddress();
+        long heapSize = availableBytes();
+        Address limit = start.add(heapSize);
+
+        FreeChunk freeChunk = currentChunkPointer.value;
+        FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure();
+
+        while (object.toAddress().isLessThan(limit)) {
+            Relocation relocation = getRelocation(object.toAddress());
+            if (relocation != null) {
+                object.classReference = relocation.classBackup | RuntimeObject.GC_MARKED;
+                object.size = relocation.sizeBackup;
+            }
+            int size = objectSize(object);
+            object = object.toAddress().add(size).toStructure();
+        }
+    }
+
+    private static void relocateObjects() {
+        Address start = heapAddress();
+        long heapSize = availableBytes();
+        Address limit = start.add(heapSize);
+
+        int freeChunks = GC.freeChunks;
+        FreeChunk freeChunk = currentChunkPointer.value;
+        FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure();
+
+        RelocationBlock relocationTarget = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks)
+                .toAddress().toStructure();
+        Address currentTarget = relocationTarget.start;
+
+        Address blockTarget = null;
+        Address blockSource = null;
+        int blockSize = 0;
+
+        objects: while (object.toAddress().isLessThan(limit)) {
+            int size = objectSize(object);
+            if ((object.classReference & RuntimeObject.GC_MARKED) != 0) {
+                object.classReference &= ~RuntimeObject.GC_MARKED;
+
+                Address nextRelocationTarget;
+                while (true) {
+                    nextRelocationTarget = currentTarget.add(size);
+                    if (nextRelocationTarget.isLessThan(relocationTarget.end)) {
+                        break;
+                    }
+
+                    if (blockSize != 0) {
+                        Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize);
+                        blockSource = null;
+                        blockSize = 0;
+                    }
+
+                    relocationTarget = Structure.add(RelocationBlock.class, relocationTarget, 1);
+                    if (lastRelocationTarget.toAddress().isLessThan(relocationTarget.toAddress())) {
+                        break objects;
+                    }
+                    currentTarget = relocationTarget.start;
+                }
+
+                if (blockSource == null) {
+                    blockSource = object.toAddress();
+                    blockTarget = currentTarget;
+                }
+                currentTarget = nextRelocationTarget;
+                blockSize += size;
+            } else if (blockSource != null) {
+                Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize);
+                blockSource = null;
+                blockSize = 0;
+            }
+
+            object = object.toAddress().add(size).toStructure();
+        }
+
+        if (blockSource != null) {
+            Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize);
+        }
+    }
+
+    private static void putNewFreeChunks() {
+        FreeChunkHolder freeChunkPointer = currentChunkPointer;
+        RelocationBlock relocationBlock = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks)
+                .toAddress().toStructure();
+        freeChunks = 0;
+        while (!lastRelocationTarget.toAddress().isLessThan(relocationBlock.toAddress())) {
+            if (!relocationBlock.current.isLessThan(relocationBlock.end)) {
+                continue;
+            }
+            FreeChunk freeChunk = relocationBlock.current.toStructure();
+            freeChunk.size = (int) (relocationBlock.end.toLong() - relocationBlock.current.toLong());
+            freeChunk.classReference = 0;
+            freeChunkPointer.value = freeChunk;
+            freeChunkPointer = Structure.add(FreeChunkHolder.class, freeChunkPointer, 1);
+            freeChunks++;
+            relocationBlock = Structure.add(RelocationBlock.class, relocationBlock, 1);
+        }
+    }
+
+    private static void sortFreeChunks() {
+        currentChunkPointer = gcStorageAddress().toStructure();
         sortFreeChunks(0, freeChunks - 1);
         currentChunk = currentChunkPointer.value;
         currentChunkLimit = currentChunk.toAddress().add(currentChunk.size);
@@ -434,23 +784,27 @@ public final class GC {
         if (object.classReference == 0) {
             return object.size;
         } else {
-            RuntimeClass cls = RuntimeClass.getClass(object.toAddress().toStructure());
+            RuntimeObject realObject = object.toAddress().toStructure();
-            if (cls.itemType == null) {
+            RuntimeClass cls = RuntimeClass.getClass(realObject);
-                return cls.size;
+            return objectSize(realObject, cls);
-            } else {
-                int itemSize = (cls.itemType.flags & RuntimeClass.PRIMITIVE) == 0
-                        ? Address.sizeOf()
-                        : cls.itemType.size;
-                RuntimeArray array = object.toAddress().toStructure();
-                Address address = Address.fromInt(Structure.sizeOf(RuntimeArray.class));
-                address = Address.align(address, itemSize);
-                address = address.add(itemSize * array.size);
-                address = Address.align(address, Address.sizeOf());
-                return address.toInt();
-            }
         }
     }
 
+    private static int objectSize(RuntimeObject object, RuntimeClass cls) {
+        if (cls.itemType == null) {
+            return cls.size;
+        }
+        int itemSize = (cls.itemType.flags & RuntimeClass.PRIMITIVE) == 0
+                ? Address.sizeOf()
+                : cls.itemType.size;
+        RuntimeArray array = object.toAddress().toStructure();
+        Address address = Address.fromInt(Structure.sizeOf(RuntimeArray.class));
+        address = Address.align(address, itemSize);
+        address = address.add(itemSize * array.size);
+        address = Address.align(address, Address.sizeOf());
+        return address.toInt();
+    }
+
     private static boolean isMarked(RuntimeObject object) {
         return (object.classReference & RuntimeObject.GC_MARKED) != 0;
     }

core/src/main/java/org/teavm/runtime/Relocation.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2019 konsoletyper.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.runtime;
+
+import org.teavm.interop.Address;
+import org.teavm.interop.Structure;
+
+class Relocation extends Structure {
+    int classBackup;
+    int sizeBackup;
+    Address newAddress;
+}

core/src/main/java/org/teavm/runtime/RelocationBlock.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2019 konsoletyper.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.runtime;
+
+import org.teavm.interop.Address;
+import org.teavm.interop.Structure;
+
+class RelocationBlock extends Structure {
+    Address start;
+    Address current;
+    Address end;
+}

interop/core/src/main/java/org/teavm/interop/Address.java

@@ -89,4 +89,8 @@ public final class Address {
     public static native int sizeOf();
 
     public native Address add(Class<? extends Structure> type, int offset);
+
+    public long diff(Address that) {
+        return toLong() - that.toLong();
+    }
 }