From 16f4d2571b8d1cf5edefa5240e8f3b346889af7d Mon Sep 17 00:00:00 2001 From: Alexey Andreev Date: Thu, 7 Nov 2019 19:05:31 +0300 Subject: [PATCH] Support generations in GC --- .../org/teavm/classlib/java/lang/TSystem.java | 3 +- .../java/org/teavm/backend/c/CTarget.java | 9 + .../c/intrinsic/AllocatorIntrinsic.java | 11 + .../backend/c/intrinsic/GCIntrinsic.java | 28 +- .../c/intrinsic/MemoryTraceIntrinsic.java | 1 + .../java/org/teavm/backend/wasm/WasmHeap.java | 8 +- .../org/teavm/backend/wasm/WasmRuntime.java | 29 +- .../org/teavm/backend/wasm/WasmTarget.java | 3 + .../wasm/intrinsics/AllocatorIntrinsic.java | 2 + .../backend/wasm/intrinsics/GCIntrinsic.java | 25 + .../java/org/teavm/dependency/Linker.java | 32 +- .../lowlevel/GCShadowStackContributor.java | 52 +- .../model/lowlevel/SpilledPhisFinder.java | 31 +- .../model/lowlevel/WriteBarrierInsertion.java | 236 ++++++ .../transformation/ClassInitInsertion.java | 168 ++++ .../org/teavm/model/util/DominatorWalker.java | 112 ++- .../model/util/DominatorWalkerCallback.java | 3 +- .../model/util/DominatorWalkerContext.java | 38 + .../java/org/teavm/runtime/Allocator.java | 6 +- core/src/main/java/org/teavm/runtime/GC.java | 745 +++++++++++++++--- .../java/org/teavm/runtime/MemoryTrace.java | 14 +- .../java/org/teavm/runtime/RuntimeObject.java | 1 + core/src/main/java/org/teavm/vm/TeaVM.java | 2 +- .../main/resources/org/teavm/backend/c/core.h | 8 +- .../org/teavm/backend/c/definitions.h | 4 + .../resources/org/teavm/backend/c/heaptrace.c | 206 ++++- .../resources/org/teavm/backend/c/heaptrace.h | 10 +- .../resources/org/teavm/backend/c/memory.c | 16 +- .../resources/org/teavm/backend/c/memory.h | 3 +- .../org/teavm/backend/c/references.c | 3 + .../org/teavm/backend/c/stringhash.c | 4 +- 31 files changed, 1599 insertions(+), 214 deletions(-) create mode 100644 core/src/main/java/org/teavm/model/lowlevel/WriteBarrierInsertion.java create mode 100644 core/src/main/java/org/teavm/model/transformation/ClassInitInsertion.java create mode 100644 core/src/main/java/org/teavm/model/util/DominatorWalkerContext.java diff --git a/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java b/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java index 5d2d4a8f5..390a00cbd 100644 --- a/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java +++ b/classlib/src/main/java/org/teavm/classlib/java/lang/TSystem.java @@ -121,6 +121,7 @@ public final class TSystem extends TObject { int itemSize = type.itemType.size; if ((type.itemType.flags & RuntimeClass.PRIMITIVE) == 0) { itemSize = Address.sizeOf(); + GC.writeBarrier(dest); } Address srcAddress = Address.align(src.toAddress().add(RuntimeArray.class, 1), itemSize); @@ -253,7 +254,7 @@ public final class TSystem extends TObject { } private static void gcLowLevel() { - GC.collectGarbage(); + GC.collectGarbageFull(); } public static void runFinalization() { diff --git a/core/src/main/java/org/teavm/backend/c/CTarget.java b/core/src/main/java/org/teavm/backend/c/CTarget.java index 85897150c..cfba2f2b4 100644 --- a/core/src/main/java/org/teavm/backend/c/CTarget.java +++ b/core/src/main/java/org/teavm/backend/c/CTarget.java @@ -118,6 +118,7 @@ import org.teavm.model.lowlevel.ClassInitializerTransformer; import org.teavm.model.lowlevel.ExportDependencyListener; import org.teavm.model.lowlevel.LowLevelNullCheckFilter; import org.teavm.model.lowlevel.ShadowStackTransformer; +import org.teavm.model.lowlevel.WriteBarrierInsertion; import org.teavm.model.optimization.InliningFilterFactory; import org.teavm.model.transformation.BoundCheckInsertion; import org.teavm.model.transformation.ClassPatch; @@ -155,6 +156,7 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { private ClassInitializerEliminator classInitializerEliminator; private ClassInitializerTransformer classInitializerTransformer; private ShadowStackTransformer shadowStackTransformer; + private WriteBarrierInsertion writeBarrierInsertion; private NullCheckInsertion nullCheckInsertion; private BoundCheckInsertion boundCheckInsertion = new BoundCheckInsertion(); private CheckInstructionTransformation checkTransformation; @@ -235,6 +237,7 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { shadowStackTransformer = new ShadowStackTransformer(characteristics, !longjmpUsed); nullCheckInsertion = new NullCheckInsertion(new LowLevelNullCheckFilter(characteristics)); checkTransformation = new CheckInstructionTransformation(); + writeBarrierInsertion = new WriteBarrierInsertion(characteristics); controller.addVirtualMethods(VIRTUAL_METHODS::contains); } @@ -272,6 +275,7 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { dependencyAnalyzer.linkMethod(new MethodReference(GC.class, "fixHeap", void.class)).use(); dependencyAnalyzer.linkMethod(new MethodReference(GC.class, "tryShrink", void.class)).use(); dependencyAnalyzer.linkMethod(new MethodReference(GC.class, "collectGarbage", void.class)).use(); + dependencyAnalyzer.linkMethod(new MethodReference(GC.class, "collectGarbageFull", void.class)).use(); dependencyAnalyzer.linkMethod(new MethodReference(ExceptionHandling.class, "throwException", Throwable.class, void.class)).use(); @@ -350,6 +354,7 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { ? this.shadowStackTransformer : new ShadowStackTransformer(characteristics, !longjmpUsed); shadowStackTransformer.apply(program, method); + writeBarrierInsertion.apply(program); } @Override @@ -392,6 +397,7 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { stringPool = new SimpleStringPool(); boolean vmAssertions = Boolean.parseBoolean(System.getProperty("teavm.c.vmAssertions", "false")); + boolean gcStats = Boolean.parseBoolean(System.getProperty("teavm.c.gcStats", "false")); GenerationContext context = new GenerationContext(vtableProvider, characteristics, controller.getDependencyInfo(), stringPool, nameProvider, controller.getDiagnostics(), classes, intrinsics, generators, asyncMethods::contains, buildTarget, @@ -417,6 +423,9 @@ public class CTarget implements TeaVMTarget, TeaVMCHost { if (obfuscated) { configHeaderWriter.println("#define TEAVM_OBFUSCATED 1"); } + if (gcStats) { + configHeaderWriter.println("#define TEAVM_GC_STATS 1"); + } ClassGenerator classGenerator = new ClassGenerator(context, tagRegistry, decompiler, controller.getCacheStatus()); diff --git a/core/src/main/java/org/teavm/backend/c/intrinsic/AllocatorIntrinsic.java b/core/src/main/java/org/teavm/backend/c/intrinsic/AllocatorIntrinsic.java index e1aea0a8b..fdb936fed 100644 --- a/core/src/main/java/org/teavm/backend/c/intrinsic/AllocatorIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/c/intrinsic/AllocatorIntrinsic.java @@ -32,6 +32,7 @@ public class AllocatorIntrinsic implements Intrinsic { switch (method.getName()) { case "fillZero": + case "fill": case "moveMemoryBlock": case "isInitialized": return true; @@ -51,6 +52,16 @@ public class AllocatorIntrinsic implements Intrinsic { context.emit(invocation.getArguments().get(1)); context.writer().print(")"); break; + case "fill": + context.includes().addInclude(""); + context.writer().print("memset("); + context.emit(invocation.getArguments().get(0)); + context.writer().print(", "); + context.emit(invocation.getArguments().get(1)); + context.writer().print(", "); + context.emit(invocation.getArguments().get(2)); + context.writer().print(")"); + break; case "moveMemoryBlock": context.includes().addInclude(""); context.writer().print("memmove("); diff --git a/core/src/main/java/org/teavm/backend/c/intrinsic/GCIntrinsic.java b/core/src/main/java/org/teavm/backend/c/intrinsic/GCIntrinsic.java index df6403354..19057b495 100644 --- a/core/src/main/java/org/teavm/backend/c/intrinsic/GCIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/c/intrinsic/GCIntrinsic.java @@ -37,6 +37,8 @@ public class GCIntrinsic implements Intrinsic { case "minAvailableBytes": case "maxAvailableBytes": case "resizeHeap": + case "cardTable": + case "writeBarrier": return true; default: return false; @@ -45,15 +47,23 @@ public class GCIntrinsic implements Intrinsic { @Override public void apply(IntrinsicContext context, InvocationExpr invocation) { - context.includes().includePath("memory.h"); - if (invocation.getMethod().getName().equals("resizeHeap")) { - context.writer().print("teavm_gc_resizeHeap("); - context.emit(invocation.getArguments().get(0)); - context.writer().print(")"); - return; - } + switch (invocation.getMethod().getName()) { + case "resizeHeap": + context.writer().print("teavm_gc_resizeHeap("); + context.emit(invocation.getArguments().get(0)); + context.writer().print(")"); + break; - context.includes().includePath("heaptrace.h"); - context.writer().print("teavm_gc_").print(invocation.getMethod().getName()); + case "writeBarrier": + context.writer().print("teavm_gc_writeBarrier("); + context.emit(invocation.getArguments().get(0)); + context.writer().print(")"); + break; + + default: + context.includes().includePath("heaptrace.h"); + context.writer().print("teavm_gc_").print(invocation.getMethod().getName()); + break; + } } } diff --git a/core/src/main/java/org/teavm/backend/c/intrinsic/MemoryTraceIntrinsic.java b/core/src/main/java/org/teavm/backend/c/intrinsic/MemoryTraceIntrinsic.java index 0f2fa2bb1..2144f8aee 100644 --- a/core/src/main/java/org/teavm/backend/c/intrinsic/MemoryTraceIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/c/intrinsic/MemoryTraceIntrinsic.java @@ -27,6 +27,7 @@ public class MemoryTraceIntrinsic implements Intrinsic { @Override public void apply(IntrinsicContext context, InvocationExpr invocation) { + context.includes().includePath("heaptrace.h"); context.writer().print("teavm_gc_").print(invocation.getMethod().getName()).print("("); if (!invocation.getArguments().isEmpty()) { context.emit(invocation.getArguments().get(0)); diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java b/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java index 12031b00b..d5ac64aab 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java @@ -24,7 +24,7 @@ import org.teavm.interop.Unmanaged; public final class WasmHeap { public static final int PAGE_SIZE = 65536; public static final int DEFAULT_STACK_SIZE = PAGE_SIZE * 4; - public static final int DEFAULT_REGION_SIZE = 32768; + public static final int DEFAULT_REGION_SIZE = 4096; public static int minHeapSize; public static int maxHeapSize; @@ -33,6 +33,7 @@ public final class WasmHeap { public static Address regionsAddress; public static int regionsCount; public static int regionsSize; + public static Address cardTable; public static Address heapAddress; public static int heapSize; public static int regionSize = DEFAULT_REGION_SIZE; @@ -79,7 +80,8 @@ public final class WasmHeap { int newRegionsSize = calculateRegionsSize(newRegionsCount); Address newRegionsAddress = WasmRuntime.align(heapAddress.add(newHeapSize), 16); - Address newStorageAddress = WasmRuntime.align(newRegionsAddress.add(newRegionsSize), 16); + Address newCardTable = WasmRuntime.align(newRegionsAddress.add(newRegionsCount), 16); + Address newStorageAddress = WasmRuntime.align(newCardTable.add(newRegionsSize), 16); Address newMemoryLimit = WasmRuntime.align(newStorageAddress.add(newStorageSize), PAGE_SIZE); if (newMemoryLimit != memoryLimit) { growMemory((int) (newMemoryLimit.toLong() - memoryLimit.toLong()) / PAGE_SIZE); @@ -89,11 +91,13 @@ public final class WasmHeap { WasmRuntime.moveMemoryBlock(storageAddress, newStorageAddress, storageSize); } if (regionsSize > 0) { + WasmRuntime.moveMemoryBlock(cardTable, newCardTable, regionsCount); WasmRuntime.moveMemoryBlock(regionsAddress, newRegionsAddress, regionsSize); } storageAddress = newStorageAddress; regionsAddress = newRegionsAddress; + cardTable = newCardTable; storageSize = newStorageSize; regionsCount = newRegionsCount; regionsSize = newRegionsSize; diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmRuntime.java b/core/src/main/java/org/teavm/backend/wasm/WasmRuntime.java index 0ba488755..6602908e7 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmRuntime.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmRuntime.java @@ -97,23 +97,30 @@ public final class WasmRuntime { public static native void printOutOfMemory(); public static void fillZero(Address address, int count) { + fill(address, (byte) 0, count); + } + + public static void fill(Address address, byte value, int count) { + int value4 = (value & 0xFF << 24) | (value & 0xFF << 16) | (value & 0xFF << 8) | (value & 0xFF); int start = address.toInt(); int alignedStart = start >>> 2 << 2; address = Address.fromInt(alignedStart); switch (start - alignedStart) { case 0: - address.putInt(0); + address.putInt(value4); break; case 1: - address.add(1).putByte((byte) 0); - address.add(2).putShort((short) 0); + address.add(1).putByte(value); + address.add(2).putByte(value); + address.add(3).putByte(value); break; case 2: - address.add(2).putShort((short) 0); + address.add(2).putByte(value); + address.add(3).putByte(value); break; case 3: - address.add(3).putByte((byte) 0); + address.add(3).putByte(value); break; } @@ -124,19 +131,21 @@ public final class WasmRuntime { case 0: break; case 1: - address.putByte((byte) 0); + address.putByte(value); break; case 2: - address.putShort((short) 0); + address.putByte(value); + address.add(1).putByte(value); break; case 3: - address.putShort((short) 0); - address.add(2).putByte((byte) 0); + address.putByte(value); + address.add(1).putByte(value); + address.add(2).putByte(value); break; } for (address = Address.fromInt(alignedStart + 4); address.toInt() < alignedEnd; address = address.add(4)) { - address.putInt(0); + address.putInt(value4); } } diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java b/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java index 81f2aa0be..12de1f6ca 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java @@ -272,6 +272,8 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "align", Address.class, int.class, Address.class)).use(); + dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "fill", Address.class, int.class, + int.class, void.class)).use(); dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "fillZero", Address.class, int.class, void.class)).use(); dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "moveMemoryBlock", Address.class, @@ -821,6 +823,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { address = WasmRuntime.align(address + WasmHeap.DEFAULT_STACK_SIZE, 16); address = WasmRuntime.align(address + maxHeapSize, 16); address = WasmRuntime.align(address + newRegionsSize, 16); + address = WasmRuntime.align(address + newRegionsCount, 16); address = WasmRuntime.align(address + newStorageSize, 16); gcIntrinsic.setRegionSize(WasmHeap.DEFAULT_REGION_SIZE); diff --git a/core/src/main/java/org/teavm/backend/wasm/intrinsics/AllocatorIntrinsic.java b/core/src/main/java/org/teavm/backend/wasm/intrinsics/AllocatorIntrinsic.java index d59f7d220..4c3d584ff 100644 --- a/core/src/main/java/org/teavm/backend/wasm/intrinsics/AllocatorIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/wasm/intrinsics/AllocatorIntrinsic.java @@ -46,6 +46,7 @@ public class AllocatorIntrinsic implements WasmIntrinsic { return false; } switch (methodReference.getName()) { + case "fill": case "fillZero": case "moveMemoryBlock": case "isInitialized": @@ -58,6 +59,7 @@ public class AllocatorIntrinsic implements WasmIntrinsic { @Override public WasmExpression apply(InvocationExpr invocation, WasmIntrinsicManager manager) { switch (invocation.getMethod().getName()) { + case "fill": case "fillZero": case "moveMemoryBlock": { MethodReference delegateMethod = new MethodReference(WasmRuntime.class.getName(), diff --git a/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java b/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java index 3ec7ea0e7..6b885ab77 100644 --- a/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java @@ -27,7 +27,11 @@ import org.teavm.backend.wasm.model.expression.WasmConversion; import org.teavm.backend.wasm.model.expression.WasmExpression; import org.teavm.backend.wasm.model.expression.WasmInt32Constant; import org.teavm.backend.wasm.model.expression.WasmInt32Subtype; +import org.teavm.backend.wasm.model.expression.WasmIntBinary; +import org.teavm.backend.wasm.model.expression.WasmIntBinaryOperation; +import org.teavm.backend.wasm.model.expression.WasmIntType; import org.teavm.backend.wasm.model.expression.WasmLoadInt32; +import org.teavm.backend.wasm.model.expression.WasmStoreInt32; import org.teavm.backend.wasm.model.expression.WasmUnreachable; import org.teavm.model.FieldReference; import org.teavm.model.MethodReference; @@ -38,6 +42,8 @@ public class GCIntrinsic implements WasmIntrinsic { WasmRuntime.class, "printOutOfMemory", void.class); private static final MethodReference RESIZE_HEAP = new MethodReference( WasmHeap.class, "resizeHeap", int.class, void.class); + private static final FieldReference CARD_TABLE = new FieldReference(WasmHeap.class.getName(), "cardTable"); + private static final FieldReference HEAP_ADDRESS = new FieldReference(WasmHeap.class.getName(), "heapAddress"); private List regionSizeExpressions = new ArrayList<>(); public void setRegionSize(int regionSize) { @@ -58,12 +64,14 @@ public class GCIntrinsic implements WasmIntrinsic { case "heapAddress": case "availableBytes": case "regionsAddress": + case "cardTable": case "regionMaxCount": case "regionSize": case "outOfMemory": case "minAvailableBytes": case "maxAvailableBytes": case "resizeHeap": + case "writeBarrier": return true; default: return false; @@ -81,6 +89,8 @@ public class GCIntrinsic implements WasmIntrinsic { return getStaticField(manager, "heapAddress"); case "regionsAddress": return getStaticField(manager, "regionsAddress"); + case "cardTable": + return getStaticField(manager, "cardTable"); case "regionMaxCount": return getStaticField(manager, "regionsCount"); case "minAvailableBytes": @@ -106,6 +116,21 @@ public class GCIntrinsic implements WasmIntrinsic { block.getBody().add(new WasmUnreachable()); return block; } + case "writeBarrier": { + WasmExpression cardTableField = new WasmInt32Constant(manager.getStaticField(CARD_TABLE)); + WasmExpression cardTable = new WasmLoadInt32(4, cardTableField, WasmInt32Subtype.INT32); + WasmExpression heapAddressField = new WasmInt32Constant(manager.getStaticField(HEAP_ADDRESS)); + WasmExpression heapAddress = new WasmLoadInt32(4, heapAddressField, WasmInt32Subtype.INT32); + WasmInt32Constant regionSize = new WasmInt32Constant(0); + regionSizeExpressions.add(regionSize); + WasmExpression offsetInHeap = new WasmIntBinary(WasmIntType.INT32, WasmIntBinaryOperation.SUB, + manager.generate(invocation.getArguments().get(0)), heapAddress); + WasmExpression cardIndex = new WasmIntBinary(WasmIntType.INT32, WasmIntBinaryOperation.DIV_SIGNED, + offsetInHeap, regionSize); + WasmExpression card = new WasmIntBinary(WasmIntType.INT32, WasmIntBinaryOperation.ADD, cardTable, + cardIndex); + return new WasmStoreInt32(1, card, new WasmInt32Constant(0), WasmInt32Subtype.INT8); + } default: throw new IllegalArgumentException(invocation.getMethod().toString()); } diff --git a/core/src/main/java/org/teavm/dependency/Linker.java b/core/src/main/java/org/teavm/dependency/Linker.java index 64a7c9d7b..e5ed36bde 100644 --- a/core/src/main/java/org/teavm/dependency/Linker.java +++ b/core/src/main/java/org/teavm/dependency/Linker.java @@ -23,23 +23,23 @@ import org.teavm.model.ElementModifier; import org.teavm.model.FieldHolder; import org.teavm.model.FieldReference; import org.teavm.model.Instruction; -import org.teavm.model.MethodDescriptor; import org.teavm.model.MethodHolder; import org.teavm.model.MethodReader; import org.teavm.model.MethodReference; import org.teavm.model.Program; import org.teavm.model.instructions.GetFieldInstruction; -import org.teavm.model.instructions.InitClassInstruction; import org.teavm.model.instructions.InvocationType; import org.teavm.model.instructions.InvokeInstruction; import org.teavm.model.instructions.PutFieldInstruction; +import org.teavm.model.transformation.ClassInitInsertion; public class Linker { - private static final MethodDescriptor clinitDescriptor = new MethodDescriptor("", void.class); private DependencyInfo dependency; + private ClassInitInsertion classInitInsertion; public Linker(DependencyInfo dependency) { this.dependency = dependency; + classInitInsertion = new ClassInitInsertion(dependency); } public void link(ClassHolder cls) { @@ -55,7 +55,7 @@ public class Linker { method.setProgram(null); } } else if (method.getProgram() != null) { - link(method.getReference(), method.getProgram()); + link(method, method.getProgram()); } } for (FieldHolder field : cls.getFields().toArray(new FieldHolder[0])) { @@ -66,7 +66,7 @@ public class Linker { } } - public void link(MethodReference method, Program program) { + public void link(MethodReader method, Program program) { for (int i = 0; i < program.basicBlockCount(); ++i) { BasicBlock block = program.basicBlockAt(i); for (Instruction insn : block) { @@ -111,36 +111,16 @@ public class Linker { getField.setField(linkedField.getReference()); } - FieldReference fieldRef = getField.getField(); - if (getField.getInstance() == null) { - insertClinit(dependency, fieldRef.getClassName(), method, insn); - } } else if (insn instanceof PutFieldInstruction) { PutFieldInstruction putField = (PutFieldInstruction) insn; FieldDependencyInfo linkedField = dependency.getField(putField.getField()); if (linkedField != null) { putField.setField(linkedField.getReference()); } - - FieldReference fieldRef = putField.getField(); - if (putField.getInstance() == null) { - insertClinit(dependency, fieldRef.getClassName(), method, insn); - } } } } - } - private void insertClinit(DependencyInfo dependency, String className, MethodReference method, Instruction insn) { - if (className.equals(method.getClassName())) { - return; - } - ClassReader cls = dependency.getClassSource().get(className); - if (cls == null || cls.getMethod(clinitDescriptor) != null) { - InitClassInstruction initInsn = new InitClassInstruction(); - initInsn.setClassName(className); - initInsn.setLocation(insn.getLocation()); - insn.insertPrevious(initInsn); - } + classInitInsertion.apply(program, method); } } diff --git a/core/src/main/java/org/teavm/model/lowlevel/GCShadowStackContributor.java b/core/src/main/java/org/teavm/model/lowlevel/GCShadowStackContributor.java index 988d0fa96..0d794264f 100644 --- a/core/src/main/java/org/teavm/model/lowlevel/GCShadowStackContributor.java +++ b/core/src/main/java/org/teavm/model/lowlevel/GCShadowStackContributor.java @@ -33,14 +33,13 @@ import org.teavm.common.Graph; import org.teavm.common.GraphBuilder; import org.teavm.common.GraphUtils; import org.teavm.model.BasicBlock; -import org.teavm.model.Incoming; import org.teavm.model.Instruction; import org.teavm.model.MethodReader; import org.teavm.model.MethodReference; -import org.teavm.model.Phi; import org.teavm.model.Program; import org.teavm.model.Variable; import org.teavm.model.instructions.AssignInstruction; +import org.teavm.model.instructions.CastInstruction; import org.teavm.model.instructions.ClassConstantInstruction; import org.teavm.model.instructions.IntegerConstantInstruction; import org.teavm.model.instructions.InvocationType; @@ -48,6 +47,7 @@ import org.teavm.model.instructions.InvokeInstruction; import org.teavm.model.instructions.NullCheckInstruction; import org.teavm.model.instructions.NullConstantInstruction; import org.teavm.model.instructions.StringConstantInstruction; +import org.teavm.model.instructions.UnwrapArrayInstruction; import org.teavm.model.util.DefinitionExtractor; import org.teavm.model.util.GraphColorer; import org.teavm.model.util.LivenessAnalyzer; @@ -98,28 +98,38 @@ public class GCShadowStackContributor { // for this phi. Graph cfg = ProgramUtils.buildControlFlowGraph(program); DominatorTree dom = GraphUtils.buildDominatorTree(cfg); - boolean[] autoSpilled = new SpilledPhisFinder(liveInInformation, dom, program).find(); + boolean[] autoSpilled = new SpilledPhisFinder(liveInInformation, dom, program, variableClasses, colors).find(); List> liveInStores = reduceGCRootStores(dom, program, usedColors, liveInInformation, - colors, autoSpilled); + colors, autoSpilled, variableClasses); putLiveInGCRoots(program, liveInStores); return usedColors; } private int[] getVariableClasses(Program program) { - DisjointSet variableClasses = new DisjointSet(); + DisjointSet disjointSet = new DisjointSet(); for (int i = 0; i < program.variableCount(); ++i) { - variableClasses.create(); + disjointSet.create(); } for (BasicBlock block : program.getBasicBlocks()) { - for (Phi phi : block.getPhis()) { - for (Incoming incoming : phi.getIncomings()) { - variableClasses.union(phi.getReceiver().getIndex(), incoming.getValue().getIndex()); + for (Instruction instruction : block) { + if (instruction instanceof AssignInstruction) { + AssignInstruction assign = (AssignInstruction) instruction; + disjointSet.union(assign.getAssignee().getIndex(), assign.getReceiver().getIndex()); + } else if (instruction instanceof NullCheckInstruction) { + NullCheckInstruction nullCheck = (NullCheckInstruction) instruction; + disjointSet.union(nullCheck.getValue().getIndex(), nullCheck.getReceiver().getIndex()); + } else if (instruction instanceof CastInstruction) { + CastInstruction cast = (CastInstruction) instruction; + disjointSet.union(cast.getValue().getIndex(), cast.getReceiver().getIndex()); + } else if (instruction instanceof UnwrapArrayInstruction) { + UnwrapArrayInstruction unwrapArray = (UnwrapArrayInstruction) instruction; + disjointSet.union(unwrapArray.getArray().getIndex(), unwrapArray.getReceiver().getIndex()); } } } - return variableClasses.pack(program.variableCount()); + return disjointSet.pack(program.variableCount()); } private List> findCallSiteLiveIns(Program program, MethodReader method) { @@ -207,7 +217,8 @@ public class GCShadowStackContributor { } private List> reduceGCRootStores(DominatorTree dom, Program program, int usedColors, - List> liveInInformation, int[] colors, boolean[] autoSpilled) { + List> liveInInformation, int[] colors, boolean[] autoSpilled, + int[] variableClasses) { class Step { private final int node; private final int[] slotStates = new int[usedColors]; @@ -228,7 +239,6 @@ public class GCShadowStackContributor { Step start = new Step(0); Arrays.fill(start.slotStates, program.variableCount()); stack[head++] = start; - int[] definitionClasses = getDefinitionClasses(program); while (head > 0) { Step step = stack[--head]; @@ -250,7 +260,7 @@ public class GCShadowStackContributor { } } - int[] updates = compareStates(previousStates, states, autoSpilled, definitionClasses); + int[] updates = compareStates(previousStates, states, autoSpilled, variableClasses); updatesByCallSite.put(callSiteLocation, updates); previousStates = states; states = states.clone(); @@ -266,22 +276,6 @@ public class GCShadowStackContributor { return slotsToUpdate; } - private int[] getDefinitionClasses(Program program) { - DisjointSet disjointSet = new DisjointSet(); - for (int i = 0; i < program.variableCount(); ++i) { - disjointSet.create(); - } - for (BasicBlock block : program.getBasicBlocks()) { - for (Instruction instruction : block) { - if (instruction instanceof NullCheckInstruction) { - NullCheckInstruction nullCheck = (NullCheckInstruction) instruction; - disjointSet.union(nullCheck.getValue().getIndex(), nullCheck.getReceiver().getIndex()); - } - } - } - return disjointSet.pack(program.variableCount()); - } - private List sortInstructions(Collection instructions, BasicBlock block) { ObjectIntMap indexes = new ObjectIntHashMap<>(); int index = 0; diff --git a/core/src/main/java/org/teavm/model/lowlevel/SpilledPhisFinder.java b/core/src/main/java/org/teavm/model/lowlevel/SpilledPhisFinder.java index 9d19320ff..8aaa16be8 100644 --- a/core/src/main/java/org/teavm/model/lowlevel/SpilledPhisFinder.java +++ b/core/src/main/java/org/teavm/model/lowlevel/SpilledPhisFinder.java @@ -37,26 +37,33 @@ class SpilledPhisFinder { int[][] variableSpilledBlocks; Phi[] definingPhis; int variableCount; + private int[] variableClasses; + private int[] colors; - SpilledPhisFinder(List> liveInInformation, DominatorTree dom, Program program) { + SpilledPhisFinder(List> liveInInformation, DominatorTree dom, Program program, + int[] variableClasses, int[] colors) { this.dom = dom; variableCount = program.variableCount(); autoSpilled = new boolean[variableCount]; status = new byte[variableCount]; - variableSpilledBlocks = variableSpilledBlocks(liveInInformation, variableCount); - definingPhis = findPhis(program); + variableSpilledBlocks = variableSpilledBlocks(liveInInformation, variableCount, variableClasses); + definingPhis = findPhis(program, variableClasses); + this.variableClasses = variableClasses; + this.colors = colors; } - private static int[][] variableSpilledBlocks(List> liveInInformation, int count) { + private static int[][] variableSpilledBlocks(List> liveInInformation, int count, + int[] variableClasses) { IntSet[] builder = new IntSet[count]; for (int b = 0; b < liveInInformation.size(); b++) { Map blockLiveIn = liveInInformation.get(b); for (BitSet liveVarsSet : blockLiveIn.values()) { for (int v = liveVarsSet.nextSetBit(0); v >= 0; v = liveVarsSet.nextSetBit(v + 1)) { - if (builder[v] == null) { - builder[v] = new IntHashSet(); + int cls = variableClasses[v]; + if (builder[cls] == null) { + builder[cls] = new IntHashSet(); } - builder[v].add(b); + builder[cls].add(b); } } } @@ -71,11 +78,11 @@ class SpilledPhisFinder { return result; } - private static Phi[] findPhis(Program program) { + private static Phi[] findPhis(Program program, int[] variableClasses) { Phi[] result = new Phi[program.variableCount()]; for (BasicBlock block : program.getBasicBlocks()) { for (Phi phi : block.getPhis()) { - result[phi.getReceiver().getIndex()] = phi; + result[variableClasses[phi.getReceiver().getIndex()]] = phi; } } return result; @@ -89,6 +96,8 @@ class SpilledPhisFinder { } private boolean isAutoSpilled(int v) { + v = variableClasses[v]; + if (status[v] == VISITED) { return autoSpilled[v]; } @@ -107,6 +116,10 @@ class SpilledPhisFinder { boolean result = true; for (Incoming incoming : definingPhi.getIncomings()) { + if (colors[incoming.getValue().getIndex()] != colors[definingPhi.getReceiver().getIndex()]) { + result = false; + break; + } if (!isAutoSpilled(incoming.getValue().getIndex())) { int[] spilledAt = variableSpilledBlocks[incoming.getValue().getIndex()]; result = false; diff --git a/core/src/main/java/org/teavm/model/lowlevel/WriteBarrierInsertion.java b/core/src/main/java/org/teavm/model/lowlevel/WriteBarrierInsertion.java new file mode 100644 index 000000000..dd3efa77c --- /dev/null +++ b/core/src/main/java/org/teavm/model/lowlevel/WriteBarrierInsertion.java @@ -0,0 +1,236 @@ +/* + * Copyright 2019 Alexey Andreev. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.model.lowlevel; + +import com.carrotsearch.hppc.IntArrayList; +import com.carrotsearch.hppc.IntHashSet; +import com.carrotsearch.hppc.IntSet; +import com.carrotsearch.hppc.cursors.IntCursor; +import org.teavm.model.BasicBlock; +import org.teavm.model.Instruction; +import org.teavm.model.MethodReference; +import org.teavm.model.Phi; +import org.teavm.model.Program; +import org.teavm.model.ValueType; +import org.teavm.model.Variable; +import org.teavm.model.instructions.AbstractInstructionVisitor; +import org.teavm.model.instructions.ArrayElementType; +import org.teavm.model.instructions.AssignInstruction; +import org.teavm.model.instructions.CastInstruction; +import org.teavm.model.instructions.CloneArrayInstruction; +import org.teavm.model.instructions.ConstructArrayInstruction; +import org.teavm.model.instructions.ConstructInstruction; +import org.teavm.model.instructions.ConstructMultiArrayInstruction; +import org.teavm.model.instructions.InitClassInstruction; +import org.teavm.model.instructions.InvocationType; +import org.teavm.model.instructions.InvokeInstruction; +import org.teavm.model.instructions.MonitorEnterInstruction; +import org.teavm.model.instructions.MonitorExitInstruction; +import org.teavm.model.instructions.NullCheckInstruction; +import org.teavm.model.instructions.PutElementInstruction; +import org.teavm.model.instructions.PutFieldInstruction; +import org.teavm.model.util.DominatorWalker; +import org.teavm.model.util.DominatorWalkerCallback; +import org.teavm.model.util.DominatorWalkerContext; +import org.teavm.runtime.GC; +import org.teavm.runtime.RuntimeObject; + +public class WriteBarrierInsertion { + private static final MethodReference BARRIER_METHOD = new MethodReference(GC.class, "writeBarrier", + RuntimeObject.class, void.class); + private Characteristics characteristics; + + public WriteBarrierInsertion(Characteristics characteristics) { + this.characteristics = characteristics; + } + + public void apply(Program program) { + if (program.basicBlockCount() == 0) { + return; + } + new DominatorWalker(program).walk(new WalkerCallbackImpl()); + } + + class WalkerCallbackImpl extends AbstractInstructionVisitor implements DominatorWalkerCallback { + private DominatorWalkerContext context; + IntHashSet installedBarriers = new IntHashSet(); + State state; + + @Override + public void setContext(DominatorWalkerContext context) { + this.context = context; + } + + @Override + public State visit(BasicBlock block) { + state = new State(); + + if (context.isExceptionHandler(block.getIndex()) || !context.allPredecessorsVisited(block.getIndex())) { + invalidateBarriers(); + } else { + for (Phi phi : block.getPhis()) { + if (phi.getIncomings().stream().allMatch( + incoming -> installedBarriers.contains(incoming.getValue().getIndex()))) { + markAsInstalled(phi.getReceiver().getIndex()); + } + } + } + + for (Instruction instruction : block) { + instruction.acceptVisitor(this); + } + + return state; + } + + @Override + public void endVisit(BasicBlock block, State state) { + if (state.oldBarriers != null) { + installedBarriers.clear(); + installedBarriers.addAll(state.oldBarriers); + } else { + for (IntCursor cursor : state.newBarriers) { + installedBarriers.remove(cursor.value); + } + } + } + + @Override + public void visit(PutFieldInstruction insn) { + if (insn.getInstance() != null && isManagedReferenceType(insn.getFieldType())) { + installBarrier(insn, insn.getInstance()); + } + } + + @Override + public void visit(InvokeInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(ConstructInstruction insn) { + invalidateBarriers(); + markAsInstalled(insn.getReceiver().getIndex()); + } + + @Override + public void visit(InitClassInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(CloneArrayInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(ConstructArrayInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(ConstructMultiArrayInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(MonitorEnterInstruction insn) { + invalidateBarriers(); + } + + @Override + public void visit(MonitorExitInstruction insn) { + invalidateBarriers(); + } + + private boolean isManagedReferenceType(ValueType type) { + if (type instanceof ValueType.Array) { + return true; + } + if (type instanceof ValueType.Object) { + return characteristics.isManaged(((ValueType.Object) type).getClassName()); + } + return false; + } + + @Override + public void visit(PutElementInstruction insn) { + if (insn.getType() == ArrayElementType.OBJECT) { + installBarrier(insn, insn.getArray()); + } + } + + @Override + public void visit(AssignInstruction insn) { + assign(insn.getAssignee(), insn.getReceiver()); + } + + @Override + public void visit(CastInstruction insn) { + assign(insn.getValue(), insn.getReceiver()); + } + + @Override + public void visit(NullCheckInstruction insn) { + assign(insn.getValue(), insn.getReceiver()); + } + + private void assign(Variable from, Variable to) { + if (installedBarriers.contains(from.getIndex())) { + markAsInstalled(to.getIndex()); + } + } + + private void installBarrier(Instruction instruction, Variable variable) { + if (markAsInstalled(variable.getIndex())) { + InvokeInstruction invoke = new InvokeInstruction(); + invoke.setType(InvocationType.SPECIAL); + invoke.setMethod(BARRIER_METHOD); + invoke.setArguments(variable); + invoke.setLocation(instruction.getLocation()); + instruction.insertPrevious(invoke); + } + } + + private boolean markAsInstalled(int index) { + if (!installedBarriers.add(index)) { + return false; + } + if (state.newBarriers != null) { + state.newBarriers.add(index); + } + return true; + } + + private void invalidateBarriers() { + if (state.newBarriers != null) { + state.oldBarriers = new IntArrayList(); + for (IntCursor cursor : installedBarriers) { + if (!state.newBarriers.contains(cursor.value)) { + state.oldBarriers.add(cursor.value); + } + } + state.newBarriers = null; + } + installedBarriers.clear(); + } + } + + static class State { + IntSet newBarriers = new IntHashSet(); + IntArrayList oldBarriers; + } +} diff --git a/core/src/main/java/org/teavm/model/transformation/ClassInitInsertion.java b/core/src/main/java/org/teavm/model/transformation/ClassInitInsertion.java new file mode 100644 index 000000000..3d23b73fd --- /dev/null +++ b/core/src/main/java/org/teavm/model/transformation/ClassInitInsertion.java @@ -0,0 +1,168 @@ +/* + * Copyright 2019 Alexey Andreev. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.model.transformation; + +import java.util.HashSet; +import java.util.Set; +import org.teavm.dependency.DependencyInfo; +import org.teavm.model.BasicBlock; +import org.teavm.model.ClassReader; +import org.teavm.model.ElementModifier; +import org.teavm.model.Instruction; +import org.teavm.model.MethodDescriptor; +import org.teavm.model.MethodReader; +import org.teavm.model.Program; +import org.teavm.model.instructions.AbstractInstructionVisitor; +import org.teavm.model.instructions.GetFieldInstruction; +import org.teavm.model.instructions.InitClassInstruction; +import org.teavm.model.instructions.InvokeInstruction; +import org.teavm.model.instructions.PutFieldInstruction; +import org.teavm.model.util.DominatorWalker; +import org.teavm.model.util.DominatorWalkerCallback; +import org.teavm.model.util.DominatorWalkerContext; + +public class ClassInitInsertion { + private static final MethodDescriptor CLINIT = new MethodDescriptor("", void.class); + private DependencyInfo dependencyInfo; + + public ClassInitInsertion(DependencyInfo dependencyInfo) { + this.dependencyInfo = dependencyInfo; + } + + public void apply(Program program, MethodReader method) { + if (program.basicBlockCount() == 0) { + return; + } + String currentClass = null; + if (method.hasModifier(ElementModifier.STATIC)) { + currentClass = method.getOwnerName(); + } + Visitor visitor = new Visitor(currentClass); + new DominatorWalker(program).walk(visitor); + } + + class Visitor extends AbstractInstructionVisitor implements DominatorWalkerCallback { + private String currentClass; + private DominatorWalkerContext context; + Set initializedClasses = new HashSet<>(); + private State state; + + Visitor(String currentClass) { + this.currentClass = currentClass; + if (currentClass != null) { + initializedClasses.add(currentClass); + } + } + + @Override + public void setContext(DominatorWalkerContext context) { + this.context = context; + } + + @Override + public State visit(BasicBlock block) { + state = new State(); + if (context.isExceptionHandler(block.getIndex())) { + markAllClassesAsNotInitialized(); + if (currentClass != null) { + markClassAsInitialized(currentClass); + } + } + + for (Instruction instruction : block) { + instruction.acceptVisitor(this); + } + + return state; + } + + @Override + public void endVisit(BasicBlock block, State state) { + if (state.oldInitializedClasses != null) { + initializedClasses.clear(); + initializedClasses.addAll(state.oldInitializedClasses); + } else { + initializedClasses.removeAll(state.newInitializedClasses); + } + } + + @Override + public void visit(GetFieldInstruction insn) { + if (insn.getInstance() == null) { + initializeClass(insn.getField().getClassName(), insn); + } + } + + @Override + public void visit(PutFieldInstruction insn) { + if (insn.getInstance() == null) { + initializeClass(insn.getField().getClassName(), insn); + } + } + + @Override + public void visit(InvokeInstruction insn) { + if (insn.getInstance() == null) { + markClassAsInitialized(insn.getMethod().getClassName()); + } + } + + @Override + public void visit(InitClassInstruction insn) { + markClassAsInitialized(insn.getClassName()); + } + + private void initializeClass(String className, Instruction instruction) { + if (markClassAsInitialized(className)) { + ClassReader cls = dependencyInfo.getClassSource().get(className); + if (cls == null || cls.getMethod(CLINIT) != null) { + InitClassInstruction initInsn = new InitClassInstruction(); + initInsn.setClassName(className); + initInsn.setLocation(instruction.getLocation()); + instruction.insertPrevious(initInsn); + } + } + } + + boolean markClassAsInitialized(String className) { + if (initializedClasses.add(className)) { + if (state.newInitializedClasses != null) { + state.newInitializedClasses.add(className); + } + return true; + } + return false; + } + + private void markAllClassesAsNotInitialized() { + if (state.newInitializedClasses != null) { + state.oldInitializedClasses = new HashSet<>(); + for (String className : initializedClasses) { + if (!state.newInitializedClasses.contains(className)) { + state.oldInitializedClasses.add(className); + } + } + state.newInitializedClasses = null; + } + initializedClasses.clear(); + } + } + + static class State { + Set newInitializedClasses = new HashSet<>(); + Set oldInitializedClasses; + } +} diff --git a/core/src/main/java/org/teavm/model/util/DominatorWalker.java b/core/src/main/java/org/teavm/model/util/DominatorWalker.java index a0d15620a..d3502b491 100644 --- a/core/src/main/java/org/teavm/model/util/DominatorWalker.java +++ b/core/src/main/java/org/teavm/model/util/DominatorWalker.java @@ -15,29 +15,70 @@ */ package org.teavm.model.util; +import com.carrotsearch.hppc.IntStack; import org.teavm.common.DominatorTree; import org.teavm.common.Graph; import org.teavm.common.GraphUtils; import org.teavm.model.BasicBlock; import org.teavm.model.Program; +import org.teavm.model.TryCatchBlock; public class DominatorWalker { private Program program; private DominatorTree dom; + private Graph cfg; private Graph domGraph; + private int[] order; public DominatorWalker(Program program) { this.program = program; - Graph cfg = ProgramUtils.buildControlFlowGraph(program); + cfg = ProgramUtils.buildControlFlowGraph(program); dom = GraphUtils.buildDominatorTree(cfg); domGraph = GraphUtils.buildDominatorGraph(dom, cfg.size()); + order = dfs(cfg); + } + + private int[] dfs(Graph graph) { + if (graph.size() == 0) { + return new int[0]; + } + + int index = 0; + int[] result = new int[graph.size()]; + IntStack stack = new IntStack(graph.size()); + byte[] state = new byte[graph.size()]; + stack.push(0); + + while (!stack.isEmpty()) { + int node = stack.pop(); + switch (state[node]) { + case 0: { + state[node] = 1; + stack.push(node); + for (int succ : graph.outgoingEdges(node)) { + if (state[succ] == 0) { + stack.push(succ); + } + } + break; + } + case 1: { + state[node] = 2; + result[node] = index++; + break; + } + } + } + + return result; } public void walk(DominatorWalkerCallback callback) { int[] stack = new int[program.basicBlockCount() * 2]; Object[] stateStack = new Object[stack.length]; boolean[] backward = new boolean[stack.length]; - callback.setDomTree(dom); + ContextImpl context = new ContextImpl(dom, cfg, findExceptionHandlers()); + callback.setContext(context); int head = 1; while (head > 0) { @@ -47,6 +88,7 @@ public class DominatorWalker { @SuppressWarnings("unchecked") T state = (T) stateStack[head]; callback.endVisit(block, state); + context.visited[block.getIndex()] = true; } else if (callback.filter(block)) { stack[head] = node; backward[head] = true; @@ -54,6 +96,7 @@ public class DominatorWalker { head++; int[] successors = domGraph.outgoingEdges(node); + sort(successors); for (int i = successors.length - 1; i >= 0; --i) { stack[head] = successors[i]; backward[head] = false; @@ -62,4 +105,69 @@ public class DominatorWalker { } } } + + private boolean[] findExceptionHandlers() { + boolean[] exceptionHandlers = new boolean[program.basicBlockCount()]; + for (BasicBlock block : program.getBasicBlocks()) { + for (TryCatchBlock tryCatch : block.getTryCatchBlocks()) { + exceptionHandlers[tryCatch.getHandler().getIndex()] = true; + } + } + return exceptionHandlers; + } + + private void sort(int[] nodes) { + for (int i = 0; i < nodes.length - 1; ++i) { + int a = nodes[i]; + int bestIndex = i; + int best = order[a]; + for (int j = i + 1; j < nodes.length; ++j) { + int b = nodes[j]; + int score = order[b]; + if (score < best) { + best = score; + a = b; + bestIndex = j; + } + } + if (i != bestIndex) { + nodes[bestIndex] = nodes[i]; + nodes[i] = a; + } + } + } + + static class ContextImpl implements DominatorWalkerContext { + private boolean[] visited; + private boolean[] isExceptionHandler; + private DominatorTree dom; + private Graph cfg; + + ContextImpl(DominatorTree dom, Graph cfg, boolean[] isExceptionHandler) { + this.dom = dom; + this.cfg = cfg; + this.isExceptionHandler = isExceptionHandler; + visited = new boolean[cfg.size()]; + } + + @Override + public DominatorTree getDominatorTree() { + return dom; + } + + @Override + public Graph getControlFlowGraph() { + return cfg; + } + + @Override + public boolean isVisited(int blockIndex) { + return visited[blockIndex]; + } + + @Override + public boolean isExceptionHandler(int blockIndex) { + return isExceptionHandler[blockIndex]; + } + } } diff --git a/core/src/main/java/org/teavm/model/util/DominatorWalkerCallback.java b/core/src/main/java/org/teavm/model/util/DominatorWalkerCallback.java index 2c8faf163..0af0b6152 100644 --- a/core/src/main/java/org/teavm/model/util/DominatorWalkerCallback.java +++ b/core/src/main/java/org/teavm/model/util/DominatorWalkerCallback.java @@ -15,7 +15,6 @@ */ package org.teavm.model.util; -import org.teavm.common.DominatorTree; import org.teavm.model.BasicBlock; /** @@ -24,7 +23,7 @@ import org.teavm.model.BasicBlock; * @param type of state that can be saved for each visited node. */ public interface DominatorWalkerCallback { - default void setDomTree(DominatorTree domTree) { + default void setContext(DominatorWalkerContext context) { } /** diff --git a/core/src/main/java/org/teavm/model/util/DominatorWalkerContext.java b/core/src/main/java/org/teavm/model/util/DominatorWalkerContext.java new file mode 100644 index 000000000..88226f2ba --- /dev/null +++ b/core/src/main/java/org/teavm/model/util/DominatorWalkerContext.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Alexey Andreev. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.model.util; + +import org.teavm.common.DominatorTree; +import org.teavm.common.Graph; + +public interface DominatorWalkerContext { + DominatorTree getDominatorTree(); + + Graph getControlFlowGraph(); + + boolean isVisited(int blockIndex); + + boolean isExceptionHandler(int blockIndex); + + default boolean allPredecessorsVisited(int blockIndex) { + for (int predecessor : getControlFlowGraph().incomingEdges(blockIndex)) { + if (!isVisited(predecessor)) { + return false; + } + } + return true; + } +} diff --git a/core/src/main/java/org/teavm/runtime/Allocator.java b/core/src/main/java/org/teavm/runtime/Allocator.java index 0a14e6c9c..dff9569ce 100644 --- a/core/src/main/java/org/teavm/runtime/Allocator.java +++ b/core/src/main/java/org/teavm/runtime/Allocator.java @@ -21,6 +21,7 @@ import org.teavm.interop.Structure; import org.teavm.interop.Unmanaged; @StaticInit +@Unmanaged public final class Allocator { private Allocator() { } @@ -62,12 +63,11 @@ public final class Allocator { return array; } - @Unmanaged public static native void fillZero(Address address, int count); - @Unmanaged + public static native void fill(Address address, byte value, int count); + public static native void moveMemoryBlock(Address source, Address target, int count); - @Unmanaged public static native boolean isInitialized(Class cls); } diff --git a/core/src/main/java/org/teavm/runtime/GC.java b/core/src/main/java/org/teavm/runtime/GC.java index e96feda9d..c7fc2fbf6 100644 --- a/core/src/main/java/org/teavm/runtime/GC.java +++ b/core/src/main/java/org/teavm/runtime/GC.java @@ -28,15 +28,23 @@ public final class GC { private GC() { } + private static final byte CARD_VALID = 1; + private static final byte CARD_YOUNG_GEN = 2; + private static final byte CARD_GAP = 4; + private static final byte CARD_RELOCATABLE = 8; + static Address currentChunkLimit; static FreeChunk currentChunk; static FreeChunkHolder currentChunkPointer; static int freeChunks; + static int totalChunks; static int freeMemory = (int) availableBytes(); static RuntimeReference firstWeakReference; static FreeChunk lastChunk; static RelocationBlock lastRelocationBlock; + static boolean isFullGC = true; + private static int youngGCCount; static native Address gcStorageAddress(); @@ -46,6 +54,8 @@ public final class GC { private static native Region regionsAddress(); + private static native Address cardTable(); + private static native int regionMaxCount(); public static native long availableBytes(); @@ -58,6 +68,8 @@ public final class GC { private static native int regionSize(); + public static native void writeBarrier(RuntimeObject object); + @Import(name = "teavm_outOfMemory") public static native void outOfMemory(); @@ -73,6 +85,14 @@ public final class GC { currentChunkPointer = gcStorageAddress().toStructure(); currentChunkPointer.value = currentChunk; freeChunks = 1; + totalChunks = 1; + + int regionCount = getRegionCount(); + Allocator.fill(cardTable(), CARD_VALID, regionCount); + } + + private static int getRegionCount() { + return (int) (availableBytes() / regionSize()) + 1; } public static RuntimeObject alloc(int size) { @@ -126,15 +146,38 @@ public final class GC { collectGarbageImpl(0); } + @Export(name = "teavm_gc_collectFull") + public static void collectGarbageFull() { + fixHeap(); + collectGarbageFullImpl(0); + } + + private static void collectGarbageFullImpl(int size) { + triggerFullGC(); + collectGarbageImpl(size); + } + + private static void triggerFullGC() { + isFullGC = true; + int regionsCount = getRegionCount(); + Allocator.fill(cardTable(), (byte) 0, getRegionCount()); + Allocator.fill(regionsAddress().toAddress(), (byte) 0, regionsCount * Structure.sizeOf(Region.class)); + } + private static void collectGarbageImpl(int size) { - MemoryTrace.gcStarted(); - mark(); - processReferences(); - sweep(); - MemoryTrace.sweepCompleted(); - defragment(); - MemoryTrace.defragCompleted(); - updateFreeMemory(); + doCollectGarbage(); + + if (!isFullGC) { + if (++youngGCCount >= 8 && isAboutToExpand(size)) { + triggerFullGC(); + doCollectGarbage(); + youngGCCount = 0; + } + } else { + youngGCCount = 0; + } + isFullGC = false; + long minRequestedSize = 0; if (!hasAvailableChunk(size)) { minRequestedSize = computeMinRequestedSize(size); @@ -142,6 +185,22 @@ public final class GC { resizeHeapIfNecessary(minRequestedSize); currentChunk = currentChunkPointer.value; currentChunkLimit = currentChunk.toAddress().add(currentChunk.size); + + Allocator.fill(cardTable(), CARD_VALID, getRegionCount()); + } + + private static void doCollectGarbage() { + MemoryTrace.gcStarted(isFullGC); + if (!isFullGC) { + storeGapsInCardTable(); + } + mark(); + processReferences(); + sweep(); + defragment(); + updateFreeMemory(); + MemoryTrace.gcCompleted(); + totalChunks = freeChunks; } private static boolean hasAvailableChunk(int size) { @@ -177,16 +236,25 @@ public final class GC { long availableBytes = availableBytes(); long occupiedMemory = availableBytes - freeMemory; if (occupiedMemory < availableBytes / 4) { - collectGarbage(); + collectGarbageFull(); } } private static void mark() { - MemoryTrace.initMark(); + MemoryTrace.markStarted(); firstWeakReference = null; - int regionsCount = (int) ((availableBytes() - 1) / regionSize()) + 1; - Allocator.fillZero(regionsAddress().toAddress(), regionsCount * Structure.sizeOf(Region.class)); + markFromStaticFields(); + markFromClasses(); + markFromStack(); + if (!isFullGC) { + markFromOldGeneration(); + } + + MemoryTrace.markCompleted(); + } + + private static void markFromStaticFields() { Address staticRoots = Mutator.getStaticGCRoots(); int staticCount = staticRoots.getInt(); staticRoots = staticRoots.add(Address.sizeOf()); @@ -197,7 +265,9 @@ public final class GC { } staticRoots = staticRoots.add(Address.sizeOf()); } + } + private static void markFromClasses() { int classCount = Mutator.getClassCount(); Address classPtr = Mutator.getClasses(); for (int i = 0; i < classCount; ++i) { @@ -213,7 +283,9 @@ public final class GC { } classPtr = classPtr.add(Address.sizeOf()); } + } + private static void markFromStack() { for (Address stackRoots = ShadowStack.getStackTop(); stackRoots != null; stackRoots = ShadowStack.getNextStackFrame(stackRoots)) { int count = ShadowStack.getStackRootCount(stackRoots); @@ -226,16 +298,90 @@ public final class GC { } } + private static void markFromOldGeneration() { + int validMask = CARD_VALID | (CARD_VALID << 8) | (CARD_VALID << 16) | (CARD_VALID << 24); + int regionsCount = getRegionCount(); + int regionSize = regionSize(); + + Address cardPtr = cardTable(); + Address regionPtr = heapAddress(); + int regionIndex; + for (regionIndex = 0; regionIndex < regionsCount - 3; regionIndex += 4) { + int n = cardPtr.getInt(); + if ((n & validMask) != validMask) { + for (int i = 0; i < 4; ++i) { + n = cardPtr.add(i).getByte(); + if ((n & CARD_VALID) == 0) { + markFromRegion(regionIndex + i); + } + } + } + cardPtr = cardPtr.add(4); + regionPtr = regionPtr.add(4 * regionSize); + } + + for (; regionIndex < regionsCount; regionIndex++) { + if ((cardPtr.getByte() & CARD_VALID) == 0) { + markFromRegion(regionIndex); + } + cardPtr = cardPtr.add(1); + } + } + + private static void markFromRegion(int regionIndex) { + Address card = cardTable().add(regionIndex); + int regionOffset = Structure.add(Region.class, regionsAddress(), regionIndex).start; + if (regionOffset == 0) { + card.putByte((byte) (card.getByte() | CARD_VALID)); + return; + } + regionOffset--; + + int regionSize = regionSize(); + Address regionStart = heapAddress().add(regionIndex * regionSize); + MemoryTrace.reportDirtyRegion(regionStart); + Address regionEnd = regionStart.add(regionSize); + FreeChunk object = regionStart.add(regionOffset).toStructure(); + Address heapLimit = heapAddress().add(availableBytes()); + if (heapLimit.isLessThan(regionEnd)) { + regionEnd = heapLimit; + } + + boolean objectMarked = false; + while (object.toAddress().isLessThan(regionEnd)) { + int header = object.classReference; + + if (header != 0 && (header & RuntimeObject.GC_OLD_GENERATION) != 0) { + objectMarked |= doMarkOldGeneration(object.toAddress().toStructure()); + } + + object = object.toAddress().add(objectSize(object)).toStructure(); + } + + if (!objectMarked) { + card.putByte((byte) (card.getByte() | CARD_VALID)); + } + } + private static void mark(RuntimeObject object) { if (object == null || isMarked(object)) { return; } - MarkQueue.init(); - MarkQueue.enqueue(object); - object.classReference |= RuntimeObject.GC_MARKED; + enqueueMark(object); + doProcessMarkQueue(); + } + + private static boolean doMarkOldGeneration(RuntimeObject object) { + MarkQueue.init(); + boolean hasObjectsFromYoungGen = markObjectData(object); + doProcessMarkQueue(); + return hasObjectsFromYoungGen; + } + + private static void doProcessMarkQueue() { while (!MarkQueue.isEmpty()) { - object = MarkQueue.dequeue(); + RuntimeObject object = MarkQueue.dequeue(); MemoryTrace.mark(object.toAddress()); long offset = object.toAddress().toLong() - heapAddress().toLong(); @@ -244,87 +390,118 @@ public final class GC { if (region.start == 0 || region.start > relativeOffset) { region.start = relativeOffset; } + Address cardTableItem = cardTable().add(offset / regionSize()); + cardTableItem.putByte((byte) (cardTableItem.getByte() | CARD_YOUNG_GEN)); - RuntimeClass cls = RuntimeClass.getClass(object); - if (cls.itemType == null) { - markObject(cls, object); - } else { - markArray(cls, (RuntimeArray) object); - } + markObjectData(object); } } - private static void markObject(RuntimeClass cls, RuntimeObject object) { + private static boolean markObjectData(RuntimeObject object) { + RuntimeClass cls = RuntimeClass.getClass(object); + if (cls.itemType == null) { + return markObject(cls, object); + } else { + return markArray(cls, (RuntimeArray) object); + } + } + + private static boolean markObject(RuntimeClass cls, RuntimeObject object) { + boolean hasObjectsFromYoungGen = false; while (cls != null) { int type = (cls.flags >> RuntimeClass.VM_TYPE_SHIFT) & RuntimeClass.VM_TYPE_MASK; switch (type) { case RuntimeClass.VM_TYPE_WEAKREFERENCE: - markWeakReference((RuntimeReference) object); + hasObjectsFromYoungGen |= markWeakReference((RuntimeReference) object); break; case RuntimeClass.VM_TYPE_REFERENCEQUEUE: - markReferenceQueue((RuntimeReferenceQueue) object); + hasObjectsFromYoungGen |= markReferenceQueue((RuntimeReferenceQueue) object); break; default: - markFields(cls, object); + hasObjectsFromYoungGen |= markFields(cls, object); break; } cls = cls.parent; } + return hasObjectsFromYoungGen; } - private static void markWeakReference(RuntimeReference object) { + private static boolean markWeakReference(RuntimeReference object) { + boolean hasObjectsFromYoungGen = false; if (object.queue != null) { - enqueueMark(object.queue); + hasObjectsFromYoungGen |= enqueueMark(object.queue); if (object.next != null && object.object != null) { - enqueueMark(object.object); + hasObjectsFromYoungGen |= enqueueMark(object.object); } } if (object.next == null && object.object != null) { object.next = firstWeakReference; firstWeakReference = object; } + return hasObjectsFromYoungGen; } - private static void markReferenceQueue(RuntimeReferenceQueue object) { + private static boolean markReferenceQueue(RuntimeReferenceQueue object) { RuntimeReference reference = object.first; + boolean hasObjectsFromYoungGen = false; while (reference != null) { - enqueueMark(reference); + hasObjectsFromYoungGen |= enqueueMark(reference); reference = reference.next; } + return hasObjectsFromYoungGen; } - private static void markFields(RuntimeClass cls, RuntimeObject object) { + private static boolean markFields(RuntimeClass cls, RuntimeObject object) { Address layout = cls.layout; - if (layout != null) { - short fieldCount = layout.getShort(); - while (fieldCount-- > 0) { - layout = layout.add(2); - int fieldOffset = layout.getShort(); - RuntimeObject reference = object.toAddress().add(fieldOffset).getAddress().toStructure(); - enqueueMark(reference); - } + if (layout == null) { + return false; } + boolean hasObjectsFromYoungGen = false; + short fieldCount = layout.getShort(); + while (fieldCount-- > 0) { + layout = layout.add(2); + int fieldOffset = layout.getShort(); + RuntimeObject reference = object.toAddress().add(fieldOffset).getAddress().toStructure(); + hasObjectsFromYoungGen |= enqueueMark(reference); + } + return hasObjectsFromYoungGen; } - private static void markArray(RuntimeClass cls, RuntimeArray array) { + private static boolean markArray(RuntimeClass cls, RuntimeArray array) { if ((cls.itemType.flags & RuntimeClass.PRIMITIVE) != 0) { - return; + return false; } Address base = Address.align(array.toAddress().add(RuntimeArray.class, 1), Address.sizeOf()); + boolean hasObjectsFromYoungGen = false; for (int i = 0; i < array.size; ++i) { RuntimeObject reference = base.getAddress().toStructure(); - enqueueMark(reference); + hasObjectsFromYoungGen |= enqueueMark(reference); base = base.add(Address.sizeOf()); } + return hasObjectsFromYoungGen; + } + + private static boolean enqueueMark(RuntimeObject object) { + if (object == null) { + return false; + } + if (!isMarked(object)) { + doEnqueueMark(object); + return true; + } else { + return (object.classReference & RuntimeObject.GC_OLD_GENERATION) == 0; + } } - private static void enqueueMark(RuntimeObject object) { - if (object != null && !isMarked(object)) { + private static void doEnqueueMark(RuntimeObject object) { + if (isFullGC) { + object.classReference |= RuntimeObject.GC_MARKED | RuntimeObject.GC_OLD_GENERATION; + } else { object.classReference |= RuntimeObject.GC_MARKED; - MarkQueue.enqueue(object); } + MarkQueue.enqueue(object); } private static void processReferences() { @@ -332,7 +509,7 @@ public final class GC { while (reference != null) { RuntimeReference next = reference.next; reference.next = null; - if ((reference.object.classReference & RuntimeObject.GC_MARKED) == 0) { + if (!isMarked(reference.object)) { reference.object = null; RuntimeReferenceQueue queue = reference.queue; if (queue != null) { @@ -349,45 +526,29 @@ public final class GC { } private static void sweep() { - FreeChunkHolder freeChunkPtr = gcStorageAddress().toStructure(); + MemoryTrace.sweepStarted(); + + currentChunkPointer = gcStorageAddress().toStructure(); freeChunks = 0; + totalChunks = 0; FreeChunk object = heapAddress().toStructure(); FreeChunk lastFreeSpace = null; long heapSize = availableBytes(); - int regionsCount = (int) ((heapSize - 1) / regionSize()) + 1; + int regionsCount = getRegionCount(); Address currentRegionEnd = null; Address limit = heapAddress().add(heapSize); loop: while (object.toAddress().isLessThan(limit)) { - if (!object.toAddress().isLessThan(currentRegionEnd)) { - int currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); - Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); - if (currentRegion.start == 0) { - if (lastFreeSpace == null) { - lastFreeSpace = object; - } - - do { - if (++currentRegionIndex == regionsCount) { - object = limit.toStructure(); - break loop; - } - currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); - } while (currentRegion.start == 0); - - Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize()); - object = newRegionStart.add(currentRegion.start - 1).toStructure(); - currentRegionEnd = newRegionStart.add(regionSize()); - } - } - int tag = object.classReference; boolean free; if (tag == 0) { free = true; } else { free = (tag & RuntimeObject.GC_MARKED) == 0; + if (free && !isFullGC && (tag & RuntimeObject.GC_OLD_GENERATION) != 0) { + free = false; + } if (!free) { tag &= ~RuntimeObject.GC_MARKED; } @@ -398,16 +559,65 @@ public final class GC { if (lastFreeSpace == null) { lastFreeSpace = object; } + + if (!object.toAddress().isLessThan(currentRegionEnd)) { + int currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) + / regionSize()); + Region currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + currentRegionEnd = heapAddress().add((currentRegionIndex + 1) * regionSize()); + if (currentRegion.start == 0) { + do { + if (++currentRegionIndex == regionsCount) { + object = limit.toStructure(); + break loop; + } + currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + } while (currentRegion.start == 0); + + Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize()); + object = newRegionStart.add(currentRegion.start - 1).toStructure(); + currentRegionEnd = newRegionStart.add(regionSize()); + continue; + } + } } else { if (lastFreeSpace != null) { - lastFreeSpace.classReference = 0; - lastFreeSpace.size = (int) (object.toAddress().toLong() - lastFreeSpace.toAddress().toLong()); - MemoryTrace.free(lastFreeSpace.toAddress(), lastFreeSpace.size); - freeChunkPtr.value = lastFreeSpace; - freeChunkPtr = Structure.add(FreeChunkHolder.class, freeChunkPtr, 1); - freeChunks++; + freeMemory(lastFreeSpace, object); lastFreeSpace = null; } + + if (!object.toAddress().isLessThan(currentRegionEnd)) { + int currentRegionIndex = (int) ((object.toAddress().toLong() - heapAddress().toLong()) + / regionSize()); + currentRegionEnd = heapAddress().add((currentRegionIndex + 1) * regionSize()); + FreeChunk formerObject = object; + + if (!isFullGC && (cardTable().add(currentRegionIndex).getByte() & CARD_YOUNG_GEN) == 0 + && (cardTable().add(currentRegionIndex).getByte() & CARD_GAP) == 0) { + // We are collecting young generation and the new region is composed entirely of old + // generation objects, so skip this one and all subsequent ones + + Region currentRegion; + do { + if (++currentRegionIndex == regionsCount) { + break; + } + currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + } while (currentRegion.start != 0 + && (cardTable().add(currentRegionIndex).getByte() & CARD_YOUNG_GEN) == 0 + && (cardTable().add(currentRegionIndex).getByte() & CARD_GAP) == 0); + + --currentRegionIndex; + currentRegion = Structure.add(Region.class, regionsAddress(), currentRegionIndex); + + Address newRegionStart = heapAddress().add(currentRegionIndex * regionSize()); + object = newRegionStart.add(currentRegion.start - 1).toStructure(); + currentRegionEnd = newRegionStart.add(regionSize()); + if (formerObject.toAddress().isLessThan(object.toAddress())) { + continue; + } + } + } } int size = objectSize(object); @@ -415,19 +625,61 @@ public final class GC { } if (lastFreeSpace != null) { - int freeSize = (int) (object.toAddress().toLong() - lastFreeSpace.toAddress().toLong()); - lastFreeSpace.classReference = 0; - lastFreeSpace.size = freeSize; - MemoryTrace.free(lastFreeSpace.toAddress(), lastFreeSpace.size); - freeChunkPtr.value = lastFreeSpace; - freeChunks++; + freeMemory(lastFreeSpace, object); } currentChunkPointer = gcStorageAddress().toStructure(); + MemoryTrace.sweepCompleted(); + } + + private static void storeGapsInCardTable() { + for (int i = 0; i < totalChunks; ++i) { + FreeChunk freeChunkStart = FreeChunkHolder.add(FreeChunkHolder.class, + gcStorageAddress().toStructure(), i).value; + long freeChunkOffset = freeChunkStart.toAddress().toLong() - heapAddress().toLong(); + long freeChunkEndOffset = freeChunkOffset + freeChunkStart.size; + int startRegion = (int) (freeChunkOffset / regionSize()); + int endRegion = (int) (freeChunkEndOffset / regionSize()); + for (int region = startRegion; region <= endRegion; ++region) { + Address card = cardTable().add(region); + card.putByte((byte) (card.getByte() | CARD_GAP)); + } + } + } + + private static void clearGapsFromCardTable() { + int gapMask = ~(CARD_GAP | (CARD_GAP << 8) | (CARD_GAP << 16) | (CARD_GAP << 24)); + int regionsCount = getRegionCount(); + + Address cardPtr = cardTable(); + int regionIndex; + for (regionIndex = 0; regionIndex < regionsCount - 3; regionIndex += 4) { + cardPtr.putInt(cardPtr.getInt() & gapMask); + cardPtr = cardPtr.add(4); + } + + for (; regionIndex < regionsCount; regionIndex++) { + cardPtr.putByte((byte) (cardPtr.getByte() & ~CARD_GAP)); + cardPtr = cardPtr.add(1); + } + } + + private static void freeMemory(FreeChunk from, FreeChunk to) { + from.classReference = 0; + from.size = (int) (to.toAddress().toLong() - from.toAddress().toLong()); + MemoryTrace.free(from.toAddress(), from.size); + currentChunkPointer.value = from; + currentChunkPointer = Structure.add(FreeChunkHolder.class, currentChunkPointer, 1); + freeChunks++; + totalChunks++; } private static void defragment() { + MemoryTrace.defragStarted(); + clearGapsFromCardTable(); + storeGapsInCardTable(); markStackRoots(); + moveNonRelocatableObjectsToOldGeneration(); calculateRelocationTargets(); updatePointersFromStaticRoots(); updatePointersFromClasses(); @@ -435,6 +687,7 @@ public final class GC { restoreObjectHeaders(); relocateObjects(); putNewFreeChunks(); + MemoryTrace.defragCompleted(); } private static void markStackRoots() { @@ -447,13 +700,47 @@ public final class GC { while (count-- > 0) { RuntimeObject obj = stackRootsPtr.getAddress().toStructure(); if (!obj.toAddress().isLessThan(relocationThreshold)) { - obj.classReference |= RuntimeObject.GC_MARKED; + if (isFullGC || (obj.classReference & RuntimeObject.GC_OLD_GENERATION) == 0) { + obj.classReference |= RuntimeObject.GC_MARKED; + } } stackRootsPtr = stackRootsPtr.add(Address.sizeOf()); } } } + private static void moveNonRelocatableObjectsToOldGeneration() { + Address limitAddress = currentChunkPointer.value.toAddress(); + long limit = limitAddress.toLong() - heapAddress().toLong(); + for (int region = 0; region * (long) regionSize() < limit; ++region) { + if ((cardTable().add(region).getByte() & CARD_YOUNG_GEN) != 0) { + moveObjectsToOldGenerationInRegion(region, limitAddress); + } + } + } + + private static void moveObjectsToOldGenerationInRegion(int region, Address limit) { + int regionOffset = Structure.add(Region.class, regionsAddress(), region).start - 1; + + int regionSize = regionSize(); + Address regionStart = heapAddress().add(region * regionSize); + Address regionEnd = regionStart.add(regionSize); + FreeChunk object = regionStart.add(regionOffset).toStructure(); + if (limit.isLessThan(regionEnd)) { + regionEnd = limit; + } + + while (object.toAddress().isLessThan(regionEnd)) { + int classRef = object.classReference; + if (classRef != 0 && (classRef & RuntimeObject.GC_OLD_GENERATION) == 0) { + classRef |= RuntimeObject.GC_OLD_GENERATION; + object.classReference = classRef; + } + int size = objectSize(object); + object = object.toAddress().add(size).toStructure(); + } + } + private static void calculateRelocationTargets() { Address start = heapAddress(); long heapSize = availableBytes(); @@ -475,6 +762,7 @@ public final class GC { Address relocations = Structure.add(FreeChunk.class, freeChunk, 1).toAddress(); Address relocationsLimit = freeChunk.toAddress().add(freeChunk.size); + Address currentRegionEnd = null; lastChunk = heapAddress().toStructure(); boolean lastWasLocked = false; @@ -482,7 +770,8 @@ public final class GC { int size = objectSize(object); if (object.classReference != 0) { Address nextRelocationTarget = null; - boolean shouldRelocateObject = (object.classReference & RuntimeObject.GC_MARKED) == 0; + boolean shouldRelocateObject = shouldRelocateObject(object); + object.classReference |= RuntimeObject.GC_OLD_GENERATION; if (shouldRelocateObject) { while (true) { nextRelocationTarget = relocationTarget.add(size); @@ -509,6 +798,32 @@ public final class GC { lastRelocationBlock.end = limit; lastWasLocked = true; } + + // Trying to skip continuous sequences of objects from old generation + if (!isFullGC && !object.toAddress().isLessThan(currentRegionEnd)) { + int region = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); + currentRegionEnd = heapAddress().add((long) regionSize() * (region + 1)); + byte card = cardTable().add(region).getByte(); + if ((card & CARD_YOUNG_GEN) == 0 && (card & CARD_GAP) == 0) { + while (true) { + region++; + card = cardTable().add(region).getByte(); + if ((card & CARD_YOUNG_GEN) != 0 || (card & CARD_GAP) != 0) { + break; + } + if (Structure.add(Region.class, regionsAddress(), region).start == 0) { + break; + } + } + region--; + currentRegionEnd = heapAddress().add((long) regionSize() * (region + 1)); + + int offset = Structure.add(Region.class, regionsAddress(), region).start - 1; + object = heapAddress().add((long) regionSize() * region).add(offset).toStructure(); + size = objectSize(object); + } + } + lastRelocationBlock.start = object.toAddress().add(size); lastRelocationBlock.count = 0; object.classReference &= ~RuntimeObject.GC_MARKED; @@ -538,6 +853,10 @@ public final class GC { object.classReference = (int) (targetAddress >>> 33) | RuntimeObject.GC_MARKED; object.size = (int) (targetAddress >> 1); relocationTarget = nextRelocationTarget; + + int region = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); + Address card = cardTable().add(region); + card.putByte((byte) (card.getByte() | CARD_RELOCATABLE)); } } else { lastWasLocked = false; @@ -550,7 +869,10 @@ public final class GC { while (object.toAddress().isLessThan(limit)) { int size = objectSize(object); if (object.classReference != 0) { - object.classReference &= ~RuntimeObject.GC_MARKED; + int classRef = object.classReference; + classRef &= ~RuntimeObject.GC_MARKED; + classRef |= RuntimeObject.GC_OLD_GENERATION; + object.classReference = classRef; } else { lastRelocationBlock = Structure.add(RelocationBlock.class, lastRelocationBlock, 1); lastRelocationBlock.start = object.toAddress(); @@ -563,6 +885,11 @@ public final class GC { GC.lastRelocationBlock = lastRelocationBlock; } + private static boolean shouldRelocateObject(FreeChunk object) { + return (object.classReference & RuntimeObject.GC_MARKED) == 0 + && (isFullGC || (object.classReference & RuntimeObject.GC_OLD_GENERATION) == 0); + } + private static void updatePointersFromStaticRoots() { Address staticRoots = Mutator.getStaticGCRoots(); int staticCount = staticRoots.getInt(); @@ -593,6 +920,14 @@ public final class GC { } private static void updatePointersFromObjects() { + if (isFullGC) { + updatePointersFromObjectsFull(); + } else { + updatePointersFromObjectsYoung(); + } + } + + private static void updatePointersFromObjectsFull() { Address start = heapAddress(); long heapSize = availableBytes(); Address limit = start.add(heapSize); @@ -619,6 +954,72 @@ public final class GC { } } + private static void updatePointersFromObjectsYoung() { + int validMask = CARD_VALID | (CARD_VALID << 8) | (CARD_VALID << 16) | (CARD_VALID << 24); + int youngMask = CARD_YOUNG_GEN | (CARD_YOUNG_GEN << 8) | (CARD_YOUNG_GEN << 16) | (CARD_YOUNG_GEN << 24); + int regionsCount = getRegionCount(); + int regionSize = regionSize(); + + Address cardPtr = cardTable(); + Address regionPtr = heapAddress(); + int regionIndex; + for (regionIndex = 0; regionIndex < regionsCount - 3; regionIndex += 4) { + int n = cardPtr.getInt(); + if ((n & validMask) != validMask || (n & youngMask) != 0) { + for (int i = 0; i < 4; ++i) { + n = cardPtr.add(i).getByte(); + if ((n & CARD_VALID) == 0 || (n & CARD_YOUNG_GEN) != 0) { + updatePointersFromRegion(regionIndex + i); + } + } + } + cardPtr = cardPtr.add(4); + regionPtr = regionPtr.add(4 * regionSize); + } + + for (; regionIndex < regionsCount; regionIndex++) { + int n = cardPtr.getByte(); + if ((n & CARD_VALID) == 0 || (n & CARD_YOUNG_GEN) != 0) { + updatePointersFromRegion(regionIndex); + } + cardPtr = cardPtr.add(1); + } + } + + private static void updatePointersFromRegion(int regionIndex) { + int regionOffset = Structure.add(Region.class, regionsAddress(), regionIndex).start - 1; + if (regionOffset < 0) { + return; + } + + int regionSize = regionSize(); + Address regionStart = heapAddress().add(regionIndex * regionSize); + Address regionEnd = regionStart.add(regionSize); + FreeChunk object = regionStart.add(regionOffset).toStructure(); + Address heapLimit = heapAddress().add(availableBytes()); + if (heapLimit.isLessThan(regionEnd)) { + regionEnd = heapLimit; + } + + while (object.toAddress().isLessThan(regionEnd)) { + int classRef = object.classReference; + int size; + if (classRef != 0) { + Relocation relocation = getRelocation(object.toAddress()); + if (relocation != null) { + classRef = relocation.classBackup; + } + RuntimeClass cls = RuntimeClass.unpack(classRef); + updatePointers(cls, object.toAddress().toStructure()); + size = objectSize(object.toAddress().toStructure(), cls); + } else { + size = object.size; + } + + object = object.toAddress().add(size).toStructure(); + } + } + private static void updatePointers(RuntimeClass cls, RuntimeObject object) { if (cls.itemType == null) { updatePointersInObject(cls, object); @@ -703,14 +1104,51 @@ public final class GC { return Address.fromLong(result).toStructure(); } + private static void restoreObjectHeaders() { - Address start = heapAddress(); - long heapSize = availableBytes(); - Address limit = start.add(heapSize); + int relocatableMask = CARD_RELOCATABLE | (CARD_RELOCATABLE << 8) | (CARD_RELOCATABLE << 16) + | (CARD_RELOCATABLE << 24); + int regionsCount = getRegionCount(); - FreeChunk freeChunk = currentChunkPointer.value; - FreeChunk object = freeChunk.toAddress().add(freeChunk.size).toStructure(); + Address cardPtr = cardTable(); + Address limit = heapAddress().add(availableBytes()); + int regionIndex; + for (regionIndex = 0; regionIndex < regionsCount - 3; regionIndex += 4) { + int n = cardPtr.getInt(); + if ((n & relocatableMask) != 0) { + for (int i = 0; i < 4; ++i) { + n = cardPtr.add(i).getByte(); + if ((n & CARD_RELOCATABLE) != 0) { + restoreObjectHeadersInRegion(regionIndex + i, limit); + } + } + } + cardPtr = cardPtr.add(4); + } + for (; regionIndex < regionsCount; regionIndex++) { + if ((cardPtr.getByte() & CARD_RELOCATABLE) != 0) { + restoreObjectHeadersInRegion(regionIndex, limit); + } + cardPtr = cardPtr.add(1); + } + } + + private static void restoreObjectHeadersInRegion(int region, Address limit) { + int regionOffset = Structure.add(Region.class, regionsAddress(), region).start - 1; + + int regionSize = regionSize(); + Address regionStart = heapAddress().add(region * regionSize); + Address regionEnd = regionStart.add(regionSize); + FreeChunk object = regionStart.add(regionOffset).toStructure(); + if (limit.isLessThan(regionEnd)) { + regionEnd = limit; + } + + restoreObjectHeadersInRange(object, regionEnd); + } + + private static void restoreObjectHeadersInRange(FreeChunk object, Address limit) { while (object.toAddress().isLessThan(limit)) { Relocation relocation = getRelocation(object.toAddress()); if (relocation != null) { @@ -739,6 +1177,8 @@ public final class GC { Address blockTarget = null; Address blockSource = null; int blockSize = 0; + Address currentRegionEnd = null; + int regionCount = getRegionCount(); while (object.toAddress().isLessThan(limit)) { int size = objectSize(object); @@ -747,8 +1187,7 @@ public final class GC { while (countInRelocationBlock == 0) { if (blockSize != 0) { - Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); - MemoryTrace.move(blockSource, blockTarget, blockSize); + moveMemoryBlock(blockSource, blockTarget, blockSize); blockSource = null; blockSize = 0; } @@ -763,14 +1202,40 @@ public final class GC { blockSource = object.toAddress(); blockTarget = relocationTarget; } + relocationTarget = relocationTarget.add(size); blockSize += size; --countInRelocationBlock; - } else if (blockSource != null) { - Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); - MemoryTrace.move(blockSource, blockTarget, blockSize); - blockSource = null; - blockSize = 0; + } else { + if (blockSource != null) { + moveMemoryBlock(blockSource, blockTarget, blockSize); + blockSource = null; + blockSize = 0; + } + + // Trying to skip continuous sequences of non-relocatable objects + if (object.classReference != 0 && !object.toAddress().isLessThan(currentRegionEnd)) { + int region = (int) ((object.toAddress().toLong() - heapAddress().toLong()) / regionSize()); + currentRegionEnd = heapAddress().add((long) regionSize() * (region + 1)); + byte card = cardTable().add(region).getByte(); + if ((card & CARD_RELOCATABLE) == 0 && (card & CARD_GAP) == 0) { + while (++region < regionCount) { + card = cardTable().add(region).getByte(); + if ((card & CARD_RELOCATABLE) != 0 || (card & CARD_GAP) != 0) { + break; + } + if (Structure.add(Region.class, regionsAddress(), region).start == 0) { + break; + } + } + region--; + currentRegionEnd = heapAddress().add((long) regionSize() * (region + 1)); + + int offset = Structure.add(Region.class, regionsAddress(), region).start - 1; + object = heapAddress().add((long) regionSize() * region).add(offset).toStructure(); + size = objectSize(object); + } + } } object = object.toAddress().add(size).toStructure(); @@ -778,11 +1243,71 @@ public final class GC { relocationBlock.start = relocationTarget; if (blockSource != null) { - Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); - MemoryTrace.move(blockSource, blockTarget, blockSize); + moveMemoryBlock(blockSource, blockTarget, blockSize); } } + private static void moveMemoryBlock(Address blockSource, Address blockTarget, int blockSize) { + long sourceStartOffset = blockSource.toLong() - heapAddress().toLong(); + int sourceStartRegionIndex = (int) (sourceStartOffset / regionSize()); + Region sourceStartRegion = Structure.add(Region.class, regionsAddress(), sourceStartRegionIndex); + + long sourceEndOffset = sourceStartOffset + blockSize; + int sourceEndRegionIndex = (int) (sourceEndOffset / regionSize()); + Region sourceEndRegion = Structure.add(Region.class, regionsAddress(), sourceEndRegionIndex); + + if (sourceStartRegion != sourceEndRegion && sourceStartOffset % regionSize() + 1 == sourceStartRegion.start) { + sourceStartRegion.start = 0; + } + for (int i = sourceStartRegionIndex + 1; i < sourceEndRegionIndex; ++i) { + Structure.add(Region.class, regionsAddress(), i).start = 0; + } + + if (sourceStartRegion != sourceEndRegion || sourceStartOffset % regionSize() + 1 == sourceStartRegion.start) { + Address heapLimit = heapAddress().add(availableBytes()); + FreeChunk objectAfterSource = blockSource.add(blockSize).toStructure(); + + if (objectAfterSource.toAddress().isLessThan(heapLimit)) { + int objectRegionIndex = sourceEndRegionIndex; + if (objectAfterSource.classReference == 0) { + objectAfterSource = objectAfterSource.toAddress().add(objectAfterSource.size).toStructure(); + objectRegionIndex = (int) ((objectAfterSource.toAddress().toLong() - heapAddress().toLong()) + / regionSize()); + } + if (objectRegionIndex != sourceEndRegionIndex || !objectAfterSource.toAddress().isLessThan(heapLimit)) { + sourceEndRegion.start = 0; + } else { + sourceEndRegion.start = (short) ((objectAfterSource.toAddress().toLong() - heapAddress().toLong()) + % regionSize() + 1); + } + } else { + sourceEndRegion.start = 0; + } + } + + Allocator.moveMemoryBlock(blockSource, blockTarget, blockSize); + + FreeChunk object = blockTarget.toStructure(); + Address blockTargetEnd = blockTarget.add(blockSize); + Address currentRegionEnd = null; + while (object.toAddress().isLessThan(blockTargetEnd)) { + if (!object.toAddress().isLessThan(currentRegionEnd)) { + long offset = object.toAddress().toLong() - heapAddress().toLong(); + int regionIndex = (int) (offset / regionSize()); + currentRegionEnd = heapAddress().add((long) regionSize() * (regionIndex + 1)); + Region region = Structure.add(Region.class, regionsAddress(), regionIndex); + int offsetInRegion = (int) (offset % regionSize()); + if (region.start == 0 || region.start - 1 > offsetInRegion) { + region.start = (short) (offsetInRegion + 1); + } + } + int size = objectSize(object); + object = object.toAddress().add(size).toStructure(); + } + + MemoryTrace.move(blockSource, blockTarget, blockSize); + } + private static void putNewFreeChunks() { FreeChunkHolder freeChunkPointer = currentChunkPointer; RelocationBlock relocationBlock = Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks) @@ -803,6 +1328,7 @@ public final class GC { } relocationBlock = Structure.add(RelocationBlock.class, relocationBlock, 1); } + totalChunks = freeChunks; } private static void updateFreeMemory() { @@ -820,7 +1346,13 @@ public final class GC { return; } if (newSize > oldSize) { + int previousRegionCount = getRegionCount(); resizeHeap(newSize); + int newRegionCount = getRegionCount(); + for (int i = previousRegionCount; i < newRegionCount; ++i) { + Structure.add(Region.class, regionsAddress(), i).start = 0; + } + if (lastChunk.classReference == 0) { lastChunk.size += (int) (newSize - oldSize); } else { @@ -830,6 +1362,7 @@ public final class GC { lastChunk.size = (int) (newSize - oldSize); Structure.add(FreeChunkHolder.class, currentChunkPointer, freeChunks).value = lastChunk; freeChunks++; + totalChunks++; } } else { long minimumSize = lastChunk.toAddress().toLong() - heapAddress().toLong(); @@ -844,6 +1377,7 @@ public final class GC { } if (newSize == minimumSize) { freeChunks--; + totalChunks--; } else { lastChunk.size -= (int) (oldSize - newSize); } @@ -854,7 +1388,7 @@ public final class GC { private static void resizeHeapIfNecessary(long requestedSize) { long availableBytes = availableBytes(); long occupiedMemory = availableBytes - freeMemory; - if (requestedSize > availableBytes || occupiedMemory > availableBytes / 2) { + if (isAboutToExpand(requestedSize)) { long newSize = max(requestedSize, (availableBytes - freeMemory) * 2); newSize = min(newSize, maxAvailableBytes()); if (newSize != availableBytes) { @@ -873,6 +1407,12 @@ public final class GC { } } + private static boolean isAboutToExpand(long requestedSize) { + long availableBytes = availableBytes(); + long occupiedMemory = availableBytes - freeMemory; + return requestedSize > availableBytes || occupiedMemory > availableBytes / 2; + } + private static long min(long a, long b) { return a < b ? a : b; } @@ -907,7 +1447,8 @@ public final class GC { } private static boolean isMarked(RuntimeObject object) { - return (object.classReference & RuntimeObject.GC_MARKED) != 0; + return (object.classReference & RuntimeObject.GC_MARKED) != 0 + || (!isFullGC && (object.classReference & RuntimeObject.GC_OLD_GENERATION) != 0); } static class Region extends Structure { diff --git a/core/src/main/java/org/teavm/runtime/MemoryTrace.java b/core/src/main/java/org/teavm/runtime/MemoryTrace.java index 58b91d5e0..41fca1099 100644 --- a/core/src/main/java/org/teavm/runtime/MemoryTrace.java +++ b/core/src/main/java/org/teavm/runtime/MemoryTrace.java @@ -29,17 +29,27 @@ public class MemoryTrace { public static native void checkIsFree(Address address, int size); - public static native void initMark(); + public static native void markStarted(); public static native void mark(Address address); + public static native void reportDirtyRegion(Address address); + + public static native void markCompleted(); + public static native void move(Address from, Address to, int size); - public static native void gcStarted(); + public static native void gcStarted(boolean full); + + public static native void sweepStarted(); public static native void sweepCompleted(); + public static native void defragStarted(); + public static native void defragCompleted(); public static native void writeHeapDump(); + + public static native void gcCompleted(); } diff --git a/core/src/main/java/org/teavm/runtime/RuntimeObject.java b/core/src/main/java/org/teavm/runtime/RuntimeObject.java index 1ae4bf34f..5f39fb8fd 100644 --- a/core/src/main/java/org/teavm/runtime/RuntimeObject.java +++ b/core/src/main/java/org/teavm/runtime/RuntimeObject.java @@ -21,6 +21,7 @@ import org.teavm.interop.Structure; @StaticInit public class RuntimeObject extends Structure { public static final int GC_MARKED = 0x80000000; + public static final int GC_OLD_GENERATION = 0x40000000; public static int nextId; diff --git a/core/src/main/java/org/teavm/vm/TeaVM.java b/core/src/main/java/org/teavm/vm/TeaVM.java index 66a671807..f14a8f3ef 100644 --- a/core/src/main/java/org/teavm/vm/TeaVM.java +++ b/core/src/main/java/org/teavm/vm/TeaVM.java @@ -961,7 +961,7 @@ public class TeaVM implements TeaVMHost, ServiceRepository { if (program == null) { program = ProgramUtils.copy(classReader.getMethod(method.getDescriptor()).getProgram()); missingItemsProcessor.processMethod(method.getReference(), program); - linker.link(method.getReference(), program); + linker.link(method, program); clinitInsertion.apply(method, program); program = optimizeMethodCacheMiss(method, program); Program finalProgram = program; diff --git a/core/src/main/resources/org/teavm/backend/c/core.h b/core/src/main/resources/org/teavm/backend/c/core.h index 06a2602a0..935012eb2 100644 --- a/core/src/main/resources/org/teavm/backend/c/core.h +++ b/core/src/main/resources/org/teavm/backend/c/core.h @@ -179,4 +179,10 @@ extern TeaVM_Class* teavm_objectClass; extern TeaVM_Class* teavm_stringClass; extern TeaVM_Class* teavm_charArrayClass; extern int32_t teavm_classReferencesCount; -extern void teavm_initClasses(); \ No newline at end of file +extern void teavm_initClasses(); + + +inline static void teavm_gc_writeBarrier(void* object) { + intptr_t offset = (intptr_t) ((char*) object - (char*) teavm_gc_heapAddress) / teavm_gc_regionSize; + ((char*) teavm_gc_cardTable)[offset] = 0; +} \ No newline at end of file diff --git a/core/src/main/resources/org/teavm/backend/c/definitions.h b/core/src/main/resources/org/teavm/backend/c/definitions.h index f3054a078..b4a485f75 100644 --- a/core/src/main/resources/org/teavm/backend/c/definitions.h +++ b/core/src/main/resources/org/teavm/backend/c/definitions.h @@ -59,6 +59,10 @@ #define TEAVM_GC_LOG 0 #endif +#ifndef TEAVM_GC_STATS + #define TEAVM_GC_STATS 0 +#endif + #ifndef TEAVM_OBFUSCATED #define TEAVM_OBFUSCATED 0 #endif \ No newline at end of file diff --git a/core/src/main/resources/org/teavm/backend/c/heaptrace.c b/core/src/main/resources/org/teavm/backend/c/heaptrace.c index 55ef06937..47cfd6fb3 100644 --- a/core/src/main/resources/org/teavm/backend/c/heaptrace.c +++ b/core/src/main/resources/org/teavm/backend/c/heaptrace.c @@ -2,6 +2,7 @@ #include "core.h" #include "definitions.h" #include "memory.h" +#include "time.h" #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #if TEAVM_WINDOWS #include @@ -33,6 +35,27 @@ void teavm_outOfMemory() { static wchar_t* teavm_gc_dumpDirectory = NULL; +#ifdef TEAVM_GC_STATS + static int32_t teavm_gc_allocationCount = 0; + static int32_t teavm_gc_freeCount = 0; + static int32_t teavm_gc_freeByteCount = 0; + static int32_t teavm_gc_markCount = 0; + static int32_t teavm_gc_dirtyRegionCount = 0; + static int32_t teavm_gc_relocatedBlocks = 0; + static int32_t teavm_gc_relocatedBytes = 0; + + static int64_t teavm_gc_startTimeMillis; + static int64_t teavm_gc_startTime; + static int64_t teavm_gc_endTime; + static int64_t teavm_gc_markStartTime; + static int64_t teavm_gc_markEndTime; + static int64_t teavm_gc_sweepStartTime; + static int64_t teavm_gc_sweepEndTime; + static int64_t teavm_gc_defragStartTime; + static int64_t teavm_gc_defragEndTime; + static bool teavm_gc_full; +#endif + #if TEAVM_MEMORY_TRACE void teavm_gc_assertSize(int32_t size) { if (size % sizeof(void*) != 0) { @@ -65,6 +88,10 @@ void teavm_gc_allocate(void* address, int32_t size) { *map++ = 2; } #endif + + #if TEAVM_GC_STATS + teavm_gc_allocationCount++; + #endif } void teavm_gc_free(void* address, int32_t size) { @@ -86,6 +113,11 @@ void teavm_gc_free(void* address, int32_t size) { uint8_t* map = teavm_gc_heapMap + offset; memset(map, 0, size); #endif + + #if TEAVM_GC_STATS + teavm_gc_freeCount++; + teavm_gc_freeByteCount += size; + #endif } void teavm_gc_assertFree(void* address, int32_t size) { @@ -106,10 +138,20 @@ void teavm_gc_assertFree(void* address, int32_t size) { #endif } -void teavm_gc_initMark() { +void teavm_gc_markStarted() { #if TEAVM_MEMORY_TRACE memset(teavm_gc_markMap, 0, teavm_gc_availableBytes / sizeof(void*)); #endif + + #if TEAVM_GC_STATS + teavm_gc_markStartTime = teavm_currentTimeNano(); + #endif +} + +void teavm_gc_markCompleted() { + #if TEAVM_GC_STATS + teavm_gc_markEndTime = teavm_currentTimeNano(); + #endif } int32_t teavm_gc_objectSize(void* address) { @@ -144,7 +186,8 @@ void teavm_gc_mark(void* address) { size /= sizeof(void*); if (*map++ != 1 || *markMap != 0) { - fprintf(stderr, "[GC] assertion failed marking object at: %d\n", (int) ((char*) address - (char*) teavm_gc_heapAddress)); + fprintf(stderr, "[GC] assertion failed marking object at: %d\n", + (int) ((char*) address - (char*) teavm_gc_heapAddress)); abort(); } *markMap++ = 1; @@ -156,6 +199,10 @@ void teavm_gc_mark(void* address) { *markMap++ = 1; } #endif + + #if TEAVM_GC_STATS + teavm_gc_markCount++; + #endif } void teavm_gc_move(void* from, void* to, int32_t size) { @@ -188,6 +235,11 @@ void teavm_gc_move(void* from, void* to, int32_t size) { } } #endif + + #if TEAVM_GC_STATS + teavm_gc_relocatedBlocks++; + teavm_gc_relocatedBytes += size; + #endif } static FILE* teavm_gc_traceFile = NULL; @@ -251,8 +303,10 @@ FILE* teavm_gc_openDumpFile(wchar_t* name) { #endif } - void teavm_gc_checkHeapConsistency() { + void teavm_gc_checkHeapConsistency(bool oldGen, bool offsets) { + int32_t lastCheckedRegion = -1; TeaVM_Object* obj = teavm_gc_heapAddress; + uint16_t* regions = (uint16_t*) teavm_gc_regionsAddress; while ((char*) obj < (char*) teavm_gc_heapAddress + teavm_gc_availableBytes) { int32_t size; if (obj->header == 0) { @@ -260,14 +314,32 @@ FILE* teavm_gc_openDumpFile(wchar_t* name) { teavm_gc_assertFree(obj, size); } else { teavm_verify(obj); + if (offsets) { + int64_t offset = (int64_t) ((char*) obj - (char*) teavm_gc_heapAddress); + int32_t objRegion = (int32_t) (offset / teavm_gc_regionSize); + if (objRegion != lastCheckedRegion) { + while (++lastCheckedRegion < objRegion) { + if (regions[lastCheckedRegion] != 0) { + abort(); + } + } + int32_t offsetInRegion = (int32_t) (offset % teavm_gc_regionSize); + if (regions[objRegion] != offsetInRegion + 1) { + abort(); + } + } + } + if (oldGen && !(obj->header & 0x40000000)) { + abort(); + } TeaVM_Class* cls = TEAVM_CLASS_OF(obj); if (cls->itemType != NULL) { if (!(cls->itemType->flags & 2)) { char* offset = NULL; offset += sizeof(TeaVM_Array); offset = TEAVM_ALIGN(offset, sizeof(void*)); - void** data = (void**)((char*)obj + (uintptr_t)offset); - int32_t size = ((TeaVM_Array*)obj)->size; + void** data = (void**) ((char*) obj + (uintptr_t) offset); + int32_t size = ((TeaVM_Array*) obj)->size; for (int32_t i = 0; i < size; ++i) { teavm_verify(data[i]); } @@ -297,26 +369,143 @@ FILE* teavm_gc_openDumpFile(wchar_t* name) { } obj = (TeaVM_Object*) ((char*) obj + size); } + + if (offsets) { + int32_t lastRegion = (int32_t) (teavm_gc_availableBytes / teavm_gc_regionSize); + while (++lastCheckedRegion <= lastRegion) { + if (regions[lastCheckedRegion] != 0) { + abort(); + } + } + } } #endif -void teavm_gc_gcStarted() { +void teavm_gc_gcStarted(int32_t full) { #if TEAVM_MEMORY_TRACE teavm_writeHeapMemory("start"); - teavm_gc_checkHeapConsistency(); + teavm_gc_checkHeapConsistency(false, false); + #endif + + #if TEAVM_GC_STATS + teavm_gc_startTime = teavm_currentTimeNano(); + teavm_gc_startTimeMillis = teavm_currentTimeMillis(); + teavm_gc_full = full; + #endif +} + +void teavm_gc_sweepStarted() { + #if TEAVM_GC_STATS + teavm_gc_sweepStartTime = teavm_currentTimeNano(); #endif } void teavm_gc_sweepCompleted() { #if TEAVM_MEMORY_TRACE teavm_writeHeapMemory("sweep"); - teavm_gc_checkHeapConsistency(); + teavm_gc_checkHeapConsistency(false, true); + #endif + + #if TEAVM_GC_STATS + teavm_gc_sweepEndTime = teavm_currentTimeNano(); + #endif +} + +void teavm_gc_defragStarted() { + #if TEAVM_GC_STATS + teavm_gc_defragStartTime = teavm_currentTimeNano(); #endif } void teavm_gc_defragCompleted() { #if TEAVM_MEMORY_TRACE teavm_writeHeapMemory("defrag"); + teavm_gc_checkHeapConsistency(true, true); + #endif + + #if TEAVM_GC_STATS + teavm_gc_defragEndTime = teavm_currentTimeNano(); + #endif +} + +#define TEAVM_GC_LOG_BUFFER_SIZE 512 + +#if TEAVM_GC_STATS + static void teavm_gc_print(wchar_t* s) { + #if TEAVM_WINDOWS_LOG + OutputDebugStringW(s); + #else + fprintf(stderr, "%ls", s); + #endif + } + + static void teavm_gc_printStats() { + wchar_t buffer[TEAVM_GC_LOG_BUFFER_SIZE]; + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Garbage collection (%ls) performed at %" PRIu64 ", took %" + PRIu64 " ns\n", teavm_gc_full ? L"full" : L"young", teavm_gc_startTimeMillis, + teavm_gc_endTime - teavm_gc_startTime); + teavm_gc_print(buffer); + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Allocations performed before GC: %" PRIu32 "\n", + teavm_gc_allocationCount); + teavm_gc_print(buffer); + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Mark phase took %" PRIu64 " ns, %" PRIu32 + " objects reached\n", teavm_gc_markEndTime - teavm_gc_markStartTime, teavm_gc_markCount); + teavm_gc_print(buffer); + + if (!teavm_gc_full) { + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Regions scanned from remembered set: %" PRIu32 "\n", + teavm_gc_dirtyRegionCount); + teavm_gc_print(buffer); + } + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Sweep phase took %" PRIu64 " ns, %" PRIu32 " regions of %" + PRIu32 " bytes freed\n", teavm_gc_sweepEndTime - teavm_gc_sweepStartTime, teavm_gc_freeCount, + teavm_gc_freeByteCount); + teavm_gc_print(buffer); + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Defrag phase took %" PRIu64 " ns\n", + teavm_gc_defragEndTime - teavm_gc_defragStartTime); + teavm_gc_print(buffer); + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Blocks relocated %" PRId32 " of total %" PRId32 " bytes\n", + teavm_gc_relocatedBlocks, teavm_gc_relocatedBytes); + teavm_gc_print(buffer); + } + + static void teavm_gc_resetStats() { + teavm_gc_allocationCount = 0; + teavm_gc_markCount = 0; + teavm_gc_dirtyRegionCount = 0; + teavm_gc_freeCount = 0; + teavm_gc_freeByteCount = 0; + teavm_gc_relocatedBlocks = 0; + teavm_gc_relocatedBytes = 0; + } +#endif + +void teavm_gc_gcCompleted() { + #if TEAVM_GC_STATS + teavm_gc_endTime = teavm_currentTimeNano(); + teavm_gc_printStats(); + teavm_gc_resetStats(); + #endif +} + +void teavm_gc_heapResized(int64_t newSize) { + #if TEAVM_GC_STATS + wchar_t buffer[TEAVM_GC_LOG_BUFFER_SIZE]; + + swprintf(buffer, TEAVM_GC_LOG_BUFFER_SIZE, L"[GC] Heap resized to %" PRIu64 " bytes\n", newSize); + teavm_gc_print(buffer); + #endif +} + +void teavm_gc_reportDirtyRegion(void* address) { + #if TEAVM_GC_STATS + teavm_gc_dirtyRegionCount++; #endif } @@ -329,4 +518,3 @@ void teavm_gc_setDumpDirectory(const wchar_t* path) { teavm_gc_dumpDirectory = malloc(bytesLen); memcpy(teavm_gc_dumpDirectory, path, bytesLen); } - diff --git a/core/src/main/resources/org/teavm/backend/c/heaptrace.h b/core/src/main/resources/org/teavm/backend/c/heaptrace.h index 429ebd6c0..814d3548e 100644 --- a/core/src/main/resources/org/teavm/backend/c/heaptrace.h +++ b/core/src/main/resources/org/teavm/backend/c/heaptrace.h @@ -12,12 +12,18 @@ extern void teavm_gc_allocate(void* address, int32_t size); extern void teavm_gc_free(void* address, int32_t size); extern void teavm_gc_assertFree(void* address, int32_t size); -extern void teavm_gc_initMark(); extern void teavm_gc_mark(void* address); extern void teavm_gc_move(void* from, void* to, int32_t size); -extern void teavm_gc_gcStarted(); +extern void teavm_gc_gcStarted(int32_t full); +extern void teavm_gc_markStarted(); +extern void teavm_gc_markCompleted(); +extern void teavm_gc_sweepStarted(); extern void teavm_gc_sweepCompleted(); +extern void teavm_gc_defragStarted(); extern void teavm_gc_defragCompleted(); +extern void teavm_gc_gcCompleted(); +extern void teavm_gc_heapResized(int64_t newSize); +extern void teavm_gc_reportDirtyRegion(void* address); extern void teavm_gc_setDumpDirectory(const wchar_t* path); extern void teavm_gc_fixHeap(); extern void teavm_gc_writeHeapDump(); diff --git a/core/src/main/resources/org/teavm/backend/c/memory.c b/core/src/main/resources/org/teavm/backend/c/memory.c index 811681031..a8caa5593 100644 --- a/core/src/main/resources/org/teavm/backend/c/memory.c +++ b/core/src/main/resources/org/teavm/backend/c/memory.c @@ -1,4 +1,5 @@ #include "memory.h" +#include "heaptrace.h" #include "definitions.h" #include #include @@ -20,7 +21,7 @@ void* teavm_gc_heapAddress = NULL; void* teavm_gc_gcStorageAddress = NULL; int32_t teavm_gc_gcStorageSize = INT32_C(0); void* teavm_gc_regionsAddress = NULL; -int32_t teavm_gc_regionSize = INT32_C(32768); +void* teavm_gc_cardTable = NULL; int32_t teavm_gc_regionMaxCount; int64_t teavm_gc_availableBytes; int64_t teavm_gc_minAvailableBytes; @@ -110,6 +111,8 @@ void teavm_gc_resizeHeap(int64_t newSize) { return; } + teavm_gc_heapResized(newSize); + int32_t workSize = teavm_gc_calculateWorkSize(newSize); int32_t regionsSize = teavm_gc_calculateRegionsSize(newSize); @@ -119,6 +122,8 @@ void teavm_gc_resizeHeap(int64_t newSize) { int64_t oldWorkSizeAligned = teavm_pageCount(teavm_gc_gcStorageSize); int64_t newRegionsSizeAligned = teavm_pageCount(regionsSize * 2); int64_t oldRegionsSizeAligned = teavm_pageCount(teavm_gc_regionMaxCount * 2); + int64_t newCardTableSizeAligned = teavm_pageCount(regionsSize); + int64_t oldCardTableSizeAligned = teavm_pageCount(teavm_gc_regionMaxCount); if (newSize > teavm_gc_availableBytes) { if (newSizeAligned > oldSizeAligned) { @@ -132,6 +137,10 @@ void teavm_gc_resizeHeap(int64_t newSize) { teavm_virtualCommit((char*) teavm_gc_regionsAddress + oldRegionsSizeAligned, newRegionsSizeAligned - oldRegionsSizeAligned); } + if (newCardTableSizeAligned > oldCardTableSizeAligned) { + teavm_virtualCommit((char*) teavm_gc_cardTable + oldCardTableSizeAligned, + newCardTableSizeAligned - oldCardTableSizeAligned); + } } else { if (newSizeAligned < oldSizeAligned) { teavm_virtualUncommit((char*) teavm_gc_heapAddress + newSizeAligned, oldSizeAligned - newSizeAligned); @@ -144,6 +153,10 @@ void teavm_gc_resizeHeap(int64_t newSize) { teavm_virtualUncommit((char*) teavm_gc_regionsAddress + newRegionsSizeAligned, oldRegionsSizeAligned - newRegionsSizeAligned); } + if (newCardTableSizeAligned < oldCardTableSizeAligned) { + teavm_virtualUncommit((char*) teavm_gc_cardTable + newCardTableSizeAligned, + oldCardTableSizeAligned - newCardTableSizeAligned); + } } teavm_gc_gcStorageSize = workSize; @@ -159,6 +172,7 @@ void teavm_initHeap(int64_t minHeap, int64_t maxHeap) { teavm_gc_heapAddress = teavm_virtualAlloc(teavm_pageCount(maxHeap)); teavm_gc_gcStorageAddress = teavm_virtualAlloc(teavm_pageCount(workSize)); teavm_gc_regionsAddress = teavm_virtualAlloc(teavm_pageCount(regionsSize * 2)); + teavm_gc_cardTable = teavm_virtualAlloc(teavm_pageCount(regionsSize)); #if TEAVM_MEMORY_TRACE int64_t heapMapSize = maxHeap / sizeof(void*); diff --git a/core/src/main/resources/org/teavm/backend/c/memory.h b/core/src/main/resources/org/teavm/backend/c/memory.h index af751d35a..1b9116aca 100644 --- a/core/src/main/resources/org/teavm/backend/c/memory.h +++ b/core/src/main/resources/org/teavm/backend/c/memory.h @@ -5,7 +5,8 @@ extern void* teavm_gc_heapAddress; extern void* teavm_gc_gcStorageAddress; extern int32_t teavm_gc_gcStorageSize; extern void* teavm_gc_regionsAddress; -extern int32_t teavm_gc_regionSize; +extern void* teavm_gc_cardTable; +#define teavm_gc_regionSize INT32_C(8192) extern int32_t teavm_gc_regionMaxCount; extern int64_t teavm_gc_availableBytes; extern int64_t teavm_gc_minAvailableBytes; diff --git a/core/src/main/resources/org/teavm/backend/c/references.c b/core/src/main/resources/org/teavm/backend/c/references.c index e29c6417c..f46a615c9 100644 --- a/core/src/main/resources/org/teavm/backend/c/references.c +++ b/core/src/main/resources/org/teavm/backend/c/references.c @@ -10,9 +10,11 @@ int32_t teavm_reference_enqueue(TeaVM_Reference* reference) { if (queue->last == NULL) { queue->first = reference; } else { + teavm_gc_writeBarrier(queue->last); queue->last->next = reference; } queue->last = reference; + teavm_gc_writeBarrier(queue); return INT32_C(1); } @@ -36,6 +38,7 @@ TeaVM_Reference* teavm_reference_poll(TeaVM_ReferenceQueue* queue) { TeaVM_Reference* reference = queue->first; queue->first = reference->next; + teavm_gc_writeBarrier(queue); if (queue->first == NULL) { queue->last = NULL; } diff --git a/core/src/main/resources/org/teavm/backend/c/stringhash.c b/core/src/main/resources/org/teavm/backend/c/stringhash.c index ad02b459c..40bd25fe0 100644 --- a/core/src/main/resources/org/teavm/backend/c/stringhash.c +++ b/core/src/main/resources/org/teavm/backend/c/stringhash.c @@ -66,8 +66,8 @@ static void teavm_rehashStrings() { } TeaVM_String* teavm_registerString(TeaVM_String* str) { - str->parent.header = TEAVM_PACK_CLASS(teavm_stringClass); - str->characters->parent.header = TEAVM_PACK_CLASS(teavm_charArrayClass); + str->parent.header = TEAVM_PACK_CLASS(teavm_stringClass) | (int32_t) INT32_C(0x80000000); + str->characters->parent.header = TEAVM_PACK_CLASS(teavm_charArrayClass) | (int32_t) INT32_C(0x80000000); if (teavm_stringHashtable == NULL) { teavm_stringHashtableSize = 256;