Wasm: fix GC

core/src/main/java/org/teavm/backend/wasm/WasmHeap.java

@@ -16,6 +16,7 @@
 package org.teavm.backend.wasm;
 
 import org.teavm.interop.Address;
+import org.teavm.interop.Import;
 import org.teavm.interop.StaticInit;
 import org.teavm.interop.Unmanaged;
 
@@ -24,7 +25,7 @@ import org.teavm.interop.Unmanaged;
 public final class WasmHeap {
     public static final int PAGE_SIZE = 65536;
     public static final int DEFAULT_STACK_SIZE = PAGE_SIZE * 4;
-    public static final int DEFAULT_REGION_SIZE = 4096;
+    public static final int DEFAULT_REGION_SIZE = 1024;
 
     public static int minHeapSize;
     public static int maxHeapSize;
@@ -50,7 +51,7 @@ public final class WasmHeap {
     }
 
     public static int calculateRegionsCount(int heapSize, int regionSize) {
-        return (heapSize + regionSize - 1) / regionSize;
+        return (heapSize / regionSize) + 1;
     }
 
     public static int calculateRegionsSize(int regionsCount) {
@@ -59,7 +60,11 @@ public final class WasmHeap {
 
     public static native void growMemory(int amount);
 
+    @Import(name = "init", module = "teavmHeapTrace")
+    private static native void initHeapTrace(int maxHeap);
+
     public static void initHeap(Address start, int minHeap, int maxHeap, int stackSize) {
+        initHeapTrace(maxHeap);
         stackAddress = start;
         stack = start;
         heapAddress = WasmRuntime.align(stackAddress.add(stackSize), 16);
@@ -80,8 +85,8 @@ public final class WasmHeap {
         int newRegionsSize = calculateRegionsSize(newRegionsCount);
 
         Address newRegionsAddress = WasmRuntime.align(heapAddress.add(newHeapSize), 16);
-        Address newCardTable = WasmRuntime.align(newRegionsAddress.add(newRegionsCount), 16);
+        Address newCardTable = WasmRuntime.align(newRegionsAddress.add(newRegionsSize), 16);
-        Address newStorageAddress = WasmRuntime.align(newCardTable.add(newRegionsSize), 16);
+        Address newStorageAddress = WasmRuntime.align(newCardTable.add(newRegionsCount), 16);
         Address newMemoryLimit = WasmRuntime.align(newStorageAddress.add(newStorageSize), PAGE_SIZE);
         if (newMemoryLimit != memoryLimit) {
             growMemory((int) (newMemoryLimit.toLong() - memoryLimit.toLong()) / PAGE_SIZE);

core/src/main/java/org/teavm/backend/wasm/WasmTarget.java

@@ -41,6 +41,7 @@ import org.teavm.backend.wasm.generate.WasmDependencyListener;
 import org.teavm.backend.wasm.generate.WasmGenerationContext;
 import org.teavm.backend.wasm.generate.WasmGenerator;
 import org.teavm.backend.wasm.generate.WasmNameProvider;
+import org.teavm.backend.wasm.generate.WasmSpecialFunctionGenerator;
 import org.teavm.backend.wasm.generate.WasmStringPool;
 import org.teavm.backend.wasm.generators.ArrayGenerator;
 import org.teavm.backend.wasm.generators.WasmMethodGenerator;
@@ -139,6 +140,7 @@ import org.teavm.model.lowlevel.Characteristics;
 import org.teavm.model.lowlevel.ClassInitializerEliminator;
 import org.teavm.model.lowlevel.ClassInitializerTransformer;
 import org.teavm.model.lowlevel.ShadowStackTransformer;
+import org.teavm.model.lowlevel.WriteBarrierInsertion;
 import org.teavm.model.optimization.InliningFilterFactory;
 import org.teavm.model.transformation.ClassPatch;
 import org.teavm.runtime.Allocator;
@@ -170,6 +172,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
     private ClassInitializerEliminator classInitializerEliminator;
     private ClassInitializerTransformer classInitializerTransformer;
     private ShadowStackTransformer shadowStackTransformer;
+    private WriteBarrierInsertion writeBarrierInsertion;
     private WasmBinaryVersion version = WasmBinaryVersion.V_0x1;
     private List<WasmIntrinsicFactory> additionalIntrinsics = new ArrayList<>();
     private int minHeapSize = 2 * 1024 * 1024;
@@ -182,6 +185,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
         classInitializerEliminator = new ClassInitializerEliminator(controller.getUnprocessedClassSource());
         classInitializerTransformer = new ClassInitializerTransformer();
         shadowStackTransformer = new ShadowStackTransformer(characteristics, true);
+        writeBarrierInsertion = new WriteBarrierInsertion(characteristics);
 
         controller.addVirtualMethods(VIRTUAL_METHODS::contains);
     }
@@ -344,6 +348,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
         classInitializerEliminator.apply(program);
         classInitializerTransformer.transform(program);
         shadowStackTransformer.apply(program, method);
+        writeBarrierInsertion.apply(program);
     }
 
     @Override
@@ -385,7 +390,9 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
         context.addIntrinsic(new ObjectIntrinsic());
         context.addIntrinsic(new ConsoleIntrinsic());
         context.addGenerator(new ArrayGenerator());
-        context.addIntrinsic(new MemoryTraceIntrinsic());
+        if (!Boolean.parseBoolean(System.getProperty("teavm.wasm.vmAssertions", "false"))) {
+            context.addIntrinsic(new MemoryTraceIntrinsic());
+        }
         context.addIntrinsic(new WasmHeapIntrinsic());
 
         IntrinsicFactoryContext intrinsicFactoryContext = new IntrinsicFactoryContext();
@@ -408,6 +415,8 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
         exceptionHandlingIntrinsic.postProcess(CallSiteDescriptor.extract(classes, classes.getClassNames()));
         generateIsSupertypeFunctions(tagRegistry, module, classGenerator);
         classGenerator.postProcess();
+        new WasmSpecialFunctionGenerator(classGenerator, gcIntrinsic.regionSizeExpressions)
+                .generateSpecialFunctions(module);
         mutatorIntrinsic.setStaticGcRootsAddress(classGenerator.getStaticGcRootsAddress());
         mutatorIntrinsic.setClassesAddress(classGenerator.getClassesAddress());
         mutatorIntrinsic.setClassCount(classGenerator.getClassCount());

core/src/main/java/org/teavm/backend/wasm/generate/WasmSpecialFunctionGenerator.java

@@ -0,0 +1,91 @@
+/*
+ *  Copyright 2020 Alexey Andreev.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.backend.wasm.generate;
+
+import java.util.List;
+import org.teavm.backend.wasm.WasmHeap;
+import org.teavm.backend.wasm.model.WasmFunction;
+import org.teavm.backend.wasm.model.WasmModule;
+import org.teavm.backend.wasm.model.WasmType;
+import org.teavm.backend.wasm.model.expression.WasmInt32Constant;
+import org.teavm.backend.wasm.model.expression.WasmInt32Subtype;
+import org.teavm.backend.wasm.model.expression.WasmLoadInt32;
+import org.teavm.backend.wasm.model.expression.WasmReturn;
+import org.teavm.model.FieldReference;
+
+public class WasmSpecialFunctionGenerator {
+    private WasmClassGenerator classGenerator;
+    private List<WasmInt32Constant> regionSizeExpressions;
+
+    public WasmSpecialFunctionGenerator(WasmClassGenerator classGenerator,
+            List<WasmInt32Constant> regionSizeExpressions) {
+        this.classGenerator = classGenerator;
+        this.regionSizeExpressions = regionSizeExpressions;
+    }
+
+    public void generateSpecialFunctions(WasmModule module) {
+        module.add(javaHeapAddress());
+        module.add(availableBytes());
+        module.add(regionsAddress());
+        module.add(regionSize());
+    }
+
+    private WasmFunction javaHeapAddress() {
+        WasmFunction function = new WasmFunction("teavm_javaHeapAddress");
+        function.setExportName("teavm_javaHeapAddress");
+        function.setResult(WasmType.INT32);
+
+        int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "heapAddress"));
+        function.getBody().add(new WasmReturn(
+                new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32)));
+        return function;
+    }
+
+
+    private WasmFunction availableBytes() {
+        WasmFunction function = new WasmFunction("teavm_availableBytes");
+        function.setExportName("teavm_availableBytes");
+        function.setResult(WasmType.INT32);
+
+        int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "heapSize"));
+        function.getBody().add(new WasmReturn(
+                new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32)));
+        return function;
+    }
+
+
+    private WasmFunction regionsAddress() {
+        WasmFunction function = new WasmFunction("teavm_regionsAddress");
+        function.setExportName("teavm_regionsAddress");
+        function.setResult(WasmType.INT32);
+
+        int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "regionsAddress"));
+        function.getBody().add(new WasmReturn(
+                new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32)));
+        return function;
+    }
+
+    private WasmFunction regionSize() {
+        WasmFunction function = new WasmFunction("teavm_regionSize");
+        function.setExportName("teavm_regionSize");
+        function.setResult(WasmType.INT32);
+
+        WasmInt32Constant constant = new WasmInt32Constant(0);
+        regionSizeExpressions.add(constant);
+        function.getBody().add(new WasmReturn(constant));
+        return function;
+    }
+}

core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java

@@ -44,7 +44,7 @@ public class GCIntrinsic implements WasmIntrinsic {
             WasmHeap.class, "resizeHeap", int.class, void.class);
     private static final FieldReference CARD_TABLE = new FieldReference(WasmHeap.class.getName(), "cardTable");
     private static final FieldReference HEAP_ADDRESS = new FieldReference(WasmHeap.class.getName(), "heapAddress");
-    private List<WasmInt32Constant> regionSizeExpressions = new ArrayList<>();
+    public final List<WasmInt32Constant> regionSizeExpressions = new ArrayList<>();
 
     public void setRegionSize(int regionSize) {
         for (WasmInt32Constant constant : regionSizeExpressions) {

core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderer.java

@@ -99,9 +99,15 @@ public class WasmCRenderer {
     }
 
     private void renderPrologue() {
+        writeResource("org/teavm/backend/wasm/wasm-runtime.c");
+        line("#define TEAVM_MEMORY_TRACE 1");
+        writeResource("org/teavm/backend/wasm/wasm-heapTrace.c");
+    }
+
+    private void writeResource(String name) {
         ClassLoader classLoader = WasmCRenderer.class.getClassLoader();
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(
-                classLoader.getResourceAsStream("org/teavm/backend/wasm/wasm-runtime.c")))) {
+                classLoader.getResourceAsStream(name)))) {
             while (true) {
                 String line = reader.readLine();
                 if (line == null) {
@@ -157,9 +163,7 @@ public class WasmCRenderer {
 
     private void renderFunctionDeclarations(WasmModule module) {
         for (WasmFunction function : module.getFunctions().values()) {
-            if (function.getImportName() == null) {
+            line(functionDeclaration(function) + ";");
-                line(functionDeclaration(function) + ";");
-            }
         }
     }
 
@@ -220,7 +224,14 @@ public class WasmCRenderer {
         StringBuilder sb = new StringBuilder();
         renderFunctionModifiers(sb, function);
         sb.append(WasmCRenderingVisitor.mapType(function.getResult())).append(' ');
-        sb.append(function.getName()).append("(");
+        if (function.getImportName() != null) {
+            sb.append(!function.getImportModule().isEmpty()
+                    ? function.getImportModule() + "_" + function.getImportName()
+                    : function.getImportName());
+        } else {
+            sb.append(function.getName());
+        }
+        sb.append("(");
         for (int i = 0; i < function.getParameters().size(); ++i) {
             if (i > 0) {
                 sb.append(", ");

core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderingVisitor.java

@@ -700,7 +700,14 @@ class WasmCRenderingVisitor implements WasmExpressionVisitor {
         WasmType type = requiredType;
 
         StringBuilder sb = new StringBuilder();
-        sb.append(expression.getFunctionName()).append('(');
+        if (expression.isImported()) {
+            sb.append(!function.getImportModule().isEmpty()
+                    ? function.getImportModule() + "_" + function.getImportName()
+                    : function.getImportName());
+        } else {
+            sb.append(expression.getFunctionName());
+        }
+        sb.append('(');
         translateArguments(expression.getArguments(), function.getParameters(), result, sb);
         sb.append(')');
         result.setText(sb.toString());

core/src/main/java/org/teavm/runtime/GC.java

@@ -114,7 +114,7 @@ public final class GC {
             return;
         }
         collectGarbageImpl(size);
-        if (currentChunk.size < size && !getNextChunkIfPossible(size)) {
+        if (currentChunk.size < size + 5 && !getNextChunkIfPossible(size)) {
             ExceptionHandling.printStack();
             outOfMemory();
         }
@@ -131,7 +131,7 @@ public final class GC {
             }
             currentChunkPointer = Structure.add(FreeChunkHolder.class, currentChunkPointer, 1);
             currentChunk = currentChunkPointer.value;
-            if (currentChunk.size >= size) {
+            if (currentChunk.size >= size + 5) {
                 currentChunkLimit = currentChunk.toAddress().add(currentChunk.size);
                 break;
             }
@@ -786,7 +786,7 @@ public final class GC {
                 if (shouldRelocateObject) {
                     while (true) {
                         nextRelocationTarget = relocationTarget.add(size);
-                        if (!relocationBlock.end.isLessThan(nextRelocationTarget)) {
+                        if (!relocationBlock.end.isLessThan(nextRelocationTarget.add(5))) {
                             break;
                         }
 
@@ -1359,6 +1359,7 @@ public final class GC {
         if (newSize > oldSize) {
             int previousRegionCount = getRegionCount();
             resizeHeap(newSize);
+            currentChunkPointer = gcStorageAddress().toStructure();
             int newRegionCount = getRegionCount();
             for (int i = previousRegionCount; i < newRegionCount; ++i) {
                 Structure.add(Region.class, regionsAddress(), i).start = 0;
@@ -1393,6 +1394,8 @@ public final class GC {
                 lastChunk.size -= (int) (oldSize - newSize);
             }
             resizeHeap(newSize);
+
+            currentChunkPointer = gcStorageAddress().toStructure();
         }
     }
 

core/src/main/java/org/teavm/runtime/MemoryTrace.java

@@ -16,40 +16,57 @@
 package org.teavm.runtime;
 
 import org.teavm.interop.Address;
+import org.teavm.interop.Import;
 
 public class MemoryTrace {
     private MemoryTrace() {
     }
 
+    @Import(name = "allocate", module = "teavmHeapTrace")
     public static native void allocate(Address address, int size);
 
+    @Import(name = "free", module = "teavmHeapTrace")
     public static native void free(Address address, int size);
 
+    @Import(name = "assertFree", module = "teavmHeapTrace")
     public static native void assertFree(Address address, int size);
 
+    @Import(name = "checkIsFree", module = "teavmHeapTrace")
     public static native void checkIsFree(Address address, int size);
 
+    @Import(name = "markStarted", module = "teavmHeapTrace")
     public static native void markStarted();
 
+    @Import(name = "mark", module = "teavmHeapTrace")
     public static native void mark(Address address);
 
+    @Import(name = "reportDirtyRegion", module = "teavmHeapTrace")
     public static native void reportDirtyRegion(Address address);
 
+    @Import(name = "markCompleted", module = "teavmHeapTrace")
     public static native void markCompleted();
 
+    @Import(name = "move", module = "teavmHeapTrace")
     public static native void move(Address from, Address to, int size);
 
+    @Import(name = "gcStarted", module = "teavmHeapTrace")
     public static native void gcStarted(boolean full);
 
+    @Import(name = "sweepStarted", module = "teavmHeapTrace")
     public static native void sweepStarted();
 
+    @Import(name = "sweepCompleted", module = "teavmHeapTrace")
     public static native void sweepCompleted();
 
+    @Import(name = "defragStarted", module = "teavmHeapTrace")
     public static native void defragStarted();
 
+    @Import(name = "defragCompleted", module = "teavmHeapTrace")
     public static native void defragCompleted();
 
+    @Import(name = "writeHeapDump", module = "teavmHeapTrace")
     public static native void writeHeapDump();
 
+    @Import(name = "gcCompleted", module = "teavmHeapTrace")
     public static native void gcCompleted();
 }

core/src/main/resources/org/teavm/backend/c/fiber.c

@@ -30,7 +30,7 @@ void teavm_initFiber() {
         sigaction(SIGRTMIN, &sigact, NULL);
 
         sigset_t signals;
-        sigemptyset(&signals );
+        sigemptyset(&signals);
         sigaddset(&signals, SIGRTMIN);
         sigprocmask(SIG_BLOCK, &signals, NULL);
 

core/src/main/resources/org/teavm/backend/wasm/wasm-heapTrace.c

@@ -0,0 +1,448 @@
+#define TEAVM_REFERENCE_SIZE 4
+
+#if TEAVM_MEMORY_TRACE
+    uint8_t* teavm_gc_heapMap = NULL;
+    uint8_t* teavm_gc_markMap = NULL;
+#endif
+
+#if TEAVM_GC_STATS
+    static int32_t teavm_gc_allocationCount = 0;
+    static int32_t teavm_gc_freeCount = 0;
+    static int32_t teavm_gc_freeByteCount = 0;
+    static int32_t teavm_gc_markCount = 0;
+    static int32_t teavm_gc_dirtyRegionCount = 0;
+    static int32_t teavm_gc_relocatedBlocks = 0;
+    static int32_t teavm_gc_relocatedBytes = 0;
+
+    static int64_t teavm_gc_startTimeMillis;
+    static int64_t teavm_gc_startTime;
+    static int64_t teavm_gc_endTime;
+    static int64_t teavm_gc_markStartTime;
+    static int64_t teavm_gc_markEndTime;
+    static int64_t teavm_gc_sweepStartTime;
+    static int64_t teavm_gc_sweepEndTime;
+    static int64_t teavm_gc_defragStartTime;
+    static int64_t teavm_gc_defragEndTime;
+    static int32_t teavm_gc_full;
+#endif
+
+#if TEAVM_MEMORY_TRACE
+    static inline void teavmHeapTrace_assertSize(int32_t size) {
+        if (size % TEAVM_REFERENCE_SIZE != 0) {
+            abort();
+        }
+    }
+
+    static inline void teavmHeapTrace_assertAddress(int32_t address) {
+        if (address % TEAVM_REFERENCE_SIZE != 0) {
+            abort();
+        }
+    }
+#endif
+
+int32_t teavm_javaHeapAddress();
+int32_t teavm_availableBytes();
+int32_t teavm_regionSize();
+int32_t teavm_regionsAddress();
+
+void teavmHeapTrace_init(int32_t maxHeap) {
+    teavm_gc_heapMap = (uint8_t*) malloc(maxHeap / TEAVM_REFERENCE_SIZE);
+    teavm_gc_markMap = (uint8_t*) malloc(maxHeap / TEAVM_REFERENCE_SIZE);
+}
+
+void teavmHeapTrace_allocate(int32_t address, int32_t size) {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_assertAddress(address);
+        teavmHeapTrace_assertSize(size);
+
+        address -= teavm_javaHeapAddress();
+        size /= TEAVM_REFERENCE_SIZE;
+        uint8_t* map = teavm_gc_heapMap + address / TEAVM_REFERENCE_SIZE;
+
+        if (*map != 0) {
+            fprintf(stderr, "[GC] trying allocate at memory in use at: %" PRId32 "\n", address);
+            abort();
+        }
+        *map++ = 1;
+
+        for (int32_t i = 1; i < size; ++i) {
+            if (*map != 0) {
+                fprintf(stderr, "[GC] trying allocate at memory in use at: %" PRId32 "\n", address);
+                abort();
+            }
+            *map++ = 2;
+        }
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_allocationCount++;
+    #endif
+}
+
+void teavmHeapTrace_free(int32_t address, int32_t size) {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_assertAddress(address);
+        teavmHeapTrace_assertSize(size);
+
+        address -= teavm_javaHeapAddress();
+        int32_t offset = address / TEAVM_REFERENCE_SIZE;
+        uint8_t* markMap = teavm_gc_markMap + offset;
+        size /= TEAVM_REFERENCE_SIZE;
+        for (int32_t i = 0; i < size; ++i) {
+            if (markMap[i] != 0) {
+                fprintf(stderr, "[GC] trying to release reachable object at: %" PRId32 "\n", address);
+                abort();
+            }
+        }
+
+        uint8_t* map = teavm_gc_heapMap + offset;
+        memset(map, 0, size);
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_freeCount++;
+        teavm_gc_freeByteCount += size;
+    #endif
+}
+
+void teavmHeapTrace_assertFree(int32_t address, int32_t size) {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_assertAddress(address);
+        teavmHeapTrace_assertSize(size);
+
+        address -= teavm_javaHeapAddress();
+        int32_t offset = address / TEAVM_REFERENCE_SIZE;
+        uint8_t* map = teavm_gc_heapMap + offset;
+        size /= TEAVM_REFERENCE_SIZE;
+        for (int32_t i = 0; i < size; ++i) {
+            if (map[i] != 0) {
+                fprintf(stderr, "[GC] memory supposed to be free at: %" PRId32 "\n", address);
+                abort();
+            }
+        }
+    #endif
+}
+
+void teavmHeapTrace_markStarted() {
+    #if TEAVM_MEMORY_TRACE
+        memset(teavm_gc_markMap, 0, teavm_availableBytes() / TEAVM_REFERENCE_SIZE);
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_markStartTime = teavm_nanoTime();
+    #endif
+}
+
+void teavmHeapTrace_markCompleted() {
+    #if TEAVM_GC_STATS
+        teavm_gc_markEndTime = teavm_nanoTime();
+    #endif
+}
+
+#define TEAVM_ALIGN(addr, alignment) ((((addr) + ((int32_t) (alignment) - 1)) / (alignment) * (alignment)))
+#define TEAVM_OBJECT_HEADER(address) (*(int32_t*) (wasm_heap + (address)))
+#define TEAVM_OBJECT_HASH(address) (*(int32_t*) (wasm_heap + (address) + 4))
+#define TEAVM_CLASS_OF(address) (TEAVM_OBJECT_HEADER(address) << 3);
+
+#define teavm_class_size(address) (*(int32_t*) (wasm_heap + ((address) + 8)))
+#define teavm_class_flags(address) (*(int32_t*) (wasm_heap + ((address) + 12)))
+#define teavm_class_itemType(address) (*(int32_t*) (wasm_heap + ((address) + 32)))
+#define teavm_class_superclass(address) (*(int32_t*) (wasm_heap + ((address) + 56)))
+#define teavm_class_layout(address) (*(int32_t*) (wasm_heap + ((address) + 72)))
+#define teavm_array_size(address) (*(int32_t*) (wasm_heap + ((address) + 8)))
+#define teavm_reference_queue(address) (*(int32_t*) (wasm_heap + ((address) + 8)))
+#define teavm_reference_object(address) (*(int32_t*) (wasm_heap + ((address) + 12)))
+#define teavm_reference_next(address) (*(int32_t*) (wasm_heap + ((address) + 16)))
+#define teavm_referenceQueue_first(address) (*(int32_t*) (wasm_heap + ((address) + 8)))
+#define teavm_referenceQueue_last(address) (*(int32_t*) (wasm_heap + ((address) + 12)))
+#define TEAVM_ARRAY_STRUCT_SIZE 12
+
+int32_t teavmHeapTrace_objectSize(int32_t address) {
+    int32_t cls = TEAVM_CLASS_OF(address);
+    int32_t itemType = teavm_class_itemType(cls);
+    if (itemType == 0) {
+        return teavm_class_size(cls);
+    }
+
+    int32_t itemSize = teavm_class_flags(itemType) & 2 ? teavm_class_size(itemType) : TEAVM_REFERENCE_SIZE;
+    int32_t size = TEAVM_ALIGN(TEAVM_ARRAY_STRUCT_SIZE, itemSize);
+    size += teavm_array_size(address) * itemSize;
+    size = TEAVM_ALIGN(size, TEAVM_REFERENCE_SIZE);
+    return size;
+}
+
+void teavmHeapTrace_mark(int32_t address) {
+    #if TEAVM_MEMORY_TRACE
+        if (address < teavm_javaHeapAddress() || address >= teavm_javaHeapAddress() + teavm_availableBytes()) {
+            return;
+        }
+
+        teavmHeapTrace_assertAddress(address);
+
+        int32_t offset = (address - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE;
+        uint8_t* map = teavm_gc_heapMap + offset;
+        uint8_t* markMap = teavm_gc_markMap + offset;
+
+        int32_t size = teavmHeapTrace_objectSize(address);
+        teavmHeapTrace_assertSize(size);
+        size /= TEAVM_REFERENCE_SIZE;
+
+        if (*map++ != 1 || *markMap != 0) {
+            fprintf(stderr, "[GC] assertion failed marking object at: %" PRId32 "\n",
+                    address - teavm_javaHeapAddress());
+            abort();
+        }
+        *markMap++ = 1;
+
+        for (int32_t i = 1; i < size; ++i) {
+            if (*map++ != 2 || *markMap != 0) {
+                abort();
+            }
+            *markMap++ = 1;
+        }
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_markCount++;
+    #endif
+}
+
+void teavmHeapTrace_move(int32_t from, int32_t to, int32_t size) {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_assertAddress(from);
+        teavmHeapTrace_assertAddress(to);
+        teavmHeapTrace_assertSize(size);
+
+        uint8_t* mapFrom = teavm_gc_heapMap + ((from - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE);
+        uint8_t* mapTo = teavm_gc_heapMap + ((to - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE);
+        size /= TEAVM_REFERENCE_SIZE;
+
+        if (mapFrom > mapTo) {
+            for (int32_t i = 0; i < size; ++i) {
+                if (mapFrom[i] == 0 || mapTo[i] != 0) {
+                    fprintf(stderr, "[GC] assertion failed moving object from: %" PRId32 " to %" PRId32 "\n",
+                            from - teavm_javaHeapAddress(), to - teavm_javaHeapAddress());
+                    abort();
+                }
+                mapTo[i] = mapFrom[i];
+                mapFrom[i] = 0;
+            }
+        } else {
+            for (int32_t i = size - 1; i >= 0; --i) {
+                if (mapFrom[i] == 0 || mapTo[i] != 0) {
+                    abort();
+                }
+                mapTo[i] = mapFrom[i];
+                mapFrom[i] = 0;
+            }
+        }
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_relocatedBlocks++;
+        teavm_gc_relocatedBytes += size;
+    #endif
+}
+
+#if TEAVM_MEMORY_TRACE
+
+    static inline int32_t teavm_verify(int32_t address) {
+        if (address >= teavm_javaHeapAddress() && address < teavm_javaHeapAddress() + teavm_availableBytes()) {
+            teavmHeapTrace_assertAddress(address);
+            uint8_t* map = teavm_gc_heapMap + ((address - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE);
+            if (*map != 1) {
+                abort();
+            }
+        }
+
+        return address;
+    }
+
+    void teavmHeapTrace_checkHeapConsistency(int32_t oldGen, int32_t offsets) {
+        int32_t lastCheckedRegion = -1;
+        int32_t obj = teavm_javaHeapAddress();
+        uint16_t* regions = (uint16_t*) (wasm_heap + teavm_regionsAddress());
+        while (obj < teavm_javaHeapAddress() + teavm_availableBytes()) {
+            int32_t size;
+            int32_t header = TEAVM_OBJECT_HEADER(obj);
+            if (header == 0) {
+                size = TEAVM_OBJECT_HASH(obj);
+                teavmHeapTrace_assertFree(obj, size);
+            } else {
+                teavm_verify(obj);
+                if (offsets) {
+                    int32_t offset = obj - teavm_javaHeapAddress();
+                    int32_t objRegion = offset / teavm_regionSize();
+                    if (objRegion != lastCheckedRegion) {
+                        while (++lastCheckedRegion < objRegion) {
+                            if (regions[lastCheckedRegion] != 0) {
+                                abort();
+                            }
+                        }
+                        int32_t offsetInRegion = offset % teavm_regionSize();
+                        if (regions[objRegion] != offsetInRegion + 1) {
+                            abort();
+                        }
+                    }
+                }
+                if (oldGen && !(header & 0x40000000)) {
+                    abort();
+                }
+                int32_t cls = TEAVM_CLASS_OF(obj);
+                int32_t itemType = teavm_class_itemType(cls);
+                if (itemType != 0) {
+                    if (!(teavm_class_flags(itemType) & 2)) {
+                        int32_t offset = 0;
+                        offset += TEAVM_ARRAY_STRUCT_SIZE;
+                        offset = TEAVM_ALIGN(offset, TEAVM_REFERENCE_SIZE);
+                        int32_t data = obj + offset;
+                        int32_t size = teavm_array_size(obj);
+                        for (int32_t i = 0; i < size; ++i) {
+                            teavm_verify(((int32_t*) (wasm_heap + data))[i]);
+                        }
+                    }
+                } else {
+                    while (cls != 0) {
+                        int32_t kind = (teavm_class_flags(cls) >> 7) & 7;
+                        if (kind == 1) {
+                            teavm_verify(teavm_reference_next(obj));
+                            teavm_verify(teavm_reference_object(obj));
+                            teavm_verify(teavm_reference_queue(obj));
+                        } else if (kind == 2) {
+                            teavm_verify(teavm_referenceQueue_first(obj));
+                            teavm_verify(teavm_referenceQueue_last(obj));
+                        } else {
+                            int32_t layoutOffset = teavm_class_layout(cls);
+                            if (layoutOffset != 0) {
+                                int16_t* layout = (int16_t*) (wasm_heap + layoutOffset);
+                                int16_t size = *layout++;
+                                for (int32_t i = 0; i < size; ++i) {
+                                    int32_t ptr = obj + *layout++;
+                                    teavm_verify(*(int32_t*) (wasm_heap + ptr));
+                                }
+                            }
+                        }
+
+                        cls = teavm_class_superclass(cls);
+                    }
+                }
+                size = teavmHeapTrace_objectSize(obj);
+            }
+
+            if (size == 0) {
+                abort();
+            }
+            obj += size;
+        }
+
+        if (offsets) {
+            int32_t lastRegion = teavm_availableBytes() / teavm_regionSize();
+            while (++lastCheckedRegion <= lastRegion) {
+                if (regions[lastCheckedRegion] != 0) {
+                    abort();
+                }
+            }
+        }
+    }
+#endif
+
+void teavmHeapTrace_gcStarted(int32_t full) {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_checkHeapConsistency(0, 0);
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_startTime = teavm_nanoTime();
+        teavm_gc_startTimeMillis = teavm_currentTimeMillis();
+        teavm_gc_full = full;
+    #endif
+}
+
+void teavmHeapTrace_sweepStarted() {
+    #if TEAVM_GC_STATS
+        teavm_gc_sweepStartTime = teavm_nanoTime();
+    #endif
+}
+
+void teavmHeapTrace_sweepCompleted() {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_checkHeapConsistency(0, 1);
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_sweepEndTime = teavm_nanoTime();
+    #endif
+}
+
+void teavmHeapTrace_defragStarted() {
+    #if TEAVM_GC_STATS
+        teavm_gc_defragStartTime = teavm_nanoTime();
+    #endif
+}
+
+void teavmHeapTrace_defragCompleted() {
+    #if TEAVM_MEMORY_TRACE
+        teavmHeapTrace_checkHeapConsistency(1, 1);
+    #endif
+
+    #if TEAVM_GC_STATS
+        teavm_gc_defragEndTime = teavm_nanoTime();
+    #endif
+}
+
+#if TEAVM_GC_STATS
+    static void teavmHeapTrace_printStats() {
+        fprintf(stderr, "[GC] Garbage collection (%s) performed at %" PRIu64 ", took %"
+                PRIu64 " ns\n", teavm_gc_full ? "full" : "young", teavm_gc_startTimeMillis,
+                teavm_gc_endTime - teavm_gc_startTime);
+
+        fprintf(stderr, "[GC]   Allocations performed before GC: %" PRIu32 "\n", teavm_gc_allocationCount);
+
+        fprintf(stderr, "[GC]   Mark phase took %" PRIu64 " ns, %" PRIu32 " objects reached\n",
+                teavm_gc_markEndTime - teavm_gc_markStartTime, teavm_gc_markCount);
+
+        if (!teavm_gc_full) {
+            fprintf(stderr, "[GC]     Regions scanned from remembered set: %" PRIu32 "\n", teavm_gc_dirtyRegionCount);
+        }
+
+        fprintf(stderr, "[GC]   Sweep phase took %" PRIu64 " ns, %" PRIu32 " regions of %"
+                PRIu32 " bytes freed\n", teavm_gc_sweepEndTime - teavm_gc_sweepStartTime, teavm_gc_freeCount,
+                teavm_gc_freeByteCount);
+
+        fprintf(stderr, "[GC]   Defrag phase took %" PRIu64 " ns\n",
+                teavm_gc_defragEndTime - teavm_gc_defragStartTime);
+
+        fprintf(stderr, "[GC]     Blocks relocated %" PRId32 " of total %" PRId32 " bytes\n",
+                teavm_gc_relocatedBlocks, teavm_gc_relocatedBytes);
+    }
+
+    static void teavmHeapTrace_resetStats() {
+        teavm_gc_allocationCount = 0;
+        teavm_gc_markCount = 0;
+        teavm_gc_dirtyRegionCount = 0;
+        teavm_gc_freeCount = 0;
+        teavm_gc_freeByteCount = 0;
+        teavm_gc_relocatedBlocks = 0;
+        teavm_gc_relocatedBytes = 0;
+    }
+#endif
+
+void teavmHeapTrace_gcCompleted() {
+    #if TEAVM_GC_STATS
+        teavm_gc_endTime = teavm_nanoTime();
+        teavmHeapTrace_printStats();
+        teavmHeapTrace_resetStats();
+    #endif
+}
+
+void teavmHeapTrace_heapResized(int64_t newSize) {
+    #if TEAVM_GC_STATS
+        fprintf(stderr, "[GC] Heap resized to %" PRIu64 " bytes\n", newSize);
+    #endif
+}
+
+void teavmHeapTrace_reportDirtyRegion(int32_t address) {
+    #if TEAVM_GC_STATS
+        teavm_gc_dirtyRegionCount++;
+    #endif
+}

core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.c

@@ -12,45 +12,61 @@
 static int8_t *wasm_heap;
 static int32_t wasm_heap_size;
 
-static inline float teavm_getNaN() {
+float teavm_teavm_getNaN() {
     return NAN;
 }
 
-static int64_t currentTimeMillis() {
+#define teavm_isnan isnan
+#define teavm_isinf isinf
+#define teavm_isfinite isfinite
+#define teavmMath_sin sin
+#define teavmMath_cos cos
+#define teavmMath_sqrt sqrt
+#define teavmMath_ceil ceil
+#define teavmMath_floor floor
+
+double teavm_currentTimeMillis() {
     struct timespec time;
     clock_gettime(CLOCK_REALTIME, &time);
 
     return time.tv_sec * 1000 + (int64_t) round(time.tv_nsec / 1000000);
 }
 
+double teavm_nanoTime() {
+    struct timespec time;
+    clock_gettime(CLOCK_REALTIME, &time);
+
+    return time.tv_sec * 1000000000 + (int64_t) round(time.tv_nsec);
+}
+
 static union { float f; int32_t i; } reinterpret_union_32;
 static union { double f; int64_t i; } reinterpret_union_64;
 
-inline static int64_t reinterpret_float64(double v) {
+int64_t reinterpret_float64(double v) {
     reinterpret_union_64.f = v;
     return reinterpret_union_64.i;
 }
 
-inline static double reinterpret_int64(int64_t v) {
+double reinterpret_int64(int64_t v) {
     reinterpret_union_64.i = v;
     return reinterpret_union_64.f;
 }
 
-inline static int32_t reinterpret_float32(double v) {
+int32_t reinterpret_float32(double v) {
     reinterpret_union_32.f = v;
     return reinterpret_union_32.i;
 }
 
-inline static float reinterpret_int32(int32_t v) {
+float reinterpret_int32(int32_t v) {
     reinterpret_union_32.i = v;
     return reinterpret_union_32.f;
 }
 
-static void logOutOfMemory() {
+void teavm_logOutOfMemory() {
     abort();
 }
 
-static void logString(int32_t string) {
+void teavm_logString(int32_t string) {
     uint32_t arrayPtr = *(uint32_t*) (wasm_heap + string + 8);
     uint32_t length = *(uint32_t*) (wasm_heap + arrayPtr + 8);
     for (int32_t i = 0; i < length; ++i) {
@@ -59,6 +75,6 @@ static void logString(int32_t string) {
     }
 }
 
-static void logInt(int32_t v) {
+void teavm_logInt(int32_t v) {
     wprintf(L"%" PRId32, v);
 }

core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.js

@@ -67,6 +67,24 @@ TeaVM.wasm = function() {
         };
 
         obj.teavmMath = Math;
+
+        obj.teavmHeapTrace = {
+            allocate: function(address, size) {},
+            free: function(address, size) {},
+            assertFree: function(address, size) {},
+            markStarted: function() {},
+            mark: function(address) {},
+            reportDirtyRegion: function(address) {},
+            markCompleted: function() {},
+            move: function(from, to, size) {},
+            gcStarted: function(full) {},
+            sweepStarted: function() {},
+            sweepCompleted: function() {},
+            defragStarted: function() {},
+            defragCompleted: function() {},
+            gcCompleted: function() {},
+            init: function(maxHeap) {}
+        };
     }
 
     function run(path, options) {

tools/core/src/main/java/org/teavm/tooling/TeaVMTool.java

@@ -331,7 +331,7 @@ public class TeaVMTool {
         webAssemblyTarget.setWastEmitted(debugInformationGenerated);
         webAssemblyTarget.setVersion(wasmVersion);
         webAssemblyTarget.setMinHeapSize(minHeapSize);
-        webAssemblyTarget.setMinHeapSize(maxHeapSize);
+        webAssemblyTarget.setMaxHeapSize(maxHeapSize);
         return webAssemblyTarget;
     }