From 7d95c0fb0476fa6c37b8423743f42b7b38eca742 Mon Sep 17 00:00:00 2001 From: Alexey Andreev Date: Fri, 21 Feb 2020 16:22:12 +0300 Subject: [PATCH] Wasm: fix GC --- .../java/org/teavm/backend/wasm/WasmHeap.java | 13 +- .../org/teavm/backend/wasm/WasmTarget.java | 11 +- .../WasmSpecialFunctionGenerator.java | 91 ++++ .../backend/wasm/intrinsics/GCIntrinsic.java | 2 +- .../backend/wasm/render/WasmCRenderer.java | 21 +- .../wasm/render/WasmCRenderingVisitor.java | 9 +- core/src/main/java/org/teavm/runtime/GC.java | 9 +- .../java/org/teavm/runtime/MemoryTrace.java | 17 + .../resources/org/teavm/backend/c/fiber.c | 2 +- .../org/teavm/backend/wasm/wasm-heapTrace.c | 448 ++++++++++++++++++ .../org/teavm/backend/wasm/wasm-runtime.c | 34 +- .../org/teavm/backend/wasm/wasm-runtime.js | 18 + .../java/org/teavm/tooling/TeaVMTool.java | 2 +- 13 files changed, 651 insertions(+), 26 deletions(-) create mode 100644 core/src/main/java/org/teavm/backend/wasm/generate/WasmSpecialFunctionGenerator.java create mode 100644 core/src/main/resources/org/teavm/backend/wasm/wasm-heapTrace.c diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java b/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java index d5ac64aab..e1fa841bf 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmHeap.java @@ -16,6 +16,7 @@ package org.teavm.backend.wasm; import org.teavm.interop.Address; +import org.teavm.interop.Import; import org.teavm.interop.StaticInit; import org.teavm.interop.Unmanaged; @@ -24,7 +25,7 @@ import org.teavm.interop.Unmanaged; public final class WasmHeap { public static final int PAGE_SIZE = 65536; public static final int DEFAULT_STACK_SIZE = PAGE_SIZE * 4; - public static final int DEFAULT_REGION_SIZE = 4096; + public static final int DEFAULT_REGION_SIZE = 1024; public static int minHeapSize; public static int maxHeapSize; @@ -50,7 +51,7 @@ public final class WasmHeap { } public static int calculateRegionsCount(int heapSize, int regionSize) { - return (heapSize + regionSize - 1) / regionSize; + return (heapSize / regionSize) + 1; } public static int calculateRegionsSize(int regionsCount) { @@ -59,7 +60,11 @@ public final class WasmHeap { public static native void growMemory(int amount); + @Import(name = "init", module = "teavmHeapTrace") + private static native void initHeapTrace(int maxHeap); + public static void initHeap(Address start, int minHeap, int maxHeap, int stackSize) { + initHeapTrace(maxHeap); stackAddress = start; stack = start; heapAddress = WasmRuntime.align(stackAddress.add(stackSize), 16); @@ -80,8 +85,8 @@ public final class WasmHeap { int newRegionsSize = calculateRegionsSize(newRegionsCount); Address newRegionsAddress = WasmRuntime.align(heapAddress.add(newHeapSize), 16); - Address newCardTable = WasmRuntime.align(newRegionsAddress.add(newRegionsCount), 16); - Address newStorageAddress = WasmRuntime.align(newCardTable.add(newRegionsSize), 16); + Address newCardTable = WasmRuntime.align(newRegionsAddress.add(newRegionsSize), 16); + Address newStorageAddress = WasmRuntime.align(newCardTable.add(newRegionsCount), 16); Address newMemoryLimit = WasmRuntime.align(newStorageAddress.add(newStorageSize), PAGE_SIZE); if (newMemoryLimit != memoryLimit) { growMemory((int) (newMemoryLimit.toLong() - memoryLimit.toLong()) / PAGE_SIZE); diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java b/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java index b86b1924c..5b68708e3 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmTarget.java @@ -41,6 +41,7 @@ import org.teavm.backend.wasm.generate.WasmDependencyListener; import org.teavm.backend.wasm.generate.WasmGenerationContext; import org.teavm.backend.wasm.generate.WasmGenerator; import org.teavm.backend.wasm.generate.WasmNameProvider; +import org.teavm.backend.wasm.generate.WasmSpecialFunctionGenerator; import org.teavm.backend.wasm.generate.WasmStringPool; import org.teavm.backend.wasm.generators.ArrayGenerator; import org.teavm.backend.wasm.generators.WasmMethodGenerator; @@ -139,6 +140,7 @@ import org.teavm.model.lowlevel.Characteristics; import org.teavm.model.lowlevel.ClassInitializerEliminator; import org.teavm.model.lowlevel.ClassInitializerTransformer; import org.teavm.model.lowlevel.ShadowStackTransformer; +import org.teavm.model.lowlevel.WriteBarrierInsertion; import org.teavm.model.optimization.InliningFilterFactory; import org.teavm.model.transformation.ClassPatch; import org.teavm.runtime.Allocator; @@ -170,6 +172,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { private ClassInitializerEliminator classInitializerEliminator; private ClassInitializerTransformer classInitializerTransformer; private ShadowStackTransformer shadowStackTransformer; + private WriteBarrierInsertion writeBarrierInsertion; private WasmBinaryVersion version = WasmBinaryVersion.V_0x1; private List additionalIntrinsics = new ArrayList<>(); private int minHeapSize = 2 * 1024 * 1024; @@ -182,6 +185,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { classInitializerEliminator = new ClassInitializerEliminator(controller.getUnprocessedClassSource()); classInitializerTransformer = new ClassInitializerTransformer(); shadowStackTransformer = new ShadowStackTransformer(characteristics, true); + writeBarrierInsertion = new WriteBarrierInsertion(characteristics); controller.addVirtualMethods(VIRTUAL_METHODS::contains); } @@ -344,6 +348,7 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { classInitializerEliminator.apply(program); classInitializerTransformer.transform(program); shadowStackTransformer.apply(program, method); + writeBarrierInsertion.apply(program); } @Override @@ -385,7 +390,9 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { context.addIntrinsic(new ObjectIntrinsic()); context.addIntrinsic(new ConsoleIntrinsic()); context.addGenerator(new ArrayGenerator()); - context.addIntrinsic(new MemoryTraceIntrinsic()); + if (!Boolean.parseBoolean(System.getProperty("teavm.wasm.vmAssertions", "false"))) { + context.addIntrinsic(new MemoryTraceIntrinsic()); + } context.addIntrinsic(new WasmHeapIntrinsic()); IntrinsicFactoryContext intrinsicFactoryContext = new IntrinsicFactoryContext(); @@ -408,6 +415,8 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost { exceptionHandlingIntrinsic.postProcess(CallSiteDescriptor.extract(classes, classes.getClassNames())); generateIsSupertypeFunctions(tagRegistry, module, classGenerator); classGenerator.postProcess(); + new WasmSpecialFunctionGenerator(classGenerator, gcIntrinsic.regionSizeExpressions) + .generateSpecialFunctions(module); mutatorIntrinsic.setStaticGcRootsAddress(classGenerator.getStaticGcRootsAddress()); mutatorIntrinsic.setClassesAddress(classGenerator.getClassesAddress()); mutatorIntrinsic.setClassCount(classGenerator.getClassCount()); diff --git a/core/src/main/java/org/teavm/backend/wasm/generate/WasmSpecialFunctionGenerator.java b/core/src/main/java/org/teavm/backend/wasm/generate/WasmSpecialFunctionGenerator.java new file mode 100644 index 000000000..7a6dff54a --- /dev/null +++ b/core/src/main/java/org/teavm/backend/wasm/generate/WasmSpecialFunctionGenerator.java @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Alexey Andreev. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.backend.wasm.generate; + +import java.util.List; +import org.teavm.backend.wasm.WasmHeap; +import org.teavm.backend.wasm.model.WasmFunction; +import org.teavm.backend.wasm.model.WasmModule; +import org.teavm.backend.wasm.model.WasmType; +import org.teavm.backend.wasm.model.expression.WasmInt32Constant; +import org.teavm.backend.wasm.model.expression.WasmInt32Subtype; +import org.teavm.backend.wasm.model.expression.WasmLoadInt32; +import org.teavm.backend.wasm.model.expression.WasmReturn; +import org.teavm.model.FieldReference; + +public class WasmSpecialFunctionGenerator { + private WasmClassGenerator classGenerator; + private List regionSizeExpressions; + + public WasmSpecialFunctionGenerator(WasmClassGenerator classGenerator, + List regionSizeExpressions) { + this.classGenerator = classGenerator; + this.regionSizeExpressions = regionSizeExpressions; + } + + public void generateSpecialFunctions(WasmModule module) { + module.add(javaHeapAddress()); + module.add(availableBytes()); + module.add(regionsAddress()); + module.add(regionSize()); + } + + private WasmFunction javaHeapAddress() { + WasmFunction function = new WasmFunction("teavm_javaHeapAddress"); + function.setExportName("teavm_javaHeapAddress"); + function.setResult(WasmType.INT32); + + int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "heapAddress")); + function.getBody().add(new WasmReturn( + new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32))); + return function; + } + + + private WasmFunction availableBytes() { + WasmFunction function = new WasmFunction("teavm_availableBytes"); + function.setExportName("teavm_availableBytes"); + function.setResult(WasmType.INT32); + + int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "heapSize")); + function.getBody().add(new WasmReturn( + new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32))); + return function; + } + + + private WasmFunction regionsAddress() { + WasmFunction function = new WasmFunction("teavm_regionsAddress"); + function.setExportName("teavm_regionsAddress"); + function.setResult(WasmType.INT32); + + int address = classGenerator.getFieldOffset(new FieldReference(WasmHeap.class.getName(), "regionsAddress")); + function.getBody().add(new WasmReturn( + new WasmLoadInt32(4, new WasmInt32Constant(address), WasmInt32Subtype.INT32))); + return function; + } + + private WasmFunction regionSize() { + WasmFunction function = new WasmFunction("teavm_regionSize"); + function.setExportName("teavm_regionSize"); + function.setResult(WasmType.INT32); + + WasmInt32Constant constant = new WasmInt32Constant(0); + regionSizeExpressions.add(constant); + function.getBody().add(new WasmReturn(constant)); + return function; + } +} diff --git a/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java b/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java index 6b885ab77..0b774d93c 100644 --- a/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java +++ b/core/src/main/java/org/teavm/backend/wasm/intrinsics/GCIntrinsic.java @@ -44,7 +44,7 @@ public class GCIntrinsic implements WasmIntrinsic { WasmHeap.class, "resizeHeap", int.class, void.class); private static final FieldReference CARD_TABLE = new FieldReference(WasmHeap.class.getName(), "cardTable"); private static final FieldReference HEAP_ADDRESS = new FieldReference(WasmHeap.class.getName(), "heapAddress"); - private List regionSizeExpressions = new ArrayList<>(); + public final List regionSizeExpressions = new ArrayList<>(); public void setRegionSize(int regionSize) { for (WasmInt32Constant constant : regionSizeExpressions) { diff --git a/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderer.java b/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderer.java index 151fe6ed7..225b060b3 100644 --- a/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderer.java +++ b/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderer.java @@ -99,9 +99,15 @@ public class WasmCRenderer { } private void renderPrologue() { + writeResource("org/teavm/backend/wasm/wasm-runtime.c"); + line("#define TEAVM_MEMORY_TRACE 1"); + writeResource("org/teavm/backend/wasm/wasm-heapTrace.c"); + } + + private void writeResource(String name) { ClassLoader classLoader = WasmCRenderer.class.getClassLoader(); try (BufferedReader reader = new BufferedReader(new InputStreamReader( - classLoader.getResourceAsStream("org/teavm/backend/wasm/wasm-runtime.c")))) { + classLoader.getResourceAsStream(name)))) { while (true) { String line = reader.readLine(); if (line == null) { @@ -157,9 +163,7 @@ public class WasmCRenderer { private void renderFunctionDeclarations(WasmModule module) { for (WasmFunction function : module.getFunctions().values()) { - if (function.getImportName() == null) { - line(functionDeclaration(function) + ";"); - } + line(functionDeclaration(function) + ";"); } } @@ -220,7 +224,14 @@ public class WasmCRenderer { StringBuilder sb = new StringBuilder(); renderFunctionModifiers(sb, function); sb.append(WasmCRenderingVisitor.mapType(function.getResult())).append(' '); - sb.append(function.getName()).append("("); + if (function.getImportName() != null) { + sb.append(!function.getImportModule().isEmpty() + ? function.getImportModule() + "_" + function.getImportName() + : function.getImportName()); + } else { + sb.append(function.getName()); + } + sb.append("("); for (int i = 0; i < function.getParameters().size(); ++i) { if (i > 0) { sb.append(", "); diff --git a/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderingVisitor.java b/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderingVisitor.java index 0593e6869..67414dd2a 100644 --- a/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderingVisitor.java +++ b/core/src/main/java/org/teavm/backend/wasm/render/WasmCRenderingVisitor.java @@ -700,7 +700,14 @@ class WasmCRenderingVisitor implements WasmExpressionVisitor { WasmType type = requiredType; StringBuilder sb = new StringBuilder(); - sb.append(expression.getFunctionName()).append('('); + if (expression.isImported()) { + sb.append(!function.getImportModule().isEmpty() + ? function.getImportModule() + "_" + function.getImportName() + : function.getImportName()); + } else { + sb.append(expression.getFunctionName()); + } + sb.append('('); translateArguments(expression.getArguments(), function.getParameters(), result, sb); sb.append(')'); result.setText(sb.toString()); diff --git a/core/src/main/java/org/teavm/runtime/GC.java b/core/src/main/java/org/teavm/runtime/GC.java index 0d670f589..56fb5a20e 100644 --- a/core/src/main/java/org/teavm/runtime/GC.java +++ b/core/src/main/java/org/teavm/runtime/GC.java @@ -114,7 +114,7 @@ public final class GC { return; } collectGarbageImpl(size); - if (currentChunk.size < size && !getNextChunkIfPossible(size)) { + if (currentChunk.size < size + 5 && !getNextChunkIfPossible(size)) { ExceptionHandling.printStack(); outOfMemory(); } @@ -131,7 +131,7 @@ public final class GC { } currentChunkPointer = Structure.add(FreeChunkHolder.class, currentChunkPointer, 1); currentChunk = currentChunkPointer.value; - if (currentChunk.size >= size) { + if (currentChunk.size >= size + 5) { currentChunkLimit = currentChunk.toAddress().add(currentChunk.size); break; } @@ -786,7 +786,7 @@ public final class GC { if (shouldRelocateObject) { while (true) { nextRelocationTarget = relocationTarget.add(size); - if (!relocationBlock.end.isLessThan(nextRelocationTarget)) { + if (!relocationBlock.end.isLessThan(nextRelocationTarget.add(5))) { break; } @@ -1359,6 +1359,7 @@ public final class GC { if (newSize > oldSize) { int previousRegionCount = getRegionCount(); resizeHeap(newSize); + currentChunkPointer = gcStorageAddress().toStructure(); int newRegionCount = getRegionCount(); for (int i = previousRegionCount; i < newRegionCount; ++i) { Structure.add(Region.class, regionsAddress(), i).start = 0; @@ -1393,6 +1394,8 @@ public final class GC { lastChunk.size -= (int) (oldSize - newSize); } resizeHeap(newSize); + + currentChunkPointer = gcStorageAddress().toStructure(); } } diff --git a/core/src/main/java/org/teavm/runtime/MemoryTrace.java b/core/src/main/java/org/teavm/runtime/MemoryTrace.java index 41fca1099..1d8f20c39 100644 --- a/core/src/main/java/org/teavm/runtime/MemoryTrace.java +++ b/core/src/main/java/org/teavm/runtime/MemoryTrace.java @@ -16,40 +16,57 @@ package org.teavm.runtime; import org.teavm.interop.Address; +import org.teavm.interop.Import; public class MemoryTrace { private MemoryTrace() { } + @Import(name = "allocate", module = "teavmHeapTrace") public static native void allocate(Address address, int size); + @Import(name = "free", module = "teavmHeapTrace") public static native void free(Address address, int size); + @Import(name = "assertFree", module = "teavmHeapTrace") public static native void assertFree(Address address, int size); + @Import(name = "checkIsFree", module = "teavmHeapTrace") public static native void checkIsFree(Address address, int size); + @Import(name = "markStarted", module = "teavmHeapTrace") public static native void markStarted(); + @Import(name = "mark", module = "teavmHeapTrace") public static native void mark(Address address); + @Import(name = "reportDirtyRegion", module = "teavmHeapTrace") public static native void reportDirtyRegion(Address address); + @Import(name = "markCompleted", module = "teavmHeapTrace") public static native void markCompleted(); + @Import(name = "move", module = "teavmHeapTrace") public static native void move(Address from, Address to, int size); + @Import(name = "gcStarted", module = "teavmHeapTrace") public static native void gcStarted(boolean full); + @Import(name = "sweepStarted", module = "teavmHeapTrace") public static native void sweepStarted(); + @Import(name = "sweepCompleted", module = "teavmHeapTrace") public static native void sweepCompleted(); + @Import(name = "defragStarted", module = "teavmHeapTrace") public static native void defragStarted(); + @Import(name = "defragCompleted", module = "teavmHeapTrace") public static native void defragCompleted(); + @Import(name = "writeHeapDump", module = "teavmHeapTrace") public static native void writeHeapDump(); + @Import(name = "gcCompleted", module = "teavmHeapTrace") public static native void gcCompleted(); } diff --git a/core/src/main/resources/org/teavm/backend/c/fiber.c b/core/src/main/resources/org/teavm/backend/c/fiber.c index 02d52639c..e4a4fd68a 100644 --- a/core/src/main/resources/org/teavm/backend/c/fiber.c +++ b/core/src/main/resources/org/teavm/backend/c/fiber.c @@ -30,7 +30,7 @@ void teavm_initFiber() { sigaction(SIGRTMIN, &sigact, NULL); sigset_t signals; - sigemptyset(&signals ); + sigemptyset(&signals); sigaddset(&signals, SIGRTMIN); sigprocmask(SIG_BLOCK, &signals, NULL); diff --git a/core/src/main/resources/org/teavm/backend/wasm/wasm-heapTrace.c b/core/src/main/resources/org/teavm/backend/wasm/wasm-heapTrace.c new file mode 100644 index 000000000..9c5acc180 --- /dev/null +++ b/core/src/main/resources/org/teavm/backend/wasm/wasm-heapTrace.c @@ -0,0 +1,448 @@ +#define TEAVM_REFERENCE_SIZE 4 + +#if TEAVM_MEMORY_TRACE + uint8_t* teavm_gc_heapMap = NULL; + uint8_t* teavm_gc_markMap = NULL; +#endif + +#if TEAVM_GC_STATS + static int32_t teavm_gc_allocationCount = 0; + static int32_t teavm_gc_freeCount = 0; + static int32_t teavm_gc_freeByteCount = 0; + static int32_t teavm_gc_markCount = 0; + static int32_t teavm_gc_dirtyRegionCount = 0; + static int32_t teavm_gc_relocatedBlocks = 0; + static int32_t teavm_gc_relocatedBytes = 0; + + static int64_t teavm_gc_startTimeMillis; + static int64_t teavm_gc_startTime; + static int64_t teavm_gc_endTime; + static int64_t teavm_gc_markStartTime; + static int64_t teavm_gc_markEndTime; + static int64_t teavm_gc_sweepStartTime; + static int64_t teavm_gc_sweepEndTime; + static int64_t teavm_gc_defragStartTime; + static int64_t teavm_gc_defragEndTime; + static int32_t teavm_gc_full; +#endif + +#if TEAVM_MEMORY_TRACE + static inline void teavmHeapTrace_assertSize(int32_t size) { + if (size % TEAVM_REFERENCE_SIZE != 0) { + abort(); + } + } + + static inline void teavmHeapTrace_assertAddress(int32_t address) { + if (address % TEAVM_REFERENCE_SIZE != 0) { + abort(); + } + } +#endif + +int32_t teavm_javaHeapAddress(); +int32_t teavm_availableBytes(); +int32_t teavm_regionSize(); +int32_t teavm_regionsAddress(); + +void teavmHeapTrace_init(int32_t maxHeap) { + teavm_gc_heapMap = (uint8_t*) malloc(maxHeap / TEAVM_REFERENCE_SIZE); + teavm_gc_markMap = (uint8_t*) malloc(maxHeap / TEAVM_REFERENCE_SIZE); +} + +void teavmHeapTrace_allocate(int32_t address, int32_t size) { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_assertAddress(address); + teavmHeapTrace_assertSize(size); + + address -= teavm_javaHeapAddress(); + size /= TEAVM_REFERENCE_SIZE; + uint8_t* map = teavm_gc_heapMap + address / TEAVM_REFERENCE_SIZE; + + if (*map != 0) { + fprintf(stderr, "[GC] trying allocate at memory in use at: %" PRId32 "\n", address); + abort(); + } + *map++ = 1; + + for (int32_t i = 1; i < size; ++i) { + if (*map != 0) { + fprintf(stderr, "[GC] trying allocate at memory in use at: %" PRId32 "\n", address); + abort(); + } + *map++ = 2; + } + #endif + + #if TEAVM_GC_STATS + teavm_gc_allocationCount++; + #endif +} + +void teavmHeapTrace_free(int32_t address, int32_t size) { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_assertAddress(address); + teavmHeapTrace_assertSize(size); + + address -= teavm_javaHeapAddress(); + int32_t offset = address / TEAVM_REFERENCE_SIZE; + uint8_t* markMap = teavm_gc_markMap + offset; + size /= TEAVM_REFERENCE_SIZE; + for (int32_t i = 0; i < size; ++i) { + if (markMap[i] != 0) { + fprintf(stderr, "[GC] trying to release reachable object at: %" PRId32 "\n", address); + abort(); + } + } + + uint8_t* map = teavm_gc_heapMap + offset; + memset(map, 0, size); + #endif + + #if TEAVM_GC_STATS + teavm_gc_freeCount++; + teavm_gc_freeByteCount += size; + #endif +} + +void teavmHeapTrace_assertFree(int32_t address, int32_t size) { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_assertAddress(address); + teavmHeapTrace_assertSize(size); + + address -= teavm_javaHeapAddress(); + int32_t offset = address / TEAVM_REFERENCE_SIZE; + uint8_t* map = teavm_gc_heapMap + offset; + size /= TEAVM_REFERENCE_SIZE; + for (int32_t i = 0; i < size; ++i) { + if (map[i] != 0) { + fprintf(stderr, "[GC] memory supposed to be free at: %" PRId32 "\n", address); + abort(); + } + } + #endif +} + +void teavmHeapTrace_markStarted() { + #if TEAVM_MEMORY_TRACE + memset(teavm_gc_markMap, 0, teavm_availableBytes() / TEAVM_REFERENCE_SIZE); + #endif + + #if TEAVM_GC_STATS + teavm_gc_markStartTime = teavm_nanoTime(); + #endif +} + +void teavmHeapTrace_markCompleted() { + #if TEAVM_GC_STATS + teavm_gc_markEndTime = teavm_nanoTime(); + #endif +} + +#define TEAVM_ALIGN(addr, alignment) ((((addr) + ((int32_t) (alignment) - 1)) / (alignment) * (alignment))) +#define TEAVM_OBJECT_HEADER(address) (*(int32_t*) (wasm_heap + (address))) +#define TEAVM_OBJECT_HASH(address) (*(int32_t*) (wasm_heap + (address) + 4)) +#define TEAVM_CLASS_OF(address) (TEAVM_OBJECT_HEADER(address) << 3); + +#define teavm_class_size(address) (*(int32_t*) (wasm_heap + ((address) + 8))) +#define teavm_class_flags(address) (*(int32_t*) (wasm_heap + ((address) + 12))) +#define teavm_class_itemType(address) (*(int32_t*) (wasm_heap + ((address) + 32))) +#define teavm_class_superclass(address) (*(int32_t*) (wasm_heap + ((address) + 56))) +#define teavm_class_layout(address) (*(int32_t*) (wasm_heap + ((address) + 72))) +#define teavm_array_size(address) (*(int32_t*) (wasm_heap + ((address) + 8))) +#define teavm_reference_queue(address) (*(int32_t*) (wasm_heap + ((address) + 8))) +#define teavm_reference_object(address) (*(int32_t*) (wasm_heap + ((address) + 12))) +#define teavm_reference_next(address) (*(int32_t*) (wasm_heap + ((address) + 16))) +#define teavm_referenceQueue_first(address) (*(int32_t*) (wasm_heap + ((address) + 8))) +#define teavm_referenceQueue_last(address) (*(int32_t*) (wasm_heap + ((address) + 12))) +#define TEAVM_ARRAY_STRUCT_SIZE 12 + +int32_t teavmHeapTrace_objectSize(int32_t address) { + int32_t cls = TEAVM_CLASS_OF(address); + int32_t itemType = teavm_class_itemType(cls); + if (itemType == 0) { + return teavm_class_size(cls); + } + + int32_t itemSize = teavm_class_flags(itemType) & 2 ? teavm_class_size(itemType) : TEAVM_REFERENCE_SIZE; + int32_t size = TEAVM_ALIGN(TEAVM_ARRAY_STRUCT_SIZE, itemSize); + size += teavm_array_size(address) * itemSize; + size = TEAVM_ALIGN(size, TEAVM_REFERENCE_SIZE); + return size; +} + +void teavmHeapTrace_mark(int32_t address) { + #if TEAVM_MEMORY_TRACE + if (address < teavm_javaHeapAddress() || address >= teavm_javaHeapAddress() + teavm_availableBytes()) { + return; + } + + teavmHeapTrace_assertAddress(address); + + int32_t offset = (address - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE; + uint8_t* map = teavm_gc_heapMap + offset; + uint8_t* markMap = teavm_gc_markMap + offset; + + int32_t size = teavmHeapTrace_objectSize(address); + teavmHeapTrace_assertSize(size); + size /= TEAVM_REFERENCE_SIZE; + + if (*map++ != 1 || *markMap != 0) { + fprintf(stderr, "[GC] assertion failed marking object at: %" PRId32 "\n", + address - teavm_javaHeapAddress()); + abort(); + } + *markMap++ = 1; + + for (int32_t i = 1; i < size; ++i) { + if (*map++ != 2 || *markMap != 0) { + abort(); + } + *markMap++ = 1; + } + #endif + + #if TEAVM_GC_STATS + teavm_gc_markCount++; + #endif +} + +void teavmHeapTrace_move(int32_t from, int32_t to, int32_t size) { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_assertAddress(from); + teavmHeapTrace_assertAddress(to); + teavmHeapTrace_assertSize(size); + + uint8_t* mapFrom = teavm_gc_heapMap + ((from - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE); + uint8_t* mapTo = teavm_gc_heapMap + ((to - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE); + size /= TEAVM_REFERENCE_SIZE; + + if (mapFrom > mapTo) { + for (int32_t i = 0; i < size; ++i) { + if (mapFrom[i] == 0 || mapTo[i] != 0) { + fprintf(stderr, "[GC] assertion failed moving object from: %" PRId32 " to %" PRId32 "\n", + from - teavm_javaHeapAddress(), to - teavm_javaHeapAddress()); + abort(); + } + mapTo[i] = mapFrom[i]; + mapFrom[i] = 0; + } + } else { + for (int32_t i = size - 1; i >= 0; --i) { + if (mapFrom[i] == 0 || mapTo[i] != 0) { + abort(); + } + mapTo[i] = mapFrom[i]; + mapFrom[i] = 0; + } + } + #endif + + #if TEAVM_GC_STATS + teavm_gc_relocatedBlocks++; + teavm_gc_relocatedBytes += size; + #endif +} + +#if TEAVM_MEMORY_TRACE + + static inline int32_t teavm_verify(int32_t address) { + if (address >= teavm_javaHeapAddress() && address < teavm_javaHeapAddress() + teavm_availableBytes()) { + teavmHeapTrace_assertAddress(address); + uint8_t* map = teavm_gc_heapMap + ((address - teavm_javaHeapAddress()) / TEAVM_REFERENCE_SIZE); + if (*map != 1) { + abort(); + } + } + + return address; + } + + void teavmHeapTrace_checkHeapConsistency(int32_t oldGen, int32_t offsets) { + int32_t lastCheckedRegion = -1; + int32_t obj = teavm_javaHeapAddress(); + uint16_t* regions = (uint16_t*) (wasm_heap + teavm_regionsAddress()); + while (obj < teavm_javaHeapAddress() + teavm_availableBytes()) { + int32_t size; + int32_t header = TEAVM_OBJECT_HEADER(obj); + if (header == 0) { + size = TEAVM_OBJECT_HASH(obj); + teavmHeapTrace_assertFree(obj, size); + } else { + teavm_verify(obj); + if (offsets) { + int32_t offset = obj - teavm_javaHeapAddress(); + int32_t objRegion = offset / teavm_regionSize(); + if (objRegion != lastCheckedRegion) { + while (++lastCheckedRegion < objRegion) { + if (regions[lastCheckedRegion] != 0) { + abort(); + } + } + int32_t offsetInRegion = offset % teavm_regionSize(); + if (regions[objRegion] != offsetInRegion + 1) { + abort(); + } + } + } + if (oldGen && !(header & 0x40000000)) { + abort(); + } + int32_t cls = TEAVM_CLASS_OF(obj); + int32_t itemType = teavm_class_itemType(cls); + if (itemType != 0) { + if (!(teavm_class_flags(itemType) & 2)) { + int32_t offset = 0; + offset += TEAVM_ARRAY_STRUCT_SIZE; + offset = TEAVM_ALIGN(offset, TEAVM_REFERENCE_SIZE); + int32_t data = obj + offset; + int32_t size = teavm_array_size(obj); + for (int32_t i = 0; i < size; ++i) { + teavm_verify(((int32_t*) (wasm_heap + data))[i]); + } + } + } else { + while (cls != 0) { + int32_t kind = (teavm_class_flags(cls) >> 7) & 7; + if (kind == 1) { + teavm_verify(teavm_reference_next(obj)); + teavm_verify(teavm_reference_object(obj)); + teavm_verify(teavm_reference_queue(obj)); + } else if (kind == 2) { + teavm_verify(teavm_referenceQueue_first(obj)); + teavm_verify(teavm_referenceQueue_last(obj)); + } else { + int32_t layoutOffset = teavm_class_layout(cls); + if (layoutOffset != 0) { + int16_t* layout = (int16_t*) (wasm_heap + layoutOffset); + int16_t size = *layout++; + for (int32_t i = 0; i < size; ++i) { + int32_t ptr = obj + *layout++; + teavm_verify(*(int32_t*) (wasm_heap + ptr)); + } + } + } + + cls = teavm_class_superclass(cls); + } + } + size = teavmHeapTrace_objectSize(obj); + } + + if (size == 0) { + abort(); + } + obj += size; + } + + if (offsets) { + int32_t lastRegion = teavm_availableBytes() / teavm_regionSize(); + while (++lastCheckedRegion <= lastRegion) { + if (regions[lastCheckedRegion] != 0) { + abort(); + } + } + } + } +#endif + +void teavmHeapTrace_gcStarted(int32_t full) { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_checkHeapConsistency(0, 0); + #endif + + #if TEAVM_GC_STATS + teavm_gc_startTime = teavm_nanoTime(); + teavm_gc_startTimeMillis = teavm_currentTimeMillis(); + teavm_gc_full = full; + #endif +} + +void teavmHeapTrace_sweepStarted() { + #if TEAVM_GC_STATS + teavm_gc_sweepStartTime = teavm_nanoTime(); + #endif +} + +void teavmHeapTrace_sweepCompleted() { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_checkHeapConsistency(0, 1); + #endif + + #if TEAVM_GC_STATS + teavm_gc_sweepEndTime = teavm_nanoTime(); + #endif +} + +void teavmHeapTrace_defragStarted() { + #if TEAVM_GC_STATS + teavm_gc_defragStartTime = teavm_nanoTime(); + #endif +} + +void teavmHeapTrace_defragCompleted() { + #if TEAVM_MEMORY_TRACE + teavmHeapTrace_checkHeapConsistency(1, 1); + #endif + + #if TEAVM_GC_STATS + teavm_gc_defragEndTime = teavm_nanoTime(); + #endif +} + +#if TEAVM_GC_STATS + static void teavmHeapTrace_printStats() { + fprintf(stderr, "[GC] Garbage collection (%s) performed at %" PRIu64 ", took %" + PRIu64 " ns\n", teavm_gc_full ? "full" : "young", teavm_gc_startTimeMillis, + teavm_gc_endTime - teavm_gc_startTime); + + fprintf(stderr, "[GC] Allocations performed before GC: %" PRIu32 "\n", teavm_gc_allocationCount); + + fprintf(stderr, "[GC] Mark phase took %" PRIu64 " ns, %" PRIu32 " objects reached\n", + teavm_gc_markEndTime - teavm_gc_markStartTime, teavm_gc_markCount); + + if (!teavm_gc_full) { + fprintf(stderr, "[GC] Regions scanned from remembered set: %" PRIu32 "\n", teavm_gc_dirtyRegionCount); + } + + fprintf(stderr, "[GC] Sweep phase took %" PRIu64 " ns, %" PRIu32 " regions of %" + PRIu32 " bytes freed\n", teavm_gc_sweepEndTime - teavm_gc_sweepStartTime, teavm_gc_freeCount, + teavm_gc_freeByteCount); + + fprintf(stderr, "[GC] Defrag phase took %" PRIu64 " ns\n", + teavm_gc_defragEndTime - teavm_gc_defragStartTime); + + fprintf(stderr, "[GC] Blocks relocated %" PRId32 " of total %" PRId32 " bytes\n", + teavm_gc_relocatedBlocks, teavm_gc_relocatedBytes); + } + + static void teavmHeapTrace_resetStats() { + teavm_gc_allocationCount = 0; + teavm_gc_markCount = 0; + teavm_gc_dirtyRegionCount = 0; + teavm_gc_freeCount = 0; + teavm_gc_freeByteCount = 0; + teavm_gc_relocatedBlocks = 0; + teavm_gc_relocatedBytes = 0; + } +#endif + +void teavmHeapTrace_gcCompleted() { + #if TEAVM_GC_STATS + teavm_gc_endTime = teavm_nanoTime(); + teavmHeapTrace_printStats(); + teavmHeapTrace_resetStats(); + #endif +} + +void teavmHeapTrace_heapResized(int64_t newSize) { + #if TEAVM_GC_STATS + fprintf(stderr, "[GC] Heap resized to %" PRIu64 " bytes\n", newSize); + #endif +} + +void teavmHeapTrace_reportDirtyRegion(int32_t address) { + #if TEAVM_GC_STATS + teavm_gc_dirtyRegionCount++; + #endif +} diff --git a/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.c b/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.c index b349ec117..647307fc1 100644 --- a/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.c +++ b/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.c @@ -12,45 +12,61 @@ static int8_t *wasm_heap; static int32_t wasm_heap_size; -static inline float teavm_getNaN() { +float teavm_teavm_getNaN() { return NAN; } -static int64_t currentTimeMillis() { +#define teavm_isnan isnan +#define teavm_isinf isinf +#define teavm_isfinite isfinite +#define teavmMath_sin sin +#define teavmMath_cos cos +#define teavmMath_sqrt sqrt +#define teavmMath_ceil ceil +#define teavmMath_floor floor + +double teavm_currentTimeMillis() { struct timespec time; clock_gettime(CLOCK_REALTIME, &time); return time.tv_sec * 1000 + (int64_t) round(time.tv_nsec / 1000000); } +double teavm_nanoTime() { + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + + return time.tv_sec * 1000000000 + (int64_t) round(time.tv_nsec); +} + static union { float f; int32_t i; } reinterpret_union_32; static union { double f; int64_t i; } reinterpret_union_64; -inline static int64_t reinterpret_float64(double v) { +int64_t reinterpret_float64(double v) { reinterpret_union_64.f = v; return reinterpret_union_64.i; } -inline static double reinterpret_int64(int64_t v) { +double reinterpret_int64(int64_t v) { reinterpret_union_64.i = v; return reinterpret_union_64.f; } -inline static int32_t reinterpret_float32(double v) { +int32_t reinterpret_float32(double v) { reinterpret_union_32.f = v; return reinterpret_union_32.i; } -inline static float reinterpret_int32(int32_t v) { +float reinterpret_int32(int32_t v) { reinterpret_union_32.i = v; return reinterpret_union_32.f; } -static void logOutOfMemory() { +void teavm_logOutOfMemory() { abort(); } -static void logString(int32_t string) { +void teavm_logString(int32_t string) { uint32_t arrayPtr = *(uint32_t*) (wasm_heap + string + 8); uint32_t length = *(uint32_t*) (wasm_heap + arrayPtr + 8); for (int32_t i = 0; i < length; ++i) { @@ -59,6 +75,6 @@ static void logString(int32_t string) { } } -static void logInt(int32_t v) { +void teavm_logInt(int32_t v) { wprintf(L"%" PRId32, v); } \ No newline at end of file diff --git a/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.js b/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.js index d91604b70..4fbd41e57 100644 --- a/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.js +++ b/core/src/main/resources/org/teavm/backend/wasm/wasm-runtime.js @@ -67,6 +67,24 @@ TeaVM.wasm = function() { }; obj.teavmMath = Math; + + obj.teavmHeapTrace = { + allocate: function(address, size) {}, + free: function(address, size) {}, + assertFree: function(address, size) {}, + markStarted: function() {}, + mark: function(address) {}, + reportDirtyRegion: function(address) {}, + markCompleted: function() {}, + move: function(from, to, size) {}, + gcStarted: function(full) {}, + sweepStarted: function() {}, + sweepCompleted: function() {}, + defragStarted: function() {}, + defragCompleted: function() {}, + gcCompleted: function() {}, + init: function(maxHeap) {} + }; } function run(path, options) { diff --git a/tools/core/src/main/java/org/teavm/tooling/TeaVMTool.java b/tools/core/src/main/java/org/teavm/tooling/TeaVMTool.java index e5a35aedb..c8f9ae9b7 100644 --- a/tools/core/src/main/java/org/teavm/tooling/TeaVMTool.java +++ b/tools/core/src/main/java/org/teavm/tooling/TeaVMTool.java @@ -331,7 +331,7 @@ public class TeaVMTool { webAssemblyTarget.setWastEmitted(debugInformationGenerated); webAssemblyTarget.setVersion(wasmVersion); webAssemblyTarget.setMinHeapSize(minHeapSize); - webAssemblyTarget.setMinHeapSize(maxHeapSize); + webAssemblyTarget.setMaxHeapSize(maxHeapSize); return webAssemblyTarget; }