diff --git a/core/src/main/java/org/teavm/backend/wasm/WasmGCTarget.java b/core/src/main/java/org/teavm/backend/wasm/WasmGCTarget.java index 093d06221..682eb31c5 100644 --- a/core/src/main/java/org/teavm/backend/wasm/WasmGCTarget.java +++ b/core/src/main/java/org/teavm/backend/wasm/WasmGCTarget.java @@ -47,6 +47,7 @@ import org.teavm.backend.wasm.intrinsics.gc.WasmGCIntrinsics; import org.teavm.backend.wasm.model.WasmCustomSection; import org.teavm.backend.wasm.model.WasmFunction; import org.teavm.backend.wasm.model.WasmLocal; +import org.teavm.backend.wasm.model.WasmMemorySegment; import org.teavm.backend.wasm.model.WasmModule; import org.teavm.backend.wasm.model.WasmTag; import org.teavm.backend.wasm.model.WasmType; @@ -99,6 +100,9 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost { private List customGeneratorFactories = new ArrayList<>(); private EntryPointTransformation entryPointTransformation = new EntryPointTransformation(); private List classConsumers = new ArrayList<>(); + private boolean enableDirectMallocSupport = false; + private int directMallocMinHeapSize = 0x10000; + private int directMallocMaxHeapSize = 0x10000000; public void setObfuscated(boolean obfuscated) { this.obfuscated = obfuscated; @@ -128,6 +132,18 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost { this.sourceMapLocation = sourceMapLocation; } + public void setEnableDirectMallocSupport(boolean enable) { + this.enableDirectMallocSupport = enable; + } + + public void setDirectMallocMinHeapSize(int minHeapSize) { + this.directMallocMinHeapSize = WasmRuntime.align(minHeapSize, WasmHeap.PAGE_SIZE); + } + + public void setDirectMallocMaxHeapSize(int maxHeapSize) { + this.directMallocMaxHeapSize = WasmRuntime.align(maxHeapSize, WasmHeap.PAGE_SIZE); + } + @Override public void addIntrinsicFactory(WasmGCIntrinsicFactory intrinsicFactory) { intrinsicFactories.add(intrinsicFactory); @@ -194,6 +210,9 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost { var deps = new WasmGCDependencies(dependencyAnalyzer); deps.contribute(); deps.contributeStandardExports(); + if(enableDirectMallocSupport) { + deps.contributeDirectMalloc(); + } } @Override @@ -285,6 +304,17 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost { moduleGenerator.generate(); customGenerators.contributeToModule(module); generateExceptionExports(declarationsGenerator); + if(enableDirectMallocSupport) { + var heapSegment = new WasmMemorySegment(); + if (!module.getSegments().isEmpty()) { + var lastSegment = module.getSegments().get(module.getSegments().size() - 1); + heapSegment.setOffset(WasmRuntime.align(lastSegment.getOffset() + lastSegment.getLength(), WasmHeap.PAGE_SIZE)); + } + heapSegment.setLength(directMallocMinHeapSize); + module.getSegments().add(heapSegment); + intrinsics.setupLaxMallocHeap(heapSegment.getOffset(), heapSegment.getOffset() + directMallocMinHeapSize, + heapSegment.getOffset() + directMallocMaxHeapSize); + } adjustModuleMemory(module); emitWasmFile(module, buildTarget, outputName, debugInfoBuilder); @@ -390,9 +420,16 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost { return; } - var pages = (memorySize - 1) / WasmHeap.PAGE_SIZE + 1; - module.setMinMemorySize(pages); - module.setMaxMemorySize(pages); + if(enableDirectMallocSupport) { + var minPages = (memorySize - 1) / WasmHeap.PAGE_SIZE + 1; + var maxPages = (memorySize - directMallocMinHeapSize + directMallocMaxHeapSize - 1) / WasmHeap.PAGE_SIZE + 1; + module.setMinMemorySize(minPages); + module.setMaxMemorySize(maxPages); + }else { + var pages = (memorySize - 1) / WasmHeap.PAGE_SIZE + 1; + module.setMinMemorySize(pages); + module.setMaxMemorySize(pages); + } } private void emitWasmFile(WasmModule module, BuildTarget buildTarget, String outputName, diff --git a/core/src/main/java/org/teavm/backend/wasm/gc/WasmGCDependencies.java b/core/src/main/java/org/teavm/backend/wasm/gc/WasmGCDependencies.java index 20ddf69d7..e491f256a 100644 --- a/core/src/main/java/org/teavm/backend/wasm/gc/WasmGCDependencies.java +++ b/core/src/main/java/org/teavm/backend/wasm/gc/WasmGCDependencies.java @@ -22,7 +22,9 @@ import org.teavm.backend.wasm.runtime.gc.WasmGCSupport; import org.teavm.dependency.AbstractDependencyListener; import org.teavm.dependency.DependencyAgent; import org.teavm.dependency.DependencyAnalyzer; +import org.teavm.interop.Address; import org.teavm.model.MethodReference; +import org.teavm.runtime.LaxMalloc; public class WasmGCDependencies { private DependencyAnalyzer analyzer; @@ -127,4 +129,10 @@ public class WasmGCDependencies { private void contributeString() { analyzer.addDependencyListener(new StringInternDependencySupport()); } + + public void contributeDirectMalloc() { + analyzer.linkMethod(new MethodReference(LaxMalloc.class, "laxMalloc", int.class, Address.class)).use(); + analyzer.linkMethod(new MethodReference(LaxMalloc.class, "laxCalloc", int.class, Address.class)).use(); + analyzer.linkMethod(new MethodReference(LaxMalloc.class, "laxFree", Address.class, void.class)).use(); + } } diff --git a/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/LaxMallocIntrinsic.java b/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/LaxMallocIntrinsic.java new file mode 100644 index 000000000..7bf4adce4 --- /dev/null +++ b/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/LaxMallocIntrinsic.java @@ -0,0 +1,95 @@ +/* + * Copyright 2024 lax1dude. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.teavm.backend.wasm.intrinsics.gc; + +import java.util.ArrayList; +import java.util.List; + +import org.teavm.ast.InvocationExpr; +import org.teavm.backend.wasm.model.expression.WasmExpression; +import org.teavm.backend.wasm.model.expression.WasmInt32Constant; +import org.teavm.backend.wasm.model.expression.WasmIntBinary; +import org.teavm.backend.wasm.model.expression.WasmIntBinaryOperation; +import org.teavm.backend.wasm.model.expression.WasmIntType; +import org.teavm.backend.wasm.model.expression.WasmMemoryGrow; + +public class LaxMallocIntrinsic implements WasmGCIntrinsic { + + private final List addressList = new ArrayList<>(); + private final List minAddrConstants = new ArrayList<>(); + private final List maxAddrConstants = new ArrayList<>(); + + @Override + public WasmExpression apply(InvocationExpr invocation, WasmGCIntrinsicContext context) { + switch (invocation.getMethod().getName()) { + case "addrHeap": { + WasmExpression value = context.generate(invocation.getArguments().get(0)); + if(value instanceof WasmInt32Constant) { + // if addrHeap is passed a constant i32, add the heap offset at compile time + final int memOffset = ((WasmInt32Constant)value).getValue(); + WasmInt32Constant ret = new WasmInt32Constant(0); + addressList.add((heapLoc) -> { + ret.setValue(heapLoc + memOffset); + }); + return ret; + }else { + WasmInt32Constant heapLocConst = new WasmInt32Constant(0); + WasmExpression calcOffset = new WasmIntBinary(WasmIntType.INT32, WasmIntBinaryOperation.ADD, heapLocConst, value); + addressList.add(heapLocConst::setValue); + return calcOffset; + } + } + case "growHeapOuter": { + return new WasmMemoryGrow(context.generate(invocation.getArguments().get(0))); + } + case "getHeapMinAddr": { + WasmInt32Constant ret = new WasmInt32Constant(0); + minAddrConstants.add(ret); + return ret; + } + case "getHeapMaxAddr": { + WasmInt32Constant ret = new WasmInt32Constant(0); + maxAddrConstants.add(ret); + return ret; + } + default: + throw new IllegalArgumentException(invocation.getMethod().toString()); + } + } + + public void setHeapLocation(int heapLoc) { + for(LaxMallocHeapMapper mapper : addressList) { + mapper.setHeapLocation(heapLoc); + } + } + + private static interface LaxMallocHeapMapper { + void setHeapLocation(int heapLoc); + } + + public void setHeapMinAddr(int heapSegmentMinAddr) { + for(WasmInt32Constant ct : minAddrConstants) { + ct.setValue(heapSegmentMinAddr); + } + } + + public void setHeapMaxAddr(int heapSegmentMaxAddr) { + for(WasmInt32Constant ct : maxAddrConstants) { + ct.setValue(heapSegmentMaxAddr); + } + } + +} diff --git a/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/WasmGCIntrinsics.java b/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/WasmGCIntrinsics.java index b100d0552..a477d8d5b 100644 --- a/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/WasmGCIntrinsics.java +++ b/core/src/main/java/org/teavm/backend/wasm/intrinsics/gc/WasmGCIntrinsics.java @@ -30,12 +30,14 @@ import org.teavm.interop.DirectMalloc; import org.teavm.model.ClassReaderSource; import org.teavm.model.MethodReference; import org.teavm.model.ValueType; +import org.teavm.runtime.LaxMalloc; public class WasmGCIntrinsics implements WasmGCIntrinsicProvider { private Map intrinsics = new HashMap<>(); private List factories; private ClassReaderSource classes; private ServiceRepository services; + private LaxMallocIntrinsic laxMallocIntrinsic; public WasmGCIntrinsics(ClassReaderSource classes, ServiceRepository services, List factories, Map customIntrinsics) { @@ -54,6 +56,7 @@ public class WasmGCIntrinsics implements WasmGCIntrinsicProvider { fillString(); fillResources(); fillDirectMalloc(); + fillLaxMalloc(); fillAddress(); for (var entry : customIntrinsics.entrySet()) { add(entry.getKey(), entry.getValue()); @@ -180,6 +183,15 @@ public class WasmGCIntrinsics implements WasmGCIntrinsicProvider { add(new MethodReference(DirectMalloc.class, "zmemset", Address.class, int.class, void.class), intrinsic); } + private void fillLaxMalloc() { + laxMallocIntrinsic = new LaxMallocIntrinsic(); + add(new MethodReference(LaxMalloc.class, "laxMalloc", int.class, Address.class), laxMallocIntrinsic); + add(new MethodReference(LaxMalloc.class, "laxCalloc", int.class, Address.class), laxMallocIntrinsic); + add(new MethodReference(LaxMalloc.class, "laxFree", Address.class, void.class), laxMallocIntrinsic); + add(new MethodReference(LaxMalloc.class, "getHeapMinSize", int.class), laxMallocIntrinsic); + add(new MethodReference(LaxMalloc.class, "getHeapMaxSize", int.class), laxMallocIntrinsic); + } + private void fillAddress() { var intrinsic = new AddressIntrinsic(); add(new MethodReference(Address.class, "add", int.class, Address.class), intrinsic); @@ -233,6 +245,12 @@ public class WasmGCIntrinsics implements WasmGCIntrinsicProvider { return result.intrinsic; } + public void setupLaxMallocHeap(int heapAddr, int heapSegmentMinAddr, int heapSegmentMaxAddr) { + laxMallocIntrinsic.setHeapLocation(heapAddr); + laxMallocIntrinsic.setHeapMinAddr(heapSegmentMinAddr); + laxMallocIntrinsic.setHeapMaxAddr(heapSegmentMaxAddr); + } + static class IntrinsicContainer { final WasmGCIntrinsic intrinsic; diff --git a/core/src/main/java/org/teavm/runtime/LaxMalloc.java b/core/src/main/java/org/teavm/runtime/LaxMalloc.java index 5aaec902e..78b762b78 100644 --- a/core/src/main/java/org/teavm/runtime/LaxMalloc.java +++ b/core/src/main/java/org/teavm/runtime/LaxMalloc.java @@ -50,12 +50,23 @@ public final class LaxMalloc { private static final int ADDR_HEAP_BUCKETS_START = 16; // Address to the list of 64 pointers to the beginnings of the 64 buckets private static final int ADDR_HEAP_DATA_START = 272; // Beginning of the first chunk of the heap + private static native Address addrHeap(int offset); // Intrinsic function to get an address in the heap segment + + private static native int growHeapOuter(int bytes); // Intrinsic function to grow the heap segment + + private static native Address getHeapMinAddr(); // Intrinsic function to get the minimum direct malloc heap segment ending address + + private static native Address getHeapMaxAddr(); // Intrinsic function to get the maximum direct malloc heap segment ending address + + @Import(name = "teavm_notifyHeapResized") + private static native void notifyHeapResized(); + static { // zero out the control region - DirectMalloc.zmemset(Address.fromInt(0), ADDR_HEAP_DATA_START); + DirectMalloc.zmemset(addrHeap(0), ADDR_HEAP_DATA_START); // initialize heap limit - Address.fromInt(ADDR_HEAP_INNER_LIMIT).putInt(ADDR_HEAP_DATA_START); - Address.fromInt(ADDR_HEAP_OUTER_LIMIT).putInt(0x10000); + addrHeap(ADDR_HEAP_INNER_LIMIT).putAddress(addrHeap(ADDR_HEAP_DATA_START)); + addrHeap(ADDR_HEAP_OUTER_LIMIT).putAddress(getHeapMinAddr()); } /** @@ -96,7 +107,7 @@ public final class LaxMalloc { } // load bitmask of buckets with free chunks - long bucketMask = Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); + long bucketMask = addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); // mask away the buckets that we know are too small for this allocation bucketMask = (bucketMask & (0xFFFFFFFFFFFFFFFFL << bucket)); @@ -125,7 +136,7 @@ public final class LaxMalloc { // quickly determine which bucket it is with bit hacks int availableBucket = Long.numberOfTrailingZeros(bucketMask); - Address bucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(availableBucket << SIZEOF_PTR_SH); + Address bucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(availableBucket << SIZEOF_PTR_SH); Address chunkPtr = bucketStartAddr.getAddress(); int chunkSize = readChunkSizeStatus(chunkPtr); Address itrChunkStart = Address.fromInt(0); @@ -149,7 +160,7 @@ public final class LaxMalloc { if(bucketMask != 0l) { // there is a bucket with a larger chunk int availableLargerBucket = Long.numberOfTrailingZeros(bucketMask); - Address largerBucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(availableLargerBucket << SIZEOF_PTR_SH); + Address largerBucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(availableLargerBucket << SIZEOF_PTR_SH); Address largerChunkPtr = largerBucketStartAddr.getAddress(); int largerChunkSize = readChunkSizeStatus(largerChunkPtr); @@ -234,10 +245,10 @@ public final class LaxMalloc { private static Address laxHugeAlloc(int sizeBytes, boolean cleared) { // check the bucket mask if bucket 63 has any chunks - if((Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).getLong() & 0x8000000000000000L) != 0) { + if((addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).getLong() & 0x8000000000000000L) != 0) { // bucket 63 address - Address bucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(63 << SIZEOF_PTR_SH); + Address bucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(63 << SIZEOF_PTR_SH); Address chunkPtr = bucketStartAddr.getAddress(); // iterate all free huge chunks @@ -302,7 +313,7 @@ public final class LaxMalloc { // set the chunk no longer in use chunkSize &= 0x7FFFFFFF; - if(Address.fromInt(ADDR_HEAP_DATA_START).isLessThan(chunkPtr)) { + if(addrHeap(ADDR_HEAP_DATA_START).isLessThan(chunkPtr)) { // check if we can merge with the previous chunk, and move it to another bucket Address prevChunkPtr = chunkPtr.add(-(chunkPtr.add(-4).getInt())); int prevChunkSize = readChunkSizeStatus(prevChunkPtr); @@ -320,7 +331,7 @@ public final class LaxMalloc { } Address nextChunkPtr = chunkPtr.add(chunkSize); - if(Address.fromInt(ADDR_HEAP_INNER_LIMIT).getAddress().isLessThan(nextChunkPtr)) { + if(addrHeap(ADDR_HEAP_INNER_LIMIT).getAddress().isLessThan(nextChunkPtr)) { // check if we can merge with the next chunk as well int nextChunkSize = readChunkSizeStatus(nextChunkPtr); if((nextChunkSize & 0x80000000) == 0) { @@ -388,8 +399,8 @@ public final class LaxMalloc { private static void linkChunkInFreeList(Address chunkPtr, int chunkSize) { int bucket = getListBucket(chunkSize - 8); // size - 2 ints - long bucketMask = Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); - Address bucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(bucket << SIZEOF_PTR_SH); + long bucketMask = addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); + Address bucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(bucket << SIZEOF_PTR_SH); // test the bucket mask if the bucket is empty if((bucketMask & (1L << bucket)) == 0l) { @@ -401,7 +412,7 @@ public final class LaxMalloc { // set the free bit in bucket mask bucketMask |= (1L << bucket); - Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).putLong(bucketMask); + addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).putLong(bucketMask); }else { @@ -434,13 +445,13 @@ public final class LaxMalloc { int chunkBucket = getListBucket(chunkSize - 8); // size - 2 ints - Address bucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(chunkBucket << SIZEOF_PTR_SH); + Address bucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(chunkBucket << SIZEOF_PTR_SH); bucketStartAddr.putAddress(Address.fromInt(0)); // remove chunk from the bucket // clear the bit in the free buckets bitmask - long bucketsFreeMask = Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); + long bucketsFreeMask = addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).getLong(); bucketsFreeMask &= ~(1L << chunkBucket); - Address.fromInt(ADDR_HEAP_BUCKETS_FREE_MASK).putLong(bucketsFreeMask); + addrHeap(ADDR_HEAP_BUCKETS_FREE_MASK).putLong(bucketsFreeMask); }else { // there are other chunks in this bucket @@ -450,7 +461,7 @@ public final class LaxMalloc { writeChunkPrevFreeAddr(nextChunkPtr, prevChunkPtr); int chunkBucket = getListBucket(chunkSize - 8); // size - 2 ints - Address bucketStartAddr = Address.fromInt(ADDR_HEAP_BUCKETS_START).add(chunkBucket << SIZEOF_PTR_SH); + Address bucketStartAddr = addrHeap(ADDR_HEAP_BUCKETS_START).add(chunkBucket << SIZEOF_PTR_SH); Address bucketStartChunk = bucketStartAddr.getAddress(); // chunk is the first in the bucket, so we also need to @@ -480,28 +491,27 @@ public final class LaxMalloc { * This is our sbrk */ private static Address growHeap(int amount) { - Address heapInnerLimit = Address.fromInt(ADDR_HEAP_INNER_LIMIT).getAddress(); - Address heapOuterLimit = Address.fromInt(ADDR_HEAP_OUTER_LIMIT).getAddress(); + Address heapInnerLimit = addrHeap(ADDR_HEAP_INNER_LIMIT).getAddress(); + Address heapOuterLimit = addrHeap(ADDR_HEAP_OUTER_LIMIT).getAddress(); Address newHeapInnerLimit = heapInnerLimit.add(amount); if(heapOuterLimit.isLessThan(newHeapInnerLimit)) { int bytesNeeded = newHeapInnerLimit.toInt() - heapOuterLimit.toInt(); bytesNeeded = (bytesNeeded + 0xFFFF) & 0xFFFF0000; - if(growHeapOuter(bytesNeeded)) { - Address.fromInt(ADDR_HEAP_INNER_LIMIT).putAddress(newHeapInnerLimit); - Address.fromInt(ADDR_HEAP_OUTER_LIMIT).putAddress(heapOuterLimit.add(bytesNeeded)); + Address newHeapOuterLimit = heapOuterLimit.add(bytesNeeded); + if(!getHeapMaxAddr().isLessThan(newHeapOuterLimit) && growHeapOuter(bytesNeeded >> 16) != -1) { + addrHeap(ADDR_HEAP_INNER_LIMIT).putAddress(newHeapInnerLimit); + addrHeap(ADDR_HEAP_OUTER_LIMIT).putAddress(newHeapOuterLimit); + notifyHeapResized(); return newHeapInnerLimit; }else { return Address.fromInt(0); } }else { - Address.fromInt(ADDR_HEAP_INNER_LIMIT).putAddress(newHeapInnerLimit); + addrHeap(ADDR_HEAP_INNER_LIMIT).putAddress(newHeapInnerLimit); return newHeapInnerLimit; } } - @Import(name = "teavm_growHeap") - private static native boolean growHeapOuter(int bytes); - /** * Note that on a free chunk, this is the size, because the status bit is 0 */