wasm: use bulk memory operations when necessary

This commit is contained in:
Alexey Andreev 2023-08-22 17:13:06 +02:00
parent e291128948
commit e3bbf12f49
16 changed files with 324 additions and 182 deletions

View File

@ -19,6 +19,7 @@ import org.teavm.backend.wasm.runtime.WasmSupport;
import org.teavm.interop.Address;
import org.teavm.interop.StaticInit;
import org.teavm.interop.Unmanaged;
import org.teavm.runtime.Allocator;
@StaticInit
@Unmanaged
@ -99,11 +100,11 @@ public final class WasmHeap {
memoryLimit = newMemoryLimit;
}
if (storageSize > 0) {
WasmRuntime.moveMemoryBlock(storageAddress, newStorageAddress, storageSize);
Allocator.moveMemoryBlock(storageAddress, newStorageAddress, storageSize);
}
if (regionsSize > 0) {
WasmRuntime.moveMemoryBlock(cardTable, newCardTable, regionsCount);
WasmRuntime.moveMemoryBlock(regionsAddress, newRegionsAddress, regionsSize);
Allocator.moveMemoryBlock(cardTable, newCardTable, regionsCount);
Allocator.moveMemoryBlock(regionsAddress, newRegionsAddress, regionsSize);
}
storageAddress = newStorageAddress;

View File

@ -35,6 +35,10 @@ public final class WasmRuntime {
return gtu(a, b) ? 1 : ltu(a, b) ? -1 : 0;
}
public static int compareUnsigned(long a, long b) {
return gtu(a, b) ? 1 : ltu(a, b) ? -1 : 0;
}
public static int compare(long a, long b) {
return gt(a, b) ? 1 : lt(a, b) ? -1 : 0;
}
@ -67,6 +71,10 @@ public final class WasmRuntime {
private static native boolean gt(long a, long b);
private static native boolean ltu(long a, long b);
private static native boolean gtu(long a, long b);
private static native boolean lt(float a, float b);
private static native boolean gt(float a, float b);
@ -113,167 +121,6 @@ public final class WasmRuntime {
}
public static void fill(Address address, byte value, int count) {
int value4 = (value & 0xFF << 24) | (value & 0xFF << 16) | (value & 0xFF << 8) | (value & 0xFF);
int start = address.toInt();
int alignedStart = start >>> 2 << 2;
address = Address.fromInt(alignedStart);
switch (start - alignedStart) {
case 0:
address.putInt(value4);
break;
case 1:
address.add(1).putByte(value);
address.add(2).putByte(value);
address.add(3).putByte(value);
break;
case 2:
address.add(2).putByte(value);
address.add(3).putByte(value);
break;
case 3:
address.add(3).putByte(value);
break;
}
int end = start + count;
int alignedEnd = end >>> 2 << 2;
address = Address.fromInt(alignedEnd);
switch (end - alignedEnd) {
case 0:
break;
case 1:
address.putByte(value);
break;
case 2:
address.putByte(value);
address.add(1).putByte(value);
break;
case 3:
address.putByte(value);
address.add(1).putByte(value);
address.add(2).putByte(value);
break;
}
for (address = Address.fromInt(alignedStart + 4); address.toInt() < alignedEnd; address = address.add(4)) {
address.putInt(value4);
}
}
public static void moveMemoryBlock(Address source, Address target, int count) {
if (count < 8) {
slowMemoryMove(source, target, count);
return;
}
int diff = source.toInt() - target.toInt();
if (diff == 0) {
return;
}
if ((diff & 3) != 0) {
slowMemoryMove(source, target, count);
return;
}
Address alignedSourceStart = Address.fromInt(source.toInt() >>> 2 << 2);
Address alignedTargetStart = Address.fromInt(target.toInt() >>> 2 << 2);
Address alignedSourceEnd = Address.fromInt((source.toInt() + count) >>> 2 << 2);
Address alignedTargetEnd = Address.fromInt((target.toInt() + count) >>> 2 << 2);
if (source.toInt() > target.toInt()) {
switch (source.toInt() - alignedSourceStart.toInt()) {
case 0:
alignedTargetStart.putInt(alignedSourceStart.getInt());
break;
case 1:
alignedTargetStart.add(1).putByte(alignedSourceStart.add(1).getByte());
alignedTargetStart.add(2).putShort(alignedSourceStart.add(2).getShort());
break;
case 2:
alignedTargetStart.add(2).putShort(alignedSourceStart.add(2).getShort());
break;
case 3:
alignedTargetStart.add(3).putByte(alignedSourceStart.add(3).getByte());
break;
}
alignedSourceStart = alignedSourceStart.add(4);
alignedTargetStart = alignedTargetStart.add(4);
while (alignedSourceStart.toInt() < alignedSourceEnd.toInt()) {
alignedTargetStart.putInt(alignedSourceStart.getInt());
alignedSourceStart = alignedSourceStart.add(4);
alignedTargetStart = alignedTargetStart.add(4);
}
switch (source.toInt() + count - alignedSourceEnd.toInt()) {
case 0:
break;
case 1:
alignedTargetEnd.putByte(alignedSourceEnd.getByte());
break;
case 2:
alignedTargetEnd.putShort(alignedSourceEnd.getShort());
break;
case 3:
alignedTargetEnd.putShort(alignedSourceEnd.getShort());
alignedTargetEnd.add(2).putByte(alignedSourceEnd.add(2).getByte());
break;
}
} else {
switch (source.toInt() + count - alignedSourceEnd.toInt()) {
case 0:
break;
case 1:
alignedTargetEnd.putByte(alignedSourceEnd.getByte());
break;
case 2:
alignedTargetEnd.putShort(alignedSourceEnd.getShort());
break;
case 3:
alignedTargetEnd.add(2).putByte(alignedSourceEnd.add(2).getByte());
alignedTargetEnd.putShort(alignedSourceEnd.getShort());
break;
}
while (alignedSourceEnd.toInt() > alignedSourceStart.toInt()) {
alignedSourceEnd = alignedSourceEnd.add(-4);
alignedTargetEnd = alignedTargetEnd.add(-4);
alignedTargetEnd.putInt(alignedSourceEnd.getInt());
}
switch (source.toInt() - alignedSourceStart.toInt()) {
case 1:
alignedTargetStart.add(-2).putShort(alignedSourceStart.add(-2).getShort());
alignedTargetStart.add(-3).putByte(alignedSourceStart.add(-3).getByte());
break;
case 2:
alignedTargetStart.add(-2).putShort(alignedSourceStart.add(-2).getShort());
break;
case 3:
alignedTargetStart.add(-1).putByte(alignedSourceStart.add(-1).getByte());
break;
}
}
}
private static void slowMemoryMove(Address source, Address target, int count) {
if (source.toInt() > target.toInt()) {
while (count-- > 0) {
target.putByte(source.getByte());
target = target.add(1);
source = source.add(1);
}
} else {
source = source.add(count);
target = target.add(count);
while (count-- > 0) {
target = target.add(-1);
source = source.add(-1);
target.putByte(source.getByte());
}
}
}
public static Address allocStack(int size) {

View File

@ -319,6 +319,11 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
var method = new MethodReference(WasmRuntime.class, "compare", type, type, int.class);
dependencyAnalyzer.linkMethod(method).use();
}
for (Class<?> type : Arrays.asList(int.class, long.class)) {
var method = new MethodReference(WasmRuntime.class, "compareUnsigned", type, type, int.class);
dependencyAnalyzer.linkMethod(method).use();
}
for (Class<?> type : Arrays.asList(float.class, double.class)) {
var method = new MethodReference(WasmRuntime.class, "remainder", type, type, type);
dependencyAnalyzer.linkMethod(method).use();
@ -326,12 +331,6 @@ public class WasmTarget implements TeaVMTarget, TeaVMWasmHost {
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "align", Address.class, int.class,
Address.class)).use();
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "fill", Address.class, int.class,
int.class, void.class)).use();
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "fillZero", Address.class, int.class,
void.class)).use();
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "moveMemoryBlock", Address.class,
Address.class, int.class, void.class)).use();
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "allocStack",
int.class, Address.class)).use();
dependencyAnalyzer.linkMethod(new MethodReference(WasmRuntime.class, "getStackTop", Address.class)).use();

View File

@ -382,6 +382,16 @@ public class DisassemblyCodeSectionListener implements AddressListener, CodeSect
writer.address(address).write("memory.grow").eol();
}
@Override
public void memoryFill() {
writer.address(address).write("memory.fill").eol();
}
@Override
public void memoryCopy() {
writer.address(address).write("memory.copy").eol();
}
private void writeMemArg(int align, int defaultAlign, int offset) {
var needsComma = false;
if (align != defaultAlign) {

View File

@ -15,12 +15,11 @@
*/
package org.teavm.backend.wasm.intrinsics;
import java.util.stream.Collectors;
import org.teavm.ast.InvocationExpr;
import org.teavm.backend.wasm.WasmRuntime;
import org.teavm.backend.wasm.generate.WasmClassGenerator;
import org.teavm.backend.wasm.model.expression.WasmCall;
import org.teavm.backend.wasm.model.expression.WasmCopy;
import org.teavm.backend.wasm.model.expression.WasmExpression;
import org.teavm.backend.wasm.model.expression.WasmFill;
import org.teavm.backend.wasm.model.expression.WasmInt32Constant;
import org.teavm.backend.wasm.model.expression.WasmInt32Subtype;
import org.teavm.backend.wasm.model.expression.WasmIntBinary;
@ -59,16 +58,26 @@ public class AllocatorIntrinsic implements WasmIntrinsic {
@Override
public WasmExpression apply(InvocationExpr invocation, WasmIntrinsicManager manager) {
switch (invocation.getMethod().getName()) {
case "fill":
case "fillZero":
case "fill": {
var fill = new WasmFill();
fill.setIndex(manager.generate(invocation.getArguments().get(0)));
fill.setValue(manager.generate(invocation.getArguments().get(1)));
fill.setCount(manager.generate(invocation.getArguments().get(2)));
return fill;
}
case "fillZero": {
var fill = new WasmFill();
fill.setIndex(manager.generate(invocation.getArguments().get(0)));
fill.setValue(new WasmInt32Constant(0));
fill.setCount(manager.generate(invocation.getArguments().get(1)));
return fill;
}
case "moveMemoryBlock": {
MethodReference delegateMethod = new MethodReference(WasmRuntime.class.getName(),
invocation.getMethod().getDescriptor());
WasmCall call = new WasmCall(manager.getNames().forMethod(delegateMethod));
call.getArguments().addAll(invocation.getArguments().stream()
.map(manager::generate)
.collect(Collectors.toList()));
return call;
var copy = new WasmCopy();
copy.setSourceIndex(manager.generate(invocation.getArguments().get(0)));
copy.setDestinationIndex(manager.generate(invocation.getArguments().get(1)));
copy.setCount(manager.generate(invocation.getArguments().get(2)));
return copy;
}
case "isInitialized": {
WasmExpression pointer = manager.generate(invocation.getArguments().get(0));

View File

@ -0,0 +1,51 @@
/*
* Copyright 2023 konsoletyper.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.backend.wasm.model.expression;
public class WasmCopy extends WasmExpression {
private WasmExpression destinationIndex;
private WasmExpression sourceIndex;
private WasmExpression count;
public WasmExpression getDestinationIndex() {
return destinationIndex;
}
public void setDestinationIndex(WasmExpression destinationIndex) {
this.destinationIndex = destinationIndex;
}
public WasmExpression getSourceIndex() {
return sourceIndex;
}
public void setSourceIndex(WasmExpression sourceIndex) {
this.sourceIndex = sourceIndex;
}
public WasmExpression getCount() {
return count;
}
public void setCount(WasmExpression count) {
this.count = count;
}
@Override
public void acceptVisitor(WasmExpressionVisitor visitor) {
visitor.visit(this);
}
}

View File

@ -185,4 +185,18 @@ public class WasmDefaultExpressionVisitor implements WasmExpressionVisitor {
public void visit(WasmMemoryGrow expression) {
expression.getAmount().acceptVisitor(this);
}
@Override
public void visit(WasmFill expression) {
expression.getIndex().acceptVisitor(this);
expression.getValue().acceptVisitor(this);
expression.getCount().acceptVisitor(this);
}
@Override
public void visit(WasmCopy expression) {
expression.getDestinationIndex().acceptVisitor(this);
expression.getSourceIndex().acceptVisitor(this);
expression.getCount().acceptVisitor(this);
}
}

View File

@ -75,4 +75,8 @@ public interface WasmExpressionVisitor {
void visit(WasmStoreFloat64 expression);
void visit(WasmMemoryGrow expression);
void visit(WasmFill expression);
void visit(WasmCopy expression);
}

View File

@ -0,0 +1,51 @@
/*
* Copyright 2023 konsoletyper.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.backend.wasm.model.expression;
public class WasmFill extends WasmExpression {
private WasmExpression index;
private WasmExpression value;
private WasmExpression count;
public WasmExpression getIndex() {
return index;
}
public void setIndex(WasmExpression index) {
this.index = index;
}
public WasmExpression getValue() {
return value;
}
public void setValue(WasmExpression value) {
this.value = value;
}
public WasmExpression getCount() {
return count;
}
public void setCount(WasmExpression count) {
this.count = count;
}
@Override
public void acceptVisitor(WasmExpressionVisitor visitor) {
visitor.visit(this);
}
}

View File

@ -227,4 +227,28 @@ public class WasmReplacingExpressionVisitor implements WasmExpressionVisitor {
expression.getAmount().acceptVisitor(this);
expression.setAmount(mapper.apply(expression.getAmount()));
}
@Override
public void visit(WasmFill expression) {
expression.getIndex().acceptVisitor(this);
expression.setIndex(mapper.apply(expression.getIndex()));
expression.getValue().acceptVisitor(this);
expression.setValue(mapper.apply(expression.getValue()));
expression.getCount().acceptVisitor(this);
expression.setCount(mapper.apply(expression.getCount()));
}
@Override
public void visit(WasmCopy expression) {
expression.getSourceIndex().acceptVisitor(this);
expression.setSourceIndex(mapper.apply(expression.getSourceIndex()));
expression.getDestinationIndex().acceptVisitor(this);
expression.setDestinationIndex(mapper.apply(expression.getDestinationIndex()));
expression.getCount().acceptVisitor(this);
expression.setCount(mapper.apply(expression.getCount()));
}
}

View File

@ -103,6 +103,12 @@ public interface CodeListener {
default void memoryGrow() {
}
default void memoryFill() {
}
default void memoryCopy() {
}
default void int32Constant(int value) {
}

View File

@ -610,12 +610,37 @@ public class CodeSectionParser {
codeListener.convert(WasmType.INT64, WasmType.FLOAT64, false, true);
break;
case 0xFC:
return parseExtExpr();
default:
return false;
}
return true;
}
private boolean parseExtExpr() {
switch (readLEB()) {
case 10: {
if (data[ptr++] != 0 || data[ptr++] != 0) {
return false;
}
codeListener.memoryCopy();
return true;
}
case 11: {
if (data[ptr++] != 0) {
return false;
}
codeListener.memoryFill();
return true;
}
default:
return false;
}
}
private boolean parseBlock(boolean isLoop) {
var type = readType();
var token = codeListener.startBlock(isLoop, type);

View File

@ -30,9 +30,11 @@ import org.teavm.backend.wasm.model.expression.WasmBreak;
import org.teavm.backend.wasm.model.expression.WasmCall;
import org.teavm.backend.wasm.model.expression.WasmConditional;
import org.teavm.backend.wasm.model.expression.WasmConversion;
import org.teavm.backend.wasm.model.expression.WasmCopy;
import org.teavm.backend.wasm.model.expression.WasmDrop;
import org.teavm.backend.wasm.model.expression.WasmExpression;
import org.teavm.backend.wasm.model.expression.WasmExpressionVisitor;
import org.teavm.backend.wasm.model.expression.WasmFill;
import org.teavm.backend.wasm.model.expression.WasmFloat32Constant;
import org.teavm.backend.wasm.model.expression.WasmFloat64Constant;
import org.teavm.backend.wasm.model.expression.WasmFloatBinary;
@ -914,6 +916,29 @@ class WasmBinaryRenderingVisitor implements WasmExpressionVisitor {
popLocation();
}
@Override
public void visit(WasmFill expression) {
pushLocation(expression);
expression.getIndex().acceptVisitor(this);
expression.getValue().acceptVisitor(this);
expression.getCount().acceptVisitor(this);
writer.writeByte(0xFC);
writer.writeLEB(11);
writer.writeByte(0);
}
@Override
public void visit(WasmCopy expression) {
pushLocation(expression);
expression.getDestinationIndex().acceptVisitor(this);
expression.getSourceIndex().acceptVisitor(this);
expression.getCount().acceptVisitor(this);
writer.writeByte(0xFC);
writer.writeLEB(10);
writer.writeByte(0);
writer.writeByte(0);
}
private int alignment(int value) {
return 31 - Integer.numberOfLeadingZeros(Math.max(1, value));
}

View File

@ -32,9 +32,11 @@ import org.teavm.backend.wasm.model.expression.WasmBreak;
import org.teavm.backend.wasm.model.expression.WasmCall;
import org.teavm.backend.wasm.model.expression.WasmConditional;
import org.teavm.backend.wasm.model.expression.WasmConversion;
import org.teavm.backend.wasm.model.expression.WasmCopy;
import org.teavm.backend.wasm.model.expression.WasmDrop;
import org.teavm.backend.wasm.model.expression.WasmExpression;
import org.teavm.backend.wasm.model.expression.WasmExpressionVisitor;
import org.teavm.backend.wasm.model.expression.WasmFill;
import org.teavm.backend.wasm.model.expression.WasmFloat32Constant;
import org.teavm.backend.wasm.model.expression.WasmFloat64Constant;
import org.teavm.backend.wasm.model.expression.WasmFloatBinary;
@ -1058,6 +1060,48 @@ class WasmCRenderingVisitor implements WasmExpressionVisitor {
value = result;
}
@Override
public void visit(WasmFill expression) {
var result = new CExpression();
expression.getIndex().acceptVisitor(this);
var dest = value;
expression.getValue().acceptVisitor(this);
var v = value;
expression.getValue().acceptVisitor(this);
var num = value;
result.getLines().addAll(dest.getLines());
result.getLines().addAll(v.getLines());
result.getLines().addAll(num.getLines());
result.addLine("memset(" + dest.getText() + ", " + v.getText() + ", " + num.getText() + ");");
value = result;
}
@Override
public void visit(WasmCopy expression) {
var result = new CExpression();
expression.getDestinationIndex().acceptVisitor(this);
var dest = value;
expression.getSourceIndex().acceptVisitor(this);
var src = value;
expression.getCount().acceptVisitor(this);
var num = value;
result.getLines().addAll(dest.getLines());
result.getLines().addAll(src.getLines());
result.getLines().addAll(num.getLines());
result.addLine("memcpy(" + dest.getText() + ", " + src.getText() + ", " + num.getText() + ");");
value = result;
}
private CExpression checkAddress(CExpression index) {
if (!memoryAccessChecked) {
return index;

View File

@ -27,10 +27,12 @@ import org.teavm.backend.wasm.model.expression.WasmBreak;
import org.teavm.backend.wasm.model.expression.WasmCall;
import org.teavm.backend.wasm.model.expression.WasmConditional;
import org.teavm.backend.wasm.model.expression.WasmConversion;
import org.teavm.backend.wasm.model.expression.WasmCopy;
import org.teavm.backend.wasm.model.expression.WasmDefaultExpressionVisitor;
import org.teavm.backend.wasm.model.expression.WasmDrop;
import org.teavm.backend.wasm.model.expression.WasmExpression;
import org.teavm.backend.wasm.model.expression.WasmExpressionVisitor;
import org.teavm.backend.wasm.model.expression.WasmFill;
import org.teavm.backend.wasm.model.expression.WasmFloat32Constant;
import org.teavm.backend.wasm.model.expression.WasmFloat64Constant;
import org.teavm.backend.wasm.model.expression.WasmFloatBinary;
@ -602,6 +604,24 @@ class WasmRenderingVisitor implements WasmExpressionVisitor {
close();
}
@Override
public void visit(WasmCopy expression) {
open().append("memory.copy");
line(expression.getDestinationIndex());
line(expression.getSourceIndex());
line(expression.getCount());
close();
}
@Override
public void visit(WasmFill expression) {
open().append("memory.fill");
line(expression.getIndex());
line(expression.getValue());
line(expression.getCount());
close();
}
private String type(WasmType type) {
switch (type) {
case INT32:

View File

@ -24,8 +24,10 @@ import org.teavm.backend.wasm.model.expression.WasmBreak;
import org.teavm.backend.wasm.model.expression.WasmCall;
import org.teavm.backend.wasm.model.expression.WasmConditional;
import org.teavm.backend.wasm.model.expression.WasmConversion;
import org.teavm.backend.wasm.model.expression.WasmCopy;
import org.teavm.backend.wasm.model.expression.WasmDrop;
import org.teavm.backend.wasm.model.expression.WasmExpressionVisitor;
import org.teavm.backend.wasm.model.expression.WasmFill;
import org.teavm.backend.wasm.model.expression.WasmFloat32Constant;
import org.teavm.backend.wasm.model.expression.WasmFloat64Constant;
import org.teavm.backend.wasm.model.expression.WasmFloatBinary;
@ -215,6 +217,16 @@ public class WasmTypeInference implements WasmExpressionVisitor {
result = WasmType.INT32;
}
@Override
public void visit(WasmFill expression) {
result = null;
}
@Override
public void visit(WasmCopy expression) {
result = null;
}
private static WasmType map(WasmIntType type) {
switch (type) {
case INT32: