Fix AST caching to support async methods

This commit is contained in:
konsoletyper 2015-03-15 17:46:01 +03:00
parent 6c57fb866f
commit 9ce9f970d7
11 changed files with 457 additions and 37 deletions

View File

@ -87,6 +87,7 @@ public final class TeaVMRunner {
.create('i'));
options.addOption(OptionBuilder
.withArgName("directory")
.hasArg()
.withDescription("Incremental build cache directory")
.withLongOpt("cachedir")
.create('c'));

View File

@ -86,6 +86,56 @@ public class AstIO {
return node;
}
public void writeAsync(DataOutput output, AsyncMethodNode method) throws IOException {
output.writeInt(packModifiers(method.getModifiers()));
output.writeShort(method.getVariables().size());
for (int var : method.getVariables()) {
output.writeShort(var);
}
output.writeShort(method.getParameterDebugNames().size());
for (Set<String> debugNames : method.getParameterDebugNames()) {
output.writeShort(debugNames != null ? debugNames.size() : 0);
if (debugNames != null) {
for (String debugName : debugNames) {
output.writeUTF(debugName);
}
}
}
try {
output.writeShort(method.getBody().size());
for (int i = 0; i < method.getBody().size(); ++i) {
method.getBody().get(i).getStatement().acceptVisitor(new NodeWriter(output));
}
} catch (IOExceptionWrapper e) {
throw new IOException("Error writing method body", e.getCause());
}
}
public AsyncMethodNode readAsync(DataInput input, MethodReference method) throws IOException {
AsyncMethodNode node = new AsyncMethodNode(method);
node.getModifiers().addAll(unpackModifiers(input.readInt()));
int varCount = input.readShort();
for (int i = 0; i < varCount; ++i) {
node.getVariables().add((int)input.readShort());
}
int paramDebugNameCount = input.readShort();
for (int i = 0; i < paramDebugNameCount; ++i) {
int debugNameCount = input.readShort();
Set<String> debugNames = new HashSet<>();
for (int j = 0; j < debugNameCount; ++j) {
debugNames.add(input.readUTF());
}
node.getParameterDebugNames().add(debugNames);
}
int partCount = input.readShort();
for (int i = 0; i < partCount; ++i) {
AsyncMethodPart part = new AsyncMethodPart();
part.setStatement(readStatement(input));
node.getBody().add(part);
}
return node;
}
private int packModifiers(Set<NodeModifier> modifiers) {
int packed = 0;
for (NodeModifier modifier : modifiers) {
@ -155,6 +205,7 @@ public class AstIO {
writeExpr(statement.getLeftValue());
}
writeExpr(statement.getRightValue());
output.writeBoolean(statement.isAsync());
} catch (IOException e) {
throw new IOExceptionWrapper(e);
}
@ -318,6 +369,7 @@ public class AstIO {
public void visit(MonitorEnterStatement statement) {
try {
output.writeByte(18);
writeLocation(statement.getLocation());
writeExpr(statement.getObjectRef());
} catch (IOException e) {
throw new IOExceptionWrapper(e);
@ -328,6 +380,7 @@ public class AstIO {
public void visit(MonitorExitStatement statement) {
try {
output.writeByte(19);
writeLocation(statement.getLocation());
writeExpr(statement.getObjectRef());
} catch (IOException e) {
throw new IOExceptionWrapper(e);
@ -549,6 +602,7 @@ public class AstIO {
}
stmt.setLeftValue(readExpr(input));
stmt.setRightValue(readExpr(input));
stmt.setAsync(input.readBoolean());
return stmt;
}
case 1: {
@ -559,6 +613,7 @@ public class AstIO {
stmt.getDebugNames().add(input.readUTF());
}
stmt.setRightValue(readExpr(input));
stmt.setAsync(input.readBoolean());
return stmt;
}
case 2: {
@ -684,6 +739,18 @@ public class AstIO {
stmt.setPart(input.readShort());
return stmt;
}
case 18: {
MonitorEnterStatement stmt = new MonitorEnterStatement();
stmt.setLocation(readLocation(input));
stmt.setObjectRef(readExpr(input));
return stmt;
}
case 19: {
MonitorExitStatement stmt = new MonitorExitStatement();
stmt.setLocation(readLocation(input));
stmt.setObjectRef(readExpr(input));
return stmt;
}
// TODO: MonitorEnter/MonitorExit
default:
throw new RuntimeException("Unexpected statement type: " + type);

View File

@ -0,0 +1,24 @@
/*
* Copyright 2015 Alexey Andreev.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.cache;
/**
*
* @author Alexey Andreev
*/
public class AsyncMethodExtractor {
}

View File

@ -17,7 +17,7 @@ package org.teavm.cache;
import java.io.*;
import java.util.*;
import org.teavm.javascript.RegularMethodNodeCache;
import org.teavm.javascript.MethodNodeCache;
import org.teavm.javascript.ast.*;
import org.teavm.model.MethodReference;
import org.teavm.parsing.ClassDateProvider;
@ -26,12 +26,14 @@ import org.teavm.parsing.ClassDateProvider;
*
* @author Alexey Andreev
*/
public class DiskRegularMethodNodeCache implements RegularMethodNodeCache {
public class DiskRegularMethodNodeCache implements MethodNodeCache {
private File directory;
private AstIO astIO;
private ClassDateProvider classDateProvider;
private Map<MethodReference, Item> cache = new HashMap<>();
private Map<MethodReference, AsyncItem> asyncCache = new HashMap<>();
private Set<MethodReference> newMethods = new HashSet<>();
private Set<MethodReference> newAsyncMethods = new HashSet<>();
public DiskRegularMethodNodeCache(File directory, SymbolTable symbolTable, SymbolTable fileTable,
ClassDateProvider classDateProvider) {
@ -46,7 +48,7 @@ public class DiskRegularMethodNodeCache implements RegularMethodNodeCache {
if (item == null) {
item = new Item();
cache.put(methodReference, item);
File file = getMethodFile(methodReference);
File file = getMethodFile(methodReference, false);
if (file.exists()) {
try (InputStream stream = new BufferedInputStream(new FileInputStream(file))) {
DataInput input = new DataInputStream(stream);
@ -79,9 +81,48 @@ public class DiskRegularMethodNodeCache implements RegularMethodNodeCache {
newMethods.add(methodReference);
}
@Override
public AsyncMethodNode getAsync(MethodReference methodReference) {
AsyncItem item = asyncCache.get(methodReference);
if (item == null) {
item = new AsyncItem();
asyncCache.put(methodReference, item);
File file = getMethodFile(methodReference, true);
if (file.exists()) {
try (InputStream stream = new BufferedInputStream(new FileInputStream(file))) {
DataInput input = new DataInputStream(stream);
int depCount = input.readShort();
boolean dependenciesChanged = false;
for (int i = 0; i < depCount; ++i) {
String depClass = input.readUTF();
Date depDate = classDateProvider.getModificationDate(depClass);
if (depDate == null || depDate.after(new Date(file.lastModified()))) {
dependenciesChanged = true;
break;
}
}
if (!dependenciesChanged) {
item.node = astIO.readAsync(input, methodReference);
}
} catch (IOException e) {
// we could not read program, just leave it empty
}
}
}
return item.node;
}
@Override
public void storeAsync(MethodReference methodReference, AsyncMethodNode node) {
AsyncItem item = new AsyncItem();
item.node = node;
asyncCache.put(methodReference, item);
newAsyncMethods.add(methodReference);
}
public void flush() throws IOException {
for (MethodReference method : newMethods) {
File file = getMethodFile(method);
File file = getMethodFile(method, true);
AstDependencyAnalyzer analyzer = new AstDependencyAnalyzer();
RegularMethodNode node = cache.get(method).node;
node.getBody().acceptVisitor(analyzer);
@ -94,11 +135,28 @@ public class DiskRegularMethodNodeCache implements RegularMethodNodeCache {
astIO.write(output, node);
}
}
for (MethodReference method : newAsyncMethods) {
File file = getMethodFile(method, true);
AstDependencyAnalyzer analyzer = new AstDependencyAnalyzer();
AsyncMethodNode node = asyncCache.get(method).node;
for (AsyncMethodPart part : node.getBody()) {
part.getStatement().acceptVisitor(analyzer);
}
analyzer.dependencies.add(method.getClassName());
try (DataOutputStream output = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) {
output.writeShort(analyzer.dependencies.size());
for (String dependency : analyzer.dependencies) {
output.writeUTF(dependency);
}
astIO.writeAsync(output, node);
}
}
}
private File getMethodFile(MethodReference method) {
private File getMethodFile(MethodReference method, boolean async) {
File dir = new File(directory, method.getClassName().replace('.', '/'));
return new File(dir, FileNameEncoder.encodeFileName(method.getDescriptor().toString()) + ".teavm-ast");
return new File(dir, FileNameEncoder.encodeFileName(method.getDescriptor().toString()) + ".teavm-ast" +
(async ? "-async" : ""));
}
static class AstDependencyAnalyzer implements StatementVisitor, ExprVisitor {
@ -275,4 +333,8 @@ public class DiskRegularMethodNodeCache implements RegularMethodNodeCache {
static class Item {
RegularMethodNode node;
}
static class AsyncItem {
AsyncMethodNode node;
}
}

View File

@ -0,0 +1,208 @@
/*
* Copyright 2015 Alexey Andreev.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.javascript;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.teavm.javascript.ast.*;
import org.teavm.model.MethodReference;
/**
*
* @author Alexey Andreev
*/
class AsyncCallsFinder implements StatementVisitor, ExprVisitor {
Set<MethodReference> asyncCalls = new HashSet<>();
Set<MethodReference> allCalls = new HashSet<>();
private void visitList(List<Statement> statements) {
for (Statement stmt : statements) {
stmt.acceptVisitor(this);
}
}
@Override
public void visit(AssignmentStatement statement) {
if (statement.getLeftValue() != null) {
statement.getLeftValue().acceptVisitor(this);
}
statement.getRightValue().acceptVisitor(this);
if (!statement.isAsync()) {
statement.getRightValue().acceptVisitor(this);
return;
}
if (!(statement.getRightValue() instanceof InvocationExpr)) {
statement.getRightValue().acceptVisitor(this);
return;
}
InvocationExpr invocation = (InvocationExpr)statement.getRightValue();
asyncCalls.add(invocation.getMethod());
}
@Override
public void visit(SequentialStatement statement) {
visitList(statement.getSequence());
}
@Override
public void visit(ConditionalStatement statement) {
visitList(statement.getConsequent());
visitList(statement.getAlternative());
statement.getCondition().acceptVisitor(this);
}
@Override
public void visit(SwitchStatement statement) {
for (SwitchClause clause : statement.getClauses()) {
visitList(clause.getBody());
}
visitList(statement.getDefaultClause());
statement.getValue().acceptVisitor(this);
}
@Override
public void visit(WhileStatement statement) {
visitList(statement.getBody());
if (statement.getCondition() != null) {
statement.getCondition().acceptVisitor(this);
}
}
@Override
public void visit(BlockStatement statement) {
visitList(statement.getBody());
}
@Override
public void visit(BreakStatement statement) {
}
@Override
public void visit(ContinueStatement statement) {
}
@Override
public void visit(ReturnStatement statement) {
if (statement.getResult() != null) {
statement.getResult().acceptVisitor(this);
}
}
@Override
public void visit(ThrowStatement statement) {
statement.getException().acceptVisitor(this);
}
@Override
public void visit(InitClassStatement statement) {
}
@Override
public void visit(TryCatchStatement statement) {
visitList(statement.getProtectedBody());
visitList(statement.getHandler());
}
@Override
public void visit(GotoPartStatement statement) {
}
@Override
public void visit(MonitorEnterStatement statement) {
statement.getObjectRef().acceptVisitor(this);
}
@Override
public void visit(MonitorExitStatement statement) {
statement.getObjectRef().acceptVisitor(this);
}
@Override
public void visit(BinaryExpr expr) {
expr.getFirstOperand().acceptVisitor(this);
expr.getSecondOperand().acceptVisitor(this);
}
@Override
public void visit(UnaryExpr expr) {
expr.getOperand().acceptVisitor(this);
}
@Override
public void visit(ConditionalExpr expr) {
expr.getCondition().acceptVisitor(this);
expr.getConsequent().acceptVisitor(this);
expr.getAlternative().acceptVisitor(this);
}
@Override
public void visit(ConstantExpr expr) {
}
@Override
public void visit(VariableExpr expr) {
}
@Override
public void visit(SubscriptExpr expr) {
expr.getArray().acceptVisitor(this);
expr.getIndex().acceptVisitor(this);
}
@Override
public void visit(UnwrapArrayExpr expr) {
expr.getArray().acceptVisitor(this);
}
@Override
public void visit(InvocationExpr expr) {
allCalls.add(expr.getMethod());
for (Expr arg : expr.getArguments()) {
arg.acceptVisitor(this);
}
}
@Override
public void visit(QualificationExpr expr) {
expr.getQualified().acceptVisitor(this);
}
@Override
public void visit(NewExpr expr) {
}
@Override
public void visit(NewArrayExpr expr) {
expr.getLength().acceptVisitor(this);
}
@Override
public void visit(NewMultiArrayExpr expr) {
for (Expr dim : expr.getDimensions()) {
dim.acceptVisitor(this);
}
}
@Override
public void visit(InstanceOfExpr expr) {
expr.getExpr().acceptVisitor(this);
}
@Override
public void visit(StaticClassExpr expr) {
}
}

View File

@ -44,7 +44,7 @@ public class Decompiler {
private RangeTree.Node parentNode;
private Map<MethodReference, Generator> generators = new HashMap<>();
private Set<MethodReference> methodsToPass = new HashSet<>();
private RegularMethodNodeCache regularMethodCache;
private MethodNodeCache regularMethodCache;
private Set<MethodReference> asyncMethods;
private Set<MethodReference> splitMethods = new HashSet<>();
@ -57,11 +57,11 @@ public class Decompiler {
splitMethods.addAll(asyncFamilyMethods);
}
public RegularMethodNodeCache getRegularMethodCache() {
public MethodNodeCache getRegularMethodCache() {
return regularMethodCache;
}
public void setRegularMethodCache(RegularMethodNodeCache regularMethodCache) {
public void setRegularMethodCache(MethodNodeCache regularMethodCache) {
this.regularMethodCache = regularMethodCache;
}
@ -198,7 +198,58 @@ public class Decompiler {
return node;
}
public RegularMethodNode decompileRegularCacheMiss(MethodHolder method) {
RegularMethodNode methodNode = new RegularMethodNode(method.getReference());
Program program = method.getProgram();
int[] targetBlocks = new int[program.basicBlockCount()];
Arrays.fill(targetBlocks, -1);
methodNode.setBody(getRegularMethodStatement(program, targetBlocks, false).getStatement());
for (int i = 0; i < program.variableCount(); ++i) {
methodNode.getVariables().add(program.variableAt(i).getRegister());
}
Optimizer optimizer = new Optimizer();
optimizer.optimize(methodNode, method.getProgram());
methodNode.getModifiers().addAll(mapModifiers(method.getModifiers()));
int paramCount = Math.min(method.getSignature().length, program.variableCount());
for (int i = 0; i < paramCount; ++i) {
Variable var = program.variableAt(i);
methodNode.getParameterDebugNames().add(new HashSet<>(var.getDebugNames()));
}
return methodNode;
}
public AsyncMethodNode decompileAsync(MethodHolder method) {
if (regularMethodCache == null) {
return decompileAsyncCacheMiss(method);
}
AsyncMethodNode node = regularMethodCache.getAsync(method.getReference());
if (node == null || !checkAsyncRelevant(node)) {
node = decompileAsyncCacheMiss(method);
regularMethodCache.storeAsync(method.getReference(), node);
}
return node;
}
private boolean checkAsyncRelevant(AsyncMethodNode node) {
AsyncCallsFinder asyncCallsFinder = new AsyncCallsFinder();
for (AsyncMethodPart part : node.getBody()) {
part.getStatement().acceptVisitor(asyncCallsFinder);
}
for (MethodReference asyncCall : asyncCallsFinder.asyncCalls) {
if (!splitMethods.contains(asyncCall)) {
return false;
}
}
asyncCallsFinder.allCalls.removeAll(asyncCallsFinder.asyncCalls);
for (MethodReference asyncCall : asyncCallsFinder.allCalls) {
if (splitMethods.contains(asyncCall)) {
return false;
}
}
return true;
}
public AsyncMethodNode decompileAsyncCacheMiss(MethodHolder method) {
AsyncMethodNode node = new AsyncMethodNode(method.getReference());
AsyncProgramSplitter splitter = new AsyncProgramSplitter(classSource, splitMethods);
splitter.split(method.getProgram());
@ -222,26 +273,6 @@ public class Decompiler {
return node;
}
public RegularMethodNode decompileRegularCacheMiss(MethodHolder method) {
RegularMethodNode methodNode = new RegularMethodNode(method.getReference());
Program program = method.getProgram();
int[] targetBlocks = new int[program.basicBlockCount()];
Arrays.fill(targetBlocks, -1);
methodNode.setBody(getRegularMethodStatement(program, targetBlocks, false).getStatement());
for (int i = 0; i < program.variableCount(); ++i) {
methodNode.getVariables().add(program.variableAt(i).getRegister());
}
Optimizer optimizer = new Optimizer();
optimizer.optimize(methodNode, method.getProgram());
methodNode.getModifiers().addAll(mapModifiers(method.getModifiers()));
int paramCount = Math.min(method.getSignature().length, program.variableCount());
for (int i = 0; i < paramCount; ++i) {
Variable var = program.variableAt(i);
methodNode.getParameterDebugNames().add(new HashSet<>(var.getDebugNames()));
}
return methodNode;
}
private AsyncMethodPart getRegularMethodStatement(Program program, int[] targetBlocks, boolean async) {
AsyncMethodPart result = new AsyncMethodPart();
lastBlockId = 1;

View File

@ -15,6 +15,7 @@
*/
package org.teavm.javascript;
import org.teavm.javascript.ast.AsyncMethodNode;
import org.teavm.javascript.ast.RegularMethodNode;
import org.teavm.model.MethodReference;
@ -22,7 +23,7 @@ import org.teavm.model.MethodReference;
*
* @author Alexey Andreev
*/
public class EmptyRegularMethodNodeCache implements RegularMethodNodeCache {
public class EmptyRegularMethodNodeCache implements MethodNodeCache {
@Override
public RegularMethodNode get(MethodReference methodReference) {
return null;
@ -31,4 +32,13 @@ public class EmptyRegularMethodNodeCache implements RegularMethodNodeCache {
@Override
public void store(MethodReference methodReference, RegularMethodNode node) {
}
@Override
public AsyncMethodNode getAsync(MethodReference methodReference) {
return null;
}
@Override
public void storeAsync(MethodReference methodReference, AsyncMethodNode node) {
}
}

View File

@ -17,6 +17,7 @@ package org.teavm.javascript;
import java.util.HashMap;
import java.util.Map;
import org.teavm.javascript.ast.AsyncMethodNode;
import org.teavm.javascript.ast.RegularMethodNode;
import org.teavm.model.MethodReference;
@ -24,8 +25,9 @@ import org.teavm.model.MethodReference;
*
* @author Alexey Andreev
*/
public class InMemoryRegularMethodNodeCache implements RegularMethodNodeCache {
public class InMemoryRegularMethodNodeCache implements MethodNodeCache {
private Map<MethodReference, RegularMethodNode> cache = new HashMap<>();
private Map<MethodReference, AsyncMethodNode> asyncCache = new HashMap<>();
@Override
public RegularMethodNode get(MethodReference methodReference) {
@ -36,4 +38,14 @@ public class InMemoryRegularMethodNodeCache implements RegularMethodNodeCache {
public void store(MethodReference methodReference, RegularMethodNode node) {
cache.put(methodReference, node);
}
@Override
public AsyncMethodNode getAsync(MethodReference methodReference) {
return null;
}
@Override
public void storeAsync(MethodReference methodReference, AsyncMethodNode node) {
asyncCache.put(methodReference, node);
}
}

View File

@ -15,6 +15,7 @@
*/
package org.teavm.javascript;
import org.teavm.javascript.ast.AsyncMethodNode;
import org.teavm.javascript.ast.RegularMethodNode;
import org.teavm.model.MethodReference;
@ -22,8 +23,12 @@ import org.teavm.model.MethodReference;
*
* @author Alexey Andreev
*/
public interface RegularMethodNodeCache {
public interface MethodNodeCache {
RegularMethodNode get(MethodReference methodReference);
void store(MethodReference methodReference, RegularMethodNode node);
AsyncMethodNode getAsync(MethodReference methodReference);
void storeAsync(MethodReference methodReference, AsyncMethodNode node);
}

View File

@ -25,7 +25,7 @@ import org.teavm.debugging.information.DebugInformation;
import org.teavm.debugging.information.DebugInformationBuilder;
import org.teavm.javascript.EmptyRegularMethodNodeCache;
import org.teavm.javascript.InMemoryRegularMethodNodeCache;
import org.teavm.javascript.RegularMethodNodeCache;
import org.teavm.javascript.MethodNodeCache;
import org.teavm.model.*;
import org.teavm.parsing.ClasspathClassHolderSource;
import org.teavm.testing.JUnitTestAdapter;
@ -58,7 +58,7 @@ public class TeaVMTestTool {
private boolean sourceFilesCopied;
private boolean incremental;
private List<SourceFileProvider> sourceFileProviders = new ArrayList<>();
private RegularMethodNodeCache astCache;
private MethodNodeCache astCache;
private ProgramCache programCache;
private SourceFilesCopier sourceFilesCopier;

View File

@ -84,7 +84,7 @@ public class TeaVM implements TeaVMHost, ServiceRepository {
private Properties properties = new Properties();
private DebugInformationEmitter debugEmitter;
private ProgramCache programCache;
private RegularMethodNodeCache astCache = new EmptyRegularMethodNodeCache();
private MethodNodeCache astCache = new EmptyRegularMethodNodeCache();
private boolean incremental;
private TeaVMProgressListener progressListener;
private boolean cancelled;
@ -181,11 +181,11 @@ public class TeaVM implements TeaVMHost, ServiceRepository {
return new Properties(properties);
}
public RegularMethodNodeCache getAstCache() {
public MethodNodeCache getAstCache() {
return astCache;
}
public void setAstCache(RegularMethodNodeCache methodAstCache) {
public void setAstCache(MethodNodeCache methodAstCache) {
this.astCache = methodAstCache;
}