Prototype implementation of selenium test runner

core/src/main/java/org/teavm/tooling/TeaVMTestCase.java

@@ -16,22 +16,29 @@
 package org.teavm.tooling;
 
 import java.io.File;
+import org.teavm.model.MethodReference;
 
 /**
  *
  * @author Alexey Andreev
  */
 public class TeaVMTestCase {
+    private MethodReference testMethod;
     private File runtimeScript;
     private File testScript;
     private File debugTable;
 
-    public TeaVMTestCase(File runtimeScript, File testScript, File debugTable) {
+    public TeaVMTestCase(MethodReference testMethod, File runtimeScript, File testScript, File debugTable) {
+        this.testMethod = testMethod;
         this.runtimeScript = runtimeScript;
         this.testScript = testScript;
         this.debugTable = debugTable;
     }
 
+    public MethodReference getTestMethod() {
+        return testMethod;
+    }
+
     public File getRuntimeScript() {
         return runtimeScript;
     }

core/src/main/java/org/teavm/tooling/TeaVMTestTool.java

@@ -400,7 +400,8 @@ public class TeaVMTestTool {
             sourceFilesCopier.addClasses(vm.getWrittenClasses());
         }
 
-        TeaVMTestCase testCase = new TeaVMTestCase(new File(outputDir, "res/runtime.js"), file, debugTableFile);
+        TeaVMTestCase testCase = new TeaVMTestCase(methodRef, new File(outputDir, "res/runtime.js"),
+                file, debugTableFile);
         for (TeaVMTestToolListener listener : listeners) {
             listener.testGenerated(testCase);
         }

tools/maven/plugin/pom.xml

@@ -60,12 +60,15 @@
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
       <artifactId>selenium-java</artifactId>
-      <version>2.47.2</version>
     </dependency>
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
       <artifactId>selenium-remote-driver</artifactId>
-      <version>2.47.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.6.2</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

tools/maven/plugin/src/main/java/org/teavm/maven/BuildJavascriptTestMojo.java

@@ -16,9 +16,7 @@
 package org.teavm.maven;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.MalformedURLException;
@@ -31,13 +29,9 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
 import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.IOUtils;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.artifact.repository.MavenArtifactRepository;
 import org.apache.maven.plugin.AbstractMojo;
@@ -50,16 +44,10 @@ import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.plugins.annotations.ResolutionScope;
 import org.apache.maven.project.MavenProject;
 import org.apache.maven.repository.RepositorySystem;
-import org.openqa.selenium.JavascriptExecutor;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.remote.DesiredCapabilities;
-import org.openqa.selenium.remote.RemoteWebDriver;
 import org.teavm.model.ClassHolderTransformer;
 import org.teavm.testing.JUnitTestAdapter;
 import org.teavm.testing.TestAdapter;
 import org.teavm.tooling.SourceFileProvider;
-import org.teavm.tooling.TeaVMTestCase;
 import org.teavm.tooling.TeaVMTestTool;
 import org.teavm.tooling.TeaVMToolException;
 
@@ -139,11 +127,10 @@ public class BuildJavascriptTestMojo extends AbstractMojo {
 
     @Parameter
     private URL seleniumURL;
-    private WebDriver webDriver;
+
+    private SeleniumTestRunner seleniumRunner;
 
     private TeaVMTestTool tool = new TeaVMTestTool();
-    private BlockingQueue<Runnable> seleniumTaskQueue = new LinkedBlockingQueue<>();
-    private volatile boolean seleniumStopped = false;
 
     public void setProject(MavenProject project) {
         this.project = project;
@@ -233,7 +220,10 @@ public class BuildJavascriptTestMojo extends AbstractMojo {
             return;
         }
 
-        detectSelenium();
+        seleniumRunner = new SeleniumTestRunner();
+        seleniumRunner.setUrl(seleniumURL);
+        seleniumRunner.setLog(getLog());
+        seleniumRunner.detectSelenium();
         try {
             final ClassLoader classLoader = prepareClassLoader();
             getLog().info("Searching for tests in the directory `" + testFiles.getAbsolutePath() + "'");
@@ -269,13 +259,35 @@ public class BuildJavascriptTestMojo extends AbstractMojo {
             if (additionalScripts != null) {
                 tool.getAdditionalScripts().addAll(Arrays.asList(additionalScripts));
             }
-            tool.addListener(testCase -> runSelenium(testCase));
+            tool.addListener(testCase -> seleniumRunner.run(testCase));
             tool.generate();
+            seleniumRunner.stopSelenium();
+            seleniumRunner.waitForSelenium();
+            processReport(seleniumRunner.getReport());
         } catch (TeaVMToolException e) {
             throw new MojoFailureException("Error occured generating JavaScript files", e);
         } finally {
-            webDriver.close();
+            seleniumRunner.stopSelenium();
-            stopSelenium();
+        }
+    }
+
+    private void processReport(List<TestResult> report) throws MojoExecutionException {
+        if (report.isEmpty()) {
+            getLog().info("No tests ran");
+            return;
+        }
+
+        int failedTests = 0;
+        for (TestResult result : report) {
+            if (result.getStatus() != TestStatus.PASSED) {
+                failedTests++;
+            }
+        }
+
+        if (failedTests > 0) {
+            throw new MojoExecutionException(failedTests + " of " + report.size() + " test(s) failed");
+        } else {
+            getLog().info("All of " + report.size() + " tests successfully passed");
         }
     }
 
@@ -454,63 +466,4 @@ public class BuildJavascriptTestMojo extends AbstractMojo {
         return transformerInstances;
     }
 
-    private void detectSelenium() {
-        if (seleniumURL == null) {
-            return;
-        }
-        ChromeDriver driver = new ChromeDriver();
-        webDriver = driver;
-        new Thread(() -> {
-            while (!seleniumStopped) {
-                Runnable task;
-                try {
-                    task = seleniumTaskQueue.poll(1, TimeUnit.SECONDS);
-                } catch (InterruptedException e) {
-                    break;
-                }
-                task.run();
-            }
-        }).start();
-    }
-
-    private void addSeleniumTask(Runnable runnable) {
-        if (seleniumURL != null) {
-            seleniumTaskQueue.add(runnable);
-        }
-    }
-
-    private void stopSelenium() {
-        addSeleniumTask(() -> seleniumStopped = true);
-    }
-
-    private void runSelenium(TeaVMTestCase testCase) {
-        if (webDriver == null) {
-            return;
-        }
-        try {
-            JavascriptExecutor js = (JavascriptExecutor) webDriver;
-            js.executeAsyncScript(
-                    readResource("teavm-selenium.js"),
-                    readFile(testCase.getRuntimeScript()),
-                    readFile(testCase.getTestScript()),
-                    readResource("teavm-selenium-adapter.js"));
-        } catch (IOException e) {
-            getLog().error(e);
-        }
-    }
-
-    private String readFile(File file) throws IOException {
-        try (InputStream input = new FileInputStream(file)) {
-            return IOUtils.toString(input, "UTF-8");
-        }
-    }
-
-    private String readResource(String resourceName) throws IOException {
-        try (InputStream input = BuildJavascriptTestMojo.class.getClassLoader().getResourceAsStream(resourceName)) {
-            if (input == null) {
-                return "";
-            }
-            return IOUtils.toString(input, "UTF-8");
-        }
-    }
 }

tools/maven/plugin/src/main/java/org/teavm/maven/SeleniumTestRunner.java

@@ -0,0 +1,172 @@
+/*
+ *  Copyright 2015 Alexey Andreev.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.maven;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.IOUtils;
+import org.apache.maven.plugin.logging.Log;
+import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebDriverException;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.teavm.tooling.TeaVMTestCase;
+
+/**
+ *
+ * @author Alexey Andreev
+ */
+public class SeleniumTestRunner {
+    private URL url;
+    private WebDriver webDriver;
+    private BlockingQueue<Runnable> seleniumTaskQueue = new LinkedBlockingQueue<>();
+    private CountDownLatch latch = new CountDownLatch(1);
+    private volatile boolean seleniumStopped = false;
+    private Log log;
+    private List<TestResult> report = new CopyOnWriteArrayList<>();
+    private ThreadLocal<List<TestResult>> localReport = new ThreadLocal<>();
+
+    public URL getUrl() {
+        return url;
+    }
+
+    public void setUrl(URL url) {
+        this.url = url;
+    }
+
+    public void setLog(Log log) {
+        this.log = log;
+    }
+
+    public void detectSelenium() {
+        if (url == null) {
+            return;
+        }
+        ChromeDriver driver = new ChromeDriver();
+        webDriver = driver;
+        new Thread(() -> {
+            localReport.set(new ArrayList<>());
+            while (!seleniumStopped) {
+                Runnable task;
+                try {
+                    task = seleniumTaskQueue.poll(1, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    break;
+                }
+                if (task != null) {
+                    task.run();
+                }
+            }
+            report.addAll(localReport.get());
+            localReport.remove();
+        }).start();
+    }
+
+    private void addSeleniumTask(Runnable runnable) {
+        if (url != null) {
+            seleniumTaskQueue.add(runnable);
+        }
+    }
+
+    public void stopSelenium() {
+        addSeleniumTask(() -> {
+            seleniumStopped = true;
+            latch.countDown();
+        });
+    }
+
+    public void waitForSelenium() {
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            return;
+        }
+    }
+
+    public void run(TeaVMTestCase testCase) {
+        addSeleniumTask(() -> runImpl(testCase));
+    }
+
+    private void runImpl(TeaVMTestCase testCase) {
+        if (webDriver == null) {
+            return;
+        }
+        webDriver.manage().timeouts().setScriptTimeout(5, TimeUnit.SECONDS);
+        JavascriptExecutor js = (JavascriptExecutor) webDriver;
+        try {
+            String result = (String) js.executeAsyncScript(
+                    readResource("teavm-selenium.js"),
+                    readFile(testCase.getRuntimeScript()),
+                    readFile(testCase.getTestScript()),
+                    readResource("teavm-selenium-adapter.js"));
+            ObjectMapper mapper = new ObjectMapper();
+            ObjectNode resultObject = (ObjectNode) mapper.readTree(result);
+            String status = resultObject.get("status").asText();
+            switch (status) {
+                case "ok":
+                    log.info("Test passed: " + testCase.getTestMethod());
+                    localReport.get().add(TestResult.passed(testCase.getTestMethod()));
+                    break;
+                case "exception": {
+                    String stack = resultObject.get("stack").asText();
+                    log.info("Test failed: " + testCase.getTestMethod());
+                    localReport.get().add(TestResult.error(testCase.getTestMethod(), stack));
+                    break;
+                }
+            }
+        } catch (IOException e) {
+            log.error(e);
+        } catch (WebDriverException e) {
+            log.error("Error occured running test " + testCase.getTestMethod(), e);
+            @SuppressWarnings("unchecked")
+            List<Object> errors = (List<Object>) js.executeScript("return window.jsErrors;");
+            for (Object error : errors) {
+                log.error("  -- additional error: " + error);
+            }
+        }
+    }
+
+    private String readFile(File file) throws IOException {
+        try (InputStream input = new FileInputStream(file)) {
+            return IOUtils.toString(input, "UTF-8");
+        }
+    }
+
+    private String readResource(String resourceName) throws IOException {
+        try (InputStream input = BuildJavascriptTestMojo.class.getClassLoader().getResourceAsStream(resourceName)) {
+            if (input == null) {
+                return "";
+            }
+            return IOUtils.toString(input, "UTF-8");
+        }
+    }
+
+    public List<TestResult> getReport() {
+        return new ArrayList<>(report);
+    }
+}

tools/maven/plugin/src/main/java/org/teavm/maven/TestResult.java

@@ -0,0 +1,54 @@
+/*
+ *  Copyright 2015 Alexey Andreev.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.maven;
+
+import org.teavm.model.MethodReference;
+
+/**
+ *
+ * @author Alexey Andreev
+ */
+public class TestResult {
+    private MethodReference method;
+    private TestStatus status;
+    private String stack;
+
+    private TestResult(MethodReference method, TestStatus status, String stack) {
+        this.method = method;
+        this.status = status;
+        this.stack = stack;
+    }
+
+    public static TestResult passed(MethodReference method) {
+        return new TestResult(method, TestStatus.PASSED, null);
+    }
+
+    public static TestResult error(MethodReference method, String stack) {
+        return new TestResult(method, TestStatus.ERROR, stack);
+    }
+
+    public MethodReference getMethod() {
+        return method;
+    }
+
+    public TestStatus getStatus() {
+        return status;
+    }
+
+    public String getStack() {
+        return stack;
+    }
+}

tools/maven/plugin/src/main/java/org/teavm/maven/TestStatus.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2015 Alexey Andreev.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.teavm.maven;
+
+/**
+ *
+ * @author Alexey Andreev
+ */
+public enum TestStatus {
+    PASSED,
+    ERROR
+}

tools/maven/plugin/src/main/resources/teavm-selenium-adapter.js

@@ -1,5 +1,6 @@
+var JUnitClient = {}
 JUnitClient.run = function() {
-    var handler = window.addEventListener("message", $rt_threadStarter(function() {
+    $rt_startThread(function() {
         var thread = $rt_nativeThread();
         var instance;
         var ptr = 0;
@@ -44,5 +45,17 @@ JUnitClient.run = function() {
             break loop;
         }}
         window.parent.postMessage(JSON.stringify(message), "*");
-    }));
+    })
+}
+
+JUnitClient.makeErrorMessage = function(message, e) {
+    message.status = "exception";
+    var stack = e.stack;
+    if (e.$javaException && e.$javaException.constructor.$meta) {
+        message.exception = e.$javaException.constructor.$meta.name;
+        message.stack = e.$javaException.constructor.$meta.name + ": ";
+        var exceptionMessage = extractException(e.$javaException);
+        message.stack += exceptionMessage ? $rt_ustr(exceptionMessage) : "";
+    }
+    message.stack += "\n" + stack;
 }

tools/maven/plugin/src/main/resources/teavm-selenium.js

@@ -1,40 +1,43 @@
 var runtimeSource = arguments[0]
 var testSource = arguments[1]
 var adapterSource = arguments[2]
+var seleniumCallback = arguments[arguments.length - 1]
 
 var iframe = document.createElement("iframe")
-document.appendChild(iframe)
+document.body.appendChild(iframe)
 var doc = iframe.contentDocument
 
-loadScripts([ runtimeSource, adapterSource, testSource ], runTest)
+window.jsErrors = []
+window.onerror = reportError
+iframe.contentWindow.onerror = reportError
+
+loadScripts([ runtimeSource, adapterSource, testSource ])
 window.addEventListener("message", handleMessage)
 
 function handleMessage(event) {
     window.removeEventListener("message", handleMessage)
-    callback(JSON.stringify(message.data))
+    document.body.removeChild(iframe)
+    seleniumCallback(event.data)
 }
 
-var handler = window.addEventListener("message", function(event) {
-    window.removeEventListener
-})
-
 function loadScript(script, callback) {
-    var elem = doc.createElement("script")
-    elem.setAttribute("type", "text/javascript")
-    elem.appendChild(doc.createTextNode(runtimeSource))
-    elem.onload = function() {
     callback()
 }
-    doc.body.appendChild(script)
-}
 
-function loadScripts(scripts, callback, index) {
+function loadScripts(scripts) {
-    index = index || 0
+    for (var i = 0; i < scripts.length; ++i) {
-    loadScript(scripts[i], function() {
+        var elem = doc.createElement("script")
-        if (++index == scripts.length) {
+        elem.type = "text/javascript"
-            callback()
+        doc.head.appendChild(elem)
-        } else {
+        elem.text = scripts[i]
-            loadScripts(scripts, callback, index)
     }
-    })
+}
+function reportError(error, url, line) {
+    window.jsErrors.push(error + " at " + line)
+}
+function report(error) {
+    window.jsErrors.push(error)
+}
+function globalEval(window, arg) {
+    eval.apply(window, [arg])
 }