Testrunner refactor to support multiple targets

tools/common_py/path.py

@@ -52,13 +52,13 @@
 # Root directory for http-parser submodule.
 HTTPPARSER_ROOT = fs.join(DEPS_ROOT, 'http-parser')
 
+# Root directory for stlink.
+STLINK_ROOT = fs.join(DEPS_ROOT, 'stlink')
+
 # checktest
 CHECKTEST_PATH = fs.join(TOOLS_ROOT, 'check_test.js')
 
 # Build configuration file path.
 BUILD_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.config')
 BUILD_MODULE_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.module')
 BUILD_TARGET_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.target')
-
-# IoT.js build information.
-BUILD_INFO_PATH = fs.join(TOOLS_ROOT, 'iotjs_build_info.js')

tools/common_py/system/executor.py

@@ -17,6 +17,10 @@
 import subprocess
 
 
+class TimeoutException(Exception):
+    pass
+
+
 class Executor(object):
     _TERM_RED = "\033[1;31m"
     _TERM_YELLOW = "\033[1;33m"
@@ -44,9 +48,15 @@ def fail(msg):
     @staticmethod
     def run_cmd(cmd, args=[], quiet=False):
         if not quiet:
+            stdout = None
+            stderr = None
             Executor.print_cmd_line(cmd, args)
+        else:
+            stdout = subprocess.PIPE
+            stderr = subprocess.STDOUT
+
         try:
-            return subprocess.call([cmd] + args)
+            return subprocess.call([cmd] + args, stdout=stdout, stderr=stderr)
         except OSError as e:
             Executor.fail("[Failed - %s] %s" % (cmd, e.strerror))
 

tools/test_runner.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+# Copyright 2016-present Samsung Electronics Co., Ltd. and other contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+from testdriver.testrunner import TestRunner
+from testdriver.target import DEVICES
+
+
+def parse_options():
+    """
+    Parse the given options.
+    """
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--build", default=False, action="store_true",
+                        help="create build for the selected target")
+    parser.add_argument("--quiet", default=False, action="store_true",
+                        help="hide the output of the test execution")
+    parser.add_argument("--skip-modules", metavar="LIST", default="",
+                        help="""specify the built-in modules that sholuld be
+                                skipped (e.g. fs,net,process)""")
+    parser.add_argument("--target", default="host", choices=DEVICES.keys(),
+                        help="""define the target where the testing happens
+                             (default: %(default)s)""")
+    parser.add_argument("--timeout", metavar="SEC", default=300, type=int,
+                        help="default timeout in sec (default: %(default)s)")
+
+    group = parser.add_argument_group("Local testing")
+
+    group.add_argument("--bin-path", metavar="PATH",
+                       help="path to the iotjs binary file")
+    group.add_argument("--coverage", default=False, action="store_true",
+                       help="""measure JavaScript coverage
+                            (only for the meausre_coverage.sh script)""")
+    group.add_argument("--valgrind", action="store_true", default=False,
+                       help="check memory management by Valgrind")
+
+    group = parser.add_argument_group("Remote testing (SSH communication)")
+
+    group.add_argument("--address", metavar="IPADDR",
+                       help="IP address of the device")
+    group.add_argument("--remote-path", metavar="PATH",
+                       help="""define the folder (absolute path) on the device
+                            where iotjs and tests are located""")
+    group.add_argument("--username", metavar="USER",
+                       help="username to login")
+
+    group = parser.add_argument_group("Remote testing (Serial communication)")
+
+    group.add_argument("--baud", default=115200, type=int,
+                       help="baud rate (default: %(default)s)")
+    group.add_argument("--port",
+                       help="serial port name (e.g. /dev/ttyACM0)")
+
+    return parser.parse_args()
+
+
+def main():
+    options = parse_options()
+
+    testrunner = TestRunner(options)
+    testrunner.run()
+
+
+if __name__ == '__main__':
+    main()

tools/testdriver/__init__.py

@@ -0,0 +1 @@
+# Required for Python to search this directory for module files

tools/testdriver/reporter.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+from common_py.system.executor import Executor as ex
+
+
+class Reporter(object):
+    @staticmethod
+    def message(msg="", color=ex._TERM_EMPTY):
+        print("%s%s%s" % (color, msg, ex._TERM_EMPTY))
+
+    @staticmethod
+    def report_testset(testset):
+        Reporter.message()
+        Reporter.message("Testset: %s" % testset, ex._TERM_BLUE)
+
+    @staticmethod
+    def report_pass(test, time):
+        Reporter.message("  PASS: %s (%ss)" % (test, time), ex._TERM_GREEN)
+
+    @staticmethod
+    def report_fail(test, time):
+        Reporter.message("  FAIL: %s (%ss)" % (test, time), ex._TERM_RED)
+
+    @staticmethod
+    def report_timeout(test):
+        Reporter.message("  TIMEOUT: %s" % test, ex._TERM_RED)
+
+    @staticmethod
+    def report_skip(test, reason):
+        skip_message = "  SKIP: %s" % test
+
+        if reason:
+            skip_message += "   (Reason: %s)" % reason
+
+        Reporter.message(skip_message, ex._TERM_YELLOW)
+
+    @staticmethod
+    def report_configuration(options):
+        Reporter.message()
+        Reporter.message("Test configuration:")
+        Reporter.message("  target:        %s" % options.target)
+
+        if options.target == "host":
+            Reporter.message("  bin path:      %s" % options.bin_path)
+            Reporter.message("  coverage:      %s" % options.coverage)
+            Reporter.message("  valgrind:      %s" % options.valgrind)
+        elif options.target == "rpi2":
+            Reporter.message("  address:       %s" % options.address)
+            Reporter.message("  username:      %s" % options.username)
+            Reporter.message("  remote path:   %s" % options.remote_path)
+        elif options.target in ["stm32f4dis", "artik053"]:
+            Reporter.message("  port:          %s" % options.port)
+            Reporter.message("  baud:          %d" % options.baud)
+
+        Reporter.message("  quiet:         %s" % options.quiet)
+        Reporter.message("  timeout:       %s" % options.timeout)
+        Reporter.message("  skip-modules:  %s" % repr(options.skip_modules))
+
+    @staticmethod
+    def report_final(results):
+        Reporter.message()
+        Reporter.message("Finished with all tests:", ex._TERM_BLUE)
+        Reporter.message("  PASS:    %d" % results["pass"], ex._TERM_GREEN)
+        Reporter.message("  FAIL:    %d" % results["fail"], ex._TERM_RED)
+        Reporter.message("  TIMEOUT: %d" % results["timeout"], ex._TERM_RED)
+        Reporter.message("  SKIP:    %d" % results["skip"], ex._TERM_YELLOW)

tools/testdriver/target/__init__.py

@@ -0,0 +1,59 @@
+# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import host
+import stm32f4dis
+
+from common_py import path
+from common_py.system.filesystem import FileSystem as fs
+from common_py.system.executor import Executor as ex
+from common_py.system.platform import Platform
+
+
+DEVICES = {
+    "host": host.Device,
+    "stm32f4dis": stm32f4dis.Device
+}
+
+
+def create_build(options):
+    if options.target == "host":
+        ex.run_cmd(fs.join(path.TOOLS_ROOT, "build.py"), ["--no-check-test"])
+
+        # Append the build path to the options.
+        target = "%s-%s" % (Platform().arch(), Platform().os())
+        options.bin_path = fs.join(path.BUILD_ROOT, target, "debug/bin/iotjs")
+
+    elif options.target == "stm32f4dis":
+        build_options = [
+            "--test=nuttx",
+            "--buildtype=release",
+            "--buildoptions=--jerry-memstat",
+            "--enable-testsuite",
+            "--flash"
+        ]
+
+        ex.run_cmd(fs.join(path.TOOLS_ROOT, "precommit.py"), build_options)
+
+    else:
+        ex.fail("Unimplemented case for building iotjs to the target.")
+
+
+def create_device(options):
+    if options.build:
+        create_build(options)
+
+    device_class = DEVICES[options.target]
+
+    return device_class(options)

tools/testdriver/target/connection/__init__.py

@@ -0,0 +1 @@
+# Required for Python to search this directory for module files

tools/testdriver/target/connection/serialcom.py

@@ -0,0 +1,123 @@
+# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import serial
+import time
+import xmodem
+
+from common_py.system.executor import TimeoutException
+
+
+class Connection(object):
+    """
+    The serial communication wrapper.
+    """
+    def __init__(self, port, baud, timeout, prompt):
+        self.port = port
+        self.baud = baud
+        self.timeout = timeout
+
+        # Defines the end of the stdout.
+        self.prompt = prompt
+
+    def get_prompt(self):
+        """
+        Get the prompt.
+        """
+        return self.prompt
+
+    def open(self):
+        """
+        Open the serial port.
+        """
+        self.serial = serial.Serial(self.port, self.baud, timeout=self.timeout)
+        self.xmodem = xmodem.XMODEM1k(self.getc, self.putc)
+
+    def close(self):
+        """
+        Close the serial port.
+        """
+        self.serial.close()
+
+    def getc(self, size, timeout=1):
+        """
+        Recevice data from the serial port.
+        """
+        time.sleep(2)
+
+        return self.serial.read(size) or None
+
+    def putc(self, data, timeout=1):
+        """
+        Send data to the serial port.
+        """
+        return self.serial.write(data)
+
+    def readline(self):
+        """
+        Read line from the serial port.
+        """
+        return self.serial.readline()
+
+    def exec_command(self, cmd):
+        """
+        Send command over the serial port.
+        """
+        self.serial.write(cmd + "\n")
+
+        receive = self.serial.read_until(self.prompt)
+
+        # Throw exception when timeout happens.
+        if self.prompt not in receive:
+            raise TimeoutException
+
+        # Note: since the received data format is
+        #
+        #   [command][CRLF][stdout][CRFL][prompt],
+        #
+        # we should process the output for the stdout.
+        return "\n".join(receive.split("\r\n")[1:-1])
+
+    def read_until(self, *args):
+        """
+        Read data until it contains args.
+        """
+        line = bytearray()
+        while True:
+            c = self.serial.read(1)
+            if c:
+                line += c
+                for stdout in args:
+                    if line[-len(stdout):] == stdout:
+                        return stdout, bytes(line)
+            else:
+                # raise utils.TimeoutException
+                raise Exception("use TimeoutException")
+
+        return False, False
+
+    def send_file(self, lpath, rpath):
+        """
+        Send file over the serial port.
+        Note: `lrzsz` package should be installed on the device.
+        """
+        self.serial.write("rm " + rpath + "\n")
+        self.serial.write("rx " + rpath + "\n")
+
+        # Receive all the data from the device except the last
+        # \x15 (NAK) byte that is needed by the xmodem protocol.
+        self.serial.read(self.serial.in_waiting - 1)
+
+        with open(lpath, "rb") as file:
+            self.xmodem.send(file)