diff --git a/tools/iotjs_build_info.js b/test/iotjs_build_info.js similarity index 100% rename from tools/iotjs_build_info.js rename to test/iotjs_build_info.js diff --git a/tools/common_py/path.py b/tools/common_py/path.py index 744f40c723..2392a38fb1 100644 --- a/tools/common_py/path.py +++ b/tools/common_py/path.py @@ -52,6 +52,9 @@ # Root directory for http-parser submodule. HTTPPARSER_ROOT = fs.join(DEPS_ROOT, 'http-parser') +# Root directory for stlink. +STLINK_ROOT = fs.join(DEPS_ROOT, 'stlink') + # checktest CHECKTEST_PATH = fs.join(TOOLS_ROOT, 'check_test.js') @@ -59,6 +62,3 @@ BUILD_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.config') BUILD_MODULE_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.module') BUILD_TARGET_CONFIG_PATH = fs.join(PROJECT_ROOT, 'build.target') - -# IoT.js build information. -BUILD_INFO_PATH = fs.join(TOOLS_ROOT, 'iotjs_build_info.js') diff --git a/tools/common_py/system/executor.py b/tools/common_py/system/executor.py index a8d721e60a..a2a6f2ec27 100644 --- a/tools/common_py/system/executor.py +++ b/tools/common_py/system/executor.py @@ -17,6 +17,10 @@ import subprocess +class TimeoutException(Exception): + pass + + class Executor(object): _TERM_RED = "\033[1;31m" _TERM_YELLOW = "\033[1;33m" @@ -44,9 +48,15 @@ def fail(msg): @staticmethod def run_cmd(cmd, args=[], quiet=False): if not quiet: + stdout = None + stderr = None Executor.print_cmd_line(cmd, args) + else: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + try: - return subprocess.call([cmd] + args) + return subprocess.call([cmd] + args, stdout=stdout, stderr=stderr) except OSError as e: Executor.fail("[Failed - %s] %s" % (cmd, e.strerror)) diff --git a/tools/test_runner.py b/tools/test_runner.py new file mode 100755 index 0000000000..3f27278de7 --- /dev/null +++ b/tools/test_runner.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright 2016-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from testdriver.testrunner import TestRunner +from testdriver.target import DEVICES + + +def parse_options(): + """ + Parse the given options. + """ + parser = argparse.ArgumentParser() + + parser.add_argument("--build", default=False, action="store_true", + help="create build for the selected target") + parser.add_argument("--quiet", default=False, action="store_true", + help="hide the output of the test execution") + parser.add_argument("--skip-modules", metavar="LIST", default="", + help="""specify the built-in modules that sholuld be + skipped (e.g. fs,net,process)""") + parser.add_argument("--target", default="host", choices=DEVICES.keys(), + help="""define the target where the testing happens + (default: %(default)s)""") + parser.add_argument("--timeout", metavar="SEC", default=300, type=int, + help="default timeout in sec (default: %(default)s)") + + group = parser.add_argument_group("Local testing") + + group.add_argument("--bin-path", metavar="PATH", + help="path to the iotjs binary file") + group.add_argument("--coverage", default=False, action="store_true", + help="""measure JavaScript coverage + (only for the meausre_coverage.sh script)""") + group.add_argument("--valgrind", action="store_true", default=False, + help="check memory management by Valgrind") + + group = parser.add_argument_group("Remote testing (SSH communication)") + + group.add_argument("--address", metavar="IPADDR", + help="IP address of the device") + group.add_argument("--remote-path", metavar="PATH", + help="""define the folder (absolute path) on the device + where iotjs and tests are located""") + group.add_argument("--username", metavar="USER", + help="username to login") + + group = parser.add_argument_group("Remote testing (Serial communication)") + + group.add_argument("--baud", default=115200, type=int, + help="baud rate (default: %(default)s)") + group.add_argument("--port", + help="serial port name (e.g. /dev/ttyACM0)") + + return parser.parse_args() + + +def main(): + options = parse_options() + + testrunner = TestRunner(options) + testrunner.run() + + +if __name__ == '__main__': + main() diff --git a/tools/testdriver/__init__.py b/tools/testdriver/__init__.py new file mode 100644 index 0000000000..ef65bee5bb --- /dev/null +++ b/tools/testdriver/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/tools/testdriver/reporter.py b/tools/testdriver/reporter.py new file mode 100644 index 0000000000..1ede6326ef --- /dev/null +++ b/tools/testdriver/reporter.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +from common_py.system.executor import Executor as ex + + +class Reporter(object): + @staticmethod + def message(msg="", color=ex._TERM_EMPTY): + print("%s%s%s" % (color, msg, ex._TERM_EMPTY)) + + @staticmethod + def report_testset(testset): + Reporter.message() + Reporter.message("Testset: %s" % testset, ex._TERM_BLUE) + + @staticmethod + def report_pass(test, time): + Reporter.message(" PASS: %s (%ss)" % (test, time), ex._TERM_GREEN) + + @staticmethod + def report_fail(test, time): + Reporter.message(" FAIL: %s (%ss)" % (test, time), ex._TERM_RED) + + @staticmethod + def report_timeout(test): + Reporter.message(" TIMEOUT: %s" % test, ex._TERM_RED) + + @staticmethod + def report_skip(test, reason): + skip_message = " SKIP: %s" % test + + if reason: + skip_message += " (Reason: %s)" % reason + + Reporter.message(skip_message, ex._TERM_YELLOW) + + @staticmethod + def report_configuration(options): + Reporter.message() + Reporter.message("Test configuration:") + Reporter.message(" target: %s" % options.target) + + if options.target == "host": + Reporter.message(" bin path: %s" % options.bin_path) + Reporter.message(" coverage: %s" % options.coverage) + Reporter.message(" valgrind: %s" % options.valgrind) + elif options.target == "rpi2": + Reporter.message(" address: %s" % options.address) + Reporter.message(" username: %s" % options.username) + Reporter.message(" remote path: %s" % options.remote_path) + elif options.target in ["stm32f4dis", "artik053"]: + Reporter.message(" port: %s" % options.port) + Reporter.message(" baud: %d" % options.baud) + + Reporter.message(" quiet: %s" % options.quiet) + Reporter.message(" timeout: %s" % options.timeout) + Reporter.message(" skip-modules: %s" % repr(options.skip_modules)) + + @staticmethod + def report_final(results): + Reporter.message() + Reporter.message("Finished with all tests:", ex._TERM_BLUE) + Reporter.message(" PASS: %d" % results["pass"], ex._TERM_GREEN) + Reporter.message(" FAIL: %d" % results["fail"], ex._TERM_RED) + Reporter.message(" TIMEOUT: %d" % results["timeout"], ex._TERM_RED) + Reporter.message(" SKIP: %d" % results["skip"], ex._TERM_YELLOW) diff --git a/tools/testdriver/target/__init__.py b/tools/testdriver/target/__init__.py new file mode 100644 index 0000000000..18b8b9eb74 --- /dev/null +++ b/tools/testdriver/target/__init__.py @@ -0,0 +1,59 @@ +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import host +import stm32f4dis + +from common_py import path +from common_py.system.filesystem import FileSystem as fs +from common_py.system.executor import Executor as ex +from common_py.system.platform import Platform + + +DEVICES = { + "host": host.Device, + "stm32f4dis": stm32f4dis.Device +} + + +def create_build(options): + if options.target == "host": + ex.run_cmd(fs.join(path.TOOLS_ROOT, "build.py"), ["--no-check-test"]) + + # Append the build path to the options. + target = "%s-%s" % (Platform().arch(), Platform().os()) + options.bin_path = fs.join(path.BUILD_ROOT, target, "debug/bin/iotjs") + + elif options.target == "stm32f4dis": + build_options = [ + "--test=nuttx", + "--buildtype=release", + "--buildoptions=--jerry-memstat", + "--enable-testsuite", + "--flash" + ] + + ex.run_cmd(fs.join(path.TOOLS_ROOT, "precommit.py"), build_options) + + else: + ex.fail("Unimplemented case for building iotjs to the target.") + + +def create_device(options): + if options.build: + create_build(options) + + device_class = DEVICES[options.target] + + return device_class(options) diff --git a/tools/testdriver/target/connection/__init__.py b/tools/testdriver/target/connection/__init__.py new file mode 100644 index 0000000000..ef65bee5bb --- /dev/null +++ b/tools/testdriver/target/connection/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/tools/testdriver/target/connection/serialcom.py b/tools/testdriver/target/connection/serialcom.py new file mode 100644 index 0000000000..08ea3ba085 --- /dev/null +++ b/tools/testdriver/target/connection/serialcom.py @@ -0,0 +1,123 @@ +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import serial +import time +import xmodem + +from common_py.system.executor import TimeoutException + + +class Connection(object): + """ + The serial communication wrapper. + """ + def __init__(self, port, baud, timeout, prompt): + self.port = port + self.baud = baud + self.timeout = timeout + + # Defines the end of the stdout. + self.prompt = prompt + + def get_prompt(self): + """ + Get the prompt. + """ + return self.prompt + + def open(self): + """ + Open the serial port. + """ + self.serial = serial.Serial(self.port, self.baud, timeout=self.timeout) + self.xmodem = xmodem.XMODEM1k(self.getc, self.putc) + + def close(self): + """ + Close the serial port. + """ + self.serial.close() + + def getc(self, size, timeout=1): + """ + Recevice data from the serial port. + """ + time.sleep(2) + + return self.serial.read(size) or None + + def putc(self, data, timeout=1): + """ + Send data to the serial port. + """ + return self.serial.write(data) + + def readline(self): + """ + Read line from the serial port. + """ + return self.serial.readline() + + def exec_command(self, cmd): + """ + Send command over the serial port. + """ + self.serial.write(cmd + "\n") + + receive = self.serial.read_until(self.prompt) + + # Throw exception when timeout happens. + if self.prompt not in receive: + raise TimeoutException + + # Note: since the received data format is + # + # [command][CRLF][stdout][CRFL][prompt], + # + # we should process the output for the stdout. + return "\n".join(receive.split("\r\n")[1:-1]) + + def read_until(self, *args): + """ + Read data until it contains args. + """ + line = bytearray() + while True: + c = self.serial.read(1) + if c: + line += c + for stdout in args: + if line[-len(stdout):] == stdout: + return stdout, bytes(line) + else: + # raise utils.TimeoutException + raise Exception("use TimeoutException") + + return False, False + + def send_file(self, lpath, rpath): + """ + Send file over the serial port. + Note: `lrzsz` package should be installed on the device. + """ + self.serial.write("rm " + rpath + "\n") + self.serial.write("rx " + rpath + "\n") + + # Receive all the data from the device except the last + # \x15 (NAK) byte that is needed by the xmodem protocol. + self.serial.read(self.serial.in_waiting - 1) + + with open(lpath, "rb") as file: + self.xmodem.send(file) diff --git a/tools/testdriver/target/connection/sshcom.py b/tools/testdriver/target/connection/sshcom.py new file mode 100644 index 0000000000..780a68c39c --- /dev/null +++ b/tools/testdriver/target/connection/sshcom.py @@ -0,0 +1,67 @@ +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paramiko +import socket + +from common_py.system.executor import TimeoutException + + +class Connection(object): + """ + The serial communication wrapper. + """ + def __init__(self, username, address, timeout): + self.username = username + self.address = address + self.timeout = timeout + + # Note: add your SSH key to the known host file + # to avoid getting password. + self.ssh = paramiko.client.SSHClient() + self.ssh.load_system_host_keys() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + def open(self): + """ + Open the serial port. + """ + self.ssh.connect(self.address, username=self.username) + + def close(self): + """ + Close the serial port. + """ + self.ssh.close() + + def exec_command(self, cmd): + """ + Send command over the serial port. + """ + try: + _, stdout, _ = self.ssh.exec_command(cmd, timeout=self.timeout) + + return stdout.readline() + + except socket.timeout: + raise TimeoutException + + def send_file(self, lpath, rpath): + """ + Send file over the Secure File Transfer Protocol. + """ + sftp = self.ssh.open_sftp() + + sftp.put(lpath, rpath) + sftp.close() diff --git a/tools/testdriver/target/host.py b/tools/testdriver/target/host.py new file mode 100644 index 0000000000..c05d75318c --- /dev/null +++ b/tools/testdriver/target/host.py @@ -0,0 +1,140 @@ + # Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import signal +import subprocess + +from common_py import path +from common_py.system.filesystem import FileSystem as fs +from common_py.system.executor import Executor as ex +from common_py.system.executor import TimeoutException + + +# Defines the folder that will contain the coverage info. +# The path must be consistent with the measure_coverage.sh script. +JS_COVERAGE_FOLDER = fs.join(path.PROJECT_ROOT, '.coverage_output') + +# This code should be applied to each testfile. +JS_COVERAGE_CODE = ( +""" +process.on('exit', function() {{ + if (typeof __coverage__ == 'undefined') + return; + + if (typeof fs == 'undefined') + var fs = require('fs'); + + if (!fs.existsSync('{folder}')) + fs.mkdirSync('{folder}'); + + var filename = '{folder}/{file}'; + fs.writeFileSync(filename, Buffer(JSON.stringify(__coverage__))); +}}) +""" +) + + +# Append coverage source to the appropriate test. +def append_coverage_code(testfile, coverage): + if not coverage: + return + + with open(testfile, 'r') as file_p: + content = file_p.read() + + with open(testfile, 'w') as file_p: + file_p.write(JS_COVERAGE_CODE.format( + folder=JS_COVERAGE_FOLDER, file=fs.basename(testfile))) + file_p.write(content) + + +# Remove coverage source from the appropriate test. +def remove_coverage_code(testfile, coverage): + if not coverage: + return + + with open(testfile, 'r') as file_p: + content = file_p.read() + index = content.find('/* Copyright') + + with open(testfile, 'w') as file_p: + file_p.write(content[index:]) + + +# Use alarm handler to handle timeout. +def alarm_handler(signum, frame): + raise TimeoutException + + +class Device(object): + """ + Device of the Host target. + """ + def __init__(self, options): + if not options.bin_path: + ex.fail("Path to the iotjs binary is not defined. " + + "Use --bin-path to define it.") + + self.iotjs = fs.abspath(options.bin_path) + self.timeout = options.timeout + self.coverage = options.coverage + self.valgrind = options.valgrind + + signal.signal(signal.SIGALRM, alarm_handler) + + def run_test(self, testset, test): + testfile = fs.join(path.TEST_ROOT, testset, test["name"]) + timeout = test.get("timeout", self.timeout) + command = [self.iotjs, testfile] + + if self.valgrind: + valgrind_options = [ + "--leak-check=full", + "--error-exitcode=5", + "--undef-value-errors=no" + ] + + command = ["valgrind"] + valgrind_options + command + + append_coverage_code(testfile, self.coverage) + + signal.alarm(timeout) + + try: + start_time = time.time() + process = subprocess.Popen(args=command, + cwd=path.TEST_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + stdout = process.communicate()[0] + retval = process.returncode + end_time = time.time() + + signal.alarm(0) + + except TimeoutException: + process.kill() + return { "exitcode": -1 } + + finally: + remove_coverage_code(testfile, self.coverage) + + return { + "exitcode": retval, + "stdout" : stdout, + "runtime": round(end_time - start_time, 2), + "mempeak" :'n/a' + } diff --git a/tools/testdriver/target/stm32f4dis.py b/tools/testdriver/target/stm32f4dis.py new file mode 100644 index 0000000000..00475a509d --- /dev/null +++ b/tools/testdriver/target/stm32f4dis.py @@ -0,0 +1,89 @@ +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import time + +from connection import serialcom + +from common_py import path +from common_py.system.filesystem import FileSystem as fs +from common_py.system.executor import Executor as ex + + +class Device(object): + """ + Device of the STM32F4-Discovery target. + """ + def __init__(self, options): + if not options.port: + ex.fail("Serial port is not defined. Use --port to define it.") + + self.serial = serialcom.Connection(options.port, options.baud, + options.timeout, prompt="nsh> ") + + def reset(self): + ex.run_cmd(fs.join(path.STLINK_ROOT, + "build/Release/st-flash"), ["reset"], quiet=True) + + # Wait a moment to boot the device. + time.sleep(5) + + def login(self): + try: + self.serial.open() + + # Press enters to start the serial communication and + # go to the test folder because some tests require resources. + self.serial.exec_command("\n\n") + self.serial.exec_command("cd /test") + + except Exception as e: + ex.fail(str(e)) + + def logout(self): + self.serial.close() + + def run_test(self, testset, test): + self.reset() + self.login() + + command = "iotjs --memstat /test/%s/%s" % (testset, test["name"]) + + start_time = time.time() + stdout = self.serial.exec_command(command.encode("utf8")) + retval = self.serial.exec_command("echo $?") + end_time = time.time() + + self.logout() + + # Process the stdout of the execution. + if stdout.rfind("Heap stat") != -1: + stdout, heap = stdout.rsplit("Heap stats", 1) + + match = re.search(r"Peak allocated = (\d+) bytes", str(heap)) + + if match: + mempeak = match.group(1) + else: + mempeak = "n/a" + else: + mempeak = "n/a" + + return { + "exitcode": int(retval), + "stdout" : stdout, + "runtime": round(end_time - start_time, 2), + "mempeak" : mempeak + } diff --git a/tools/testdriver/testrunner.py b/tools/testdriver/testrunner.py new file mode 100755 index 0000000000..6bdebbc6c5 --- /dev/null +++ b/tools/testdriver/testrunner.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import json +import target +from reporter import Reporter + +from collections import OrderedDict +from common_py import path +from common_py.system.filesystem import FileSystem as fs +from common_py.system.executor import Executor as ex +from common_py.system.platform import Platform + + +class TestRunner(object): + def __init__(self, options): + #Create target specific build if neccessary. + self.options = options + self.quiet = options.quiet + self.skip_modules = options.skip_modules + self.device = target.create_device(options) + self.results = {} + + # Select the appropriate os. + if options.target == 'host': + self.os = Platform().os() + elif options.target == 'stm32f4dis': + self.os = 'nuttx' + + # Process the iotjs build information. + iotjs_build_info = { "name": "iotjs_build_info.js" } + iotjs_output = self.device.run_test(".", iotjs_build_info) + build_info = json.loads(iotjs_output["stdout"]) + + self.builtins = build_info["builtins"] + self.stability = build_info["stability"] + + def run(self): + Reporter.report_configuration(self.options) + + self.results = { + "pass": 0, + "fail": 0, + "skip": 0, + "timeout": 0 + } + + with open(fs.join(path.TEST_ROOT, "testsets.json")) as testsets_file: + testsets = json.load(testsets_file, object_pairs_hook=OrderedDict) + + for testset, tests in testsets.items(): + self.run_testset(testset, tests) + + Reporter.report_final(self.results) + + def run_testset(self, testset, tests): + Reporter.report_testset(testset) + + for test in tests: + if self.skip_test(test): + Reporter.report_skip(test["name"], test.get("reason")) + self.results["skip"] += 1 + continue + + # Run the test on the device. + result = self.device.run_test(testset, test) + expected_failure = test.get("expected-failure", False) + + # Timeout happened. + if result["exitcode"] == -1: + Reporter.report_timeout(test["name"]) + self.results["timeout"] += 1 + continue + + # Show the output. + if not self.quiet: + print(result["stdout"], end="") + + if (bool(result["exitcode"]) == expected_failure): + Reporter.report_pass(test["name"], result["runtime"]) + self.results["pass"] += 1 + else: + Reporter.report_fail(test["name"], result["runtime"]) + self.results["fail"] += 1 + + def skip_test(self, test): + skip_list = test.get("skip", []) + + # Skip by the `skip` attribute in testsets.json file. + for i in ["all", self.os, self.stability]: + if i in skip_list: + return True + + name_parts = test["name"][0:-3].split('_') + + # Test filename does not start with 'test_' so we'll just + # assume we support it. + if name_parts[0] != 'test': + return False + + tested_module = name_parts[1] + + # Skip the test if it requires a module that is defined by + # the `--skip-modules` flag. + if tested_module in self.skip_modules: + test["reason"] = "the required module is skipped by testrunner" + return True + + # Skip the test if it requires a module that is not + # compiled into the binary. + if tested_module not in self.builtins: + test["reason"] = "unsupported module by iotjs build" + return True + + return False diff --git a/tools/testrunner.py b/tools/testrunner.py deleted file mode 100755 index c7f6a0ba24..0000000000 --- a/tools/testrunner.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017-present Samsung Electronics Co., Ltd. and other contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import argparse -import json -import signal -import subprocess -import time - -from collections import OrderedDict -from common_py import path -from common_py.system.filesystem import FileSystem as fs -from common_py.system.executor import Executor as ex -from common_py.system.platform import Platform - - -# Defines the folder that will contain the coverage info. -# The path must be consistent with the measure_coverage.sh script. -JS_COVERAGE_FOLDER = fs.join(path.PROJECT_ROOT, '.coverage_output') - -# This code should be applied to each testfile. -JS_COVERAGE_CODE = ( -""" -process.on('exit', function() {{ - if (typeof __coverage__ == 'undefined') - return; - - if (typeof fs == 'undefined') - var fs = require('fs'); - - if (!fs.existsSync('{folder}')) - fs.mkdirSync('{folder}'); - - var filename = '{folder}/{file}'; - fs.writeFileSync(filename, Buffer(JSON.stringify(__coverage__))); -}}) -""" -) - - -# Append coverage source to the appropriate test. -def append_coverage_code(testfile, coverage): - if not coverage: - return - - with open(testfile, 'r') as file_p: - content = file_p.read() - - with open(testfile, 'w') as file_p: - file_p.write(JS_COVERAGE_CODE.format( - folder=JS_COVERAGE_FOLDER, file=fs.basename(testfile))) - file_p.write(content) - - -# Remove coverage source from the appropriate test. -def remove_coverage_code(testfile, coverage): - if not coverage: - return - - with open(testfile, 'r') as file_p: - content = file_p.read() - index = content.find('/* Copyright') - - with open(testfile, 'w') as file_p: - file_p.write(content[index:]) - - -class Reporter(object): - @staticmethod - def message(msg="", color=ex._TERM_EMPTY): - print("%s%s%s" % (color, msg, ex._TERM_EMPTY)) - - @staticmethod - def report_testset(testset): - Reporter.message() - Reporter.message("Testset: %s" % testset, ex._TERM_BLUE) - - @staticmethod - def report_pass(test, time): - Reporter.message(" PASS: %s (%ss)" % (test, time), ex._TERM_GREEN) - - @staticmethod - def report_fail(test, time): - Reporter.message(" FAIL: %s (%ss)" % (test, time), ex._TERM_RED) - - @staticmethod - def report_timeout(test): - Reporter.message(" TIMEOUT: %s" % test, ex._TERM_RED) - - @staticmethod - def report_skip(test, reason): - skip_message = " SKIP: %s" % test - - if reason: - skip_message += " (Reason: %s)" % reason - - Reporter.message(skip_message, ex._TERM_YELLOW) - - @staticmethod - def report_configuration(testrunner): - Reporter.message() - Reporter.message("Test configuration:") - Reporter.message(" iotjs: %s" % testrunner.iotjs) - Reporter.message(" quiet: %s" % testrunner.quiet) - Reporter.message(" timeout: %d sec" % testrunner.timeout) - Reporter.message(" valgrind: %s" % testrunner.valgrind) - Reporter.message(" skip-modules: %s" % testrunner.skip_modules) - - @staticmethod - def report_final(results): - Reporter.message() - Reporter.message("Finished with all tests:", ex._TERM_BLUE) - Reporter.message(" PASS: %d" % results["pass"], ex._TERM_GREEN) - Reporter.message(" FAIL: %d" % results["fail"], ex._TERM_RED) - Reporter.message(" TIMEOUT: %d" % results["timeout"], ex._TERM_RED) - Reporter.message(" SKIP: %d" % results["skip"], ex._TERM_YELLOW) - - -class TimeoutException(Exception): - pass - - -def alarm_handler(signum, frame): - raise TimeoutException - - -class TestRunner(object): - def __init__(self, options): - self.iotjs = fs.abspath(options.iotjs) - self.quiet = options.quiet - self.timeout = options.timeout - self.valgrind = options.valgrind - self.coverage = options.coverage - self.skip_modules = [] - self.results = {} - - if options.skip_modules: - self.skip_modules = options.skip_modules.split(",") - - # Process the iotjs build information. - iotjs_output = ex.run_cmd_output(self.iotjs, [path.BUILD_INFO_PATH]) - build_info = json.loads(iotjs_output) - - self.builtins = build_info["builtins"] - self.stability = build_info["stability"] - - # Define own alarm handler to handle timeout. - signal.signal(signal.SIGALRM, alarm_handler) - - def run(self): - Reporter.report_configuration(self) - - self.results = { - "pass": 0, - "fail": 0, - "skip": 0, - "timeout": 0 - } - - with open(fs.join(path.TEST_ROOT, "testsets.json")) as testsets_file: - testsets = json.load(testsets_file, object_pairs_hook=OrderedDict) - - for testset, tests in testsets.items(): - self.run_testset(testset, tests) - - Reporter.report_final(self.results) - - def run_testset(self, testset, tests): - Reporter.report_testset(testset) - - for test in tests: - testfile = fs.join(path.TEST_ROOT, testset, test["name"]) - timeout = test.get("timeout", self.timeout) - - if self.skip_test(test): - Reporter.report_skip(test["name"], test.get("reason")) - self.results["skip"] += 1 - continue - - append_coverage_code(testfile, self.coverage) - - exitcode, output, runtime = self.run_test(testfile, timeout) - expected_failure = test.get("expected-failure", False) - - remove_coverage_code(testfile, self.coverage) - - # Timeout happened. - if exitcode == -1: - Reporter.report_timeout(test["name"]) - self.results["timeout"] += 1 - continue - - # Show the output. - if not self.quiet: - print(output, end="") - - if (bool(exitcode) == expected_failure): - Reporter.report_pass(test["name"], runtime) - self.results["pass"] += 1 - else: - Reporter.report_fail(test["name"], runtime) - self.results["fail"] += 1 - - - def run_test(self, testfile, timeout): - command = [self.iotjs, testfile] - - if self.valgrind: - valgrind_options = [ - "--leak-check=full", - "--error-exitcode=5", - "--undef-value-errors=no" - ] - - command = ["valgrind"] + valgrind_options + command - - signal.alarm(timeout) - - try: - start = time.time() - process = subprocess.Popen(args=command, - cwd=path.TEST_ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - stdout = process.communicate()[0] - exitcode = process.returncode - runtime = round((time.time() - start), 2) - - signal.alarm(0) - - except TimeoutException: - process.kill() - return -1, None, None - - return exitcode, stdout, runtime - - def skip_test(self, test): - skip_list = test.get("skip", []) - - # Skip by the `skip` attribute in testsets.json file. - for i in ["all", Platform().os(), self.stability]: - if i in skip_list: - return True - - name_parts = test["name"][0:-3].split('_') - - # Test filename does not start with 'test_' so we'll just - # assume we support it. - if name_parts[0] != 'test': - return False - - tested_module = name_parts[1] - - # Skip the test if it requires a module that is defined by - # the `--skip-modules` flag. - if tested_module in self.skip_modules: - test["reason"] = "the required module is skipped by testrunner" - return True - - # Skip the test if it requires a module that is not - # compiled into the binary. - if tested_module not in self.builtins: - test["reason"] = "unsupported module by iotjs build" - return True - - return False - - -def get_args(): - parser = argparse.ArgumentParser() - - parser.add_argument("iotjs", action="store", - help="path to the iotjs binary file") - parser.add_argument("--quiet", action="store_true", default=False, - help="show or hide the output of the tests") - parser.add_argument("--skip-modules", action="store", metavar='list', - help="module list to skip test of specific modules") - parser.add_argument("--timeout", action="store", default=300, type=int, - help="default timeout for the tests in seconds") - parser.add_argument("--valgrind", action="store_true", default=False, - help="check tests with Valgrind") - parser.add_argument("--coverage", action="store_true", default=False, - help="measure JavaScript coverage") - - return parser.parse_args() - - -def main(): - options = get_args() - - testrunner = TestRunner(options) - testrunner.run() - - -if __name__ == "__main__": - main()