xtf-runner: Basic test runner

Currently just accepts a list of full test-$ENV-$NAME, runs them in order, and
provides a results summary at the end.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>

diff --git a/Makefile b/Makefile
index fd8c3e0af26c30cb948c381c46ed91d746da0d0a..1a8099dca3da848cace06c2499b849a69da0f4c6 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,7 @@
 MAKEFLAGS += -r
+ROOT := $(abspath $(CURDIR))
+DESTDIR ?= $(ROOT)/dist
+PREFIX ?= $(ROOT)
 
 .PHONY: all
 all:
@@ -9,6 +12,8 @@ all:
 
 .PHONY: install
 install:
+       @mkdir -p $(DESTDIR)
+       install -m775 xtf-runner $(DESTDIR)
        @for D in $(wildcard tests/*); do \
                [ ! -e $$D/Makefile ] && continue; \
                $(MAKE) -C $$D install; \
@@ -33,3 +38,7 @@ distclean: clean
 .PHONY: doxygen
 doxygen: Doxyfile
        doxygen Doxyfile > /dev/null
+
+.PHONY: pylint
+pylint:
+       -pylint --rcfile=.pylintrc xtf-runner

diff --git a/xtf-runner b/xtf-runner
new file mode 100755 (executable)
index 0000000..080dbed
--- /dev/null
+++ b/xtf-runner
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+xtf-runner - A utility for enumerating and running XTF tests.
+
+Currently assumes the presence and availability of the `xl` toolstack.
+"""
+
+import sys, os, os.path as path
+
+from optparse import OptionParser
+from subprocess import Popen, PIPE, call as subproc_call
+
+class RunnerError(Exception):
+    """ Errors relating to xtf-runner itself """
+
+def run_test(test):
+    """ Run a specific test """
+
+    _, _, name = test.split('-', 2)
+
+    cfg = path.join("tests", name, test + ".cfg")
+
+    cmd = ['xl', 'create', '-p', cfg]
+
+    print "Executing '%s'" % (" ".join(cmd), )
+    rc = subproc_call(cmd)
+    if rc:
+        raise RunnerError("Failed to create VM")
+
+    cmd = ['xl', 'console', test]
+    print "Executing '%s'" % (" ".join(cmd), )
+    console = Popen(cmd, stdout = PIPE)
+
+    cmd = ['xl', 'unpause', test]
+    print "Executing '%s'" % (" ".join(cmd), )
+    rc = subproc_call(cmd)
+    if rc:
+        raise RunnerError("Failed to unpause VM")
+
+    stdout, _ = console.communicate()
+
+    if console.returncode:
+        raise RunnerError("Failed to obtain VM console")
+
+    lines = stdout.splitlines()
+
+    if lines:
+        print "\n".join(lines)
+
+    else:
+        return "ERROR"
+
+    test_result = lines[-1]
+    if not "Test result:" in test_result:
+        return "ERROR"
+
+    for res in ("SUCCESS", "SKIP", "FAILURE"):
+
+        if res in test_result:
+            return res
+
+    return "ERROR"
+
+
+def run_tests(tests):
+    """ Run tests """
+
+    if not len(tests):
+        raise RunnerError("No tests to run")
+
+    rc = 0
+    results = []
+
+    for test in tests:
+
+        parts = test.split('-', 2)
+        if len(parts) != 3:
+            raise RunnerError("Unexpected test '%s'" % (test, ))
+
+        res = run_test(test)
+        if res != "SUCCESS":
+            rc = 1
+
+        results.append(res)
+
+    print "\nCombined test results:"
+
+    for test, res in zip(tests, results):
+        print "%-40s %s" % (test, res)
+
+    return rc
+
+
+def main():
+    """ Main entrypoint """
+
+    # Change stdout to be line-buffered.
+    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)
+
+    # Normalise $CWD to the directory this script is in
+    os.chdir(path.dirname(path.abspath(sys.argv[0])))
+
+    # Avoid wrapping the epilog text
+    OptionParser.format_epilog = lambda self, formatter: self.epilog
+
+    parser = OptionParser(
+        usage = "%prog <TEST>*",
+        description = "Xen Test Framework enumeration and running tool",
+        epilog = ("\n"
+                  "Examples:\n"
+                  "  Running tests:\n"
+                  "    ./xtf-runner test-hvm32-example test-pv64-example\n"
+                  "      <console ouput>\n"
+                  "    Combined test results:\n"
+                  "    test-hvm32-example                       SUCCESS\n"
+                  "    test-pv64-example                        SUCCESS\n"
+                  ),
+    )
+
+    _, args = parser.parse_args()
+
+    return run_tests(args)
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except RunnerError, e:
+        print >>sys.stderr, "Error:", e
+        sys.exit(1)
+    except KeyboardInterrupt:
+        sys.exit(1)