From 89f866a1ffb858a961142182e5a697e92259f835 Mon Sep 17 00:00:00 2001
From: Andrew Cooper <andrew.cooper3@citrix.com>
Date: Mon, 22 Feb 2016 12:43:54 +0000
Subject: [PATCH] xtf-runner: Basic test runner
MIME-Version: 1.0
Content-Type: text/plain; charset=utf8
Content-Transfer-Encoding: 8bit

Currently just accepts a list of full test-$ENV-$NAME, runs them in order, and
provides a results summary at the end.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
 Makefile   |   9 ++++
 xtf-runner | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 143 insertions(+)
 create mode 100755 xtf-runner

diff --git a/Makefile b/Makefile
index fd8c3e0..1a8099d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,7 @@
 MAKEFLAGS += -r
+ROOT := $(abspath $(CURDIR))
+DESTDIR ?= $(ROOT)/dist
+PREFIX ?= $(ROOT)
 
 .PHONY: all
 all:
@@ -9,6 +12,8 @@ all:
 
 .PHONY: install
 install:
+	@mkdir -p $(DESTDIR)
+	install -m775 xtf-runner $(DESTDIR)
 	@for D in $(wildcard tests/*); do \
 		[ ! -e $$D/Makefile ] && continue; \
 		$(MAKE) -C $$D install; \
@@ -33,3 +38,7 @@ distclean: clean
 .PHONY: doxygen
 doxygen: Doxyfile
 	doxygen Doxyfile > /dev/null
+
+.PHONY: pylint
+pylint:
+	-pylint --rcfile=.pylintrc xtf-runner
diff --git a/xtf-runner b/xtf-runner
new file mode 100755
index 0000000..080dbed
--- /dev/null
+++ b/xtf-runner
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+xtf-runner - A utility for enumerating and running XTF tests.
+
+Currently assumes the presence and availability of the `xl` toolstack.
+"""
+
+import sys, os, os.path as path
+
+from optparse import OptionParser
+from subprocess import Popen, PIPE, call as subproc_call
+
+class RunnerError(Exception):
+    """ Errors relating to xtf-runner itself """
+
+def run_test(test):
+    """ Run a specific test """
+
+    _, _, name = test.split('-', 2)
+
+    cfg = path.join("tests", name, test + ".cfg")
+
+    cmd = ['xl', 'create', '-p', cfg]
+
+    print "Executing '%s'" % (" ".join(cmd), )
+    rc = subproc_call(cmd)
+    if rc:
+        raise RunnerError("Failed to create VM")
+
+    cmd = ['xl', 'console', test]
+    print "Executing '%s'" % (" ".join(cmd), )
+    console = Popen(cmd, stdout = PIPE)
+
+    cmd = ['xl', 'unpause', test]
+    print "Executing '%s'" % (" ".join(cmd), )
+    rc = subproc_call(cmd)
+    if rc:
+        raise RunnerError("Failed to unpause VM")
+
+    stdout, _ = console.communicate()
+
+    if console.returncode:
+        raise RunnerError("Failed to obtain VM console")
+
+    lines = stdout.splitlines()
+
+    if lines:
+        print "\n".join(lines)
+
+    else:
+        return "ERROR"
+
+    test_result = lines[-1]
+    if not "Test result:" in test_result:
+        return "ERROR"
+
+    for res in ("SUCCESS", "SKIP", "FAILURE"):
+
+        if res in test_result:
+            return res
+
+    return "ERROR"
+
+
+def run_tests(tests):
+    """ Run tests """
+
+    if not len(tests):
+        raise RunnerError("No tests to run")
+
+    rc = 0
+    results = []
+
+    for test in tests:
+
+        parts = test.split('-', 2)
+        if len(parts) != 3:
+            raise RunnerError("Unexpected test '%s'" % (test, ))
+
+        res = run_test(test)
+        if res != "SUCCESS":
+            rc = 1
+
+        results.append(res)
+
+    print "\nCombined test results:"
+
+    for test, res in zip(tests, results):
+        print "%-40s %s" % (test, res)
+
+    return rc
+
+
+def main():
+    """ Main entrypoint """
+
+    # Change stdout to be line-buffered.
+    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)
+
+    # Normalise $CWD to the directory this script is in
+    os.chdir(path.dirname(path.abspath(sys.argv[0])))
+
+    # Avoid wrapping the epilog text
+    OptionParser.format_epilog = lambda self, formatter: self.epilog
+
+    parser = OptionParser(
+        usage = "%prog <TEST>*",
+        description = "Xen Test Framework enumeration and running tool",
+        epilog = ("\n"
+                  "Examples:\n"
+                  "  Running tests:\n"
+                  "    ./xtf-runner test-hvm32-example test-pv64-example\n"
+                  "      <console ouput>\n"
+                  "    Combined test results:\n"
+                  "    test-hvm32-example                       SUCCESS\n"
+                  "    test-pv64-example                        SUCCESS\n"
+                  ),
+    )
+
+    _, args = parser.parse_args()
+
+    return run_tests(args)
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except RunnerError, e:
+        print >>sys.stderr, "Error:", e
+        sys.exit(1)
+    except KeyboardInterrupt:
+        sys.exit(1)
-- 
2.39.5