runner: Use virt caps to identify whether a test can run

Right now, --host filters a test selection using xen_caps to exclude full
environments.  Furthermore, without passing --host, xtf-runner will try to
start a test even when we know it can't be constructed, leaving an xl error to
the user and reporting CRASH.

The virt capabilities are more fine grained, and include other information
such as the availability of hap and shadow paging.

Split run_test() out of run_tests() and use the test's required caps to short
circuit to SKIP.

Rework the existing --host filtering in terms of virt caps, which simplifies
the invocation of tests_from_selection().

This causes ~hap and ~shadow variation tests to SKIP rather than CRASH when
run on hardware without HAP (or with HAP disabled in firmware), or with Shadow
compiled out.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>

diff --git a/xtf-runner b/xtf-runner
index 41431312324ada92d1821945a455430e08db524c..4a6e5904de53b795c11aa482348d019c682a287c 100755 (executable)
--- a/xtf-runner
+++ b/xtf-runner
@@ -296,7 +296,7 @@ def get_virt_caps():
     return _virt_caps
 
 
-def tests_from_selection(cats, envs, tests):
+def tests_from_selection(cats, envs, tests, caps):
     """Given a selection of possible categories, environment and tests, return
     all tests within the provided parameters.
 
@@ -360,6 +360,9 @@ def tests_from_selection(cats, envs, tests):
         else:
             res = tests
 
+    if caps:
+        res = [ x for x in res if x.req_caps.issubset(caps) ]
+
     # Sort the results.  Variation third, Env second and Name fist.
     res = sorted(res, key = lambda test: test.variation or "")
     res = sorted(res, key = lambda test: test.env)
@@ -418,35 +421,12 @@ def interpret_selection(opts):
 
         tests.extend(instances)
 
-    selection = tests_from_selection(cats, envs, set(tests))
-
-    # If the caller passed --host, filter out the unsupported environments
-    if selection and opts.host:
-
-        host_envs = []
-
-        for line in check_output(['xl', 'info']).splitlines():
-            if not line.startswith("xen_caps"):
-                continue
-
-            caps = line.split()[2:]
-
-            if "xen-3.0-x86_64" in caps:
-                host_envs.append("pv64")
-            if "xen-3.0-x86_32p" in caps:
-                host_envs.append("pv32pae")
-            for cap in caps:
-                if cap.startswith("hvm"):
-                    host_envs.extend(hvm_environments)
-                    break
-
-            break
+    # Third, if --host is passed, also filter by capabilities
+    caps = None
+    if opts.host:
+        caps = get_virt_caps()
 
-        selection = tests_from_selection(cats = set(),
-                                         envs = set(host_envs),
-                                         tests = selection)
-
-    return selection
+    return tests_from_selection(cats, envs, set(tests), caps)
 
 
 def list_tests(opts):
@@ -571,6 +551,21 @@ def run_test_logfile(opts, test):
     return interpret_result(line)
 
 
+def run_test(opts, test):
+    """ Run a single test instance """
+
+    # If caps say the test can't run, short circuit to SKIP
+    if not test.req_caps.issubset(get_virt_caps()):
+        return "SKIP"
+
+    fn = {
+        "console": run_test_console,
+        "logfile": run_test_logfile,
+    }[opts.results_mode]
+
+    return fn(opts, test)
+
+
 def run_tests(opts):
     """ Run tests """
 
@@ -578,13 +573,6 @@ def run_tests(opts):
     if not tests:
         raise RunnerError("No tests to run")
 
-    run_test = { "console": run_test_console,
-                 "logfile": run_test_logfile,
-    }.get(opts.results_mode, None)
-
-    if run_test is None:
-        raise RunnerError("Unknown mode '{0}'".format(opts.mode))
-
     rc = all_results.index('SUCCESS')
     results = []