Introduce test variations

In certain cases it will be useful to run the same test binary with different
configurations, providing logically separate results.

Introduce the concept of a test variation, which generates a different set of
configuraiton files referring to the same binary.  xtf-runner is updated to
understand variations.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>

diff --git a/build/gen.mk b/build/gen.mk
index 72f1f696811b94e0941b1f133fbece5cab69aebe..c9ff6962e17b1b64e2af3eb4d3ea53891a4c47f6 100644 (file)
--- a/build/gen.mk
+++ b/build/gen.mk
@@ -21,12 +21,18 @@ ifneq ($(filter-out $(ALL_CATEGORIES),$(CATEGORY)),)
 $(error Unrecognised category '$(filter-out $(ALL_CATEGORIES),$(CATEGORY))')
 endif
 
+ifneq ($(VARY-CFG),)
+TEST-CFGS := $(foreach env,$(TEST-ENVS),$(foreach vary,$(VARY-CFG),test-$(env)-$(NAME)~$(vary).cfg))
+else
+TEST-CFGS := $(foreach env,$(TEST-ENVS),test-$(env)-$(NAME).cfg)
+endif
+
 .PHONY: build
-build: $(foreach env,$(TEST-ENVS),test-$(env)-$(NAME) test-$(env)-$(NAME).cfg)
+build: $(foreach env,$(TEST-ENVS),test-$(env)-$(NAME)) $(TEST-CFGS)
 build: info.json
 
 info.json: $(ROOT)/build/mkinfo.py FORCE
-       @$(PYTHON) $< $@.tmp "$(NAME)" "$(CATEGORY)" "$(TEST-ENVS)"
+       @$(PYTHON) $< $@.tmp "$(NAME)" "$(CATEGORY)" "$(TEST-ENVS)" "$(VARY-CFG)"
        @$(call move-if-changed,$@.tmp,$@)
 
 .PHONY: install install-each-env
@@ -50,8 +56,18 @@ endif
 
 cfg-$(1) ?= $(defcfg-$(1))
 
-test-$(1)-$(NAME).cfg: $(ROOT)/build/mkcfg.py $$(cfg-$(1)) $(TEST-EXTRA-CFG) FORCE
-       @$(PYTHON) $$< $$@.tmp "$$(cfg-$(1))" "$(TEST-EXTRA-CFG)"
+cfg-default-deps := $(ROOT)/build/mkcfg.py $$(cfg-$(1)) $(TEST-EXTRA-CFG) FORCE
+
+test-$(1)-$(NAME).cfg: $$(cfg-default-deps)
+       $(PYTHON) $$< $$@.tmp "$$(cfg-$(1))" "$(TEST-EXTRA-CFG)" ""
+       @$(call move-if-changed,$$@.tmp,$$@)
+
+test-$(1)-$(NAME)~%.cfg: $$(cfg-default-deps) %.cfg.in
+       $(PYTHON) $$< $$@.tmp "$$(cfg-$(1))" "$(TEST-EXTRA-CFG)" "$$*.cfg.in"
+       @$(call move-if-changed,$$@.tmp,$$@)
+
+test-$(1)-$(NAME)~%.cfg: $$(cfg-default-deps) $(ROOT)/config/%.cfg.in
+       $(PYTHON) $$< $$@.tmp "$$(cfg-$(1))" "$(TEST-EXTRA-CFG)" "$(ROOT)/config/$$*.cfg.in"
        @$(call move-if-changed,$$@.tmp,$$@)
 
 -include $$(link-$(1):%.lds=%.d)
@@ -62,7 +78,7 @@ install-$(1): test-$(1)-$(NAME)
        @$(INSTALL_DIR) $(DESTDIR)$(xtftestdir)/$(NAME)
        $(INSTALL_PROGRAM) $$< $(DESTDIR)$(xtftestdir)/$(NAME)
 
-install-$(1).cfg: test-$(1)-$(NAME).cfg
+install-$(1).cfg: $(TEST-CFGS)
        @$(INSTALL_DIR) $(DESTDIR)$(xtftestdir)/$(NAME)
        $(INSTALL_DATA) $$< $(DESTDIR)$(xtftestdir)/$(NAME)
 
@@ -74,7 +90,7 @@ $(foreach env,$(TEST-ENVS),$(eval $(call PERENV_build,$(env))))
 .PHONY: clean
 clean:
        find $(ROOT) \( -name "*.o" -o -name "*.d" \) -delete
-       rm -f $(foreach env,$(TEST-ENVS),test-$(env)-$(NAME) test-$(env)-$(NAME).cfg)
+       rm -f $(foreach env,$(TEST-ENVS),test-$(env)-$(NAME) test-$(env)-$(NAME)*.cfg)
 
 .PHONY: %var
 %var:

diff --git a/build/mkcfg.py b/build/mkcfg.py
index a0143c21bc3bb9244b64febd8ff35d39933cabd2..2ee7dfe09ae02b418186368170e1d2da65b0e02c 100644 (file)
--- a/build/mkcfg.py
+++ b/build/mkcfg.py
@@ -8,18 +8,25 @@ substitue variables appropriately.
 
 import sys, os
 
-# Usage: mkcfg.py $OUT $DEFAULT-CFG $EXTRA-CFG
-_, out, defcfg, extracfg = sys.argv
+# Usage: mkcfg.py $OUT $DEFAULT-CFG $EXTRA-CFG $VARY-CFG
+_, out, defcfg, extracfg, varycfg = sys.argv
 
 # Evaluate environment and name from $OUT
 _, env, name = out.split('.')[0].split('-', 2)
 
+# Possibly split apart the variation suffix
+variation = ''
+if '~' in name:
+    parts = name.split('~', 1)
+    name, variation = parts[0], '~' + parts[1]
+
 def expand(text):
     """ Expand certain variables in text """
     return (text
             .replace("@@NAME@@",   name)
             .replace("@@ENV@@",    env)
             .replace("@@XTFDIR@@", os.environ["xtfdir"])
+            .replace("@@VARIATION@@", variation)
         )
 
 config = open(defcfg).read()
@@ -28,6 +35,10 @@ if extracfg:
     config += "\n# Test Extra Configuration:\n"
     config += open(extracfg).read()
 
+if varycfg:
+    config += "\n# Test Variation Configuration:\n"
+    config += open(varycfg).read()
+
 cfg = expand(config)
 
 open(out, "w").write(cfg)

diff --git a/build/mkinfo.py b/build/mkinfo.py
index e4b5fe22a393227f3ea2a8a6c68312b2ed9d3896..02567e518f71009ed1158afdd9e3f704968df39a 100644 (file)
--- a/build/mkinfo.py
+++ b/build/mkinfo.py
@@ -3,13 +3,14 @@
 
 import sys, os, json
 
-# Usage: mkcfg.py $OUT $NAME $CATEGORY $ENVS
-_, out, name, cat, envs = sys.argv
+# Usage: mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS
+_, out, name, cat, envs, variations = sys.argv
 
 template = {
     "name": name,
     "category": cat,
     "environments": envs.split(" "),
+    "variations": variations.split(" "),
     }
 
 open(out, "w").write(

diff --git a/config/default-hvm.cfg.in b/config/default-hvm.cfg.in
index 7d29e0c0eaced207ed44b32f0d42b554008ef0d0..0de2546384ce4cdec92a82b4113de362e4f6c24c 100644 (file)
--- a/config/default-hvm.cfg.in
+++ b/config/default-hvm.cfg.in
@@ -1,4 +1,4 @@
-name="test-@@ENV@@-@@NAME@@"
+name="test-@@ENV@@-@@NAME@@@@VARIATION@@"
 builder="hvm"
 memory=128
 firmware_override="@@XTFDIR@@/tests/@@NAME@@/test-@@ENV@@-@@NAME@@"

diff --git a/config/default-pv.cfg.in b/config/default-pv.cfg.in
index 166e4647282da67788cb973d18e327827194bf4c..daedf4903edbadc7a54c37a47157579ee69c7b17 100644 (file)
--- a/config/default-pv.cfg.in
+++ b/config/default-pv.cfg.in
@@ -1,4 +1,4 @@
-name="test-@@ENV@@-@@NAME@@"
+name="test-@@ENV@@-@@NAME@@@@VARIATION@@"
 loader="generic"
 memory=128
 kernel="@@XTFDIR@@/tests/@@NAME@@/test-@@ENV@@-@@NAME@@"

diff --git a/docs/introduction.dox b/docs/introduction.dox
index 34f472c1f9c6c1d516ea561fe71049113df60ecf..5d2807d70a5165084f75c2bcfcdd2dac9fbee9ed 100644 (file)
--- a/docs/introduction.dox
+++ b/docs/introduction.dox
@@ -4,13 +4,17 @@
 
 @section overview Overview
 
-A test has three important attributes.
+A test has several important attributes, and are often referred to in the form:
 
+    test-$ENVIRONMENT-$NAME[~$VARIATION]
+
+This identifies a single test instance, with the purpose of executing and
+providing a single result (See @ref running).
 
 @subsection attr-name Name
 
-A tests name is unique identifier.  The directory `tests/<name>/` contains all
-files related to a specific test.
+A tests name is a unique identifier.  The directory `tests/$NAME/` contains
+all files related to a specific test.  A test has one or more test instances.
 
 
 @subsection attr-category Category
@@ -46,6 +50,14 @@ Some more specific collections for environments are also available:
 An individual test, compiled for more than one environment, will end up with a
 individual microkernel binary for each specified environment.
 
+@subsection Variations
+
+Variations are optional.  Some tests may only want to run the binary once, but
+some tests want to run the same binary multiple times in different
+configurations, reporting separate results.
+
+The variation suffix (if present) is arbitrary and defined by the test itself.
+
 
 @section build Building
 

diff --git a/xtf-runner b/xtf-runner
index fcae3e3ce4d9d6decabe4c121a74563aa738ad95..98d2d53ed7f893719ac6b05187c90e0ab89f70dc 100755 (executable)
--- a/xtf-runner
+++ b/xtf-runner
@@ -50,11 +50,15 @@ class TestInstance(object):
 
     def __init__(self, arg):
         """ Parse and verify 'arg' as a test instance. """
-        self.env, self.name = parse_test_instance_string(arg)
+        self.env, self.name, self.variation = parse_test_instance_string(arg)
 
         if self.env is None:
             raise RunnerError("No environment for '%s'" % (arg, ))
 
+        if self.variation is None and get_all_test_info()[self.name].variations:
+            raise RunnerError("Test '%s' has variations, but none specified"
+                              % (self.name, ))
+
     def vm_name(self):
         """ Return the VM name as `xl` expects it. """
         return repr(self)
@@ -64,7 +68,10 @@ class TestInstance(object):
         return path.join("tests", self.name, repr(self) + ".cfg")
 
     def __repr__(self):
-        return "test-%s-%s" % (self.env, self.name)
+        if not self.variation:
+            return "test-%s-%s" % (self.env, self.name)
+        else:
+            return "test-%s-%s~%s" % (self.env, self.name, self.variation)
 
     def __hash__(self):
         return hash(repr(self))
@@ -107,7 +114,13 @@ class TestInfo(object):
                 raise ValueError("Unknown environments '%s'" % (env, ))
         self.envs = envs
 
-    def all_instances(self, env_filter = None):
+        variations = test_json["variations"]
+        if not isinstance(variations, list):
+            raise TypeError("Expected list for 'variations', got '%s'"
+                            % (type(variations), ))
+        self.variations = variations
+
+    def all_instances(self, env_filter = None, vary_filter = None):
         """Return a list of TestInstances, for each supported environment.
         Optionally filtered by env_filter.  May return an empty list if
         the filter doesn't match any supported environment.
@@ -118,7 +131,21 @@ class TestInfo(object):
         else:
             envs = self.envs
 
-        return [ TestInstance("test-%s-%s" % (env, self.name)) for env in envs ]
+        if vary_filter:
+            variations = set(vary_filter).intersection(self.variations)
+        else:
+            variations = self.variations
+
+        res = []
+        if variations:
+            for env in envs:
+                for vary in variations:
+                    res.append(TestInstance("test-%s-%s~%s"
+                                            % (env, self.name, vary)))
+        else:
+            res = [ TestInstance("test-%s-%s" % (env, self.name))
+                    for env in envs ]
+        return res
 
     def __repr__(self):
         return "TestInfo(%s)" % (self.name, )
@@ -127,21 +154,27 @@ class TestInfo(object):
 def parse_test_instance_string(arg):
     """Parse a test instance string.
 
-    Has the form: '[[test-]$ENV-]$NAME'
+    Has the form: '[[test-]$ENV-]$NAME[~$VARIATION]'
 
     Optional 'test-' prefix
     Optional $ENV environment part
     Mandatory $NAME
+    Optional ~$VARIATION suffix
 
     Verifies:
       - $NAME is valid
       - if $ENV, it is valid for $NAME
+      - if $VARIATION, it is valid for $NAME
 
-    Returns: tuple($ENV or None, $NAME)
+    Returns: tuple($ENV or None, $NAME, $VARIATION or None)
     """
 
     all_tests = get_all_test_info()
 
+    variation = None
+    if '~' in arg:
+        arg, variation = arg.split('~', 1)
+
     parts = arg.split('-', 2)
     parts_len = len(parts)
 
@@ -174,7 +207,15 @@ def parse_test_instance_string(arg):
         raise RunnerError("Test '%s' has no environment '%s'"
                           % (name, env))
 
-    return env, name
+    # If a variation has been given, check it is valid
+    if variation is not None:
+        if not info.variations:
+            raise RunnerError("Test '%s' has no variations" % (name, ))
+        elif not variation in info.variations:
+            raise RunnerError("No variation '%s' for test '%s'"
+                              % (variation, name))
+
+    return env, name, variation
 
 
 # Cached test json from disk
@@ -319,10 +360,11 @@ def interpret_selection(opts):
 
     # Second, sanity check others as full or partial test names
     for arg in others:
-        env, name = parse_test_instance_string(arg)
+        env, name, vary = parse_test_instance_string(arg)
 
         instances = all_tests[name].all_instances(
             env_filter = env and [env] or None,
+            vary_filter = vary and [vary] or None,
         )
 
         if not instances: