ia64/xen-unstable

changeset 9236:000d5ccb2bec

HG Patch

Add feature to layer xm-test tests, grouping of tests by casename and
execution of tests by user specified lists.

Signed-off-by: Woody Marvel <marvel@us.ibm.com>
author root@dyn9047022116.beaverton.ibm.com
date Sat Mar 11 11:46:38 2006 +0100 (2006-03-11)
parents ed1afb12106e
children 65dfd26801af
files tools/xm-test/README tools/xm-test/grouptest/create tools/xm-test/grouptest/default tools/xm-test/grouptest/quick tools/xm-test/runtest.sh
line diff
     1.1 --- a/tools/xm-test/README	Sat Mar 11 11:45:10 2006 +0100
     1.2 +++ b/tools/xm-test/README	Sat Mar 11 11:46:38 2006 +0100
     1.3 @@ -120,8 +120,9 @@ To run the full test suite, do the follo
     1.4     # ./runtest.sh <logfile>
     1.5  
     1.6  This will run all tests, as well as generate and submit a report at
     1.7 -the end.  All output files will begin with "<logfile>."  If you wish to
     1.8 -prevent submission of a report, add "-d" to the command line like this:
     1.9 +the end. All output files will begin with "<logfile>."
    1.10 +If you wish to prevent submission of a report, add "-d" to the 
    1.11 +command line like this:
    1.12  
    1.13     # ./runtest.sh -d <logfile>
    1.14  
    1.15 @@ -131,15 +132,19 @@ submit the report at a later time.  To d
    1.16  
    1.17     # ./runtest.sh -s <logfile>
    1.18  
    1.19 -For people needing a quick test run instead the full suite, a quick
    1.20 -mode has been added that will attempt to run a representative subset
    1.21 -of tests.  This is not a substitute for the whole suite, but will
    1.22 -verify that some of the major functions of xen and xm are working:
    1.23 +Group test sets are supported in xm-test. This is form of layering of 
    1.24 +tests groups/cases/tests. In the framework directory "grouptest",
    1.25 +files exist for group processing. The user can add groups, casenames
    1.26 +and test lists as required. Default group run is "grouptest/default".
    1.27 +
    1.28 +   # ./runtest.sh -g <groupname> <logfile>
    1.29  
    1.30 -   # ./runtest.sh -q <logfile>
    1.31 +* NOTE: There is a quick set of tests in group mode, that was added to
    1.32 +run certain casenames and tests. It is not a substitute for the full
    1.33 +xm-test test suite.
    1.34 +   # ./runtest.sh -g quick <logfile>
    1.35  
    1.36 -Because of the current structure of the reporting software, submission
    1.37 -of quick test run results is not supported.
    1.38 +
    1.39  
    1.40  It may be desirable to run a specific test group.  This can be
    1.41  accomplished by doing the following:
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/tools/xm-test/grouptest/create	Sat Mar 11 11:46:38 2006 +0100
     2.3 @@ -0,0 +1,1 @@
     2.4 +create
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/tools/xm-test/grouptest/default	Sat Mar 11 11:46:38 2006 +0100
     3.3 @@ -0,0 +1,30 @@
     3.4 +block-create
     3.5 +block-destroy
     3.6 +block-list
     3.7 +console
     3.8 +create
     3.9 +destroy
    3.10 +dmesg
    3.11 +domid
    3.12 +domname
    3.13 +enforce_dom0_cpus
    3.14 +help
    3.15 +info
    3.16 +list
    3.17 +memmax
    3.18 +memset
    3.19 +migrate
    3.20 +network
    3.21 +network-attach
    3.22 +pause
    3.23 +reboot
    3.24 +restore
    3.25 +save
    3.26 +sedf
    3.27 +shutdown
    3.28 +sysrq
    3.29 +unpause
    3.30 +vcpu-disable
    3.31 +vcpu-pin
    3.32 +vtpm
    3.33 +
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/tools/xm-test/grouptest/quick	Sat Mar 11 11:46:38 2006 +0100
     4.3 @@ -0,0 +1,4 @@
     4.4 +create 01_create_basic_pos.test 07_create_mem64_pos.test 10_create_fastdestroy.test 14_create_blockroot_pos.test
     4.5 +unpause 01_unpause_basic_pos.test
     4.6 +memset 01_memset_basic_pos.test 03_memset_random_pos.test
     4.7 +help 06_help_allcmds.test
     5.1 --- a/tools/xm-test/runtest.sh	Sat Mar 11 11:45:10 2006 +0100
     5.2 +++ b/tools/xm-test/runtest.sh	Sat Mar 11 11:46:38 2006 +0100
     5.3 @@ -1,4 +1,4 @@
     5.4 -#!/bin/sh
     5.5 +#!/bin/sh 
     5.6  
     5.7  ##
     5.8  ## Test driver script
     5.9 @@ -11,7 +11,7 @@ usage() {
    5.10      echo "  Where opts are:"
    5.11      echo "  -d          : do not submit a report for this run"
    5.12      echo "  -b          : do not ask any questions (batch mode)"
    5.13 -    echo "  -q          : run a quick test set"
    5.14 +    echo "  -g          : run a group test set"
    5.15      echo "  -e <email>  : set email address for report"
    5.16      echo "  -s <report> : just submit report <report>"
    5.17      echo "  -h | --help : show this help"
    5.18 @@ -92,12 +92,13 @@ runnable_tests() {
    5.19      echo "Running sanity checks..."
    5.20      make -C tests/_sanity check 2>&1 | grep REASON
    5.21      if [ $? -eq 0 ]; then
    5.22 -	echo "Sanity checks failed"
    5.23 -	exit 1
    5.24 +        echo "Sanity checks failed"
    5.25 +        exit 1
    5.26      fi
    5.27  
    5.28  }
    5.29  
    5.30 +
    5.31  # Get contact info if needed
    5.32  get_contact_info() {
    5.33      
    5.34 @@ -128,26 +129,21 @@ get_contact_info() {
    5.35  
    5.36  # Run the tests
    5.37  run_tests() {
    5.38 -    output=$1
    5.39 -    echo Running real tests...
    5.40 -    TEST_VERBOSE=1 make -k check > $output 2>&1
    5.41 -}
    5.42 -
    5.43 -run_tests_quick() {
    5.44 -
    5.45 -    output=$1
    5.46 +    groupentered=$1
    5.47 +    output=$2
    5.48  
    5.49 -    create_tests="01_create_basic_pos.test 07_create_mem64_pos.test 10_create_fastdestroy.test 14_create_blockroot_pos.test"
    5.50 -    unpause_tests="01_unpause_basic_pos.test"
    5.51 -    memset_tests="01_memset_basic_pos.test 03_memset_random_pos.test"
    5.52 -    help_tests="06_help_allcmds.test"
    5.53 -    testgroups="create unpause memset help"
    5.54 +    exec <  grouptest/$groupentered
    5.55 +    while read casename testlist; do
    5.56 +       echo Running $casename tests...
    5.57 +       echo "*** case $casename from group $groupentered" >> $output
    5.58 +       if [ -z "$testlist" ]; then
    5.59 +          echo "*** Running tests for case $casename" >> $output
    5.60 +          (cd tests/$casename && TEST_VERBOSE=1 make -k check) >> $output 2>&1
    5.61 +       else
    5.62 +          echo "*** Running tests $testlist from case $casename" >> $output
    5.63 +          (cd tests/$casename && TEST_VERBOSE=1 make -k check TESTS="$testlist") >> $output 2>&1
    5.64 +       fi
    5.65  
    5.66 -    echo "*** Quick test" > $output
    5.67 -    for group in $testgroups; do
    5.68 -	eval $(echo list=\$${group}_tests)
    5.69 -	echo "*** Running tests [$list] from $group"
    5.70 -	(cd tests/$group && TEST_VERBOSE=1 make -k check TESTS="$list") >> $output 2>&1
    5.71      done
    5.72  
    5.73  }
    5.74 @@ -195,6 +191,7 @@ MAXFAIL=10
    5.75  report=yes
    5.76  batch=no
    5.77  run=yes
    5.78 +GROUPENTERED=default
    5.79  
    5.80  # Resolve options
    5.81  while [ $# -gt 0 ]
    5.82 @@ -213,8 +210,13 @@ while [ $# -gt 0 ]
    5.83  	  echo $1 > contact_info
    5.84  	  echo "(Email set to $1)"
    5.85  	  ;;
    5.86 -      -q)
    5.87 -	  run=quick
    5.88 +      -g)
    5.89 +	  shift
    5.90 +          GROUPENTERED=$1
    5.91 +          if [ ! -f grouptest/$GROUPENTERED ]; then
    5.92 +             echo "No file for group $GROUPENTERED"
    5.93 +             exit 1
    5.94 +          fi
    5.95  	  ;;
    5.96        -s)
    5.97  	  run=no
    5.98 @@ -265,18 +267,15 @@ fi
    5.99  if [ "$run" != "no" ]; then
   5.100      runnable_tests
   5.101      make_environment_report $OSREPORTTEMP $PROGREPORTTEMP
   5.102 -    if [ "$run" = "yes" ]; then
   5.103 -	run_tests $OUTPUT
   5.104 -    else
   5.105 -	run_tests_quick $OUTPUT
   5.106 -    fi
   5.107 +    run_tests $GROUPENTERED $OUTPUT
   5.108      make_text_reports $PASSFAIL $FAILURES $OUTPUT $TXTREPORT
   5.109      make_result_report $OUTPUT $RESULTREPORTTEMP
   5.110      cat $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP > $XMLREPORT
   5.111      rm $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP
   5.112 +
   5.113  fi
   5.114  
   5.115 -if [ "$report" = "yes" ] && [ "$run" = "yes" ]; then
   5.116 +if [ "$report" = "yes" ]; then
   5.117      if [ ! -f "$XMLREPORT" ]; then
   5.118  	echo "No such file: $XMLREPORT"
   5.119  	exit 1