1.1 --- a/tools/xm-test/runtest.sh	Thu Nov 10 11:48:05 2005 +0100
     1.2 +++ b/tools/xm-test/runtest.sh	Thu Nov 10 11:48:05 2005 +0100
     1.3 @@ -1,215 +1,259 @@
     1.4  #!/bin/sh
     1.5  
     1.6 -usage() {
     1.7 -    echo "Usage: $0 [opts] <logfile>"
     1.8 -    echo "  Where opts are:"
     1.9 -    echo "  -d         : do not submit a report for this run"
    1.10 -    echo "  -b         : do not ask any questions (batch mode)"
    1.11 -    echo "  -e <email> : set email address for report"
    1.12 -}
    1.13  ##
    1.14  ## Test driver script
    1.15  ##
    1.16  
    1.17 -#
    1.18 -# Defaults
    1.19 -#
    1.20 -MAXFAIL=10
    1.21 -report=yes
    1.22 -batch=no
    1.23 +usage() {
    1.24 +    echo "Usage: $0 [opts] <report>"
    1.25 +    echo "  Where report is a name that will be used for report files"
    1.26 +    echo ""
    1.27 +    echo "  Where opts are:"
    1.28 +    echo "  -d          : do not submit a report for this run"
    1.29 +    echo "  -b          : do not ask any questions (batch mode)"
    1.30 +    echo "  -q          : run a quick test set"
    1.31 +    echo "  -e <email>  : set email address for report"
    1.32 +    echo "  -s <report> : just submit report <report>"
    1.33 +}
    1.34 +
    1.35 +# Just submit the report
    1.36 +submit_report() {
    1.37  
    1.38 -#
    1.39 -# Resolve options
    1.40 -#
    1.41 -while [ $# -gt 0 ]
    1.42 -do
    1.43 -    case "$1" in
    1.44 -    -d)
    1.45 -        echo "(Skipping report submission)"
    1.46 -        report=no
    1.47 -        ;;
    1.48 -    -b)
    1.49 -        echo "(Batch mode)"
    1.50 -        batch=yes
    1.51 -        ;;
    1.52 -    -e)
    1.53 -        shift
    1.54 -        echo $1 > contact_info
    1.55 -        echo "(Email set to $1)"
    1.56 -        ;;
    1.57 -    *)
    1.58 -        LOGFILE=$1
    1.59 -        break
    1.60 -        ;;
    1.61 -    esac
    1.62 -    shift
    1.63 -done
    1.64 +    reportfile=$1
    1.65 +
    1.66 +    ./lib/XmTestReport/Report.py $reportfile
    1.67 +}
    1.68 +
    1.69 +# Generate XML result report from output file
    1.70 +make_result_report() {
    1.71 +    output=$1
    1.72 +    reportfile=$2
    1.73 +    if ! ./lib/XmTestReport/ResultReport.py $output > $reportfile; then
    1.74 +	echo "Unable to generate clean ResultReport"
    1.75 +	echo "Take a look at $report"
    1.76 +	exit 1
    1.77 +    fi
    1.78 +}
    1.79  
    1.80 -#
    1.81 -# Usage
    1.82 -#
    1.83 -if [ -z $LOGFILE ]; then
    1.84 -	usage
    1.85 +# Collect environment information for XML report
    1.86 +make_environment_report() {
    1.87 +    os=$1
    1.88 +    prog=$2
    1.89 +    if ! ./lib/XmTestReport/OSReport.py > $os; then
    1.90 +	echo "Unable to generate clean OSReport"
    1.91 +	echo "Take a look at $os"
    1.92  	exit 1
    1.93 -fi
    1.94 +    fi
    1.95 +    if ! ./lib/XmTestReport/ProgReport.py > $prog; then
    1.96 +	echo "Unable to generate clean ProgReport"
    1.97 +	echo "Take a look at $prog"
    1.98 +	exit 1
    1.99 +    fi
   1.100 +}
   1.101  
   1.102 -#
   1.103 -# Output files
   1.104 -#
   1.105 -OSREPORTTEMP=${LOGFILE}.os.xml
   1.106 -PROGREPORTTEMP=${LOGFILE}.prog.xml
   1.107 -RESULTREPORTTEMP=${LOGFILE}.result.xml
   1.108 -OUTPUT=${LOGFILE}.output
   1.109 -SUMMARY=${LOGFILE}.summary
   1.110 -PASSFAIL=${LOGFILE}.passfail
   1.111 -REPORT=${LOGFILE}.report
   1.112 -FAILURES=${LOGFILE}.failures
   1.113 -		
   1.114 -#
   1.115 -# Make sure we're root
   1.116 -#
   1.117 -uid=$(id -u)
   1.118 -if [ $uid != 0 ]; then 
   1.119 -    echo "ERROR: I must be run as root!"
   1.120 -    exit 1
   1.121 -fi
   1.122 +# Check conditions needed to actually run the tests
   1.123 +runnable_tests() {	
   1.124 +    # Make sure we're root
   1.125 +    uid=$(id -u)
   1.126 +    if [ $uid != 0 ]; then 
   1.127 +	echo "ERROR: I must be run as root!"
   1.128 +	exit 1
   1.129 +    fi
   1.130  
   1.131 -#
   1.132 -# See if the ramdisk has been built
   1.133 -#
   1.134 -rdsize=$(stat -c %s ramdisk/initrd.img 2>/dev/null)
   1.135 -if [ -z "$rdsize" ] || [ $rdsize -le 16384 ]; then
   1.136 -    echo "Cannot find a valid ramdisk.  You need to run \"make\" or"
   1.137 -    echo "copy in a previously-built ramdisk to the ramdisk/ directory"
   1.138 -    exit 1
   1.139 -fi
   1.140 +    # See if the ramdisk has been built
   1.141 +    rdsize=$(stat -c %s ramdisk/initrd.img 2>/dev/null)
   1.142 +    if [ -z "$rdsize" ] || [ $rdsize -le 16384 ]; then
   1.143 +	echo "Cannot find a valid ramdisk.  You need to run \"make\" or"
   1.144 +	echo "copy in a previously-built ramdisk to the ramdisk/ directory"
   1.145 +	exit 1
   1.146 +    fi
   1.147 +
   1.148 +    # See if xend is running
   1.149 +    if ! xm list >/dev/null 2>&1; then
   1.150 +	echo "'xm list' failed: is xend running?"
   1.151 +	exit 1
   1.152 +    fi
   1.153 +    
   1.154 +}
   1.155  
   1.156 -#
   1.157 -# See if xend is running
   1.158 -#
   1.159 -if ! xm list >/dev/null 2>&1; then
   1.160 -    echo "'xm list' failed: is xend running?"
   1.161 -    exit 1
   1.162 -fi
   1.163 +# Get contact info if needed
   1.164 +get_contact_info() {
   1.165 +    
   1.166 +    if [ ! -f contact_info ]; then
   1.167 +	if [ "$batch" = "yes" ]; then
   1.168 +	    echo "Unable to read contact_info!"
   1.169 +	    echo "Please run me once interactively before using batch mode!"
   1.170 +	    exit 1
   1.171 +	else
   1.172 +	    echo "Please provide your email address so that we can "
   1.173 +	    echo "contact you if we need further information concerning"
   1.174 +	    echo "your results.  Any information provided will be"
   1.175 +	    echo "kept private.  If you wish to remain anonymous, please"
   1.176 +	    echo "hit [ENTER] now."
   1.177 +	    
   1.178 +	    while ! echo "$EMAIL" | grep -q '@'; do
   1.179 +		echo
   1.180 +		echo -n "Your email address: "
   1.181 +		read EMAIL
   1.182 +		if [ -z $EMAIL ]; then
   1.183 +		    EMAIL="anonymous@somewhere.com"
   1.184 +		fi
   1.185 +	    done
   1.186 +	    echo $EMAIL > contact_info
   1.187 +	fi
   1.188 +    fi
   1.189 +}
   1.190  
   1.191 -#
   1.192 -#  Make sure permissions are correct
   1.193 -#
   1.194 -chmod a+x lib/XmTestReport/*
   1.195 -chmod a+x mkreport mergereport
   1.196 +# Run the tests
   1.197 +run_tests() {
   1.198 +    output=$1
   1.199 +    echo Running tests...
   1.200 +    TEST_VERBOSE=1 make -k check > $output 2>&1
   1.201 +}
   1.202  
   1.203 -#
   1.204 -# Get contact info if needed
   1.205 -#
   1.206 -if [ ! -f contact_info ]; then
   1.207 -    if [ "$batch" = "yes" ]; then
   1.208 -	echo "Unable to read contact_info!"
   1.209 -	echo "Please run me once interactively before using batch mode!"
   1.210 -	exit 1
   1.211 -    else
   1.212 -	echo "Please provide your email address so that we can "
   1.213 -	echo "contact you if we need further information concerning"
   1.214 -	echo "your results.  Any information provided will be"
   1.215 -	echo "kept private.  If you wish to remain anonymous, please"
   1.216 -	echo "hit [ENTER] now."
   1.217 -	
   1.218 -	while ! echo "$EMAIL" | grep -q '@'; do
   1.219 -	    echo
   1.220 -	    echo -n "Your email address: "
   1.221 -	    read EMAIL
   1.222 -	    if [ -z $EMAIL ]; then
   1.223 -		EMAIL="anonymous@somewhere.com"
   1.224 -	    fi
   1.225 -	done
   1.226 -	echo $EMAIL > contact_info
   1.227 -    fi
   1.228 -fi
   1.229 +run_tests_quick() {
   1.230 +
   1.231 +    output=$1
   1.232 +
   1.233 +    create_tests="01_create_basic_pos.test 07_create_mem64_pos.test 10_create_fastdestroy.test 14_create_blockroot_pos.test"
   1.234 +    unpause_tests="01_unpause_basic_pos.test"
   1.235 +    memset_tests="01_memset_basic_pos.test 03_memset_random_pos.test"
   1.236 +    help_tests="06_help_allcmds.test"
   1.237 +    testgroups="create unpause memset help"
   1.238 +
   1.239 +    echo "*** Quick test" > $output
   1.240 +    for group in $testgroups; do
   1.241 +	eval $(echo list=\$${group}_tests)
   1.242 +	echo "*** Running tests [$list] from $group"
   1.243 +	(cd tests/$group && TEST_VERBOSE=1 make -k check TESTS="$list") >> $output 2>&1
   1.244 +    done
   1.245 +
   1.246 +}
   1.247  
   1.248 -#
   1.249 -# Collect environment information for XML report
   1.250 -#
   1.251 -if ! ./lib/XmTestReport/OSReport.py > $OSREPORTTEMP; then
   1.252 -    echo "Unable to generate clean OSReport"
   1.253 -    echo "Take a look at $OSREPORTTEMP"
   1.254 -    exit 1
   1.255 -fi
   1.256 -if ! ./lib/XmTestReport/ProgReport.py > $PROGREPORTTEMP; then
   1.257 -    echo "Unable to generate clean ProgReport"
   1.258 -    echo "Take a look at $PROGREPORTTEMP"
   1.259 -    exit 1
   1.260 -fi
   1.261 -
   1.262 -#
   1.263 -# Run the tests
   1.264 -#
   1.265 -export TEST_VERBOSE=1
   1.266 -echo Running tests...
   1.267 -make -k check > $OUTPUT 2>&1
   1.268 -
   1.269 -#
   1.270  # Generate some plain-text reports
   1.271 -#
   1.272 -echo "Making PASS/FAIL report ($PASSFAIL)..."
   1.273 -cat $OUTPUT | egrep '(REASON|PASS|FAIL|XPASS|XFAIL|SKIP)' | perl -pe 's/^(PASS|FAIL|XPASS|XFAIL)(.+)$/$1$2\n/' > $PASSFAIL
   1.274 -
   1.275 -echo "Making FAIL report ($FAILURES)..."
   1.276 -cat $PASSFAIL | egrep '(REASON|FAIL)' > $FAILURES
   1.277 -
   1.278 -NUMPASS=`grep -c PASS $OUTPUT`
   1.279 -NUMFAIL=`grep -c FAIL $OUTPUT`
   1.280 -NUMXPASS=`grep -c XPASS $OUTPUT`
   1.281 -NUMXFAIL=`grep -c XFAIL $OUTPUT`
   1.282 -cat > $SUMMARY << EOF
   1.283 +make_text_reports() {
   1.284 +    passfail=$1
   1.285 +    failures=$2
   1.286 +    output=$3
   1.287 +    reportfile=$4
   1.288 +    summary=summary.tmp
   1.289 +    echo "Making PASS/FAIL report ($passfail)..."
   1.290 +    cat $OUTPUT | egrep '(REASON|PASS|FAIL|XPASS|XFAIL|SKIP)' | perl -pe 's/^(PASS|FAIL|XPASS|XFAIL)(.+)$/$1$2\n/' > $passfail
   1.291 +    
   1.292 +    echo "Making FAIL report ($failures)..."
   1.293 +    cat $passfail | egrep '(REASON|FAIL)' > $failures
   1.294 +    
   1.295 +    NUMPASS=`grep -c PASS $output`
   1.296 +    NUMFAIL=`grep -c FAIL $output`
   1.297 +    NUMXPASS=`grep -c XPASS $output`
   1.298 +    NUMXFAIL=`grep -c XFAIL $output`
   1.299 +    cat > $summary << EOF
   1.300  Xm-test execution summary:
   1.301    PASS:  $NUMPASS
   1.302    FAIL:  $NUMFAIL
   1.303    XPASS: $NUMXPASS
   1.304    XFAIL: $NUMXFAIL
   1.305  EOF
   1.306 +    
   1.307 +    cat $summary > $reportfile
   1.308 +    
   1.309 +    echo -e '\n\nDetails:\n' >> $reportfile
   1.310 +    
   1.311 +    ./mkreport $passfail >> $reportfile
   1.312  
   1.313 -cat $SUMMARY > $REPORT
   1.314 +    rm $summary
   1.315 +}
   1.316  
   1.317 -echo -e '\n\nDetails:\n' >> $REPORT
   1.318 - 
   1.319 -./mkreport $PASSFAIL >> $REPORT
   1.320 +############
   1.321 +### Main ###
   1.322 +############
   1.323 +
   1.324 +# Defaults
   1.325 +MAXFAIL=10
   1.326 +report=yes
   1.327 +batch=no
   1.328 +run=yes
   1.329  
   1.330 -#
   1.331 -# Check to see if it's worth reporting these results
   1.332 -#
   1.333 -#if  [ "$batch"  =   "no" ] && 
   1.334 -#    [ "$report" =   "yes" ] && 
   1.335 -#    [ $NUMFAIL  -gt $MAXFAIL ]; then
   1.336 -#    echo "NOTE: $NUMFAIL tests failed, which may be erroneous.  It may"
   1.337 -#    echo "be a good idea to review the report before sending.  If you"
   1.338 -#    echo "choose not to submit the report, it will be saved for your review"
   1.339 -#    echo "and later submission."
   1.340 -#    echo
   1.341 -#    echo -n "Submit anyway? [y/n] "
   1.342 -#    read ANSWER
   1.343 -#    if [ "$ANSWER" = "n" ]; then
   1.344 -#	report=no
   1.345 -#    fi
   1.346 -#fi
   1.347 +# Resolve options
   1.348 +while [ $# -gt 0 ]
   1.349 +  do
   1.350 +  case "$1" in
   1.351 +      -d)
   1.352 +	  echo "(Skipping report submission)"
   1.353 +	  report=no
   1.354 +	  ;;
   1.355 +      -b)
   1.356 +	  echo "(Batch mode)"
   1.357 +	  batch=yes
   1.358 +	  ;;
   1.359 +      -e)
   1.360 +	  shift
   1.361 +	  echo $1 > contact_info
   1.362 +	  echo "(Email set to $1)"
   1.363 +	  ;;
   1.364 +      -q)
   1.365 +	  run=quick
   1.366 +	  ;;
   1.367 +      -s)
   1.368 +	  run=no
   1.369 +	  ;;
   1.370 +      *)
   1.371 +	  REPORT=$1
   1.372 +	  break
   1.373 +	  ;;
   1.374 +  esac
   1.375 +  shift
   1.376 +done
   1.377  
   1.378 -#
   1.379 -# Generate the XML result report
   1.380 -#
   1.381 -if ! ./lib/XmTestReport/ResultReport.py $OUTPUT > $RESULTREPORTTEMP; then
   1.382 -    echo "Unable to generate clean ResultReport"
   1.383 -    echo "Take a look at $RESULTREPORTTEMP"
   1.384 -    exit 1
   1.385 +# Usage
   1.386 +if [ -z $REPORT ]; then
   1.387 +	usage
   1.388 +	exit 1
   1.389  fi
   1.390  
   1.391 -#
   1.392 -# Maybe submit report and save the combined XML file
   1.393 -#
   1.394 -if [ "$report" = "yes" ]; then
   1.395 -    echo "Sending report..."
   1.396 -    ./lib/XmTestReport/Report.py -D $OSREPORTTEMP $PROGREPORTTEMP \
   1.397 -                                    $RESULTREPORTTEMP > $1.xml
   1.398 -    echo "Report also saved in $1.xml"
   1.399 -else
   1.400 -    echo "Saving report to $1.xml..."
   1.401 -    ./lib/XmTestReport/Report.py -d $OSREPORTTEMP $PROGREPORTTEMP \
   1.402 -                                    $RESULTREPORTTEMP > $1.xml
   1.403 +# Output files
   1.404 +OSREPORTTEMP=${REPORT}.os.xml
   1.405 +PROGREPORTTEMP=${REPORT}.prog.xml
   1.406 +RESULTREPORTTEMP=${REPORT}.result.xml
   1.407 +XMLREPORT=${REPORT}.xml
   1.408 +OUTPUT=${REPORT}.output
   1.409 +SUMMARY=${REPORT}.summary
   1.410 +PASSFAIL=${REPORT}.passfail
   1.411 +TXTREPORT=${REPORT}.report
   1.412 +FAILURES=${REPORT}.failures
   1.413 +	
   1.414 +#  Make sure permissions are correct
   1.415 +chmod a+x lib/XmTestReport/*
   1.416 +chmod a+x mkreport mergereport
   1.417 +
   1.418 +if [ ! -f contact_info ]; then
   1.419 +    if [ "$batch" = "yes" ]; then
   1.420 +	echo "Unable to read contact_info"
   1.421 +	echo "You must run me interactively once!"
   1.422 +	exit 1
   1.423 +    else
   1.424 +	get_contact_info
   1.425 +    fi
   1.426  fi
   1.427 +
   1.428 +if [ "$run" != "no" ]; then
   1.429 +    runnable_tests
   1.430 +    make_environment_report $OSREPORTTEMP $PROGREPORTTEMP
   1.431 +    if [ "$run" = "yes" ]; then
   1.432 +	run_tests $OUTPUT
   1.433 +    else
   1.434 +	run_tests_quick $OUTPUT
   1.435 +    fi
   1.436 +    make_text_reports $PASSFAIL $FAILURES $OUTPUT $TXTREPORT
   1.437 +    make_result_report $OUTPUT $RESULTREPORTTEMP
   1.438 +    cat $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP > $XMLREPORT
   1.439 +    rm $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP
   1.440 +fi
   1.441 +
   1.442 +if [ "$report" = "yes" ] && [ "$run" = "yes" ]; then
   1.443 +    if [ ! -f "$XMLREPORT" ]; then
   1.444 +	echo "No such file: $XMLREPORT"
   1.445 +	exit 1
   1.446 +    fi
   1.447 +    submit_report $XMLREPORT
   1.448 +fi