build: add configure option to disable gnulib tests

The gnulib testsuite is relatively stable - the only times it is
likely to have a test change from pass to fail is on a gnulib
submodule update or a major system change (such as moving from
Fedora 18 to 19, or other large change to libc).  While it is an
important test for end users on arbitrary machines (to make sure
that the portability glue works for their machine), it mostly
wastes time for development testing (as most developers aren't
making any of the major changes that would cause gnulib tests
to alter behavior).  Thus, it pays to make the tests optional
at configure time, defaulting to off for development, on for
tarballs, with autobuilders requesting it to be on.  It also
helps to allow a make-time override, via VIR_TEST_EXPENSIVE=[01]
(much the way automake sets up V=[01] for overriding the configure
time default of how verbose to be).

Automake has some pretty hard-coded magic with regards to the
TESTS variable; I had quite a job figuring out how to keep
'make distcheck' passing regardless of the configure option
setting in use, while still disabling the tests at runtime
when I did not configure them on and did not use the override
variable.  Thankfully, we require GNU make, which lets me
hide some information from Automake's magic handling of TESTS.

* bootstrap.conf (bootstrap_epilogue): Munge gnulib test variable.
* configure.ac (--enable-expensive-tests): Add new enable switch.
(VIR_TEST_EXPENSIVE_DEFAULT, WITH_EXPENSIVE_TESTS): Set new
witnesses.
* gnulib/tests/Makefile.am (TESTS): Make tests conditional on
configure settings and the VIR_TEST_EXPENSIVE variable.
* tests/Makefile.am (TESTS_ENVIRONMENT): Expose VIR_TEST_EXPENSIVE
to all tests.
* autobuild.sh: Enable all tests during autobuilds.
* libvirt.spec.in (%configure): Likewise.
* mingw-libvirt.spec.in (%mingw_configure): Likewise.
* docs/hacking.html.in: Document the option.
* HACKING: Regenerate.

Signed-off-by: Eric Blake <eblake@redhat.com>

diff --git a/HACKING b/HACKING
index 207b9edc9109db6d60e7e357536c80e5b18c8c48..256e8aeb84ead045a2152f9f15a7fc8c81046eba 100644 (file)
--- a/HACKING
+++ b/HACKING
@@ -107,6 +107,15 @@ and run the tests:
 Valgrind <http://valgrind.org/> is a test that checks for memory management
 issues, such as leaks or use of uninitialized variables.
 
+Some tests are skipped by default in a development environment, based on the
+time they take in comparison to the likelihood that those tests will turn up
+problems during incremental builds. These tests default to being run when when
+building from a tarball or with the configure option --enable-expensive-tests;
+you can also force a one-time toggle of these tests by setting
+VIR_TEST_EXPENSIVE to 0 or 1 at make time, as in:
+
+  make check VIR_TEST_EXPENSIVE=1
+
 If you encounter any failing tests, the VIR_TEST_DEBUG environment variable
 may provide extra information to debug the failures. Larger values of
 VIR_TEST_DEBUG may provide larger amounts of information:

diff --git a/autobuild.sh b/autobuild.sh
index 7da8cb5ed298c97116ef91d0fe980d5cc8594821..e5aa35ca33c2bfef3c726abddfa4c5d00d949613 100755 (executable)
--- a/autobuild.sh
+++ b/autobuild.sh
@@ -18,6 +18,7 @@ cd build
 # Run with options not normally exercised by the rpm build, for
 # more complete code coverage.
 ../autogen.sh --prefix="$AUTOBUILD_INSTALL_ROOT" \
+  --enable-expensive-tests \
   --enable-test-coverage \
   --disable-nls \
   --enable-werror \
@@ -76,6 +77,7 @@ if test -x /usr/bin/i686-w64-mingw32-gcc ; then
     --build=$(uname -m)-w64-linux \
     --host=i686-w64-mingw32 \
     --prefix="$AUTOBUILD_INSTALL_ROOT/i686-w64-mingw32/sys-root/mingw" \
+    --enable-expensive-tests \
     --enable-werror \
     --without-libvirtd \
     --without-python
@@ -96,6 +98,7 @@ if test -x /usr/bin/x86_64-w64-mingw32-gcc ; then
     --build=$(uname -m)-w64-linux \
     --host=x86_64-w64-mingw32 \
     --prefix="$AUTOBUILD_INSTALL_ROOT/x86_64-w64-mingw32/sys-root/mingw" \
+    --enable-expensive-tests \
     --enable-werror \
     --without-libvirtd \
     --without-python

diff --git a/bootstrap.conf b/bootstrap.conf
index f166a53eecd77d789b155ba7d9051d77ecf11c52..a1d1f07a724b0d9a1f09d926ea0deafd9191dd53 100644 (file)
--- a/bootstrap.conf
+++ b/bootstrap.conf
@@ -244,9 +244,10 @@ gnulib_extra_files="
 bootstrap_epilogue()
 {
   # Change paths in gnulib/tests/gnulib.mk from "../../.." to "../..",
+  # and make tests conditional by changing "TESTS" to "GNULIB_TESTS",
   # then ensure that gnulib/tests/Makefile.in is up-to-date.
   m=gnulib/tests/gnulib.mk
-  sed 's,\.\./\.\./\.\.,../..,g' $m > $m-t
+  sed 's,\.\./\.\./\.\.,../..,g; s/^TESTS /GNULIB_TESTS /' $m > $m-t
   mv -f $m-t $m
   ${AUTOMAKE-automake} gnulib/tests/Makefile
 }

diff --git a/configure.ac b/configure.ac
index 6cfb7e96c1ce6dfeed728b5fc2cbf808d0dc154a..ac8cfa179ec467c936c717559d91a955a6d7532e 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -2020,6 +2020,27 @@ fi
 AC_MSG_RESULT([$withval])
 AM_CONDITIONAL([WITH_TESTS], [test "$withval" = "yes"])
 
+AC_ARG_ENABLE([expensive-tests],
+  [AC_HELP_STRING([--enable-expensive-tests],
+    [set the default for enabling expensive tests (gnulib and long timeouts) ]
+    [@<:@default=check@:>@; use VIR_TEST_EXPENSIVE to override during make])],
+  [case $enableval in
+    0|no) VIR_TEST_EXPENSIVE_DEFAULT=0 ;;
+    1|yes) VIR_TEST_EXPENSIVE_DEFAULT=1 ;;
+    check) ;;
+    *) AC_MSG_ERROR([bad value ${enableval} for enable-expensive-tests option])
+      ;;
+  esac], [enableval=check])
+if test "$enableval" = check; then
+  if test -d $srcdir/.git ; then
+    VIR_TEST_EXPENSIVE_DEFAULT=0
+  else
+    VIR_TEST_EXPENSIVE_DEFAULT=1
+  fi
+fi
+AC_SUBST([VIR_TEST_EXPENSIVE_DEFAULT])
+AM_CONDITIONAL([WITH_EXPENSIVE_TESTS], [test $VIR_TEST_EXPENSIVE_DEFAULT = 1])
+
 AC_ARG_ENABLE([test-coverage],
   AC_HELP_STRING([--enable-test-coverage], [turn on code coverage instrumentation @<:@default=no@:>@]),
 [case "${enableval}" in

diff --git a/docs/hacking.html.in b/docs/hacking.html.in
index 8120b198024663c35c4149bb4623375043ad1556..0892b731afe03c0d08aa72c31a93dfe77210bc35 100644 (file)
--- a/docs/hacking.html.in
+++ b/docs/hacking.html.in
@@ -118,6 +118,18 @@
         variables.
         </p>
 
+        <p>
+          Some tests are skipped by default in a development environment,
+          based on the time they take in comparison to the likelihood
+          that those tests will turn up problems during incremental builds.
+          These tests default to being run when when building from a
+          tarball or with the configure option --enable-expensive-tests;
+          you can also force a one-time toggle of these tests by
+          setting VIR_TEST_EXPENSIVE to 0 or 1 at make time, as in:
+        </p>
+<pre>
+  make check VIR_TEST_EXPENSIVE=1
+</pre>
         <p>
           If you encounter any failing tests, the VIR_TEST_DEBUG
           environment variable may provide extra information to debug

diff --git a/gnulib/tests/Makefile.am b/gnulib/tests/Makefile.am
index 6a2f51bfd59f887687ffaf2dec6d59f0405d7c53..74d71e9f619e6e21d5e911c80c8de52953d5650e 100644 (file)
--- a/gnulib/tests/Makefile.am
+++ b/gnulib/tests/Makefile.am
@@ -1,4 +1,4 @@
-## Makefile for gnulib/lib                             -*-Makefile-*-
+## Makefile for gnulib/lib
 
 ## Copyright (C) 2011, 2013 Red Hat, Inc.
 ##
@@ -19,3 +19,16 @@
 include gnulib.mk
 
 INCLUDES = $(GETTEXT_CPPFLAGS)
+
+GNULIB_TESTS0 =
+GNULIB_TESTS1 = $(GNULIB_TESTS)
+if WITH_EXPENSIVE_TESTS
+## Automake requires that at least one conditional call out all tests to
+## be run, for those tests to be shipped in the tarball
+TESTS = $(GNULIB_TESTS)
+endif
+## However, we want to change the set of tests based on the make environment,
+## where the default was set at configure time.  Use GNU make constructs to
+## hide our actions from Automake, so we don't get it too confused.
+VIR_TEST_EXPENSIVE ?= $(VIR_TEST_EXPENSIVE_DEFAULT)
+$(eval TESTS=$(GNULIB_TESTS$(VIR_TEST_EXPENSIVE)))

diff --git a/libvirt.spec.in b/libvirt.spec.in
index 054d9b89833c365c0fdd2e88669d5b2c2bc4ff82..85881ae5f0f461cc5cc00c271a4d63db140ccf32 100644 (file)
--- a/libvirt.spec.in
+++ b/libvirt.spec.in
@@ -1410,6 +1410,7 @@ of recent versions of Linux (and other OSes).
            --with-qemu-user=%{qemu_user} \
            --with-qemu-group=%{qemu_group} \
            %{?enable_werror} \
+           --enable-expensive-tests \
            %{init_scripts}
 make %{?_smp_mflags}
 gzip -9 ChangeLog

diff --git a/mingw-libvirt.spec.in b/mingw-libvirt.spec.in
index aa3923104d48850920367571c80b1a4d76725a9b..e13407e96a5c90fc5338619e0faca25fae5953dc 100644 (file)
--- a/mingw-libvirt.spec.in
+++ b/mingw-libvirt.spec.in
@@ -156,8 +156,8 @@ autoreconf -if
   --without-parallels \
   --without-netcf \
   --without-audit \
-  --without-dtrace
-
+  --without-dtrace \
+  --enable-expensive-tests
 
 %mingw_make %{?_smp_mflags}
 

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4aa02dd74c85a516bb0b095a755eeec3a702bc8a..6f43b09edcbfa024f4e36f559fa8404422a3c3f6 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -307,6 +307,7 @@ lv_abs_top_builddir=`cd '$(top_builddir)'; pwd`
 path_add = $(subst :,$(PATH_SEPARATOR),\
        $(subst !,$(lv_abs_top_builddir)/,!daemon:!tools:!tests))
 
+VIR_TEST_EXPENSIVE ?= $(VIR_TEST_EXPENSIVE_DEFAULT)
 TESTS_ENVIRONMENT =                            \
   abs_top_builddir=$(lv_abs_top_builddir)      \
   abs_top_srcdir=`cd '$(top_srcdir)'; pwd`     \
@@ -318,6 +319,7 @@ TESTS_ENVIRONMENT =                         \
   LIBVIRT_DRIVER_DIR="$(abs_top_builddir)/src/.libs" \
   LIBVIRT_AUTOSTART=0                          \
   LC_ALL=C                                     \
+  VIR_TEST_EXPENSIVE=$(VIR_TEST_EXPENSIVE)     \
   $(VG)