osstest: use a locally built pkg repository for FreeBSD

This removes the dependency on the official pkg repository, which is
dangerous when not testing stable branches, since the ABI of the
development branch is not stable, and thus it's easy for packages to
get out of sync, or for osstest to test an old FreeBSD version that
has an ABI different than the one used to build the official
packages.

The output of the package build test is tested together with the newly
built image, and if there are no regressions both are anointed in
lockstep in order to prevent temporary discrepancies between the
installer and the package repository.

Note that in order to bootstrap the first run it's possible to
manually set the package repository to use with an environment
variable:

 - FREEBSD_PACKAGES_<arch>_BUILDJOB: points to the flight.job that
   contains the binary package repository.
 - FREEBSD_PACKAGES: points to a local directory that contains the
   binary repository.

It's also possible to set the directory that contains the package
repository in the configuration file using FreeBSDPackages.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>

diff --git a/cr-daily-branch b/cr-daily-branch
index 971f4c014b457e7719b045566702abfeabf87564..28cbd61ae6fa1855d5632801864f18caa5c1f9c5 100755 (executable)
--- a/cr-daily-branch
+++ b/cr-daily-branch
@@ -321,15 +321,26 @@ freebsd)
 esac
 
 IFS=$'\n'
+count=0
 for anointed in \
-    `./mg-anoint list-prepared "freebsd build $freebsd_branch *"`; do
+    `./mg-anoint list-prepared "freebsd* build $freebsd_branch *"`; do
     # Retrieve previous successful FreeBSD build for each arch.
     freebsd_arch=${anointed##* }
-    freebsd_envvar="FREEBSD_${freebsd_arch^^}_BUILDJOB"
+    freebsd_name=${anointed%% *}
+    freebsd_name=${freebsd_name/-/_}
+    freebsd_envvar="${freebsd_name^^}_${freebsd_arch^^}_BUILDJOB"
     if [ "x${!freebsd_envvar}" = "x" ]; then
-        flight_job=`./mg-anoint retrieve "$anointed"`
-        export ${freebsd_envvar}=${flight_job/ /.}
+       envvars[$count]="$freebsd_envvar"
+       refkeys[$count]="$anointed"
+       count=$((count+1))
+    fi
+done
+count=0
+for flight_job in `./mg-anoint retrieve ${refkeys[@]}`; do
+    if [ "$flight_job" != "ERROR" ]; then
+       export ${envvars[$count]}=${flight_job/ /.}
     fi
+    count=$((count+1))
 done
 unset IFS
 
@@ -542,17 +553,23 @@ freebsd-*)
        [ "x$OSSTEST_BLESSING" == "xreal" ]; then
         IFS=$'\n'
         for anointed in `./mg-anoint list-prepared \
-                                     "freebsd build $freebsd_branch *"`; do
+                                     "freebsd* build $freebsd_branch *"`; do
             # Update anointed versions
             # NB: failure to update an anointed build for a specific arch
             # should not be fatal, and it's not an issue if one of the
             # arches gets slightly out of sync with the other ones.
             freebsd_arch=${anointed##* }
-            if ./mg-anoint anoint "$anointed" \
-                           $flight build-$freebsd_arch-freebsd; then
-                echo "Anointed artifacts from build-$freebsd_arch-freebsd"
-            fi
+            freebsd_name=${anointed%% *}
+           # Rely on the fact that the job suffix is the same as the
+           # anointment refkey. Ie:
+           # refkey: freebsd          job: build-<arch>-freebsd
+           # refkey: freebsd-packages job: build-<arch>-freebsd-packages
+            anoint="$anoint \"$anointed\" $flight \
+                    build-$freebsd_arch-$freebsd_name"
         done
+       if ./mg-anoint anoint $anoint; then
+               echo "Anointed build artifacts from flight"
+       fi
         unset IFS
     fi
     ;;

diff --git a/make-freebsd-flight b/make-freebsd-flight
index fc3d2d83f34321cfde7f45d407ea98fc5840471d..0458a33b665c5f851e22ad731e990fb18d87d1c2 100755 (executable)
--- a/make-freebsd-flight
+++ b/make-freebsd-flight
@@ -45,12 +45,14 @@ for arch in "$arches"; do
                      recipe_testinstall=true"
     create_freebsd_pkg_build_job build-$arch-freebsd-packages
 
-    # Create an identical job that's going to use the build output from
-    # the previous one.
+    # Create a build job that going to use the output of both the jobs
+    # above in order to test the newly built FreeBSD and packages
+    freebsd_runvars="$freebsd_runvars \
+                     freebsdpackagesbuildjob=build-$arch-freebsd-packages"
     create_freebsd_build_job build-$arch-freebsd-again
 
-    # Create a Xen build job that's going to use the output from the first
-    # FreeBSD build job.
+    # Create a Xen build job that's going to use the output from the
+    # FreeBSD build jobs.
     create_xen_build_job build-$arch-xen-freebsd build-xen-freebsd      \
         host_hostflags=arch-$arch,purpose-build all_host_os=freebsd     \
         $freebsd_runvars

diff --git a/mfi-common b/mfi-common
index 83d3c7130c3c5cd29fbefb2e178395c03fb858f7..12cde85fbc377458995d34e5985d09d52121685a 100644 (file)
--- a/mfi-common
+++ b/mfi-common
@@ -156,7 +156,6 @@ set_freebsd_runvars () {
     # 4. Look for an anointed build of FreeBSD `master' (Executive only)
     #
     local no_hostflags=$1
-    local envvar="FREEBSD_${arch^^}_BUILDJOB"
 
     if [ x$no_hostflags != xtrue ]; then
         # osstest doesn't yet know how to install FreeBSD on UEFI hosts, so
@@ -164,27 +163,59 @@ set_freebsd_runvars () {
         freebsd_runvars="all_hostflags,=PropEq:Firmware:bios:bios"
     fi
 
-    if [ -n "${!envvar}" ]; then
-        freebsd_runvars="$freebsd_runvars freebsdbuildjob=${!envvar}"
-        return
-    fi
-    if [ -n "$FREEBSD_DIST" ] && [ -n "$FREEBSD_VERSION" ]; then
-        freebsd_runvars="$freebsd_runvars freebsd_distpath=$FREEBSD_DIST/$arch \
-                         freebsd_version=$FREEBSD_VERSION"
-        return
-    fi
-    local distpath=`getconfig "FreeBSDDist"`
-    if [ -n "$distpath" ]; then
-        local version=`getconfig "FreeBSDVersion"`
-        freebsd_runvars="$freebsd_runvars freebsd_distpath=$distpath/$arch \
-                         freebsd_version=$version"
-        return
-    fi
-    local anointment="freebsd build master $arch"
-    local flightjob=`./mg-anoint retrieve --tolerate-unprepared "$anointment"`
-    if [ -n "$flightjob" ]; then
-        freebsd_runvars="$freebsd_runvars freebsdbuildjob=${flightjob/ /.}"
-        return
+    # Check if the packages are provided externally, or else assume they
+    # are provided by the same flight as the installer binaries.
+    local pkgpath=`getconfig "FreeBSDPackages"`
+    counter=0
+    IFS=$'\n'
+    for flightjob in `./mg-anoint retrieve --tolerate-unprepared \
+                      "freebsd build master $arch" \
+                      "freebsd-packages build master $arch"`; do
+        if [ $counter -eq 0 ]; then
+            # Anointed FreeBSD installer
+            local envvar="FREEBSD_${arch^^}_BUILDJOB"
+            local distpath=`getconfig "FreeBSDDist"`
+            if [ -n "${!envvar}" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsdbuildjob=${!envvar}"
+            elif [ -n "$FREEBSD_DIST" ] && [ -n "$FREEBSD_VERSION" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsd_distpath=$FREEBSD_DIST/$arch \
+                                 freebsd_version=$FREEBSD_VERSION"
+            elif [ -n "$distpath" ]; then
+                local version=`getconfig "FreeBSDVersion"`
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsd_distpath=$distpath/$arch \
+                                 freebsd_version=$version"
+            elif [ "$flightjob" != "ERROR" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsdbuildjob=${flightjob/ /.}"
+            fi
+        elif [ $counter -eq 1 ]; then
+            # Anointed package repository
+            local envvar="FREEBSD_PACKAGES_${arch^^}_BUILDJOB"
+            local pkgpath=`getconfig "FreeBSDPackages"`
+            if [ -n "${!envvar}" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsdpackagesbuildjob=${!envvar}"
+            elif [ -n "$FREEBSD_PACKAGES" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsd_packages=$FREEBSD_PACKAGES/$arch"
+            elif [ -n "$pkgpath" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsd_packages=$pkgpath/$arch"
+            elif [ "$flightjob" != "ERROR" ]; then
+                freebsd_runvars="$freebsd_runvars \
+                                 freebsdpackagesbuildjob=${flightjob/ /.}"
+            fi
+        fi
+        counter=$((counter+1))
+    done
+    unset IFS
+
+    # Make sure we got exactly 2 results.
+    if [ $counter -ne 2 ]; then
+        exit 1
     fi
 }
 
@@ -202,6 +233,25 @@ create_freebsd_build_job () {
     $freebsd_runvars
 }
 
+create_freebsd_pkg_build_job () {
+  repos=`getrepos`
+  local name=$1
+  local svnrev=`repo_tree_git2svn_rev freebsd-ports $TREE_FREEBSD_PORTS \
+                                      $REVISION_FREEBSD_PORTS`
+
+  job_create_build $name build-pkg-freebsd                              \
+    arch=$arch                                                          \
+    $RUNVARS $BUILD_RUNVARS $BUILD_FREEBSD_RUNVARS                      \
+    $arch_runvars                                                       \
+    tree_freebsdports=$TREE_FREEBSD_PORTS                               \
+    revision_freebsdports=$REVISION_FREEBSD_PORTS                       \
+    svntree_freebsdports=$TREE_FREEBSD_PORTS_SVN                        \
+    svnrevision_freebsdports=$svnrev                                    \
+    host_hostflags=arch-$arch,purpose-build                             \
+    all_host_os=freebsd                                                 \
+    $freebsd_runvars recipe_skipbuildprep=true
+}
+
 create_xen_build_job () {
   local name=$1; shift
   local recipe=$1; shift

diff --git a/ts-build-prep-freebsd b/ts-build-prep-freebsd
index ef8805039ce3edd46418611c696e50c269256890..ec428227dbd6e8b95dbc03d22c8025af94924930 100755 (executable)
--- a/ts-build-prep-freebsd
+++ b/ts-build-prep-freebsd
@@ -30,6 +30,48 @@ $whhost ||= 'host';
 our $ho= selecthost($whhost);
 exit 0 if $ho->{SharedReady};
 
+# NB: the packages are built as part of the build-<arch>-freebsd-again
+# job, which installs the host using the newly compiled FreeBSD version.
+our $path_packages = $r{"freebsd_packages"} ||
+                     get_stashed("path_freebsdpackagesdist",
+                                 $r{"freebsdpackagesbuildjob"});
+
+sub bootstrap () {
+    # Create a host-specific link to the pkg binaries to use.
+    my $pkg_repo_url = create_weblink($ho, "packages", $path_packages);
+
+    target_cmd_root($ho, <<END, 600);
+set -ex
+# Set proxy for the pkg manager
+mkdir -p /usr/local/etc/
+cat << ENDPKG >> /usr/local/etc/pkg.conf
+pkg_env: { http_proxy = $c{HttpProxy} }
+default_always_yes: true
+assume_always_yes: true
+ENDPKG
+
+# Setup custom pkg repository
+mkdir -p /usr/local/etc/pkg/repos/
+cat << ENDREPO > /usr/local/etc/pkg/repos/osstest.conf
+osstest: {
+    url: "$pkg_repo_url",
+    enabled: yes,
+}
+ENDREPO
+# Disable default official FreeBSD mirror
+cat << ENDREPO > /usr/local/etc/pkg/repos/FreeBSD.conf
+FreeBSD: {
+    enabled: no,
+}
+ENDREPO
+
+# Bootstap the package manager
+export HTTP_PROXY=$c{HttpProxy}
+export ASSUME_ALWAYS_YES=yes
+pkg bootstrap
+END
+}
+
 sub install_deps () {
     my @packages = qw(git glib pkgconf yajl gmake pixman markdown gettext
                       python argp-standalone lzo2 git gcc binutils);
@@ -37,6 +79,7 @@ sub install_deps () {
     target_install_packages($ho, @packages);
 }
 
+bootstrap();
 install_deps();
 gitcache_setup($ho);
 

diff --git a/ts-freebsd-host-install b/ts-freebsd-host-install
index 3c3e9c34298436dd23f166ef09c7d753c4ec55c9..e8977e951f7133e9d7c043ec7ac750c41002fddb 100755 (executable)
--- a/ts-freebsd-host-install
+++ b/ts-freebsd-host-install
@@ -194,19 +194,6 @@ sysrc sendmail_submit_enable=NO
 sysrc sendmail_outbound_enable=NO
 sysrc sendmail_msp_queue_enable=NO
 
-# Set proxy for the pkg manager
-mkdir -p /usr/local/etc/
-cat << ENDPKG >> /usr/local/etc/pkg.conf
-pkg_env: { http_proxy = $c{HttpProxy} }
-default_always_yes: true
-assume_always_yes: true
-ENDPKG
-
-# Bootstap the package manager
-export HTTP_PROXY=$c{HttpProxy}
-export ASSUME_ALWAYS_YES=yes
-pkg bootstrap
-
 # Allow root user login and setup ssh keys
 chsh -s /bin/sh root
 echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config