From: Roger Pau Monne <roger.pau@citrix.com>
Date: Wed, 13 Jun 2018 07:40:15 +0000 (+0200)
Subject: osstest: introduce a FreeBSD build script
X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=55e096b528b6147663c67d00b973254818fce49e;p=people%2Froyger%2Fosstest.git

osstest: introduce a FreeBSD build script

In order to generate the FreeBSD installer image and the install
media.

The install sets are the vanilla ones generated by the 'ftp' release
target.  The installer image is handcrafted based on the filesystem
created by the 'bootonly' target, which is then populated with the ssh
host keys, and setup in order to use the serial console. The other
difference from upstream FreeBSD installer images is that the one
built by osstest uses a ramdisk instead of relying on the installer
media to be somehow attached, either on a CD or USB drive. This is
required in order to boot the image from pxelinux (where no CD or USB
is actually attached to the host, and everything is fetched from
tftp).

Due to the nature of the FreeBSD build, the outputs are different from
what osstest expects from a buildjob, more specifically
path_freebsddist points to a folder that contains the several outputs
form this buildjob.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
Changes since v4:
 - Remove some useless comments.
 - Use qw(git) instead of 'git'.
 - Replace unicode nonbreaking space.
 - Remove trail ','.
 - Improve the way to fetch the version number.

Changes since v3:
 - Introduce two helpers to perform the build process.
 - Install packages using target_install_packages.

Changes since v2:
 - Increase build target timeouts.
 - Use sysrc instead echo to set rc.conf options.

Changes since v1:
 - Remove the ts-build-check FreeBSD hack.
 - Use pkg-static instead of pkg.
 - Introduce buildcmd_stamped_logged_root and target_cmd_build_root.
 - Use target_cmd_build_root and buildcmd_stamped_logged_root in the
   ts-freebsd-build script.
 - Fix the script snippets to use <<END.<<'END' in order to avoid
   escaping the shell variables.
 - Set path_freebsddist runvar to point to the folder where the build
   files are stashed.
 - Add a comment at the top of the file describing what runvars are
   consumed/produced by the build script.
---

diff --git a/ts-freebsd-build b/ts-freebsd-build
new file mode 100755
index 0000000..54cb590
--- /dev/null
+++ b/ts-freebsd-build
@@ -0,0 +1,239 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2017 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Consumes the following input runvars:
+#
+# revision_freebsd: FreeBSD commit ID to generate the install media from.
+# tree_freebsd: FreeBSD git tree to fetch the source code from.
+#
+# Produces the following output:
+#
+# Compressed install sets: kernel.txz, base.txz
+# Compressed debug symbols for the kernel: kernel-dbg.txz
+# Manifest file (checkums): MANIFEST
+# Installer image: install.img
+#
+# Sets the following runvar:
+#
+# path_freebsddist: points to the folder where the above files are stored.
+# freebsd_buildversion: version of FreeBSD built.
+
+use strict qw(vars);
+use DBI;
+use POSIX;
+
+unshift @INC, qw(.);
+use Osstest;
+use Osstest::TestSupport;
+use Osstest::BuildSupport;
+
+tsreadconfig();
+
+selectbuildhost(\@ARGV);
+builddirsprops();
+
+sub install_deps () {
+    target_install_packages($ho, qw(git));
+}
+
+sub checkout () {
+    prepbuilddirs();
+
+    # Remove the directory as root, there might be files owned by root
+    target_cmd_build_root($ho, 300, $builddir, <<'END');
+# chflags will return error if the directory doesn't exist
+chflags -fR noschg freebsd || true
+rm -rf freebsd
+END
+
+    logm("Checkout the FreeBSD source tree");
+    build_clone($ho, 'freebsd', $builddir, 'freebsd');
+}
+
+sub build_target ($$$) {
+    my ($target, $prefix, $time) = @_;
+
+    buildcmd_stamped_logged($time, 'freebsd', $target,
+                            $prefix, <<END, '');
+make $makeflags $target
+END
+}
+
+sub build_release ($$$) {
+    my ($target, $prefix, $time) = @_;
+
+    buildcmd_stamped_logged_root($time, 'freebsd', "release-$target",
+                                 $prefix, <<END, '');
+make -C release $target
+END
+}
+
+sub build () {
+    my $authkeys = authorized_keys();
+    my $target = "bootonly";
+    my $prefix = <<END;
+export MAKEOBJDIRPREFIX=$builddir/obj
+export TARGET=$r{arch}
+END
+
+    # Build process as documented in the handbook:
+    # https://www.freebsd.org/doc/handbook/updating-src.html
+
+    logm("Cleaning up previous builds");
+    build_target('cleanworld', $prefix, 300);
+
+    logm("Building world");
+    build_target('buildworld', $prefix, 25200);
+
+    logm("Building kernel");
+    build_target('buildkernel', $prefix, 3600);
+
+    # NB: the steps below need to be done as root or the permissions
+    # of the files won't be properly set (and the target will fail).
+    logm("Creating the install sets");
+    build_release('ftp', $prefix, 3600);
+
+    logm("Populating the installer image");
+    build_release($target, $prefix, 3600);
+
+    logm("Placing ssh host keys");
+    foreach my $file (<$c{OverlayLocal}/etc/ssh/ssh_host_*_key*>) {
+        target_putfile_root($ho, 30, $file,
+                            "$builddir/freebsd/release/$target/etc/ssh/");
+    }
+
+    logm("Configuring the installer image");
+    target_cmd_build_root($ho, 30, $builddir, <<END.<<'END');
+authkeys="$authkeys"
+bauds="$c{Baud}"
+cd freebsd/release/$target
+END
+sysrc -f etc/rc.conf sshd_enable=YES
+
+echo 'PermitRootLogin yes' >> etc/ssh/sshd_config
+mkdir -p root/.ssh
+cat << ENDKEYS > root/.ssh/authorized_keys
+$authkeys
+ENDKEYS
+
+chown root:wheel etc/ssh/ssh_host_*_key*
+chmod 0600 etc/ssh/ssh_host_*_key
+chmod 0644 etc/ssh/ssh_host_*_key.pub
+
+# Setup serial console output for stage1
+printf "%s" "-h -S$bauds" >> boot.config
+cat << ENDBOOT >> boot/loader.conf
+# Serial console configuration
+boot_serial="YES"
+comconsole_speed="$bauds"
+console="comconsole"
+boot_verbose="YES"
+beastie_disable="YES"
+
+# mfs boot parameters
+mfs_load="YES"
+mfs_type="mfs_root"
+mfs_name="/mfsroot"
+vfs.root.mountfrom="ufs:/dev/ufs/FreeBSD_Install"
+ENDBOOT
+
+sysrc -f etc/rc.conf ifconfig_DEFAULT=DHCP
+
+# Remove the local script that launches the installer by default
+rm -rf etc/rc.local
+
+# Create a temporary fstab with the root dir
+echo '/dev/ufs/FreeBSD_Install / ufs rw 1 1' > etc/fstab
+
+# Remove the linked resolv.conf
+rm -rf etc/resolv.conf
+END
+
+    logm("Create the installer");
+    target_cmd_build_root($ho, 900, $builddir, <<END.<<'END');
+target="freebsd/release/$target"
+output="install.img"
+END
+mkdir -p $output.tmp
+
+# Do some pruning
+rm -rf $target/usr/share/man
+rm -rf $target/usr/share/examples
+rm -rf $target/usr/share/doc
+rm -rf $target/usr/share/dtrace
+
+# Create a mfs root image
+makefs -b 10% -B little -o label=FreeBSD_Install $output.tmp/mfsroot $target
+gzip $output.tmp/mfsroot
+
+# Copy boot to the staging dir
+cp -r $target/boot $output.tmp/
+cp $target/boot.config $output.tmp/
+
+# The loader doesn't need any modules in order to boot into the mfsroot.
+# The rest of the modules can be loaded from the mfs root itself.
+rm -f $output.tmp/boot/kernel/*.ko
+
+gzip $output.tmp/boot/kernel/kernel
+
+makefs -B little $output.part $output.tmp
+
+mkimg -s gpt -b $target/boot/pmbr -p efi:=$target/boot/boot1.efifat \
+    -p freebsd-boot:=$target/boot/gptboot -p freebsd-ufs:=$output.part \
+    -p freebsd-swap::1M -o $output
+
+rm $output.part
+rm -rf $output.tmp
+END
+}
+
+sub stash () {
+    my @sets = qw(MANIFEST base.txz kernel.txz);
+    my @symbols = qw(kernel-dbg.txz);
+
+    logm("Stashing FreeBSD build output");
+    foreach my $set (@sets) {
+        built_stash_file($ho, $builddir, $set,
+                         "freebsd/release/ftp/$set", 0);
+    }
+    foreach my $symbol (@symbols) {
+        built_stash_debugfile($ho, $builddir, $symbol,
+                              "freebsd/release/ftp/$symbol", 0);
+    }
+    built_stash_file($ho, $builddir, "install.img", "install.img", 0);
+
+    my $srcversion = target_cmd_output_root($ho, <<END, 30);
+set -e
+cd $builddir/freebsd
+eval `make buildenvvars`
+test -n "\$SRCRELDATE"
+expr "\$SRCRELDATE" / 100000
+END
+    store_runvar("freebsd_buildversion", "$srcversion");
+
+    # Set path_freebsddist to point to the build output folder
+    # in order to make ts-build-check happy.
+    store_runvar("path_freebsddist", "build/");
+}
+
+install_deps();
+checkout();
+build();
+stash();
+
+logm("FreeBSD build successful");
+