DATA_PATH=/root/os-ext-data
# Steps to reinstall:
-# 1) Create new Ubuntu 14.04 server (copy password)
-# 1a) 7.5GB Compute v1 flavor
-# 1b) Enable monitoring and security updates
-# 2) Disable password authentication
+# 1) Log in to mycloud.rackspace.com using credentials from os-ext-data/xenlibvirt-nodepool
+# 1a) Create new Ubuntu 14.04 server (copy password), hostname 'jenkins-libvirt'
+# 7.5GB Compute v1 flavor
+# Enable monitoring and security updates
+# 1b) Save the password for use in step 2a
+# 1c) Add key from os-ext-data/xenproject_jenkins.pub with the name 'xenproject-nodepool'
+# 2) Disable password authentication on jenkins server
# 2a) ssh-copy-id to copy a key to the server
# 2b) edit /etc/sshd_config to set "PermitRootLogin without-password"
+# 2c) service ssh restart
# 3) Copy the secret credentials dir (http://hg.uk.xensource.com/openstack/infrastructure.hg/os-ext-data) to /root
-# 4) Clone this repo git:
-# 4a) git clone https://github.com/citrix-openstack/os-ext-testing.git
-# 4b) cd os-ext-testing; git checkout common_ci
-# ?) Follow steps below
+# 4) Clone this repo:
+# 4a) apt-get install git
+# 4b) git clone https://github.com/citrix-openstack/os-ext-testing.git
+# 4c) cd os-ext-testing; git checkout common_ci
+# 5) Run below commands (or just this script) to do the 'standard' install
+# 6) The jobs need an additional plugin in Jenkins to generate correctly, so:
+# 6a) Install Post-Build Script jenkins plugin (including restarting Jenkins)
+# 6b) Regenerate jenkins jobs: jenkins-jobs update --delete-old /etc/jenkins_jobs/config
# ?) Set up monitoring checks https://intelligence.rackspace.com/cloud/entities/enWCIYVVnt
# Copied from the following URL Feb 2016
sed -i -e 's/^\(127\.0\.0\.1.*\)$/\1 jenkins/' /etc/hosts
sudo puppet apply --verbose /etc/puppet/manifests/site.pp
+
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from common import run_local
+
+DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
+CACHEDIR = os.path.expanduser('~/cache/files')
+
+# Some jobs might require newer distro packages, so we can pre-cache
+# deb packages from specified Ubuntu Cloud Archive pockets.
+UCA_POCKETS = []
+
+
+def git_branches():
+ branches = []
+ for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
+ branch = branch.strip()
+ if not branch.startswith('remotes/origin'):
+ continue
+ branches.append(branch)
+ return branches
+
+
+def tokenize(fn, tokens, distribution, comment=None):
+ for line in open(fn):
+ if 'dist:' in line and ('dist:%s' % distribution not in line):
+ continue
+ if 'qpid' in line:
+ continue # TODO: explain why this is here
+ if comment and comment in line:
+ line = line[:line.rfind(comment)]
+ line = line.strip()
+ if line and line not in tokens:
+ tokens.append(line)
+
+
+def _legacy_find_images(basedir):
+ """Divine what images we should use based on parsing stackrc."""
+ images = []
+ for line in open(os.path.join(basedir, 'stackrc')):
+ line = line.strip()
+ if line.startswith('IMAGE_URLS'):
+ if '#' in line:
+ line = line[:line.rfind('#')]
+ if line.endswith(';;'):
+ line = line[:-2]
+ line = line.split('=', 1)[1].strip()
+ if line.startswith('${IMAGE_URLS:-'):
+ line = line[len('${IMAGE_URLS:-'):]
+ if line.endswith('}'):
+ line = line[:-1]
+ if not line:
+ continue
+ if line[0] == line[-1] == '"':
+ line = line[1:-1]
+ # Add image to the list to be downloaded, but
+ # skip downloading giant vmware images
+ images += [x.strip() for x in line.split(',')
+ if not x.strip().endswith('vmdk')]
+ return images
+
+
+def _find_images(basedir):
+ images = []
+ image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
+ if os.path.exists(image_tool):
+ returncode, out = run_local(image_tool, status=True)
+ if returncode:
+ print "%s failed" % image_tool
+ print "Exit: %s, Output: %s" % (returncode, out)
+ # reset images so we'll fall back
+ images = []
+ else:
+ images = out.split('\n')
+ return images
+
+
+def local_prep(distribution):
+ branches = []
+ for branch in git_branches():
+ # Ignore branches of the form 'somestring -> someotherstring'
+ # as this denotes a symbolic reference and the entire string
+ # as is cannot be checked out. We can do this safely as the
+ # reference will refer to one of the other branches returned
+ # by git_branches.
+ if ' -> ' in branch:
+ continue
+ branch_data = {'name': branch}
+ print 'Branch: ', branch
+ run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
+ run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
+
+ if os.path.exists('/usr/bin/apt-get'):
+ debs = []
+ debdir = os.path.join(DEVSTACK, 'files', 'debs')
+ if not os.path.exists(debdir):
+ debdir = os.path.join(DEVSTACK, 'files', 'apts')
+ for fn in os.listdir(debdir):
+ fn = os.path.join(debdir, fn)
+ tokenize(fn, debs, distribution, comment='#')
+ branch_data['debs'] = debs
+
+ if os.path.exists('/usr/bin/yum'):
+ rpms = []
+ rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
+ for fn in os.listdir(rpmdir):
+ fn = os.path.join(rpmdir, fn)
+ tokenize(fn, rpms, distribution, comment='#')
+ branch_data['rpms'] = rpms
+
+ images = _find_images(DEVSTACK)
+ if not images:
+ images = _legacy_find_images(DEVSTACK)
+
+ branch_data['images'] = images
+ branches.append(branch_data)
+ return branches
+
+
+def download(url, fname):
+ run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
+
+
+def cache_debs(debs, uca_pocket=None):
+ """Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
+ Archive pocket. If a UCA pocket is specified, it is enabled temporarily
+ for caching only.
+ """
+ if uca_pocket:
+ # Note this will install the ubuntu-cloud-keyring package which
+ # contains the required GPG key.
+ run_local(['sudo', 'add-apt-repository', '-y',
+ 'cloud-archive:%s' % uca_pocket])
+ run_local(['sudo', 'apt-get', 'update'])
+ run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
+ if uca_pocket:
+ run_local(['sudo', 'rm', '-f',
+ '/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
+ run_local(['sudo', 'apt-get', 'update'])
+
+
+def main():
+ distribution = sys.argv[1]
+
+ branches = local_prep(distribution)
+ image_filenames = []
+ for branch_data in branches:
+ if branch_data.get('debs'):
+ cache_debs(branch_data['debs'])
+ for uca in sorted(UCA_POCKETS):
+ cache_debs(branch_data['debs'], uca)
+ elif branch_data.get('rpms'):
+ run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
+ branch_data['rpms'])
+ else:
+ sys.exit('No supported package data found.')
+
+ for url in branch_data['images']:
+ fname = url.split('/')[-1]
+ if fname in image_filenames:
+ continue
+ image_filenames.append(fname)
+ download(url, fname)
+
+ # cache get-pip, because upstream network connection fails more
+ # often than you might imagine.
+ download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
+
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import re
+import shutil
+import sys
+import urllib2
+
+from common import run_local
+
+URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
+ 'plain/gerrit/projects.yaml')
+PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
+
+# Not using an arg libraries in order to avoid module imports that
+# are not available across all python versions
+if len(sys.argv) > 1:
+ GIT_BASE = sys.argv[1]
+else:
+ GIT_BASE = 'git://git.openstack.org'
+
+
+def clone_repo(project):
+ remote = '%s/%s.git' % (GIT_BASE, project)
+
+ # Clear out any existing target directory first, in case of a retry.
+ try:
+ shutil.rmtree(os.path.join('/opt/git', project))
+ except OSError:
+ pass
+
+ # Try to clone the requested git repository.
+ (status, out) = run_local(['git', 'clone', remote, project],
+ status=True, cwd='/opt/git')
+
+ # If it claims to have worked, make sure we can list branches.
+ if status == 0:
+ (status, moreout) = run_local(['git', 'branch', '-a'], status=True,
+ cwd=os.path.join('/opt/git', project))
+ out = '\n'.join((out, moreout))
+
+ # If that worked, try resetting to HEAD to make sure it's there.
+ if status == 0:
+ (status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
+ status=True,
+ cwd=os.path.join('/opt/git', project))
+ out = '\n'.join((out, moreout))
+
+ # Status of 0 imples all the above worked, 1 means something failed.
+ return (status, out)
+
+
+def main():
+ # TODO(jeblair): use gerrit rest api when available
+ data = urllib2.urlopen(URL).read()
+ for line in data.split('\n'):
+ # We're regex-parsing YAML so that we don't have to depend on the
+ # YAML module which is not in the stdlib.
+ m = PROJECT_RE.match(line)
+ if m:
+ (status, out) = clone_repo(m.group(1))
+ print out
+ if status != 0:
+ print 'Retrying to clone %s' % m.group(1)
+ (status, out) = clone_repo(m.group(1))
+ print out
+ if status != 0:
+ raise Exception('Failed to clone %s' % m.group(1))
+
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+
+def run_local(cmd, status=False, cwd='.', env={}):
+ print "Running:", cmd
+ newenv = os.environ
+ newenv.update(env)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
+ stderr=subprocess.STDOUT, env=newenv)
+ (out, nothing) = p.communicate()
+ if status:
+ return (p.returncode, out.strip())
+ return out.strip()
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /etc/nodepool/provider
+
+# Generate the AFS Slug from the host system.
+source /usr/local/jenkins/slave_scripts/afs-slug.sh
+
+NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org}
+NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]')
+NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple}
+NODEPOOL_WHEEL_MIRROR=${NODEPOOL_WHEEL_MIRROR:-http://$NODEPOOL_MIRROR_HOST/wheel/$AFS_SLUG}
+NODEPOOL_UBUNTU_MIRROR=${NODEPOOL_UBUNTU_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu}
+
+cat >/tmp/pip.conf <<EOF
+[global]
+timeout = 60
+index-url = $NODEPOOL_PYPI_MIRROR
+trusted-host = $NODEPOOL_MIRROR_HOST
+extra-index-url = $NODEPOOL_WHEEL_MIRROR
+EOF
+sudo mv /tmp/pip.conf /etc/pip.conf
+
+cat >/home/jenkins/.pydistutils.cfg <<EOF
+[easy_install]
+index_url = $NODEPOOL_PYPI_MIRROR
+allow_hosts = *.openstack.org
+EOF
+
+# Double check that when the node is made ready it is able
+# to resolve names against DNS.
+host git.openstack.org
+host $NODEPOOL_MIRROR_HOST
+
+LSBDISTID=$(lsb_release -is)
+LSBDISTCODENAME=$(lsb_release -cs)
+if [ "$LSBDISTID" == "Ubuntu" ] ; then
+ sudo dd of=/etc/apt/sources.list <<EOF
+deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME main universe
+deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-updates main universe
+deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-backports main universe
+deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-security main universe
+EOF
+ if [ "$LSBDISTCODENAME" != 'precise' ] ; then
+ # Turn off multi-arch
+ sudo dpkg --remove-architecture i386
+ fi
+ # Turn off checking of GPG signatures
+ sudo dd of=/etc/apt/apt.conf.d/99unauthenticated <<EOF
+APT::Get::AllowUnauthenticated "true";
+EOF
+fi
--- /dev/null
+#!/bin/bash
+
+# Copyright (C) 2011-2014 OpenStack Foundation
+# Copyright (c) 2014 Citrix Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Installation
+# ~~~~~~~~~~~~
+# 1.) Start an Ubuntu HVM instance in the Rackspace cloud.
+# 2.) Copy the files from this directory to the instance.
+# 3.) Execute this script on the instance with 3 parameters:
+# - First parameter is the password for the XenServer
+# - Second parameter is a URL to an xva appliance to be installed. For
+# building such an appliance, see:
+# https://github.com/citrix-openstack/openstack-xenapi-testing-xva
+# - Third parameter should be the name-label for the appliance.
+# 4.) Poll the public IP through ssh, and Wait until the file
+# "$FILE_TO_TOUCH_ON_COMPLETION" exists
+#
+#
+# Snapshots
+# ~~~~~~~~~
+# 1.) Delete "$FILE_TO_TOUCH_ON_COMPLETION"
+# 2.) Create snapshot
+# 3.) When booting instances from the snapshot, poll
+# "$FILE_TO_TOUCH_ON_COMPLETION"
+#
+#
+# Details
+# ~~~~~~~
+# WARINING: This script is a workaround, and should only be used for test
+# purposes.
+#
+# This script will install itself as a boot-time script, and will be executed
+# on each system startup. It shrinks the actual Ubuntu installation, and will
+# install a XenServer on the remaining space. This script will also be
+# installed as a startup script in XenServer. After the installation, whenever
+# the instance is restarted, Ubuntu will boot, the networking parameters will
+# be copied to a file, and the instance will be automatically rebooted into
+# XenServer, where the networking will be adjusted, according to the saved
+# parameters. This trickery is needed, because config drive does not include
+# the networking parameters.
+#
+# The appliance will be accessible through the public IP of the instance,
+# and dom0 could be accessed from the instance, on the IP address:
+# 192.168.33.2
+#
+# Logging
+# ~~~~~~~
+# All the output of this script is logged to $0.log
+
+set -eux
+
+THIS_FILE="$(readlink -f $0)"
+THIS_DIR="$(dirname $THIS_FILE)"
+INSTALL_DIR="$(dirname $THIS_FILE)"
+STATE_FILE="${THIS_FILE}.state"
+LOG_FILE="${THIS_FILE}.log"
+ADDITIONAL_PARAMETERS="$@"
+
+XENSERVER_PASSWORD="$1"
+STAGING_APPLIANCE_URL="$2"
+APPLIANCE_NAME="$3"
+XENSERVER_ISO_URL="$4"
+FILE_TO_TOUCH_ON_COMPLETION="/var/run/xenserver.ready"
+
+# It is assumed, that the appliance has a user, DOMZERO_USER, and by
+# writing the /local/domain/$DOMID/authorized_keys/$DOMZERO_USER xenstore
+# key, the key's contents will be copied into this user's authorized_keys file
+DOMZERO_USER=domzero
+
+# Do not change this variable, as this path is hardcoded into XenServer's
+# installer. The installer will look for this directory on the device
+# specified by the boot parameter make-ramdisk, and copy the contents of this
+# directory to ramdisk.
+XSINST_DIRECTORY="/xsinst"
+
+function main {
+ case "$(get_state)" in
+ "START")
+ dump_disk_config
+ run_this_script_on_each_boot
+ download_xenserver_files /root/xenserver.iso
+ download_appliance "$STAGING_APPLIANCE_URL"
+ create_ramdisk_contents /root/xenserver.iso
+ extract_xs_installer /root/xenserver.iso /opt/xs-install
+ generate_xs_installer_grub_config /opt/xs-install \
+ file:///tmp/ramdisk/answerfile.xml
+ configure_grub
+ update-grub
+ create_resizing_initramfs_config
+ update-initramfs -u
+ set_state "SETUP_INSTALLER"
+ reboot
+ ;;
+ "SETUP_INSTALLER")
+ dump_disk_config
+ delete_resizing_initramfs_config
+ update-initramfs -u
+ store_cloud_settings "$XSINST_DIRECTORY/cloud-settings"
+ store_authorized_keys "$XSINST_DIRECTORY/authorized_keys"
+ set_xenserver_installer_as_nextboot
+ set_state "XENSERVER_FIRSTBOOT"
+ reboot
+ ;;
+ "XENSERVER_FIRSTBOOT")
+ wait_for_xapi
+ forget_networking
+ transfer_settings_to_appliance "/root/cloud-settings"
+ add_boot_config_for_ubuntu /mnt/ubuntu/boot /boot/
+ start_ubuntu_on_next_boot /boot/
+ set_state "UBUNTU"
+ sync
+ create_done_file_on_appliance
+ ;;
+ "UBUNTU")
+ mount_dom0_fs /mnt/dom0
+ wait_for_networking
+ store_cloud_settings /mnt/dom0/root/cloud-settings
+ store_authorized_keys /mnt/dom0/root/.ssh/authorized_keys
+ start_xenserver_on_next_boot /mnt/dom0/boot
+ set_state "XENSERVER"
+ reboot
+ ;;
+ "XENSERVER")
+ wait_for_xapi
+ forget_networking
+ transfer_settings_to_appliance "/root/cloud-settings"
+ start_ubuntu_on_next_boot /boot/
+ set_state "UBUNTU"
+ sync
+ create_done_file_on_appliance
+ ;;
+ esac
+}
+
+function set_state {
+ local state
+
+ state="$1"
+
+ echo "$state" > $STATE_FILE
+}
+
+function get_state {
+ if [ -e "$STATE_FILE" ]; then
+ cat $STATE_FILE
+ else
+ echo "START"
+ fi
+}
+
+function create_resizing_initramfs_config {
+ cp "$THIS_DIR/xenserver_helper_initramfs_hook" \
+ /usr/share/initramfs-tools/hooks/resize
+ chmod +x /usr/share/initramfs-tools/hooks/resize
+
+ cp "$THIS_DIR/xenserver_helper_initramfs_premount" \
+ /usr/share/initramfs-tools/scripts/local-premount/resize
+ chmod +x /usr/share/initramfs-tools/scripts/local-premount/resize
+}
+
+function delete_resizing_initramfs_config {
+ rm -f /usr/share/initramfs-tools/hooks/resize
+ rm -f /usr/share/initramfs-tools/scripts/local-premount/resize
+}
+
+function run_this_script_on_each_boot {
+ cat > /etc/init/xenserver.conf << EOF
+start on stopped rc RUNLEVEL=[2345]
+
+task
+
+script
+ /bin/bash $THIS_FILE $ADDITIONAL_PARAMETERS >> $LOG_FILE 2>&1
+end script
+EOF
+}
+
+function create_done_file_on_appliance {
+ while ! echo "sudo touch $FILE_TO_TOUCH_ON_COMPLETION" | bash_on_appliance; do
+ sleep 1
+ done
+}
+
+function download_xenserver_files {
+ local tgt
+
+ tgt="$1"
+
+ wget -qO "$tgt" "$XENSERVER_ISO_URL"
+}
+
+function download_appliance {
+ local appliance_url
+
+ appliance_url="$1"
+
+ wget -qO /root/staging_vm.xva "$appliance_url"
+}
+
+function print_answerfile {
+ local repository
+ local postinst
+ local xenserver_pass
+
+ repository="$1"
+ postinst="$2"
+ xenserver_pass="$3"
+
+ cat << EOF
+<?xml version="1.0"?>
+<installation srtype="ext">
+<primary-disk preserve-first-partition="true">sda</primary-disk>
+<keymap>us</keymap>
+<root-password>$xenserver_pass</root-password>
+<source type="url">$repository</source>
+<admin-interface name="eth0" proto="static">
+<ip>192.168.34.2</ip>
+<subnet-mask>255.255.255.0</subnet-mask>
+<gateway>192.168.34.1</gateway>
+</admin-interface>
+<timezone>UTC</timezone>
+<script stage="filesystem-populated" type="url">$postinst</script>
+</installation>
+EOF
+}
+
+function print_postinst_file {
+ local rclocal
+ rclocal="$1"
+
+ cat << EOF
+#!/bin/sh
+touch \$1/tmp/postinst.sh.executed
+cp \$1/etc/rc.d/rc.local \$1/etc/rc.d/rc.local.backup
+cat $rclocal >> \$1/etc/rc.d/rc.local
+cp /tmp/ramdisk/cloud-settings \$1/root/
+cp /tmp/ramdisk/authorized_keys \$1/root/.ssh/
+EOF
+}
+
+function print_rclocal {
+ cat << EOF
+# This is the contents of the rc.local file on XenServer
+mkdir -p /mnt/ubuntu
+mount /dev/sda1 /mnt/ubuntu
+mkdir -p $(dirname $INSTALL_DIR)
+[ -L $INSTALL_DIR ] || ln -s /mnt/ubuntu${INSTALL_DIR} $INSTALL_DIR
+/bin/bash $THIS_FILE $ADDITIONAL_PARAMETERS >> $LOG_FILE 2>&1
+EOF
+}
+
+function create_ramdisk_contents {
+ local isofile
+ local target_dir
+
+ isofile="$1"
+ target_dir="$XSINST_DIRECTORY"
+
+ mkdir "$target_dir"
+ ln "$isofile" "$target_dir/xenserver.iso"
+ print_rclocal > "$target_dir/rclocal"
+ print_postinst_file "/tmp/ramdisk/rclocal" > "$target_dir/postinst.sh"
+ print_answerfile \
+ "file:///tmp/ramdisk" \
+ "file:///tmp/ramdisk/postinst.sh" \
+ "$XENSERVER_PASSWORD" > "$target_dir/answerfile.xml"
+}
+
+function extract_xs_installer {
+ local isofile
+ local targetpath
+
+ isofile="$1"
+ targetpath="$2"
+
+ local mountdir
+
+ mountdir=$(mktemp -d)
+ mount -o loop $isofile $mountdir
+ mkdir -p $targetpath
+ cp \
+ $mountdir/install.img \
+ $mountdir/boot/xen.gz \
+ $mountdir/boot/vmlinuz \
+ $targetpath
+ umount $mountdir
+}
+
+function generate_xs_installer_grub_config {
+ local bootfiles
+ local answerfile
+
+ bootfiles="$1"
+ answerfile="$2"
+
+ cat > /etc/grub.d/45_xs-install << EOF
+#!/bin/sh
+exec tail -n +3 \$0
+menuentry 'XenServer installer' {
+ multiboot $bootfiles/xen.gz dom0_max_vcpus=1-2 dom0_mem=max:752M com1=115200,8n1 console=com1,vga
+ module $bootfiles/vmlinuz xencons=hvc console=tty0 make-ramdisk=/dev/sda1 answerfile=$answerfile install
+ module $bootfiles/install.img
+}
+EOF
+ chmod +x /etc/grub.d/45_xs-install
+}
+
+function configure_grub {
+ sed -ie 's/^GRUB_HIDDEN_TIMEOUT/#GRUB_HIDDEN_TIMEOUT/g' /etc/default/grub
+ sed -ie 's/^GRUB_HIDDEN_TIMEOUT_QUIET/#GRUB_HIDDEN_TIMEOUT_QUIET/g' \
+ /etc/default/grub
+ # sed -ie 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=-1/g' /etc/default/grub
+ sed -ie 's/^.*GRUB_TERMINAL=.*$/GRUB_TERMINAL=console/g' /etc/default/grub
+ sed -ie 's/GRUB_DEFAULT=0/GRUB_DEFAULT=saved/g' /etc/default/grub
+}
+
+function set_xenserver_installer_as_nextboot {
+ grub-set-default "XenServer installer"
+}
+
+function store_cloud_settings {
+ local targetpath
+
+ targetpath="$1"
+
+ cat > $targetpath << EOF
+ADDRESS=$(grep -m 1 "address" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2)
+NETMASK=$(grep -m 1 "netmask" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2)
+GATEWAY=$(grep -m 1 "gateway" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2)
+MACADDRESS=$(ifconfig eth0 | sed -ne 's/.*HWaddr \(.*\)$/\1/p' | tr -d " ")
+NAMESERVERS=$(cat /etc/resolv.conf | grep nameserver | cut -d " " -f 2 | sort | uniq | tr '\n' , | sed -e 's/,$//g')
+EOF
+}
+
+function store_authorized_keys {
+ local targetpath
+
+ targetpath="$1"
+
+ cp /root/.ssh/authorized_keys $1
+}
+
+function wait_for_xapi {
+ while ! [ -e /var/run/xapi_init_complete.cookie ]; do
+ sleep 1
+ done
+}
+
+function forget_networking {
+ xe host-management-disable
+ IFS=,
+ for vlan in $(xe vlan-list --minimal); do
+ xe vlan-destroy uuid=$vlan
+ done
+
+ unset IFS
+ IFS=,
+ for pif in $(xe pif-list --minimal); do
+ xe pif-forget uuid=$pif
+ done
+ unset IFS
+}
+
+function add_boot_config_for_ubuntu {
+ local ubuntu_bootfiles
+ local bootfiles
+
+ ubuntu_bootfiles="$1"
+ bootfiles="$2"
+
+ local kernel
+ local initrd
+
+ kernel=$(ls -1c $ubuntu_bootfiles/vmlinuz-* | head -1)
+ initrd=$(ls -1c $ubuntu_bootfiles/initrd.img-* | head -1)
+
+ cp $kernel $bootfiles/vmlinuz-ubuntu
+ cp $initrd $bootfiles/initrd-ubuntu
+
+ cat >> $bootfiles/extlinux.conf << UBUNTU
+label ubuntu
+ LINUX $bootfiles/vmlinuz-ubuntu
+ APPEND root=/dev/xvda1 ro quiet splash
+ INITRD $bootfiles/initrd-ubuntu
+UBUNTU
+}
+
+function start_ubuntu_on_next_boot {
+ local bootfiles
+
+ bootfiles="$1"
+
+ sed -ie 's,default xe,default ubuntu,g' $bootfiles/extlinux.conf
+
+ log_extlinux $bootfiles/extlinux.conf
+}
+
+function start_xenserver_on_next_boot {
+ local bootfiles
+
+ bootfiles="$1"
+
+ sed -ie 's,default ubuntu,default xe,g' $bootfiles/extlinux.conf
+
+ log_extlinux $bootfiles/extlinux.conf
+}
+
+function log_extlinux {
+ local extlinux_conf
+
+ extlinux_conf="$1"
+
+ echo "ACTUAL STATE OF EXTLINUX IS"
+ cat $extlinux_conf
+}
+
+function mount_dom0_fs {
+ local target
+
+ target="$1"
+
+ mkdir -p $target
+ mount /dev/xvda2 $target
+}
+
+function wait_for_networking {
+ while ! ping -c 1 xenserver.org > /dev/null 2>&1; do
+ sleep 1
+ done
+}
+
+function bash_on_appliance {
+ local vm_ip
+ local vm
+
+ vm=$(xe vm-list name-label="$APPLIANCE_NAME" --minimal)
+
+ [ -n "$vm" ]
+
+ # Wait until appliance is accessible
+ while ! ping -c 1 "${vm_ip:-}" > /dev/null 2>&1; do
+ vm_ip=$(xe vm-param-get param-name=networks uuid=$vm | sed -e 's,^.*0/ip: ,,g' | sed -e 's,;.*$,,g')
+ sleep 1
+ done
+
+ ssh \
+ -q \
+ -i /root/dom0key \
+ -o UserKnownHostsFile=/dev/null \
+ -o StrictHostKeyChecking=no \
+ -o BatchMode=yes \
+ "$DOMZERO_USER@$vm_ip" bash -e -u -s -x -- "$@"
+}
+
+function configure_networking {
+ local network_settings
+
+ network_settings="$1"
+
+ . "$network_settings"
+
+ xe pif-introduce \
+ device=eth0 host-uuid=$(xe host-list --minimal) mac=$MACADDRESS
+
+ PIF=$(xe pif-list device=eth0 --minimal)
+ HOST_INT_NET=$(xe network-list name-label="Host internal management network" --minimal)
+
+ ORIGINAL_MGT_NET=$(xe pif-param-get param-name=network-uuid uuid=$PIF)
+ NEW_MGT_NET=$(xe network-create name-label=mgt name-description=mgt)
+ NEW_MGT_VLAN=$(xe vlan-create vlan=100 pif-uuid=$PIF network-uuid=$NEW_MGT_NET)
+ NEW_PIF=$(xe pif-list VLAN=100 device=eth0 --minimal)
+ VM=$(xe vm-list name-label="$APPLIANCE_NAME" --minimal)
+ APP_IMPORTED_NOW="false"
+ if [ -z "$VM" ]; then
+ VM=$(xe vm-import filename=/mnt/ubuntu/root/staging_vm.xva)
+ xe vm-param-set name-label="$APPLIANCE_NAME" uuid=$VM
+ xe vm-param-set VCPUs-max=6 uuid=$VM
+ xe vm-param-set VCPUs-at-startup=6 uuid=$VM
+ APP_IMPORTED_NOW="true"
+ fi
+ DNS_ADDRESSES=$(echo "$NAMESERVERS" | sed -e "s/,/ /g")
+
+ xe pif-reconfigure-ip \
+ uuid=$PIF \
+ mode=static \
+ IP=0.0.0.0 \
+ netmask=0.0.0.0
+
+ xe pif-reconfigure-ip \
+ uuid=$NEW_PIF \
+ mode=static \
+ IP=192.168.33.2 \
+ netmask=255.255.255.0 \
+ gateway=192.168.33.1 \
+ DNS=192.168.33.1
+
+ xe host-management-reconfigure pif-uuid=$NEW_PIF
+
+ # Purge all vifs of appliance
+ IFS=,
+ for vif in $(xe vif-list vm-uuid=$VM --minimal); do
+ xe vif-destroy uuid=$vif
+ done
+ unset IFS
+
+ # Create vifs for the appliance
+ xe vif-create vm-uuid=$VM network-uuid=$HOST_INT_NET device=0
+ xe vif-create vm-uuid=$VM network-uuid=$ORIGINAL_MGT_NET mac=$MACADDRESS \
+ device=1
+ xe vif-create vm-uuid=$VM network-uuid=$NEW_MGT_NET device=2
+
+ xe vm-start uuid=$VM
+
+ # Wait until appliance is accessible
+ while ! ping -c 1 "${VM_IP:-}" > /dev/null 2>&1; do
+ VM_IP=$(xe vm-param-get param-name=networks uuid=$VM | sed -e 's,^.*0/ip: ,,g' | sed -e 's,;.*$,,g')
+ sleep 1
+ done
+
+ if [ "$APP_IMPORTED_NOW" = "true" ]; then
+ rm -f /root/dom0key
+ rm -f /root/dom0key.pub
+ ssh-keygen -f /root/dom0key -P "" -C "dom0"
+ DOMID=$(xe vm-param-get param-name=dom-id uuid=$VM)
+
+ # Authenticate temporary key to appliance
+ xenstore-write /local/domain/$DOMID/authorized_keys/$DOMZERO_USER \
+ "$(cat /root/dom0key.pub)"
+ xenstore-chmod -u /local/domain/$DOMID/authorized_keys/$DOMZERO_USER r$DOMID
+
+ while ! echo "true" | bash_on_appliance; do
+ echo "waiting for key to be activated"
+ sleep 1
+ done
+
+ # Remove authorized_keys updater
+ echo "echo \"\" | crontab -" | bash_on_appliance
+
+ # Create an ssh key for domzero user
+ echo "ssh-keygen -f /home/$DOMZERO_USER/.ssh/id_rsa -C $DOMZERO_USER@appliance -N \"\" -q" | bash_on_appliance
+ fi
+
+ # Update network configuration
+ {
+ cat << EOF
+sudo tee /etc/network/interfaces
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet dhcp
+
+auto eth1
+iface eth1 inet static
+ address $ADDRESS
+ netmask $NETMASK
+ gateway $GATEWAY
+ dns-nameservers $DNS_ADDRESSES
+
+auto eth2
+ iface eth2 inet static
+ address 192.168.33.1
+ netmask 255.255.255.0
+EOF
+ } | bash_on_appliance
+
+ tmpdomzerokey=$(mktemp)
+
+ # Enable domzero user to log in to dom0
+ echo "cat /home/$DOMZERO_USER/.ssh/id_rsa.pub" | bash_on_appliance > $tmpdomzerokey
+
+ # Update ssh keys and reboot, so settings applied
+ {
+ echo "sudo tee /root/.ssh/authorized_keys"
+ cat /root/.ssh/authorized_keys
+ } | bash_on_appliance
+
+ echo "sudo reboot" | bash_on_appliance
+
+ cat $tmpdomzerokey >> /root/.ssh/authorized_keys
+}
+
+function transfer_settings_to_appliance {
+ local network_settings
+
+ network_settings="$1"
+
+ configure_networking "$network_settings"
+ /opt/xensource/libexec/interface-reconfigure rewrite
+}
+
+function dump_disk_config {
+ echo "DUMPING Primary disk's configuration"
+ sfdisk -d /dev/xvda
+}
+
+main
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p ~/cache/files
+mkdir -p ~/cache/pip
+
+if [ -f /usr/bin/yum ]; then
+ sudo yum -y install python-devel python3-devel make automake gcc gcc-c++ \
+ kernel-devel redhat-lsb-core
+elif [ -f /usr/bin/apt-get ]; then
+ if [ "$(lsb_release -c -s)" = "precise" ]; then
+ # temporary - remove the breaking -70 kernel and
+ # reinstall -69 to prevent openvswitch breakage
+ sudo DEBIAN_FRONTEND=noninteractive apt-get \
+ --option "Dpkg::Options::=--force-confold" \
+ --assume-yes remove linux-headers-3.2.0-70 \
+ linux-headers-3.2.0-70-virtual \
+ linux-image-3.2.0-70-virtual \
+ linux-headers-3.2.0-70-generic \
+ linux-image-3.2.0-70-generic
+ sudo DEBIAN_FRONTEND=noninteractive apt-get \
+ --option "Dpkg::Options::=--force-confold" \
+ --assume-yes install --reinstall linux-headers-3.2.0-69 \
+ linux-headers-3.2.0-69-virtual \
+ linux-image-3.2.0-69-virtual \
+ python-software-properties build-essential python-dev python3-dev
+ else
+ sudo DEBIAN_FRONTEND=noninteractive apt-get \
+ --option "Dpkg::Options::=--force-confold" \
+ --assume-yes install build-essential python-dev python3-dev \
+ python-software-properties linux-headers-virtual \
+ linux-headers-$(uname -r)
+ fi
+else
+ echo "Unsupported distro."
+ exit 1
+fi
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+# Copyright (c) 2014 Citrix Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+XENSERVER_XVA_URL=${NODEPOOL_XENSERVER_XVA_URL:-http://downloads.vmd.citrix.com/OpenStack/xenapi-in-the-cloud-appliances/prod_ci}
+XENSERVER_ISO_URL=${NODEPOOL_XENSERVER_ISO_URL:-http://downloadns.citrix.com.edgesuite.net/akdlm/8159/XenServer-6.2.0-install-cd.iso}
+
+./convert_node_to_xenserver.sh \
+ password \
+ "$XENSERVER_XVA_URL" \
+ devstack \
+ "$XENSERVER_ISO_URL"
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export PATH=$PATH:/usr/local/sbin:/usr/sbin
+
+for ip in $(cat /etc/nodepool/primary_node /etc/nodepool/sub_nodes /etc/nodepool/primary_node_private /etc/nodepool/sub_nodes_private | sort -u); do
+ sudo iptables -I openstack-INPUT 1 -s $ip -j ACCEPT
+done
+
+echo "" >> /home/jenkins/.ssh/authorized_keys
+cat /etc/nodepool/id_rsa.pub >> /home/jenkins/.ssh/authorized_keys
+echo "" >> /home/jenkins/.ssh/authorized_keys
+
+ROLE=$(cat /etc/nodepool/role)
+if [ $ROLE == "primary" ]; then
+ cp /etc/nodepool/id_rsa /home/jenkins/.ssh/id_rsa
+ chmod 0600 /home/jenkins/.ssh/id_rsa
+else
+ rm /etc/nodepool/id_rsa
+fi
+
+sudo chown -R root:root /etc/nodepool
+sudo chmod 0755 /etc/nodepool
+sudo chmod 0444 /etc/nodepool/*
+
+/opt/nodepool-scripts/configure_mirror.sh
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd /opt/nodepool-scripts/
+./install_devstack_dependencies.sh
+DISTRIB_CODENAME=$(lsb_release -sc)
+python ./cache_devstack.py $DISTRIB_CODENAME
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2014 - Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install the virt-preview repository that includes bleeding edge versions of
+# libvirt and qemu. The packages from this repo will get installed via
+# prepare_devstack.sh.
+wget http://fedorapeople.org/groups/virt/virt-preview/fedora-virt-preview.repo
+sudo mv fedora-virt-preview.repo /etc/yum.repos.d/
+
+/opt/nodepool-scripts/prepare_devstack.sh
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOSTNAME=$1
+
+SUDO=${SUDO:-true}
+THIN=${THIN:-true}
+ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false}
+GIT_BASE=${GIT_BASE:-git://git.openstack.org}
+
+sudo hostname $HOSTNAME
+if [ -n "$HOSTNAME" ] && ! grep -q $HOSTNAME /etc/hosts ; then
+ echo "127.0.1.1 $HOSTNAME" | sudo tee -a /etc/hosts
+fi
+
+echo $HOSTNAME > /tmp/image-hostname.txt
+sudo mv /tmp/image-hostname.txt /etc/image-hostname.txt
+
+if [ ! -f /etc/redhat-release ]; then
+ # Cloud provider apt repos break us - so stop using them
+ LSBDISTID=$(lsb_release -is)
+ LSBDISTCODENAME=$(lsb_release -cs)
+ if [ "$LSBDISTID" == "Ubuntu" ] ; then
+ sudo dd of=/etc/apt/sources.list <<EOF
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME main restricted
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME main restricted
+
+## Major bug fix updates produced after the final release of the
+## distribution.
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates main restricted
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates main restricted
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team. Also, please note that software in universe WILL NOT receive any
+## review or updates from the Ubuntu security team.
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME universe
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME universe
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates universe
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates universe
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team, and may not be under a free licence. Please satisfy yourself as to
+## your rights to use the software. Also, please note that software in
+## multiverse WILL NOT receive any review or updates from the Ubuntu
+## security team.
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME multiverse
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates multiverse
+
+## N.B. software from this repository may not have been tested as
+## extensively as that contained in the main release, although it includes
+## newer versions of some applications which may provide useful features.
+## Also, please note that software in backports WILL NOT receive any review
+## or updates from the Ubuntu security team.
+deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-backports main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-backports main restricted universe multiverse
+
+deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security main restricted
+deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security main restricted
+deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security universe
+deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security universe
+deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security multiverse
+deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security multiverse
+EOF
+ fi
+fi
+
+# Fedora image doesn't come with wget
+if [ -f /usr/bin/yum ]; then
+ sudo yum -y install wget
+fi
+wget https://git.openstack.org/cgit/openstack-infra/system-config/plain/install_puppet.sh
+sudo bash -xe install_puppet.sh
+
+sudo git clone --depth=1 $GIT_BASE/openstack-infra/system-config.git \
+ /root/system-config
+sudo /bin/bash /root/system-config/install_modules.sh
+
+set +e
+if [ -z "$NODEPOOL_SSH_KEY" ] ; then
+ sudo puppet apply --detailed-exitcodes --color=false \
+ --modulepath=/root/system-config/modules:/etc/puppet/modules \
+ -e "class {'openstack_project::single_use_slave':
+ sudo => $SUDO,
+ thin => $THIN,
+ all_mysql_privs => $ALL_MYSQL_PRIVS,
+ }"
+ PUPPET_RET_CODE=$?
+else
+ sudo puppet apply --detailed-exitcodes --color=false \
+ --modulepath=/root/system-config/modules:/etc/puppet/modules \
+ -e "class {'openstack_project::single_use_slave':
+ install_users => false,
+ sudo => $SUDO,
+ thin => $THIN,
+ all_mysql_privs => $ALL_MYSQL_PRIVS,
+ ssh_key => '$NODEPOOL_SSH_KEY',
+ }"
+ PUPPET_RET_CODE=$?
+fi
+# Puppet doesn't properly return exit codes. Check here the values that
+# indicate failure of some sort happened. 0 and 2 indicate success.
+if [ "$PUPPET_RET_CODE" -eq "4" ] || [ "$PUPPET_RET_CODE" -eq "6" ] ; then
+ exit $PUPPET_RET_CODE
+fi
+set -e
+
+# The puppet modules should install unbound. Set up some nameservers.
+cat >/tmp/forwarding.conf <<EOF
+forward-zone:
+ name: "."
+ forward-addr: 8.8.8.8
+EOF
+sudo mv /tmp/forwarding.conf /etc/unbound/
+sudo chown root:root /etc/unbound/forwarding.conf
+sudo chmod a+r /etc/unbound/forwarding.conf
+# HPCloud has selinux enabled by default, Rackspace apparently not.
+# Regardless, apply the correct context.
+if [ -x /sbin/restorecon ] ; then
+ sudo chcon system_u:object_r:named_conf_t:s0 /etc/unbound/forwarding.conf
+fi
+
+# Overwrite /etc/resolv.conf at boot
+sudo dd of=/etc/rc.local <<EOF
+#!/bin/bash
+set -o xtrace
+
+# Some providers inject dynamic network config statically. Work around this
+# for DNS nameservers. This is expected to fail on some nodes so remove -e.
+set +e
+sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
+set -e
+
+echo 'nameserver 127.0.0.1' > /etc/resolv.conf
+
+exit 0
+EOF
+
+# hpcloud has started mounting ephemeral /dev/vdb at /mnt.
+# devstack-gate wants to partition the ephemeral disk, add some swap
+# and mount it at /opt. get rid of the mount.
+#
+# note this comes down from the cloud-init metadata; which we setup to
+# ignore below.
+sudo sed -i '/^\/dev\/vdb/d' /etc/fstab
+
+
+# Make all cloud-init data sources match rackspace- only attempt to look
+# at ConfigDrive, not at metadata service. This is not needed if there
+# is no cloud-init
+if [ -d /etc/cloud/cloud.cfg.d ] ; then
+sudo dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg <<EOF
+datasource_list: [ ConfigDrive, None ]
+EOF
+fi
+
+# reset cloud-init
+sudo rm -rf /var/lib/cloud/instances
+
+sudo bash -c "echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf"
+if [ -e /etc/init.d/unbound ] ; then
+ sudo /etc/init.d/unbound restart
+elif [ -e /usr/lib/systemd/system/unbound.service ] ; then
+ sudo systemctl restart unbound
+else
+ echo "Can't discover a method to restart \"unbound\""
+ exit 1
+fi
+
+# Make sure DNS works.
+dig git.openstack.org
+
+# Cache all currently known gerrit repos.
+sudo mkdir -p /opt/git
+sudo -i python /opt/nodepool-scripts/cache_git_repos.py $GIT_BASE
+
+# We don't always get ext4 from our clouds, mount ext3 as ext4 on the next
+# boot (eg when this image is used for testing).
+sudo sed -i 's/ext3/ext4/g' /etc/fstab
+
+# Remove additional sources used to install puppet or special version of pypi.
+# We do this because leaving these sources in place causes every test that
+# does an apt-get update to hit those servers which may not have the uptime
+# of our local mirrors.
+OS_FAMILY=$(facter osfamily)
+if [ "$OS_FAMILY" == "Debian" ] ; then
+ sudo rm -f /etc/apt/sources.list.d/*
+ sudo apt-get update
+elif [ "$OS_FAMILY" == "RedHat" ] ; then
+ # Can't delete * in yum.repos.d since all of the repos are listed there.
+ # Be specific instead.
+ if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then
+ sudo rm -f /etc/yum.repos.d/puppetlabs.repo
+ fi
+fi
+
+# Remove cron jobs
+# We create fresh servers for these hosts, and they are used once. They don't
+# need to do things like update the locatedb or the mandb or rotate logs
+# or really any of those things. We only want code running here that we want
+# here.
+sudo rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/*
+
+# Install Zuul into a virtualenv
+# This is in /usr instead of /usr/local due to this bug on precise:
+# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
+git clone /opt/git/openstack-infra/zuul /tmp/zuul
+sudo virtualenv /usr/zuul-env
+sudo -H /usr/zuul-env/bin/pip install /tmp/zuul
+sudo rm -fr /tmp/zuul
+
+# Create a virtualenv for zuul-swift-logs
+# This is in /usr instead of /usr/local due to this bug on precise:
+# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
+sudo -H virtualenv /usr/zuul-swift-logs-env
+sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
+ requests glob2
+
+# Create a virtualenv for os-testr (which contains subunit2html)
+# this is in /usr instead of /usr/loca/ due to this bug on precise:
+# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
+sudo -H virtualenv /usr/os-testr-env
+sudo -H /usr/os-testr-env/bin/pip install os-testr
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOSTNAME=$1
+
+export SUDO='true'
+export THIN='false'
+export ALL_MYSQL_PRIVS='true'
+
+./prepare_node.sh "$HOSTNAME"
+
+./restrict_memory.sh
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOSTNAME=$1
+
+export SUDO='true'
+export THIN='true'
+TEMPEST_DIR=${TEMPEST_DIR:-/opt/git/openstack/tempest}
+
+./prepare_node.sh "$HOSTNAME"
+sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack.sh "$HOSTNAME"
+
+# Setup venv and install deps for prepare_tempest_testrepository.py
+sudo virtualenv -p python2 /opt/git/subunit2sql-env
+sudo -H /opt/git/subunit2sql-env/bin/pip install -U testrepository \
+ subunit2sql PyMySQL
+
+# Pre-seed tempest testrepository with data from subunit2sql
+sudo -i env PATH=/opt/git/subunit2sql-env/bin:$PATH \
+ /opt/git/subunit2sql-env/bin/python2 \
+ /opt/nodepool-scripts/prepare_tempest_testrepository.py \
+ $TEMPEST_DIR
+
+sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
+
+# Delete the venv after the script is called
+sudo rm -rf /opt/git/subunit2sql-env
+
+./restrict_memory.sh
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2014 - Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOSTNAME=$1
+
+export SUDO='true'
+export THIN='true'
+
+./prepare_node.sh "$HOSTNAME"
+sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack_virt_preview.sh \
+ "$HOSTNAME"
+
+./restrict_memory.sh
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOSTNAME=$1
+
+export SUDO='true'
+export THIN='true'
+
+# Workaround bug 1270646 during node bootstrapping.
+sudo ip link set mtu 1458 dev eth0
+./prepare_node.sh "$HOSTNAME"
+sudo -u jenkins -i /opt/nodepool-scripts/prepare_tripleo.sh "$HOSTNAME"
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+HOSTNAME=$1
+
+function tsfilter {
+ $@ 2>&1 | awk '
+ {
+ cmd ="date +\"%Y-%m-%d %H:%M:%S.%3N | \""
+ cmd | getline now
+ close("date +\"%Y-%m-%d %H:%M:%S.%3N | \"")
+ sub(/^/, now)
+ print
+ fflush()
+ }'
+}
+
+set -o pipefail
+tsfilter ./prepare_node_devstack.sh "$HOSTNAME"
+
+# After the node has been prepared, the hypervisor needs to be halted to make
+# sure that the filesystem is in a consistent state.
+sudo -u domzero \
+ ssh \
+ -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ root@192.168.33.2 \
+ halt -p </dev/null
--- /dev/null
+#!/usr/bin/env python2
+#
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+from subunit2sql.db import api
+from subunit2sql import shell
+from subunit2sql import write_subunit
+
+
+DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
+
+if len(sys.argv) == 2:
+ TEMPEST_PATH = sys.argv[1]
+elif len(sys.argv) > 2:
+ TEMPEST_PATH = sys.argv[1]
+ DB_URI = sys.argv[2]
+else:
+ TEMPEST_PATH = '/opt/stack/new/tempest'
+
+
+def main():
+ shell.parse_args([])
+ shell.CONF.set_override('connection', DB_URI, group='database')
+ session = api.get_session()
+ run_ids = api.get_recent_successful_runs(num_runs=10,
+ session=session)
+ session.close()
+ preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams')
+ os.mkdir(preseed_path)
+ for run in run_ids:
+ with open(os.path.join(preseed_path, run + '.subunit'), 'w') as fd:
+ write_subunit.sql2subunit(run, fd)
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+#!/bin/bash -xe
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+# Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Enable precise-backports so we can install jq
+if [ -f /usr/bin/apt-get ]; then
+ sudo sed -i -e 's/# \(deb .*precise-backports main \)/\1/g' \
+ /etc/apt/sources.list
+ sudo apt-get update
+fi
+
+cd /opt/nodepool-scripts/
+./install_devstack_dependencies.sh
+
+# toci scripts use both of these
+sudo -H pip install gear os-apply-config
+
+# tripleo-gate runs with two networks - the public access network and eth1
+# pointing at the in-datacentre L2 network where we can talk to the test
+# environments directly. We need to enable DHCP on eth1 though.
+# Note that we don't bring it up during prepare - it's only needed to run
+# tests.
+
+if [ -d /etc/sysconfig/network-scripts ]; then
+ sudo dd of=/etc/sysconfig/network-scripts/ifcfg-eth1 << EOF
+DEVICE="eth1"
+BOOTPROTO="dhcp"
+ONBOOT="yes"
+TYPE="Ethernet"
+PEERDNS="no"
+EOF
+
+elif [ -f /etc/network/interfaces ]; then
+ sudo dd of=/etc/network/interfaces oflag=append conv=notrunc << EOF
+auto eth1
+iface eth1 inet dhcp
+EOF
+
+# Workaround bug 1270646 for actual slaves
+ sudo dd of=/etc/network/interfaces.d/eth0.cfg oflag=append conv=notrunc << EOF
+ post-up ip link set mtu 1458 dev eth0
+EOF
+
+else
+ echo "Unsupported distro."
+ exit 1
+fi
--- /dev/null
+#!/bin/bash -xe
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Limit all test slaves to 8GB of memory so that larger flavors with more
+# cpu resources can be used without the risk of becoming dependent on more
+# memory.
+if [ -f /etc/default/grub ] ; then
+ sudo sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' \
+ -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=9023M /g' \
+ /etc/default/grub
+ if which update-grub &> /dev/null ; then
+ sudo update-grub
+ else
+ # If update-grub isn't available, use grub2-mkconfig directly
+ sudo /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ fi
+elif [ -f /boot/grub/grub.conf ] ; then
+ sudo sed -i -e 's/^timeout=[0-9]\+/timeout=0/' \
+ -e 's/\(^\s\+kernel.*\)/\1 mem=9023M/' /boot/grub/grub.conf
+fi
--- /dev/null
+#!/bin/sh
+
+set -e
+
+PREREQ=""
+
+prereqs () {
+ echo "${PREREQ}"
+}
+
+case "${1}" in
+ prereqs)
+ prereqs
+ exit 0
+ ;;
+esac
+
+. /usr/share/initramfs-tools/hook-functions
+
+copy_exec /sbin/resize2fs
+copy_exec /sbin/e2fsck
+copy_exec /usr/bin/expr
+copy_exec /sbin/tune2fs
+copy_exec /bin/grep
+copy_exec /usr/bin/tr
+copy_exec /usr/bin/cut
+copy_exec /sbin/sfdisk
+copy_exec /sbin/partprobe
+copy_exec /bin/sed
--- /dev/null
+#!/bin/sh -e
+
+set -ex
+
+PREREQ=""
+
+# Output pre-requisites
+prereqs()
+{
+ echo "$PREREQ"
+}
+
+case "$1" in
+ prereqs)
+ prereqs
+ exit 0
+ ;;
+esac
+
+. /scripts/functions
+
+log_begin_msg "Resize started"
+touch /etc/mtab
+
+tune2fs -O ^has_journal /dev/xvda1
+e2fsck -fp /dev/xvda1
+resize2fs /dev/xvda1 4G
+
+# Number of 4k blocks
+NUMBER_OF_BLOCKS=$(tune2fs -l /dev/xvda1 | grep "Block count" | tr -d " " | cut -d":" -f 2)
+
+# Convert them to 512 byte sectors
+SIZE_OF_PARTITION=$(expr $NUMBER_OF_BLOCKS \* 8)
+
+sfdisk -d /dev/xvda | sed -e "s,[0-9]\{8\},$SIZE_OF_PARTITION,g" > /tmp/new_layout
+
+while ! cat /tmp/new_layout | sfdisk /dev/xvda; do
+ sleep 1
+done
+
+while ! partprobe /dev/xvda; do
+ sleep 1
+done
+
+tune2fs -j /dev/xvda1
+
+sync
+
+log_end_msg "Resize finished"
projects / openstack / ci-loop-config.git / commitdiff