From 94457760272fd8acfa1562b17b113cb4525edde1 Mon Sep 17 00:00:00 2001 From: Bob Ball Date: Fri, 12 Feb 2016 16:31:54 +0000 Subject: [PATCH] Add nodepool scripts from infra/project-config c93c0dfd10906a8cf5b4b9d67c07983d4315edd8 --- install_master.sh | 25 +- nodepool/scripts/cache_devstack.py | 189 ++++++ nodepool/scripts/cache_git_repos.py | 88 +++ nodepool/scripts/common.py | 32 + nodepool/scripts/configure_mirror.sh | 67 ++ nodepool/scripts/convert_node_to_xenserver.sh | 612 ++++++++++++++++++ .../scripts/install_devstack_dependencies.sh | 52 ++ nodepool/scripts/install_xenserver.sh | 27 + nodepool/scripts/multinode_setup.sh | 41 ++ nodepool/scripts/prepare_devstack.sh | 22 + .../scripts/prepare_devstack_virt_preview.sh | 25 + nodepool/scripts/prepare_node.sh | 241 +++++++ nodepool/scripts/prepare_node_bare.sh | 27 + nodepool/scripts/prepare_node_devstack.sh | 44 ++ .../prepare_node_devstack_virt_preview.sh | 28 + nodepool/scripts/prepare_node_tripleo.sh | 27 + nodepool/scripts/prepare_node_xenserver.sh | 44 ++ .../scripts/prepare_tempest_testrepository.py | 50 ++ nodepool/scripts/prepare_tripleo.sh | 62 ++ nodepool/scripts/restrict_memory.sh | 34 + .../scripts/xenserver_helper_initramfs_hook | 29 + .../xenserver_helper_initramfs_premount | 49 ++ 22 files changed, 1807 insertions(+), 8 deletions(-) create mode 100755 nodepool/scripts/cache_devstack.py create mode 100755 nodepool/scripts/cache_git_repos.py create mode 100644 nodepool/scripts/common.py create mode 100755 nodepool/scripts/configure_mirror.sh create mode 100755 nodepool/scripts/convert_node_to_xenserver.sh create mode 100755 nodepool/scripts/install_devstack_dependencies.sh create mode 100755 nodepool/scripts/install_xenserver.sh create mode 100755 nodepool/scripts/multinode_setup.sh create mode 100755 nodepool/scripts/prepare_devstack.sh create mode 100755 nodepool/scripts/prepare_devstack_virt_preview.sh create mode 100755 nodepool/scripts/prepare_node.sh create mode 100755 nodepool/scripts/prepare_node_bare.sh create mode 100755 nodepool/scripts/prepare_node_devstack.sh create mode 100755 nodepool/scripts/prepare_node_devstack_virt_preview.sh create mode 100755 nodepool/scripts/prepare_node_tripleo.sh create mode 100755 nodepool/scripts/prepare_node_xenserver.sh create mode 100755 nodepool/scripts/prepare_tempest_testrepository.py create mode 100755 nodepool/scripts/prepare_tripleo.sh create mode 100755 nodepool/scripts/restrict_memory.sh create mode 100644 nodepool/scripts/xenserver_helper_initramfs_hook create mode 100644 nodepool/scripts/xenserver_helper_initramfs_premount diff --git a/install_master.sh b/install_master.sh index ddc1098..9ab5880 100644 --- a/install_master.sh +++ b/install_master.sh @@ -5,17 +5,25 @@ THIS_DIR=`pwd` DATA_PATH=/root/os-ext-data # Steps to reinstall: -# 1) Create new Ubuntu 14.04 server (copy password) -# 1a) 7.5GB Compute v1 flavor -# 1b) Enable monitoring and security updates -# 2) Disable password authentication +# 1) Log in to mycloud.rackspace.com using credentials from os-ext-data/xenlibvirt-nodepool +# 1a) Create new Ubuntu 14.04 server (copy password), hostname 'jenkins-libvirt' +# 7.5GB Compute v1 flavor +# Enable monitoring and security updates +# 1b) Save the password for use in step 2a +# 1c) Add key from os-ext-data/xenproject_jenkins.pub with the name 'xenproject-nodepool' +# 2) Disable password authentication on jenkins server # 2a) ssh-copy-id to copy a key to the server # 2b) edit /etc/sshd_config to set "PermitRootLogin without-password" +# 2c) service ssh restart # 3) Copy the secret credentials dir (http://hg.uk.xensource.com/openstack/infrastructure.hg/os-ext-data) to /root -# 4) Clone this repo git: -# 4a) git clone https://github.com/citrix-openstack/os-ext-testing.git -# 4b) cd os-ext-testing; git checkout common_ci -# ?) Follow steps below +# 4) Clone this repo: +# 4a) apt-get install git +# 4b) git clone https://github.com/citrix-openstack/os-ext-testing.git +# 4c) cd os-ext-testing; git checkout common_ci +# 5) Run below commands (or just this script) to do the 'standard' install +# 6) The jobs need an additional plugin in Jenkins to generate correctly, so: +# 6a) Install Post-Build Script jenkins plugin (including restarting Jenkins) +# 6b) Regenerate jenkins jobs: jenkins-jobs update --delete-old /etc/jenkins_jobs/config # ?) Set up monitoring checks https://intelligence.rackspace.com/cloud/entities/enWCIYVVnt # Copied from the following URL Feb 2016 @@ -48,3 +56,4 @@ cp /root/os-ext-data/single_node_ci_data.yaml /etc/puppet/environments/common.ya sed -i -e 's/^\(127\.0\.0\.1.*\)$/\1 jenkins/' /etc/hosts sudo puppet apply --verbose /etc/puppet/manifests/site.pp + diff --git a/nodepool/scripts/cache_devstack.py b/nodepool/scripts/cache_devstack.py new file mode 100755 index 0000000..379d13a --- /dev/null +++ b/nodepool/scripts/cache_devstack.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from common import run_local + +DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack') +CACHEDIR = os.path.expanduser('~/cache/files') + +# Some jobs might require newer distro packages, so we can pre-cache +# deb packages from specified Ubuntu Cloud Archive pockets. +UCA_POCKETS = [] + + +def git_branches(): + branches = [] + for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"): + branch = branch.strip() + if not branch.startswith('remotes/origin'): + continue + branches.append(branch) + return branches + + +def tokenize(fn, tokens, distribution, comment=None): + for line in open(fn): + if 'dist:' in line and ('dist:%s' % distribution not in line): + continue + if 'qpid' in line: + continue # TODO: explain why this is here + if comment and comment in line: + line = line[:line.rfind(comment)] + line = line.strip() + if line and line not in tokens: + tokens.append(line) + + +def _legacy_find_images(basedir): + """Divine what images we should use based on parsing stackrc.""" + images = [] + for line in open(os.path.join(basedir, 'stackrc')): + line = line.strip() + if line.startswith('IMAGE_URLS'): + if '#' in line: + line = line[:line.rfind('#')] + if line.endswith(';;'): + line = line[:-2] + line = line.split('=', 1)[1].strip() + if line.startswith('${IMAGE_URLS:-'): + line = line[len('${IMAGE_URLS:-'):] + if line.endswith('}'): + line = line[:-1] + if not line: + continue + if line[0] == line[-1] == '"': + line = line[1:-1] + # Add image to the list to be downloaded, but + # skip downloading giant vmware images + images += [x.strip() for x in line.split(',') + if not x.strip().endswith('vmdk')] + return images + + +def _find_images(basedir): + images = [] + image_tool = os.path.join(basedir, 'tools', 'image_list.sh') + if os.path.exists(image_tool): + returncode, out = run_local(image_tool, status=True) + if returncode: + print "%s failed" % image_tool + print "Exit: %s, Output: %s" % (returncode, out) + # reset images so we'll fall back + images = [] + else: + images = out.split('\n') + return images + + +def local_prep(distribution): + branches = [] + for branch in git_branches(): + # Ignore branches of the form 'somestring -> someotherstring' + # as this denotes a symbolic reference and the entire string + # as is cannot be checked out. We can do this safely as the + # reference will refer to one of the other branches returned + # by git_branches. + if ' -> ' in branch: + continue + branch_data = {'name': branch} + print 'Branch: ', branch + run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK) + run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) + + if os.path.exists('/usr/bin/apt-get'): + debs = [] + debdir = os.path.join(DEVSTACK, 'files', 'debs') + if not os.path.exists(debdir): + debdir = os.path.join(DEVSTACK, 'files', 'apts') + for fn in os.listdir(debdir): + fn = os.path.join(debdir, fn) + tokenize(fn, debs, distribution, comment='#') + branch_data['debs'] = debs + + if os.path.exists('/usr/bin/yum'): + rpms = [] + rpmdir = os.path.join(DEVSTACK, 'files', 'rpms') + for fn in os.listdir(rpmdir): + fn = os.path.join(rpmdir, fn) + tokenize(fn, rpms, distribution, comment='#') + branch_data['rpms'] = rpms + + images = _find_images(DEVSTACK) + if not images: + images = _legacy_find_images(DEVSTACK) + + branch_data['images'] = images + branches.append(branch_data) + return branches + + +def download(url, fname): + run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)]) + + +def cache_debs(debs, uca_pocket=None): + """Cache a list of deb packages, optionally pulling from an Ubuntu Cloud + Archive pocket. If a UCA pocket is specified, it is enabled temporarily + for caching only. + """ + if uca_pocket: + # Note this will install the ubuntu-cloud-keyring package which + # contains the required GPG key. + run_local(['sudo', 'add-apt-repository', '-y', + 'cloud-archive:%s' % uca_pocket]) + run_local(['sudo', 'apt-get', 'update']) + run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs) + if uca_pocket: + run_local(['sudo', 'rm', '-f', + '/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket]) + run_local(['sudo', 'apt-get', 'update']) + + +def main(): + distribution = sys.argv[1] + + branches = local_prep(distribution) + image_filenames = [] + for branch_data in branches: + if branch_data.get('debs'): + cache_debs(branch_data['debs']) + for uca in sorted(UCA_POCKETS): + cache_debs(branch_data['debs'], uca) + elif branch_data.get('rpms'): + run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] + + branch_data['rpms']) + else: + sys.exit('No supported package data found.') + + for url in branch_data['images']: + fname = url.split('/')[-1] + if fname in image_filenames: + continue + image_filenames.append(fname) + download(url, fname) + + # cache get-pip, because upstream network connection fails more + # often than you might imagine. + download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py') + + +if __name__ == '__main__': + main() diff --git a/nodepool/scripts/cache_git_repos.py b/nodepool/scripts/cache_git_repos.py new file mode 100755 index 0000000..bea36c9 --- /dev/null +++ b/nodepool/scripts/cache_git_repos.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import re +import shutil +import sys +import urllib2 + +from common import run_local + +URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/' + 'plain/gerrit/projects.yaml') +PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$') + +# Not using an arg libraries in order to avoid module imports that +# are not available across all python versions +if len(sys.argv) > 1: + GIT_BASE = sys.argv[1] +else: + GIT_BASE = 'git://git.openstack.org' + + +def clone_repo(project): + remote = '%s/%s.git' % (GIT_BASE, project) + + # Clear out any existing target directory first, in case of a retry. + try: + shutil.rmtree(os.path.join('/opt/git', project)) + except OSError: + pass + + # Try to clone the requested git repository. + (status, out) = run_local(['git', 'clone', remote, project], + status=True, cwd='/opt/git') + + # If it claims to have worked, make sure we can list branches. + if status == 0: + (status, moreout) = run_local(['git', 'branch', '-a'], status=True, + cwd=os.path.join('/opt/git', project)) + out = '\n'.join((out, moreout)) + + # If that worked, try resetting to HEAD to make sure it's there. + if status == 0: + (status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'], + status=True, + cwd=os.path.join('/opt/git', project)) + out = '\n'.join((out, moreout)) + + # Status of 0 imples all the above worked, 1 means something failed. + return (status, out) + + +def main(): + # TODO(jeblair): use gerrit rest api when available + data = urllib2.urlopen(URL).read() + for line in data.split('\n'): + # We're regex-parsing YAML so that we don't have to depend on the + # YAML module which is not in the stdlib. + m = PROJECT_RE.match(line) + if m: + (status, out) = clone_repo(m.group(1)) + print out + if status != 0: + print 'Retrying to clone %s' % m.group(1) + (status, out) = clone_repo(m.group(1)) + print out + if status != 0: + raise Exception('Failed to clone %s' % m.group(1)) + + +if __name__ == '__main__': + main() diff --git a/nodepool/scripts/common.py b/nodepool/scripts/common.py new file mode 100644 index 0000000..3ccfedf --- /dev/null +++ b/nodepool/scripts/common.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + + +def run_local(cmd, status=False, cwd='.', env={}): + print "Running:", cmd + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() diff --git a/nodepool/scripts/configure_mirror.sh b/nodepool/scripts/configure_mirror.sh new file mode 100755 index 0000000..6f46419 --- /dev/null +++ b/nodepool/scripts/configure_mirror.sh @@ -0,0 +1,67 @@ +#!/bin/bash -xe + +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +source /etc/nodepool/provider + +# Generate the AFS Slug from the host system. +source /usr/local/jenkins/slave_scripts/afs-slug.sh + +NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} +NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') +NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple} +NODEPOOL_WHEEL_MIRROR=${NODEPOOL_WHEEL_MIRROR:-http://$NODEPOOL_MIRROR_HOST/wheel/$AFS_SLUG} +NODEPOOL_UBUNTU_MIRROR=${NODEPOOL_UBUNTU_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu} + +cat >/tmp/pip.conf </home/jenkins/.pydistutils.cfg < $STATE_FILE +} + +function get_state { + if [ -e "$STATE_FILE" ]; then + cat $STATE_FILE + else + echo "START" + fi +} + +function create_resizing_initramfs_config { + cp "$THIS_DIR/xenserver_helper_initramfs_hook" \ + /usr/share/initramfs-tools/hooks/resize + chmod +x /usr/share/initramfs-tools/hooks/resize + + cp "$THIS_DIR/xenserver_helper_initramfs_premount" \ + /usr/share/initramfs-tools/scripts/local-premount/resize + chmod +x /usr/share/initramfs-tools/scripts/local-premount/resize +} + +function delete_resizing_initramfs_config { + rm -f /usr/share/initramfs-tools/hooks/resize + rm -f /usr/share/initramfs-tools/scripts/local-premount/resize +} + +function run_this_script_on_each_boot { + cat > /etc/init/xenserver.conf << EOF +start on stopped rc RUNLEVEL=[2345] + +task + +script + /bin/bash $THIS_FILE $ADDITIONAL_PARAMETERS >> $LOG_FILE 2>&1 +end script +EOF +} + +function create_done_file_on_appliance { + while ! echo "sudo touch $FILE_TO_TOUCH_ON_COMPLETION" | bash_on_appliance; do + sleep 1 + done +} + +function download_xenserver_files { + local tgt + + tgt="$1" + + wget -qO "$tgt" "$XENSERVER_ISO_URL" +} + +function download_appliance { + local appliance_url + + appliance_url="$1" + + wget -qO /root/staging_vm.xva "$appliance_url" +} + +function print_answerfile { + local repository + local postinst + local xenserver_pass + + repository="$1" + postinst="$2" + xenserver_pass="$3" + + cat << EOF + + +sda +us +$xenserver_pass +$repository + +192.168.34.2 +255.255.255.0 +192.168.34.1 + +UTC + + +EOF +} + +function print_postinst_file { + local rclocal + rclocal="$1" + + cat << EOF +#!/bin/sh +touch \$1/tmp/postinst.sh.executed +cp \$1/etc/rc.d/rc.local \$1/etc/rc.d/rc.local.backup +cat $rclocal >> \$1/etc/rc.d/rc.local +cp /tmp/ramdisk/cloud-settings \$1/root/ +cp /tmp/ramdisk/authorized_keys \$1/root/.ssh/ +EOF +} + +function print_rclocal { + cat << EOF +# This is the contents of the rc.local file on XenServer +mkdir -p /mnt/ubuntu +mount /dev/sda1 /mnt/ubuntu +mkdir -p $(dirname $INSTALL_DIR) +[ -L $INSTALL_DIR ] || ln -s /mnt/ubuntu${INSTALL_DIR} $INSTALL_DIR +/bin/bash $THIS_FILE $ADDITIONAL_PARAMETERS >> $LOG_FILE 2>&1 +EOF +} + +function create_ramdisk_contents { + local isofile + local target_dir + + isofile="$1" + target_dir="$XSINST_DIRECTORY" + + mkdir "$target_dir" + ln "$isofile" "$target_dir/xenserver.iso" + print_rclocal > "$target_dir/rclocal" + print_postinst_file "/tmp/ramdisk/rclocal" > "$target_dir/postinst.sh" + print_answerfile \ + "file:///tmp/ramdisk" \ + "file:///tmp/ramdisk/postinst.sh" \ + "$XENSERVER_PASSWORD" > "$target_dir/answerfile.xml" +} + +function extract_xs_installer { + local isofile + local targetpath + + isofile="$1" + targetpath="$2" + + local mountdir + + mountdir=$(mktemp -d) + mount -o loop $isofile $mountdir + mkdir -p $targetpath + cp \ + $mountdir/install.img \ + $mountdir/boot/xen.gz \ + $mountdir/boot/vmlinuz \ + $targetpath + umount $mountdir +} + +function generate_xs_installer_grub_config { + local bootfiles + local answerfile + + bootfiles="$1" + answerfile="$2" + + cat > /etc/grub.d/45_xs-install << EOF +#!/bin/sh +exec tail -n +3 \$0 +menuentry 'XenServer installer' { + multiboot $bootfiles/xen.gz dom0_max_vcpus=1-2 dom0_mem=max:752M com1=115200,8n1 console=com1,vga + module $bootfiles/vmlinuz xencons=hvc console=tty0 make-ramdisk=/dev/sda1 answerfile=$answerfile install + module $bootfiles/install.img +} +EOF + chmod +x /etc/grub.d/45_xs-install +} + +function configure_grub { + sed -ie 's/^GRUB_HIDDEN_TIMEOUT/#GRUB_HIDDEN_TIMEOUT/g' /etc/default/grub + sed -ie 's/^GRUB_HIDDEN_TIMEOUT_QUIET/#GRUB_HIDDEN_TIMEOUT_QUIET/g' \ + /etc/default/grub + # sed -ie 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=-1/g' /etc/default/grub + sed -ie 's/^.*GRUB_TERMINAL=.*$/GRUB_TERMINAL=console/g' /etc/default/grub + sed -ie 's/GRUB_DEFAULT=0/GRUB_DEFAULT=saved/g' /etc/default/grub +} + +function set_xenserver_installer_as_nextboot { + grub-set-default "XenServer installer" +} + +function store_cloud_settings { + local targetpath + + targetpath="$1" + + cat > $targetpath << EOF +ADDRESS=$(grep -m 1 "address" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2) +NETMASK=$(grep -m 1 "netmask" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2) +GATEWAY=$(grep -m 1 "gateway" /etc/network/interfaces | sed -e 's,^ *,,g' | cut -d " " -f 2) +MACADDRESS=$(ifconfig eth0 | sed -ne 's/.*HWaddr \(.*\)$/\1/p' | tr -d " ") +NAMESERVERS=$(cat /etc/resolv.conf | grep nameserver | cut -d " " -f 2 | sort | uniq | tr '\n' , | sed -e 's/,$//g') +EOF +} + +function store_authorized_keys { + local targetpath + + targetpath="$1" + + cp /root/.ssh/authorized_keys $1 +} + +function wait_for_xapi { + while ! [ -e /var/run/xapi_init_complete.cookie ]; do + sleep 1 + done +} + +function forget_networking { + xe host-management-disable + IFS=, + for vlan in $(xe vlan-list --minimal); do + xe vlan-destroy uuid=$vlan + done + + unset IFS + IFS=, + for pif in $(xe pif-list --minimal); do + xe pif-forget uuid=$pif + done + unset IFS +} + +function add_boot_config_for_ubuntu { + local ubuntu_bootfiles + local bootfiles + + ubuntu_bootfiles="$1" + bootfiles="$2" + + local kernel + local initrd + + kernel=$(ls -1c $ubuntu_bootfiles/vmlinuz-* | head -1) + initrd=$(ls -1c $ubuntu_bootfiles/initrd.img-* | head -1) + + cp $kernel $bootfiles/vmlinuz-ubuntu + cp $initrd $bootfiles/initrd-ubuntu + + cat >> $bootfiles/extlinux.conf << UBUNTU +label ubuntu + LINUX $bootfiles/vmlinuz-ubuntu + APPEND root=/dev/xvda1 ro quiet splash + INITRD $bootfiles/initrd-ubuntu +UBUNTU +} + +function start_ubuntu_on_next_boot { + local bootfiles + + bootfiles="$1" + + sed -ie 's,default xe,default ubuntu,g' $bootfiles/extlinux.conf + + log_extlinux $bootfiles/extlinux.conf +} + +function start_xenserver_on_next_boot { + local bootfiles + + bootfiles="$1" + + sed -ie 's,default ubuntu,default xe,g' $bootfiles/extlinux.conf + + log_extlinux $bootfiles/extlinux.conf +} + +function log_extlinux { + local extlinux_conf + + extlinux_conf="$1" + + echo "ACTUAL STATE OF EXTLINUX IS" + cat $extlinux_conf +} + +function mount_dom0_fs { + local target + + target="$1" + + mkdir -p $target + mount /dev/xvda2 $target +} + +function wait_for_networking { + while ! ping -c 1 xenserver.org > /dev/null 2>&1; do + sleep 1 + done +} + +function bash_on_appliance { + local vm_ip + local vm + + vm=$(xe vm-list name-label="$APPLIANCE_NAME" --minimal) + + [ -n "$vm" ] + + # Wait until appliance is accessible + while ! ping -c 1 "${vm_ip:-}" > /dev/null 2>&1; do + vm_ip=$(xe vm-param-get param-name=networks uuid=$vm | sed -e 's,^.*0/ip: ,,g' | sed -e 's,;.*$,,g') + sleep 1 + done + + ssh \ + -q \ + -i /root/dom0key \ + -o UserKnownHostsFile=/dev/null \ + -o StrictHostKeyChecking=no \ + -o BatchMode=yes \ + "$DOMZERO_USER@$vm_ip" bash -e -u -s -x -- "$@" +} + +function configure_networking { + local network_settings + + network_settings="$1" + + . "$network_settings" + + xe pif-introduce \ + device=eth0 host-uuid=$(xe host-list --minimal) mac=$MACADDRESS + + PIF=$(xe pif-list device=eth0 --minimal) + HOST_INT_NET=$(xe network-list name-label="Host internal management network" --minimal) + + ORIGINAL_MGT_NET=$(xe pif-param-get param-name=network-uuid uuid=$PIF) + NEW_MGT_NET=$(xe network-create name-label=mgt name-description=mgt) + NEW_MGT_VLAN=$(xe vlan-create vlan=100 pif-uuid=$PIF network-uuid=$NEW_MGT_NET) + NEW_PIF=$(xe pif-list VLAN=100 device=eth0 --minimal) + VM=$(xe vm-list name-label="$APPLIANCE_NAME" --minimal) + APP_IMPORTED_NOW="false" + if [ -z "$VM" ]; then + VM=$(xe vm-import filename=/mnt/ubuntu/root/staging_vm.xva) + xe vm-param-set name-label="$APPLIANCE_NAME" uuid=$VM + xe vm-param-set VCPUs-max=6 uuid=$VM + xe vm-param-set VCPUs-at-startup=6 uuid=$VM + APP_IMPORTED_NOW="true" + fi + DNS_ADDRESSES=$(echo "$NAMESERVERS" | sed -e "s/,/ /g") + + xe pif-reconfigure-ip \ + uuid=$PIF \ + mode=static \ + IP=0.0.0.0 \ + netmask=0.0.0.0 + + xe pif-reconfigure-ip \ + uuid=$NEW_PIF \ + mode=static \ + IP=192.168.33.2 \ + netmask=255.255.255.0 \ + gateway=192.168.33.1 \ + DNS=192.168.33.1 + + xe host-management-reconfigure pif-uuid=$NEW_PIF + + # Purge all vifs of appliance + IFS=, + for vif in $(xe vif-list vm-uuid=$VM --minimal); do + xe vif-destroy uuid=$vif + done + unset IFS + + # Create vifs for the appliance + xe vif-create vm-uuid=$VM network-uuid=$HOST_INT_NET device=0 + xe vif-create vm-uuid=$VM network-uuid=$ORIGINAL_MGT_NET mac=$MACADDRESS \ + device=1 + xe vif-create vm-uuid=$VM network-uuid=$NEW_MGT_NET device=2 + + xe vm-start uuid=$VM + + # Wait until appliance is accessible + while ! ping -c 1 "${VM_IP:-}" > /dev/null 2>&1; do + VM_IP=$(xe vm-param-get param-name=networks uuid=$VM | sed -e 's,^.*0/ip: ,,g' | sed -e 's,;.*$,,g') + sleep 1 + done + + if [ "$APP_IMPORTED_NOW" = "true" ]; then + rm -f /root/dom0key + rm -f /root/dom0key.pub + ssh-keygen -f /root/dom0key -P "" -C "dom0" + DOMID=$(xe vm-param-get param-name=dom-id uuid=$VM) + + # Authenticate temporary key to appliance + xenstore-write /local/domain/$DOMID/authorized_keys/$DOMZERO_USER \ + "$(cat /root/dom0key.pub)" + xenstore-chmod -u /local/domain/$DOMID/authorized_keys/$DOMZERO_USER r$DOMID + + while ! echo "true" | bash_on_appliance; do + echo "waiting for key to be activated" + sleep 1 + done + + # Remove authorized_keys updater + echo "echo \"\" | crontab -" | bash_on_appliance + + # Create an ssh key for domzero user + echo "ssh-keygen -f /home/$DOMZERO_USER/.ssh/id_rsa -C $DOMZERO_USER@appliance -N \"\" -q" | bash_on_appliance + fi + + # Update network configuration + { + cat << EOF +sudo tee /etc/network/interfaces +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp + +auto eth1 +iface eth1 inet static + address $ADDRESS + netmask $NETMASK + gateway $GATEWAY + dns-nameservers $DNS_ADDRESSES + +auto eth2 + iface eth2 inet static + address 192.168.33.1 + netmask 255.255.255.0 +EOF + } | bash_on_appliance + + tmpdomzerokey=$(mktemp) + + # Enable domzero user to log in to dom0 + echo "cat /home/$DOMZERO_USER/.ssh/id_rsa.pub" | bash_on_appliance > $tmpdomzerokey + + # Update ssh keys and reboot, so settings applied + { + echo "sudo tee /root/.ssh/authorized_keys" + cat /root/.ssh/authorized_keys + } | bash_on_appliance + + echo "sudo reboot" | bash_on_appliance + + cat $tmpdomzerokey >> /root/.ssh/authorized_keys +} + +function transfer_settings_to_appliance { + local network_settings + + network_settings="$1" + + configure_networking "$network_settings" + /opt/xensource/libexec/interface-reconfigure rewrite +} + +function dump_disk_config { + echo "DUMPING Primary disk's configuration" + sfdisk -d /dev/xvda +} + +main diff --git a/nodepool/scripts/install_devstack_dependencies.sh b/nodepool/scripts/install_devstack_dependencies.sh new file mode 100755 index 0000000..a1c713e --- /dev/null +++ b/nodepool/scripts/install_devstack_dependencies.sh @@ -0,0 +1,52 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p ~/cache/files +mkdir -p ~/cache/pip + +if [ -f /usr/bin/yum ]; then + sudo yum -y install python-devel python3-devel make automake gcc gcc-c++ \ + kernel-devel redhat-lsb-core +elif [ -f /usr/bin/apt-get ]; then + if [ "$(lsb_release -c -s)" = "precise" ]; then + # temporary - remove the breaking -70 kernel and + # reinstall -69 to prevent openvswitch breakage + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + --option "Dpkg::Options::=--force-confold" \ + --assume-yes remove linux-headers-3.2.0-70 \ + linux-headers-3.2.0-70-virtual \ + linux-image-3.2.0-70-virtual \ + linux-headers-3.2.0-70-generic \ + linux-image-3.2.0-70-generic + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + --option "Dpkg::Options::=--force-confold" \ + --assume-yes install --reinstall linux-headers-3.2.0-69 \ + linux-headers-3.2.0-69-virtual \ + linux-image-3.2.0-69-virtual \ + python-software-properties build-essential python-dev python3-dev + else + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + --option "Dpkg::Options::=--force-confold" \ + --assume-yes install build-essential python-dev python3-dev \ + python-software-properties linux-headers-virtual \ + linux-headers-$(uname -r) + fi +else + echo "Unsupported distro." + exit 1 +fi diff --git a/nodepool/scripts/install_xenserver.sh b/nodepool/scripts/install_xenserver.sh new file mode 100755 index 0000000..ed264e3 --- /dev/null +++ b/nodepool/scripts/install_xenserver.sh @@ -0,0 +1,27 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# Copyright (c) 2014 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +XENSERVER_XVA_URL=${NODEPOOL_XENSERVER_XVA_URL:-http://downloads.vmd.citrix.com/OpenStack/xenapi-in-the-cloud-appliances/prod_ci} +XENSERVER_ISO_URL=${NODEPOOL_XENSERVER_ISO_URL:-http://downloadns.citrix.com.edgesuite.net/akdlm/8159/XenServer-6.2.0-install-cd.iso} + +./convert_node_to_xenserver.sh \ + password \ + "$XENSERVER_XVA_URL" \ + devstack \ + "$XENSERVER_ISO_URL" diff --git a/nodepool/scripts/multinode_setup.sh b/nodepool/scripts/multinode_setup.sh new file mode 100755 index 0000000..7043511 --- /dev/null +++ b/nodepool/scripts/multinode_setup.sh @@ -0,0 +1,41 @@ +#!/bin/bash -xe + +# Copyright (C) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +export PATH=$PATH:/usr/local/sbin:/usr/sbin + +for ip in $(cat /etc/nodepool/primary_node /etc/nodepool/sub_nodes /etc/nodepool/primary_node_private /etc/nodepool/sub_nodes_private | sort -u); do + sudo iptables -I openstack-INPUT 1 -s $ip -j ACCEPT +done + +echo "" >> /home/jenkins/.ssh/authorized_keys +cat /etc/nodepool/id_rsa.pub >> /home/jenkins/.ssh/authorized_keys +echo "" >> /home/jenkins/.ssh/authorized_keys + +ROLE=$(cat /etc/nodepool/role) +if [ $ROLE == "primary" ]; then + cp /etc/nodepool/id_rsa /home/jenkins/.ssh/id_rsa + chmod 0600 /home/jenkins/.ssh/id_rsa +else + rm /etc/nodepool/id_rsa +fi + +sudo chown -R root:root /etc/nodepool +sudo chmod 0755 /etc/nodepool +sudo chmod 0444 /etc/nodepool/* + +/opt/nodepool-scripts/configure_mirror.sh diff --git a/nodepool/scripts/prepare_devstack.sh b/nodepool/scripts/prepare_devstack.sh new file mode 100755 index 0000000..16e05bb --- /dev/null +++ b/nodepool/scripts/prepare_devstack.sh @@ -0,0 +1,22 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /opt/nodepool-scripts/ +./install_devstack_dependencies.sh +DISTRIB_CODENAME=$(lsb_release -sc) +python ./cache_devstack.py $DISTRIB_CODENAME diff --git a/nodepool/scripts/prepare_devstack_virt_preview.sh b/nodepool/scripts/prepare_devstack_virt_preview.sh new file mode 100755 index 0000000..f86e202 --- /dev/null +++ b/nodepool/scripts/prepare_devstack_virt_preview.sh @@ -0,0 +1,25 @@ +#!/bin/bash -xe + +# Copyright (C) 2014 - Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install the virt-preview repository that includes bleeding edge versions of +# libvirt and qemu. The packages from this repo will get installed via +# prepare_devstack.sh. +wget http://fedorapeople.org/groups/virt/virt-preview/fedora-virt-preview.repo +sudo mv fedora-virt-preview.repo /etc/yum.repos.d/ + +/opt/nodepool-scripts/prepare_devstack.sh diff --git a/nodepool/scripts/prepare_node.sh b/nodepool/scripts/prepare_node.sh new file mode 100755 index 0000000..c039dcc --- /dev/null +++ b/nodepool/scripts/prepare_node.sh @@ -0,0 +1,241 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTNAME=$1 + +SUDO=${SUDO:-true} +THIN=${THIN:-true} +ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false} +GIT_BASE=${GIT_BASE:-git://git.openstack.org} + +sudo hostname $HOSTNAME +if [ -n "$HOSTNAME" ] && ! grep -q $HOSTNAME /etc/hosts ; then + echo "127.0.1.1 $HOSTNAME" | sudo tee -a /etc/hosts +fi + +echo $HOSTNAME > /tmp/image-hostname.txt +sudo mv /tmp/image-hostname.txt /etc/image-hostname.txt + +if [ ! -f /etc/redhat-release ]; then + # Cloud provider apt repos break us - so stop using them + LSBDISTID=$(lsb_release -is) + LSBDISTCODENAME=$(lsb_release -cs) + if [ "$LSBDISTID" == "Ubuntu" ] ; then + sudo dd of=/etc/apt/sources.list < $SUDO, + thin => $THIN, + all_mysql_privs => $ALL_MYSQL_PRIVS, + }" + PUPPET_RET_CODE=$? +else + sudo puppet apply --detailed-exitcodes --color=false \ + --modulepath=/root/system-config/modules:/etc/puppet/modules \ + -e "class {'openstack_project::single_use_slave': + install_users => false, + sudo => $SUDO, + thin => $THIN, + all_mysql_privs => $ALL_MYSQL_PRIVS, + ssh_key => '$NODEPOOL_SSH_KEY', + }" + PUPPET_RET_CODE=$? +fi +# Puppet doesn't properly return exit codes. Check here the values that +# indicate failure of some sort happened. 0 and 2 indicate success. +if [ "$PUPPET_RET_CODE" -eq "4" ] || [ "$PUPPET_RET_CODE" -eq "6" ] ; then + exit $PUPPET_RET_CODE +fi +set -e + +# The puppet modules should install unbound. Set up some nameservers. +cat >/tmp/forwarding.conf < /etc/resolv.conf + +exit 0 +EOF + +# hpcloud has started mounting ephemeral /dev/vdb at /mnt. +# devstack-gate wants to partition the ephemeral disk, add some swap +# and mount it at /opt. get rid of the mount. +# +# note this comes down from the cloud-init metadata; which we setup to +# ignore below. +sudo sed -i '/^\/dev\/vdb/d' /etc/fstab + + +# Make all cloud-init data sources match rackspace- only attempt to look +# at ConfigDrive, not at metadata service. This is not needed if there +# is no cloud-init +if [ -d /etc/cloud/cloud.cfg.d ] ; then +sudo dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg <> /etc/unbound/unbound.conf" +if [ -e /etc/init.d/unbound ] ; then + sudo /etc/init.d/unbound restart +elif [ -e /usr/lib/systemd/system/unbound.service ] ; then + sudo systemctl restart unbound +else + echo "Can't discover a method to restart \"unbound\"" + exit 1 +fi + +# Make sure DNS works. +dig git.openstack.org + +# Cache all currently known gerrit repos. +sudo mkdir -p /opt/git +sudo -i python /opt/nodepool-scripts/cache_git_repos.py $GIT_BASE + +# We don't always get ext4 from our clouds, mount ext3 as ext4 on the next +# boot (eg when this image is used for testing). +sudo sed -i 's/ext3/ext4/g' /etc/fstab + +# Remove additional sources used to install puppet or special version of pypi. +# We do this because leaving these sources in place causes every test that +# does an apt-get update to hit those servers which may not have the uptime +# of our local mirrors. +OS_FAMILY=$(facter osfamily) +if [ "$OS_FAMILY" == "Debian" ] ; then + sudo rm -f /etc/apt/sources.list.d/* + sudo apt-get update +elif [ "$OS_FAMILY" == "RedHat" ] ; then + # Can't delete * in yum.repos.d since all of the repos are listed there. + # Be specific instead. + if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then + sudo rm -f /etc/yum.repos.d/puppetlabs.repo + fi +fi + +# Remove cron jobs +# We create fresh servers for these hosts, and they are used once. They don't +# need to do things like update the locatedb or the mandb or rotate logs +# or really any of those things. We only want code running here that we want +# here. +sudo rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/* + +# Install Zuul into a virtualenv +# This is in /usr instead of /usr/local due to this bug on precise: +# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 +git clone /opt/git/openstack-infra/zuul /tmp/zuul +sudo virtualenv /usr/zuul-env +sudo -H /usr/zuul-env/bin/pip install /tmp/zuul +sudo rm -fr /tmp/zuul + +# Create a virtualenv for zuul-swift-logs +# This is in /usr instead of /usr/local due to this bug on precise: +# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 +sudo -H virtualenv /usr/zuul-swift-logs-env +sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \ + requests glob2 + +# Create a virtualenv for os-testr (which contains subunit2html) +# this is in /usr instead of /usr/loca/ due to this bug on precise: +# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 +sudo -H virtualenv /usr/os-testr-env +sudo -H /usr/os-testr-env/bin/pip install os-testr diff --git a/nodepool/scripts/prepare_node_bare.sh b/nodepool/scripts/prepare_node_bare.sh new file mode 100755 index 0000000..5caa0a9 --- /dev/null +++ b/nodepool/scripts/prepare_node_bare.sh @@ -0,0 +1,27 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTNAME=$1 + +export SUDO='true' +export THIN='false' +export ALL_MYSQL_PRIVS='true' + +./prepare_node.sh "$HOSTNAME" + +./restrict_memory.sh diff --git a/nodepool/scripts/prepare_node_devstack.sh b/nodepool/scripts/prepare_node_devstack.sh new file mode 100755 index 0000000..d48c5e7 --- /dev/null +++ b/nodepool/scripts/prepare_node_devstack.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTNAME=$1 + +export SUDO='true' +export THIN='true' +TEMPEST_DIR=${TEMPEST_DIR:-/opt/git/openstack/tempest} + +./prepare_node.sh "$HOSTNAME" +sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack.sh "$HOSTNAME" + +# Setup venv and install deps for prepare_tempest_testrepository.py +sudo virtualenv -p python2 /opt/git/subunit2sql-env +sudo -H /opt/git/subunit2sql-env/bin/pip install -U testrepository \ + subunit2sql PyMySQL + +# Pre-seed tempest testrepository with data from subunit2sql +sudo -i env PATH=/opt/git/subunit2sql-env/bin:$PATH \ + /opt/git/subunit2sql-env/bin/python2 \ + /opt/nodepool-scripts/prepare_tempest_testrepository.py \ + $TEMPEST_DIR + +sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams + +# Delete the venv after the script is called +sudo rm -rf /opt/git/subunit2sql-env + +./restrict_memory.sh diff --git a/nodepool/scripts/prepare_node_devstack_virt_preview.sh b/nodepool/scripts/prepare_node_devstack_virt_preview.sh new file mode 100755 index 0000000..8e9dfe8 --- /dev/null +++ b/nodepool/scripts/prepare_node_devstack_virt_preview.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xe + +# Copyright (C) 2014 - Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTNAME=$1 + +export SUDO='true' +export THIN='true' + +./prepare_node.sh "$HOSTNAME" +sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack_virt_preview.sh \ + "$HOSTNAME" + +./restrict_memory.sh diff --git a/nodepool/scripts/prepare_node_tripleo.sh b/nodepool/scripts/prepare_node_tripleo.sh new file mode 100755 index 0000000..075e047 --- /dev/null +++ b/nodepool/scripts/prepare_node_tripleo.sh @@ -0,0 +1,27 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTNAME=$1 + +export SUDO='true' +export THIN='true' + +# Workaround bug 1270646 during node bootstrapping. +sudo ip link set mtu 1458 dev eth0 +./prepare_node.sh "$HOSTNAME" +sudo -u jenkins -i /opt/nodepool-scripts/prepare_tripleo.sh "$HOSTNAME" diff --git a/nodepool/scripts/prepare_node_xenserver.sh b/nodepool/scripts/prepare_node_xenserver.sh new file mode 100755 index 0000000..1fc8905 --- /dev/null +++ b/nodepool/scripts/prepare_node_xenserver.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + + +HOSTNAME=$1 + +function tsfilter { + $@ 2>&1 | awk ' + { + cmd ="date +\"%Y-%m-%d %H:%M:%S.%3N | \"" + cmd | getline now + close("date +\"%Y-%m-%d %H:%M:%S.%3N | \"") + sub(/^/, now) + print + fflush() + }' +} + +set -o pipefail +tsfilter ./prepare_node_devstack.sh "$HOSTNAME" + +# After the node has been prepared, the hypervisor needs to be halted to make +# sure that the filesystem is in a consistent state. +sudo -u domzero \ + ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + root@192.168.33.2 \ + halt -p 2: + TEMPEST_PATH = sys.argv[1] + DB_URI = sys.argv[2] +else: + TEMPEST_PATH = '/opt/stack/new/tempest' + + +def main(): + shell.parse_args([]) + shell.CONF.set_override('connection', DB_URI, group='database') + session = api.get_session() + run_ids = api.get_recent_successful_runs(num_runs=10, + session=session) + session.close() + preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams') + os.mkdir(preseed_path) + for run in run_ids: + with open(os.path.join(preseed_path, run + '.subunit'), 'w') as fd: + write_subunit.sql2subunit(run, fd) + +if __name__ == '__main__': + main() diff --git a/nodepool/scripts/prepare_tripleo.sh b/nodepool/scripts/prepare_tripleo.sh new file mode 100755 index 0000000..214456d --- /dev/null +++ b/nodepool/scripts/prepare_tripleo.sh @@ -0,0 +1,62 @@ +#!/bin/bash -xe + +# Copyright (C) 2011-2013 OpenStack Foundation +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Enable precise-backports so we can install jq +if [ -f /usr/bin/apt-get ]; then + sudo sed -i -e 's/# \(deb .*precise-backports main \)/\1/g' \ + /etc/apt/sources.list + sudo apt-get update +fi + +cd /opt/nodepool-scripts/ +./install_devstack_dependencies.sh + +# toci scripts use both of these +sudo -H pip install gear os-apply-config + +# tripleo-gate runs with two networks - the public access network and eth1 +# pointing at the in-datacentre L2 network where we can talk to the test +# environments directly. We need to enable DHCP on eth1 though. +# Note that we don't bring it up during prepare - it's only needed to run +# tests. + +if [ -d /etc/sysconfig/network-scripts ]; then + sudo dd of=/etc/sysconfig/network-scripts/ifcfg-eth1 << EOF +DEVICE="eth1" +BOOTPROTO="dhcp" +ONBOOT="yes" +TYPE="Ethernet" +PEERDNS="no" +EOF + +elif [ -f /etc/network/interfaces ]; then + sudo dd of=/etc/network/interfaces oflag=append conv=notrunc << EOF +auto eth1 +iface eth1 inet dhcp +EOF + +# Workaround bug 1270646 for actual slaves + sudo dd of=/etc/network/interfaces.d/eth0.cfg oflag=append conv=notrunc << EOF + post-up ip link set mtu 1458 dev eth0 +EOF + +else + echo "Unsupported distro." + exit 1 +fi diff --git a/nodepool/scripts/restrict_memory.sh b/nodepool/scripts/restrict_memory.sh new file mode 100755 index 0000000..f5843b7 --- /dev/null +++ b/nodepool/scripts/restrict_memory.sh @@ -0,0 +1,34 @@ +#!/bin/bash -xe +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Limit all test slaves to 8GB of memory so that larger flavors with more +# cpu resources can be used without the risk of becoming dependent on more +# memory. +if [ -f /etc/default/grub ] ; then + sudo sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' \ + -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=9023M /g' \ + /etc/default/grub + if which update-grub &> /dev/null ; then + sudo update-grub + else + # If update-grub isn't available, use grub2-mkconfig directly + sudo /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg + fi +elif [ -f /boot/grub/grub.conf ] ; then + sudo sed -i -e 's/^timeout=[0-9]\+/timeout=0/' \ + -e 's/\(^\s\+kernel.*\)/\1 mem=9023M/' /boot/grub/grub.conf +fi diff --git a/nodepool/scripts/xenserver_helper_initramfs_hook b/nodepool/scripts/xenserver_helper_initramfs_hook new file mode 100644 index 0000000..328b197 --- /dev/null +++ b/nodepool/scripts/xenserver_helper_initramfs_hook @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +PREREQ="" + +prereqs () { + echo "${PREREQ}" +} + +case "${1}" in + prereqs) + prereqs + exit 0 + ;; +esac + +. /usr/share/initramfs-tools/hook-functions + +copy_exec /sbin/resize2fs +copy_exec /sbin/e2fsck +copy_exec /usr/bin/expr +copy_exec /sbin/tune2fs +copy_exec /bin/grep +copy_exec /usr/bin/tr +copy_exec /usr/bin/cut +copy_exec /sbin/sfdisk +copy_exec /sbin/partprobe +copy_exec /bin/sed diff --git a/nodepool/scripts/xenserver_helper_initramfs_premount b/nodepool/scripts/xenserver_helper_initramfs_premount new file mode 100644 index 0000000..3f7dc9b --- /dev/null +++ b/nodepool/scripts/xenserver_helper_initramfs_premount @@ -0,0 +1,49 @@ +#!/bin/sh -e + +set -ex + +PREREQ="" + +# Output pre-requisites +prereqs() +{ + echo "$PREREQ" +} + +case "$1" in + prereqs) + prereqs + exit 0 + ;; +esac + +. /scripts/functions + +log_begin_msg "Resize started" +touch /etc/mtab + +tune2fs -O ^has_journal /dev/xvda1 +e2fsck -fp /dev/xvda1 +resize2fs /dev/xvda1 4G + +# Number of 4k blocks +NUMBER_OF_BLOCKS=$(tune2fs -l /dev/xvda1 | grep "Block count" | tr -d " " | cut -d":" -f 2) + +# Convert them to 512 byte sectors +SIZE_OF_PARTITION=$(expr $NUMBER_OF_BLOCKS \* 8) + +sfdisk -d /dev/xvda | sed -e "s,[0-9]\{8\},$SIZE_OF_PARTITION,g" > /tmp/new_layout + +while ! cat /tmp/new_layout | sfdisk /dev/xvda; do + sleep 1 +done + +while ! partprobe /dev/xvda; do + sleep 1 +done + +tune2fs -j /dev/xvda1 + +sync + +log_end_msg "Resize finished" -- 2.39.5