From: Bob Ball Date: Thu, 11 Feb 2016 11:52:34 +0000 (+0000) Subject: Added libvirt+xen config X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=def12068149b6f30f9dcffdc8098a82e94a556eb;p=openstack%2Fci-loop-config.git Added libvirt+xen config --- diff --git a/install_master.sh b/install_master.sh new file mode 100644 index 0000000..7433f09 --- /dev/null +++ b/install_master.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -eux + +THIS_DIR=`pwd` +DATA_PATH=/root/os-ext-data + +# Steps to reinstall: +# 1) Create new Ubuntu 14.04 server (copy password) +# 1a) 7.5GB Compute v1 flavor +# 1b) Enable monitoring and security updates +# 2) Disable password authentication +# 2a) ssh-copy-id to copy a key to the server +# 2b) edit /etc/sshd_config to set "PermitRootLogin without-password" +# 3) Copy the secret credentials dir (http://hg.uk.xensource.com/openstack/infrastructure.hg/os-ext-data) to /root +# 4) Clone this repo git: +# 4a) git clone https://github.com/citrix-openstack/os-ext-testing.git +# 4b) cd os-ext-testing; git checkout common_ci +# ?) Follow steps below +# ?) Set up monitoring checks https://intelligence.rackspace.com/cloud/entities/enWCIYVVnt + +# Copied from the following URL Feb 2016 +# http://docs.openstack.org/infra/openstackci/third_party_ci.html + +sudo su - +# Install puppet +[ -e install_puppet.sh ] && rm install_puppet.sh +wget https://git.openstack.org/cgit/openstack-infra/system-config/plain/install_puppet.sh +bash install_puppet.sh + +# Install puppet modules to /etc/puppet/modules +git clone https://git.openstack.org/openstack-infra/system-config +cd system-config +./install_modules.sh +exit + +sudo su - +# Setup the site we're deploying +cp /etc/puppet/modules/openstackci/contrib/single_node_ci_site.pp /etc/puppet/manifests/site.pp + +# And the secret credentials store +cp /etc/puppet/modules/openstackci/contrib/hiera.yaml /etc/puppet + +# Verify that our config file matches the latest template file +# If this check fails, copy the new template to os-ext-data and create a new secrets file +diff -q /root/os-ext-data/single_node_ci_data_orig.yaml /etc/puppet/modules/openstackci/contrib/single_node_ci_data.yaml +# Since the template hasn't changed, just use the existing secrets +cp /root/os-ext-data/single_node_ci_data.yaml /etc/puppet/environments/common.yaml + +# Add 'jenkins' to the hostname so Apache is happy +sed -i -e 's/^\(127\.0\.0\.1.*\)$/\1 jenkins/' /etc/hosts +exit + +sudo puppet apply --verbose /etc/puppet/manifests/site.pp diff --git a/jenkins/jobs/jobs.yaml b/jenkins/jobs/jobs.yaml new file mode 100644 index 0000000..fe262ff --- /dev/null +++ b/jenkins/jobs/jobs.yaml @@ -0,0 +1,57 @@ +- job-template: + name: 'dsvm-tempest-xen' + node: '{node}' + + wrappers: + - timeout: + timeout: 185 # Timeout in *minutes* + fail: true # A job run that exceeds the timeout will cause a failure + - timestamps + + builders: + - net-info + - devstack-checkout + - shell: | + #!/bin/bash -xe + + function pre_test_hook {{ + echo "Install thirdparty libraries" + + # Install libvirt here to ensure it is + # not replaced devstack + sudo dpkg -i /opt/nodepool-scripts/*.deb + sudo apt-get install -y -f --fix-missing + }} + + function cleanup_host_hook {{ + echo "Copying custom logs" + + # Xen logs + if [ -d /var/log/xen ]; then + sudo cp -r /var/log/xen $BASE/logs/ + fi + }} + + export -f pre_test_hook + export -f cleanup_host_hook + + if [ -z $ZUUL_PROJECT ]; then + export ZUUL_PROJECT=openstack-dev/sandbox + fi + if [ -z $ZUUL_BRANCH ]; then + export ZUUL_BRANCH=master + fi + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_VIRT_DRIVER=libvirt + export DEVSTACK_GATE_LIBVIRT_TYPE=xen + export DEVSTACK_GATE_TIMEOUT=180 + export DEVSTACK_GATE_TEMPEST_REGEX='(?!.*\[.*\bslow\b.*\]|.*test_volume_boot_pattern)(^tempest\.(api|scenario|thirdparty))' + export RE_EXEC=true + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + - link-logs # In macros.yaml from os-ext-testing + + publishers: + - osci-swift-upload-logs-with-console: + upload_source: 'logs' diff --git a/jenkins/jobs/macros-common.yaml b/jenkins/jobs/macros-common.yaml index 10a7806..02222d3 100644 --- a/jenkins/jobs/macros-common.yaml +++ b/jenkins/jobs/macros-common.yaml @@ -93,3 +93,39 @@ ip -6 route show echo "Network neighbors..." ip neighbor show + +- builder: + name: osci-swift-upload-logs-with-console + builders: + - shell: "/usr/local/jenkins/slave_scripts/grab_console_log.sh" + - shell: | + #!/bin/sh + virtualenv osci-env + . osci-env/bin/activate + git clone https://github.com/citrix-openstack/openstack-citrix-ci openstack-citrix-ci + pip install -r openstack-citrix-ci/requirements.txt + pip install -e openstack-citrix-ci + # CA bundle in 2.5.2 is broken + pip install --upgrade requests>2.5.2 + sudo mkdir -p /etc/osci/ + cat | sudo tee /etc/osci/osci.config >/dev/null << EOF + SWIFT_API_KEY=<%= @swift_api_key %> + SWIFT_USERNAME=<%= @swift_api_user %> + SWIFT_REGION=IAD + SWIFT_CONTAINER=XenLogs + EOF + osci-upload /tmp/console.html {upload_source} $LOG_PATH + + +- publisher: + name: osci-swift-upload-logs-with-console + publishers: + - postbuildscript: + builders: + - osci-swift-upload-logs-with-console: + upload_source: '{upload_source}' + # These flags really mean only if on * when + # checked. When both set to false the post script + # should run on every job regardless of status. + onfailure: False + onsuccess: False diff --git a/nodepool/nodepool.yaml b/nodepool/nodepool.yaml index 21f7f97..b43e59f 100644 --- a/nodepool/nodepool.yaml +++ b/nodepool/nodepool.yaml @@ -15,7 +15,7 @@ cron: # root@jenkins:~# echo "0 $((RANDOM%23+1)) * * *" # 0 16 * * * # This references the hour of the day when images will be built. - image-update: '0 0 * * *' + image-update: '0 11 * * *' zmq-publishers: - tcp://localhost:8888 @@ -24,40 +24,73 @@ gearman-servers: - host: 127.0.0.1 labels: - - name: d-p-c - image: dpc - min-ready: 1 + - name: ds-xen + image: ds-xen + min-ready: 2 providers: - - name: local_01 - -diskimages: - - name: dpc - elements: - - ubuntu - - vm - - openstack-repos - - puppet - - nodepool-base - - node-devstack - release: trusty - env-vars: - TMPDIR: /opt/dib_tmp - DIB_IMAGE_CACHE: /opt/dib_cache + - name: rax-iad providers: - - name: local_01 -#TODO: Update the provider username, password, and authurl + - name: rax-iad + region-name: 'IAD' + service-type: 'compute' + service-name: 'cloudServersOpenStack' username: '<%= provider_username %>' password: '<%= provider_password %>' - auth-url: 'http://:5000/v2.0' - project-name: 'admin' - max-servers: 2 + project-id: '874240' + auth-url: 'https://identity.api.rackspacecloud.com/v2.0/' + boot-timeout: 600 + max-servers: 5 + rate: 10.0 + keypair: 'xenproject-nodepool' + template-hostname: '{image.name}-{timestamp}.template.xen.org' images: - - name: dpc + - name: ds-xen + # This is the image name to use found in this provider's `glance image-list` + base-image: 'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)' min-ram: 8192 - diskimage: dpc - username: jenkins - private-key: '/home/nodepool/.ssh/id_rsa' + name-filter: 'Performance' + # This is the script that will be used to prepare the image + setup: prepare_node_devstack_xen.sh + #Set the public key part only of the jenkin's key (no whitespace) to the NODEPOOL_SSH_KEY environment variable in order to + #inject it into the image's jenkins user .ssh/authorized_keys + #This key will be used to log in and setup jenkins on the target VM +# private-key: '/home/nodepool/.ssh/id_rsa' + config-drive: true + meta: + xenapi_use_agent: 'false' +# - name: rax-ord +# region-name: 'ORD' +# username: '<%= provider_username %>' +# password: '<%= provider_password %>' +# project-id: '874240' +# auth-url: 'https://identity.api.rackspacecloud.com/v2.0/' +# boot-timeout: 600 +# max-servers: 5 +# rate: 10.0 +# keypair: 'xenproject-nodepool' +# template-hostname: '{image.name}-{timestamp}.template.xen.org' +# images: +# - name: ds-xen +# # This is the image name to use found in this provider's `glance image-list` +# base-image: 'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)' +# min-ram: 8192 +# name-filter: 'Performance' +# # This is the script that will be used to prepare the image +# setup: prepare_node_devstack_xen.sh +# #Set the public key part only of the jenkin's key (no whitespace) to the NODEPOOL_SSH_KEY environment variable in order to +# #inject it into the image's jenkins user .ssh/authorized_keys +# #This key will be used to log in and setup jenkins on the target VM +# private-key: '/home/nodepool/.ssh/id_rsa' +# config-drive: true +# meta: +# xenapi_use_agent: 'false' + targets: - name: jenkins1 +# jenkins: +# url: 'http://localhost:8080/' +# user: '<%= jenkins_api_user %>' +# apikey: '<%= jenkins_api_key %>' +# credentials-id: '<%= jenkins_credentials_id %>' diff --git a/nodepool/scripts/libvirt-bin_1.2.15-anthony_amd64.deb b/nodepool/scripts/libvirt-bin_1.2.15-anthony_amd64.deb new file mode 100644 index 0000000..2b18fa0 Binary files /dev/null and b/nodepool/scripts/libvirt-bin_1.2.15-anthony_amd64.deb differ diff --git a/nodepool/scripts/libvirt-dev_1.2.15-anthony_amd64.deb b/nodepool/scripts/libvirt-dev_1.2.15-anthony_amd64.deb new file mode 100644 index 0000000..49aea62 Binary files /dev/null and b/nodepool/scripts/libvirt-dev_1.2.15-anthony_amd64.deb differ diff --git a/nodepool/scripts/libvirt0-dbg_1.2.15-anthony_amd64.deb b/nodepool/scripts/libvirt0-dbg_1.2.15-anthony_amd64.deb new file mode 100644 index 0000000..8f71051 Binary files /dev/null and b/nodepool/scripts/libvirt0-dbg_1.2.15-anthony_amd64.deb differ diff --git a/nodepool/scripts/libvirt0_1.2.15-anthony_amd64.deb b/nodepool/scripts/libvirt0_1.2.15-anthony_amd64.deb new file mode 100644 index 0000000..d5a5642 Binary files /dev/null and b/nodepool/scripts/libvirt0_1.2.15-anthony_amd64.deb differ diff --git a/nodepool/scripts/libxen-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/libxen-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..81dd333 Binary files /dev/null and b/nodepool/scripts/libxen-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/nodepool/scripts/libxen-dev_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/libxen-dev_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..2c68d90 Binary files /dev/null and b/nodepool/scripts/libxen-dev_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/nodepool/scripts/libxenstore3.0_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/libxenstore3.0_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..11959ad Binary files /dev/null and b/nodepool/scripts/libxenstore3.0_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/nodepool/scripts/prepare_node_devstack_xen.sh b/nodepool/scripts/prepare_node_devstack_xen.sh new file mode 100644 index 0000000..b4aedeb --- /dev/null +++ b/nodepool/scripts/prepare_node_devstack_xen.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -xe + +./prepare_node_devstack.sh + +# Install Xen and update the kernel +sudo apt-get -y install htop vim git pwgen xen-hypervisor-4.4 + +sudo mv /etc/default/grub{,.bak} +sudo tee /etc/default/grub << EOC +GRUB_DEFAULT=0 +GRUB_TIMEOUT=30 +GRUB_DISTRIBUTOR=\`lsb_release -i -s 2> /dev/null || echo Debian\` +GRUB_CMDLINE_LINUX_DEFAULT="debug loglevel=7" +GRUB_CMDLINE_LINUX="" +GRUB_TERMINAL=console +GRUB_DISABLE_LINUX_UUID="false" +EOC + +sudo mv /etc/default/grub.d/xen.cfg{,.bak} +sudo tee /etc/default/grub.d/xen.cfg << EOC +echo "Including Xen overrides from /etc/default/grub.d/xen.cfg" +GRUB_CMDLINE_XEN="dom0_mem=7680M:max=7680M dom0_max_vcpus=4 max-console=vga" +GRUB_DEFAULT="Ubuntu GNU/Linux, with Xen hypervisor" +EOC + +sudo update-grub + +# Disable cloud-init - it doesn't work in dom0 Xen +for i in /etc/init/cloud-*; do echo manual | sudo tee ${i%.conf}.override; done + +# Install Monty's config drive network setup (from https://review.openstack.org/#/c/154132/5) +# ./read-vendor-json/install.d/05-read-vendor-json +sudo cp read-vendor-json.py /usr/local/bin +sudo cp static-network-config /etc/init.d + +# ./read-vendor-json/install.d/10-static-network-config +sudo tee /etc/init/write-network-interfaces.conf </etc/sysconfig/network-scripts/ifcfg-$interface </dev/null 2>&1 ; then + exit 0 + fi + for interface in eth0 eth1 ; do + cat >>/etc/network/interfaces </dev/null 2>&1 || exit 0 + +python /usr/local/bin/read-vendor-json.py + +if [ -f /etc/network/interfaces ] ; then + ifdown -a ; ifup -a +else + for interface in /etc/sysconfig/network-scripts/ifcfg-eth* ; do + name=$(echo $interface | cut -f3 -d-) + ifdown $name ; ifup $name + done +fi diff --git a/nodepool/scripts/xen-hypervisor-4.4-amd64_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/xen-hypervisor-4.4-amd64_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..3a2ccc1 Binary files /dev/null and b/nodepool/scripts/xen-hypervisor-4.4-amd64_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/nodepool/scripts/xen-utils-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/xen-utils-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..df793bd Binary files /dev/null and b/nodepool/scripts/xen-utils-4.4_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/nodepool/scripts/xen-utils-common_4.4.2-0ubuntu0.14.04.2.anthony_all.deb b/nodepool/scripts/xen-utils-common_4.4.2-0ubuntu0.14.04.2.anthony_all.deb new file mode 100644 index 0000000..72dc855 Binary files /dev/null and b/nodepool/scripts/xen-utils-common_4.4.2-0ubuntu0.14.04.2.anthony_all.deb differ diff --git a/nodepool/scripts/xenstore-utils_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb b/nodepool/scripts/xenstore-utils_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb new file mode 100644 index 0000000..7645809 Binary files /dev/null and b/nodepool/scripts/xenstore-utils_4.4.2-0ubuntu0.14.04.2.anthony_amd64.deb differ diff --git a/zuul/layout.yaml b/zuul/layout.yaml index fba83a2..3a87990 100644 --- a/zuul/layout.yaml +++ b/zuul/layout.yaml @@ -5,7 +5,7 @@ includes: pipelines: - name: check description: Newly uploaded patchsets enter this pipeline to receive an initial +/-1 Verified vote from Jenkins. - failure-message: "Build failed. For 3rd party CI contact info: https://wiki.openstack.org/wiki/ThirdPartySystems" + failure-message: "Build failed. To recheck use 'xen: recheck'. For 3rd party ci contact info: https://wiki.openstack.org/wiki/ThirdPartySystems" manager: IndependentPipelineManager trigger: gerrit: @@ -20,23 +20,20 @@ pipelines: gerrit: verified: 1 smtp: - #TODO: Update these to real e-mail addresses - to: third_party_ci@example.com - from: zuul@example.com + to: openstack-ci@xenproject.org + from: zuul@zuul.openstack.xenproject.org subject: 'Silent check of {change.project} {change.number},{change.patchset} passed' failure: gerrit: verified: -1 smtp: - #TODO: Update these to real e-mail addresses - to: third_party_ci@example.com - from: zuul@example.com + to: openstack-ci@xenproject.org + from: zuul@zuul.openstack.xenproject.org subject: 'Silent check of {change.project} {change.number},{change.patchset} failed' merge-failure: smtp: - #TODO: Update these to real e-mail addresses - to: third_party_ci@example.com - from: zuul@example.com + to: openstack-ci@xenproject.org + from: zuul@zuul.openstack.xenproject.org subject: Upstream change {change} has a merge failure @@ -51,29 +48,43 @@ pipelines: comment: (?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*recheck myvendor\s*$ success: smtp: - #TODO: Update these to real e-mail addresses - to: third_party_ci@example.com - from: zuul@example.com + to: openstack-ci@xenproject.org + from: zuul@zuul.openstack.xenproject.org subject: 'Silent check of {change.project} {change.number},{change.patchset} passed' failure: smtp: - #TODO: Update these to real e-mail addresses - to: third_party_ci@example.com - from: zuul@example.com + to: openstack-ci@xenproject.org + from: zuul@zuul.openstack.xenproject.org subject: 'Silent check of {change.project} {change.number},{change.patchset} failed' jobs: - name: ^dsvm-tempest.*$ + skip-if: + - project: ^openstack/nova$ + branch: ^master$ + all-files-match-any: + - ^.*\.rst$ + - ^doc/.*$ + - ^nova/tests/.*$ + - ^nova/virt/baremetal/.*$ + - ^nova/virt/hyperv/.*$ + - ^nova/virt/ironic/.*$ + - ^nova/virt/vmwareapi/.*$ + - ^nova/virt/xenapi/.*$ + - ^tools/.*$ + - ^tox.ini$ parameter-function: single_use_node + branch: ^master.*$ + swift: + - name: logs + projects: - name: openstack-dev/ci-sandbox check: - # Remove this after successfully verifying communication with upstream - # and seeing a posted successful review. - - noop-check-communication - silent: - # Uncomment this job when you have a jenkins slave running and want to - # test a full Tempest run within devstack. - - dsvm-tempest-full + - dsvm-tempest-xen + + - name: openstack/nova +# check: +# - dsvm-tempest-xen