]> xenbits.xensource.com Git - people/gdunlap/mlib.git/commitdiff
Initial commit
authorGeorge Dunlap <george.dunlap@citrix.com>
Tue, 17 May 2016 16:59:19 +0000 (17:59 +0100)
committerGeorge Dunlap <george.dunlap@citrix.com>
Tue, 17 May 2016 16:59:19 +0000 (17:59 +0100)
README [new file with mode: 0644]
core.sh [new file with mode: 0644]
ml [new file with mode: 0755]
release.sh [new file with mode: 0644]
security.sh [new file with mode: 0644]

diff --git a/README b/README
new file mode 100644 (file)
index 0000000..7825748
--- /dev/null
+++ b/README
@@ -0,0 +1,5 @@
+To set up:
+
+ln -s $PWD $HOME/ml
+
+export PATH="$PATH:$HOME/ml"
\ No newline at end of file
diff --git a/core.sh b/core.sh
new file mode 100644 (file)
index 0000000..cb364a6
--- /dev/null
+++ b/core.sh
@@ -0,0 +1,384 @@
+#!/bin/bash
+
+: ${dry_run:=false}
+
+TESTLIB_HELP=()
+
+# arg-parse debug
+_apd=false
+
+arg_parse_cmd=\
+"local -a args;
+local _a;
+local _vn;
+local _m;
+local CLVL=\$((\$CLVL+1))
+
+_m=true;
+
+for _a in \"\$@\" ; do
+    $_apd && echo \"Evaluating \${_a} [[ \"\${_a/=}\" = \"\${_a}\" ]]\";
+    if \$_m && [[ \"\${_a/=}\" != \"\${_a}\" ]] ; then
+        $_apd && echo Parameter;
+        _vn=\${_a%%=*};
+        eval \"local \$_vn\";
+        eval \"\$_a\";
+    elif \$_m && [[ \"\${_a}\" == \"--\" ]] ; then
+        $_apd && echo Separator;
+        _m=false;
+    else
+        $_apd && echo Argument;
+        _m=false;
+        args+=(\"\$_a\");
+    fi;
+done"
+
+arg_parse="eval $arg_parse_cmd"
+
+# Pass in either the current function name, or the name of the script
+requireargs="eval _func=\"\$FUNCNAME\" ; eval [[ -n \\\"\$_func\\\" ]] || _func=\$0 ; eval _require-args \$_func"
+
+function _require-args()
+{
+    local _arg
+    local _args
+    local fail_popup
+
+    _args=($@)
+
+    fail_popup=false
+
+    for _arg in ${_args[@]:1} ; do
+       eval "[[ -n \"\${$_arg}\" ]] || fail \"${_args[0]}: Missing $_arg\""
+    done
+}
+
+function default()
+{
+    # l0: eval i="5"
+    # l1: default_post="eval $1=\"$2\""
+    # l3: eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval \$1=\\\"$2\\\"\" ; fi"
+    eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval local \$1=\\\"$2\\\"\" ; else unset default_post ; fi"
+}
+
+function default-test-scratch()
+{
+    eval "i=\"5\""
+    echo "i is $i"
+
+    
+    a="i" 
+    b="6"
+    default_post="eval $a=\\\"$b\\\""
+    echo "default_post: $default_post"
+    $default_post
+    echo "i is $i"
+
+    unset i
+    b=7
+    eval "if [[ -z \"\$$a\" ]] ; then default_post=\"eval \$a=\\\\\\\"$b\\\\\\\"\" ; fi"
+    echo "default_post: $default_post"
+    $default_post
+    echo "i is $i"
+}
+
+function default-test()
+{
+    $arg_parse
+
+    default i "Testing testing" ; $default_post
+    
+    info "default_post $default_post"
+    info "i $i"
+}
+
+# FIXME: Get rid of this?
+function pop()
+{
+    local _var
+    _var="$1";
+
+    eval "${_var}=(\"\${${_var}[@]:1}\")"
+}
+
+function die()
+{
+   echo FATAL $@
+   exit 1
+}
+
+function fail()
+{
+   echo FATAL $@
+   [[ -n "$fail_cleanup" ]] && $fail_cleanup
+   exit 1
+}
+
+function info()
+{
+   echo INFO $CLVL $@ 1>&2
+}
+
+function error()
+{
+   echo ERROR $@ 1>&2
+}
+
+function status()
+{
+   echo STATUS $CLVL $@ 1>&2
+   #$status_popup && (zenity --info --text="$@" &)
+   return 0
+}
+
+function parse-separator-array()
+{
+    local _pca_array_var="$2"
+    local _pca_list="$3"
+    local -a _pca_internal
+    local OLD_IFS
+
+    OLD_IFS=${IFS}
+    IFS="$1"
+    _pca_internal=($_pca_list)
+    IFS="${OLD_IFS}"
+
+    eval "${_pca_array_var}=(${_pca_internal[@]})"
+}
+
+function parse-comma-array()
+{
+    parse-separator-array "," "$1" "$2"
+}
+
+function test-parse-comma-array()
+{
+    local vars
+    local out
+    local i
+
+    vars="a,b,c"
+    parse-comma-array out $vars
+
+    for i in ${out[@]} ; do
+       echo "X $i";
+    done
+}
+
+function test-parse-colon-array()
+{
+    local vars
+    local out
+    local i
+
+    vars="kodo2:c6-test:c6-vm"
+    parse-separator-array ":" out $vars
+
+    for i in ${out[@]} ; do
+       echo "X $i";
+    done
+}
+
+function parse-config-array()
+{
+    local _pcfga_array_var="$1"
+    local _pcfga_reset_array_var="$2"
+    local _pcfga_list="$3"
+    local -a _pcfga_internal
+    local -a _pcfgr_internal
+    local _j
+    local _varname
+    local _value
+
+    echo "About to parse list"
+    parse-comma-array _pcfga_internal ${_pcfga_list}
+
+    echo Reset values
+    for _j in ${_pcfga_internal[@]} ; do
+       _varname=$(expr match "${_j}" '\([^=]*\)')
+       _value=$(eval echo \$$_varname)
+       echo " $_varname=$_value"
+       _pcfgr_internal=(${_pcfgr_internal[@]} "${_varname}=${_value}")
+    done
+
+    eval "${_pcfga_array_var}=(${_pcfga_internal[@]})"
+    eval "${_pcfga_reset_array_var}=(${_pcfgr_internal[@]})"
+}
+
+# Used to make a local cfg_ variable overriding the global one if
+# a given argument is given; otherwise, set the local variable to the
+# global one.
+#
+# For example, you can pass isosr_path to host-install, and it will
+# create a local variable cfg_isosr_path and set it to iso-sr, so that
+# host-install-post will get that value without having to pass it in;
+# Alternately, you can set iso_sr to cfg_isosr_path.
+#
+# Use like this: cfg_override [global_var] [local_var] ; eval $ret_eval
+function cfg_override()
+{
+    unset ret_eval
+    if eval "[[ -n \"\$$2\" ]]" ; then
+       ret_eval="local $1=\$$2"
+    else
+       ret_eval="$2=\$$1"
+    fi
+}
+
+function report-result()
+{
+    if [[ -n "$var" ]] ; then
+       eval "${var}=\"$1\""
+    else
+       if [[ -n "$1" ]] ; then
+           echo "$1"
+       else
+           echo "(empty)"
+       fi
+    fi
+}
+
+function test-report-result()
+{
+    report-result "This should be printed"
+    echo "test: $test"
+
+    echo 'The following should print "(empty)"'
+    report-result
+
+    var=test
+    report-result "This should be in a variable"
+    echo "test: $test"
+
+    echo 'The following should have nothing'
+    report-result
+    echo "test: $test"
+}
+
+# Call a function remotely, passing in variables when appropriate.
+#
+# tgt-helper should already have been called appropriately before this.
+# 
+# Usage: remote-call function-name [var1 var2 ...]
+function remote-call()
+{
+    local _args=("$@")
+    local _r
+    local _fn="${_args[0]}"
+    local _v
+    
+    for _v in "${_args[@]:1}" ; do 
+       #echo "[[ -n \"\$$_v\" ]] && $_r=\"\$$_r $_v=\$$_v\""
+       eval "[[ -n \"\$$_v\" ]] && _r=\"\$_r $_v=\$$_v\""
+    done
+
+    #eval "echo \$$_r"
+
+    info "Calling hl $_fn $_r"
+    ssh-cmd -- -t "hl $_fn $_r"
+}
+
+# Used to run commands that must be done inside a network through a gateway
+# 
+# To use, set TESTLIB_REMOTE=true and TESTLIB_GATEWAY to the ssh gateway to use.
+: ${TESTLIB_REMOTE:=false}
+function gateway-cmd
+{
+    $requireargs TESTLIB_GATEWAY
+
+    echo ssh "${TESTLIB_GATEWAY}" tl $_cmd "$@"
+    ssh "${TESTLIB_GATEWAY}" tl $_cmd "$@"
+}
+
+gateway_cmd="if \"\${TESTLIB_REMOTE}\" ; then _cmd=\"\$FUNCNAME\" ; gateway-cmd \"\$@\" ; return ; fi"
+gateway_override="eval $gateway_cmd"
+
+# Retry a command until it succeeds, or until the number of retries / timeout is exceeded.
+#
+# Condition can be inverted by passing invert=true
+function retry()
+{
+    local invert
+    local ret
+    local start
+    local now
+    local elapsed
+    local tries="0"
+    local bang
+
+    $arg_parse
+
+    [[ -n "$retries" ]] || [[ -n "$timeout" ]] || info "WARNING: $FUNCNAME: No retries or timeout, will loop forever"
+
+    if [[ "$timeout" = "0" ]] ; then
+       local timeout
+    fi
+
+    default invert "false" ; $default_post
+    default interval "1" ; $default_post
+    default dot "true" ; $default_post
+
+    $invert && bang="!";
+
+    start=$(date +%s)
+
+    while true ; do
+       #info "Running ${args[@]}"
+       retry_result=$(${args[@]})
+       ret="$?"
+       ret=$(($bang $ret))
+
+       if [[ "$ret" = "0" ]] ; then
+           break
+       fi
+
+       tries=$(($tries+1))
+       if [[ $tries -eq "1" ]] ; then
+           echo "Waiting for ${condition}(${bang}${args[@]})"
+       fi
+
+       [[ -n "$retries" && $tries -ge $retries ]] && break
+
+       if [[ -n "$timeout" ]] ; then
+           now=$(date +%s)
+           elapsed=$(($now-$start))
+           [[ $elapsed -gt $timeout ]] && break
+       fi
+
+       $dot && echo -n . 1>&2
+
+       sleep $interval
+    done
+
+    $dot && echo 1>&2
+
+    now=$(date +%s)
+    retry_time=$(($now-$start))
+
+    return $ret
+}
+
+function help()
+{
+    for i in "${TESTLIB_HELP[@]}" ; do
+       echo "$i"
+    done
+}
+
+function cmdline()
+{
+    local cmd;
+
+    if [[ "$#" -eq "0" ]] ; then
+       echo "Usage: $0 function-name [arguments...]"
+       exit 1
+    fi
+
+    $arg_parse
+    info Running "${args[0]}"
+    "${args[0]}" "${args[@]:1}" || exit 1
+
+    if ! [[ -z "$RET" ]] ; then
+       echo $RET
+    fi
+}
diff --git a/ml b/ml
new file mode 100755 (executable)
index 0000000..7d6089b
--- /dev/null
+++ b/ml
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Include your defaults
+if [[ -e "./mlconfig" ]] ; then
+    . ./mlconfig
+elif [[ -e "$HOME/.mlconfig" ]] ; then
+    . $HOME/.mlconfig
+fi
+
+# To use this library, include the following line:
+[[ -z "$MLIB_PATH" ]] && MLIB_PATH="$HOME/ml"
+
+# Then as many as the sub-libraries as you need
+. ${MLIB_PATH}/core.sh
+. ${MLIB_PATH}/release.sh
+. ${MLIB_PATH}/security.sh
+
+# And do your own thing rather than running commands
+# I suggest defining a "main" function of your own and running it like this.
+cmdline "$@"
diff --git a/release.sh b/release.sh
new file mode 100644 (file)
index 0000000..e558399
--- /dev/null
@@ -0,0 +1,186 @@
+function xen-make-prefix-config() {
+    $arg_parse
+    
+    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
+
+    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk > .config || fail "Generating .config"
+    cat .config
+}
+
+function release-make-tarball()
+{
+    $arg_parse
+
+    $requireargs v
+
+    git fetch || fail "git fetch"
+    
+    git checkout $t || fail "Checking out tag $t"
+
+    git clean -ffdx
+
+    xen-make-prefix-config
+
+    ./configure || fail "Configuring"
+    
+    # FIXME: >= 4.5
+    if $isrc ; then
+       make src-tarball || fail "Making src-tarball"
+    else
+       make src-tarball-release || fail "Making src-tarball"
+    fi
+
+    rm -rf $tdir/$v
+
+    mkdir -p $tdir/$v
+
+    cp dist/xen-$v.tar.gz $tdir/$v
+}
+
+function release-buildtest-tarball() {
+    $arg_parse
+
+    $requireargs tdir v
+    
+    if [[ -z "$bdir" ]] ; then
+       default bdir "/volatile/iwj/d" ; $default_post
+
+       if [[ ! -e $bdir ]] ; then
+           info "$bdir doesn't exist, using /tmp"
+           bdir="/tmp"
+       fi
+    fi
+
+    cd $bdir || fail "cd $bdir"
+
+    rm -rf build-$v
+    mkdir build-$v || fail "mkdir"
+
+    cd build-$v
+
+    tar xfz $tdir/$v/xen-$v.tar.gz || fail "Untar"
+
+    cd xen-$v || fail "cd"
+
+    xen-make-prefix-config
+    # FIXME: >= 4.2
+    info "Testing build (tail -f $bdir/build-$v/log.$v)..."
+    (./configure && make -j4 && touch $tdir/$v/build-tested && echo OK) 2>&1 > ../log.$v
+
+    [[ -e $tdir/$v/build-tested ]] || fail "Build failed; log at $bdir/build-$v/log.$v"
+}
+
+function release-set-tdir() {
+    if [[ -z "$tdir" ]] ; then
+       default tdir "/volatile/iwj/website-thing/xen.org/oss-xen/release" ; $default_post
+
+       if [[ ! -e $tdir ]] ; then
+           info "$tdir doesn't exist, using /tmp"
+           tdir="/tmp"
+       fi
+    fi
+}
+
+function release-sign-tarball() {
+    $arg_parse
+
+    $requireargs v rtgz
+
+    default key "57E82BD9" ; $default_post
+
+    if ! gpg --list-secret-keys | grep $key ; then
+       info "Signature required; please run the following command with the public key available"
+       info " gpg --detach-sign -u 'xen tree' $rtgz"
+    fi
+
+    gpg --detach-sign -u $key $rtgz || fail "Signing $rtgz"
+}
+
+function release-tarball-fetch-sign-push() {
+    $arg_parse
+
+    $requireargs v
+    
+    release-set-tdir
+
+    scp xenbits:$tdir/$v/xen-$v.tar.gz /tmp || fail "Fetching tarball"
+
+    rtgz=/tmp/xen-$v.tar.gz
+
+    release-sign-tarball
+
+    scp /tmp/xen-$v.tar.gz.sig xenbits:$tdir/$v/ || fail "Copying sig"
+}
+
+function release-tarball-checksig() {
+    gpg --verify $rtgz.sig || fail "Signature failed"
+}
+
+function release-tarball-cvs-checkin-and-post() {
+    $arg_parse
+
+    $requireargs v
+
+    default cvsdir "/volatile/iwj/website-thing/xen.org/" ; $default_post
+
+    if [[ ! -e $cvsdir ]] ; then
+       fail "$cvsdir does not exist"
+    fi
+
+    cd $cvsdir || fail "cd"
+
+    cvs add -kb oss-xen/release/$v/ || fail "cvs add release directory"
+
+    cd oss-xen/release/$v || fail "cd"
+    
+    cvs add -kb xen-$v.tar.gz || fail "cvs add tarball"
+    cvs add -kb xen-$v.tar.gz.sig || fail "cvs add sig"
+    
+    cd ../../..
+
+    cvs ci -m $v || fail "cvs checkin"
+    ./upload.sh || fail "Upload"
+}
+
+function release() {
+    $arg_parse
+
+    $requireargs v
+
+    release-set-tdir
+    
+    isrc="false"
+    
+    if [[ "${v%%rc*}" != "${v}" ]] ; then
+       t=$v
+       isrc="true"
+    else
+       t=RELEASE-$v
+    fi
+
+    info "Using tag $t"
+
+    rdir=$tdir/$v
+
+    rtgz=$tdir/$v/xen-$v.tar.gz
+
+    if [[ ! -e $rtgz ]] ; then
+       info "$rtgz not present, generating"
+       release-make-tarball
+    fi
+
+    info "Tarball created"
+
+    if [[ ! -e $rdir/build-tested ]] ; then
+       release-buildtest-tarball
+    fi
+
+    info "Build tested"
+
+    if [[ ! -e $rtgz.sig ]] ; then
+       release-sign-tarball
+    else
+       release-tarball-checksig
+    fi
+}
+
diff --git a/security.sh b/security.sh
new file mode 100644 (file)
index 0000000..f467821
--- /dev/null
@@ -0,0 +1,40 @@
+function sec-make-patches() {
+    local patchdir
+    
+    $arg_parse
+
+    default base "staging"; $default_post
+    default makedirectory "false"; $default_post
+
+    $requireargs xsa version
+
+    git-check
+
+    [[ -e ../xsa.git ]] || fail "Unexpected directory layout: no xsa.git!"
+
+    local xsadir
+
+    xsadir=$PWD/../xsa.git
+
+    if $makedirectory ; then
+       patchdir=$xsadir/xsa${xsa}-${version}
+    else
+       patchdir=/tmp/xsa-tmp
+    fi
+    
+    rm -rf $patchdir
+       
+    mkdir $patchdir
+       
+    git format-patch -o$patchdir $base
+
+    if ! $makedirectory ; then
+       pushd $patchdir
+       
+       for p in *.patch ; do
+           mv $p $xsadir/xsa${xsa}-${version}-$p
+       done
+       
+       popd
+    fi
+}