]> xenbits.xensource.com Git - people/andrewcoop/hwloc.git/commitdiff
Add hwloc-compress-dir utility
authorBrice Goglin <brice.goglin@inria.fr>
Tue, 24 Sep 2013 14:15:45 +0000 (14:15 +0000)
committerBrice Goglin <brice.goglin@inria.fr>
Tue, 24 Sep 2013 14:15:45 +0000 (14:15 +0000)
It takes a directory of XML topologies and compresses it by using
hwloc-diff to store as many topologies as diffs as possible.

This commit was SVN r5815.

NEWS
config/hwloc_internal.m4
doc/hwloc.doxy
utils/Makefile.am
utils/hwloc-compress-dir.1in [new file with mode: 0644]
utils/hwloc-compress-dir.in [new file with mode: 0755]
utils/hwloc-diff.1in
utils/hwloc-patch.1in

diff --git a/NEWS b/NEWS
index ddb9b4f208bfa9614653fd44c17f103ff0ccb01a..4b919d8a85e4a6506bc945ce8a0c87ecffd021be 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,8 @@ Version 1.8.0
   + Add hwloc_topology_dup() for duplicating an entire topology.
 * Tools
   + Add hwloc-diff and hwloc-patch tools together with the new diff API.
+  + Add hwloc-compress-dir to compress an entire directory of XML files
+    using hwloc-diff.
   + Object colors in the graphical output of lstopo may be changed by adding
     a "lstopoStyle" info attribute. See CUSTOM COLORS in the lstopo(1) manpage
     for details. Thanks to Jirka Hladky for discussing the idea.
index 5a86825474b9e2079075272d320b081ce2631412..3932f6b039ccb18d0ba31f0d25a0afc341a751b2 100644 (file)
@@ -409,6 +409,7 @@ int foo(void) {
         hwloc_config_prefix[tests/xml/test-topology.sh]
         hwloc_config_prefix[tests/wrapper.sh]
         hwloc_config_prefix[utils/hwloc-assembler-remote]
+        hwloc_config_prefix[utils/hwloc-compress-dir]
         hwloc_config_prefix[utils/test-hwloc-annotate.sh]
         hwloc_config_prefix[utils/test-hwloc-assembler.sh]
         hwloc_config_prefix[utils/test-hwloc-calc.sh]
@@ -419,7 +420,7 @@ int foo(void) {
         hwloc_config_prefix[utils/test-hwloc-ls.sh]
         hwloc_config_prefix[utils/test-fake-plugin.sh])
 
-    AC_CONFIG_COMMANDS([chmoding-scripts], [chmod +x ]hwloc_config_prefix[tests/linux/test-topology.sh ]hwloc_config_prefix[tests/xml/test-topology.sh ]hwloc_config_prefix[tests/linux/hwloc-gather-topology ]hwloc_config_prefix[tests/linux/gather/test-gather-topology.sh ]hwloc_config_prefix[tests/wrapper.sh ]hwloc_config_prefix[utils/hwloc-assembler-remote ]hwloc_config_prefix[utils/test-hwloc-annotate.sh ]hwloc_config_prefix[utils/test-hwloc-assembler.sh ]hwloc_config_prefix[utils/test-hwloc-calc.sh ]hwloc_config_prefix[utils/test-hwloc-diffpatch.sh ]hwloc_config_prefix[utils/test-hwloc-distances.sh ]hwloc_config_prefix[utils/test-hwloc-distrib.sh ]hwloc_config_prefix[utils/test-hwloc-info.sh ]hwloc_config_prefix[utils/test-hwloc-ls.sh ]hwloc_config_prefix[utils/test-fake-plugin.sh])
+    AC_CONFIG_COMMANDS([chmoding-scripts], [chmod +x ]hwloc_config_prefix[tests/linux/test-topology.sh ]hwloc_config_prefix[tests/xml/test-topology.sh ]hwloc_config_prefix[tests/linux/hwloc-gather-topology ]hwloc_config_prefix[tests/linux/gather/test-gather-topology.sh ]hwloc_config_prefix[tests/wrapper.sh ]hwloc_config_prefix[utils/hwloc-assembler-remote ]hwloc_config_prefix[utils/hwloc-compress-dir ]hwloc_config_prefix[utils/test-hwloc-annotate.sh ]hwloc_config_prefix[utils/test-hwloc-assembler.sh ]hwloc_config_prefix[utils/test-hwloc-calc.sh ]hwloc_config_prefix[utils/test-hwloc-diffpatch.sh ]hwloc_config_prefix[utils/test-hwloc-distances.sh ]hwloc_config_prefix[utils/test-hwloc-distrib.sh ]hwloc_config_prefix[utils/test-hwloc-info.sh ]hwloc_config_prefix[utils/test-hwloc-ls.sh ]hwloc_config_prefix[utils/test-fake-plugin.sh])
 
     # These links are only needed in standalone mode.  It would
     # be nice to m4 foreach this somehow, but whenever I tried
index 0f9a99063cc80603442a439175fb69e6e25c42f1..559529a403e28db3ddf132a750e1db89316dc6ad 100644 (file)
@@ -903,6 +903,12 @@ and outputs it to another XML file.
 hwloc-patch reads such a difference file and applies to
 another topology.
 
+\section cli_hwloc_compressdir hwloc-compress-dir
+
+hwloc-compress-dir compresses an entire directory of XML
+files by using hwloc-diff to save the differences between
+topologies instead of entire topologies.
+
 \section cli_hwloc_assembler hwloc-assembler
 
 hwloc-assembler combines several XML topology files into a single
index 3a0642200d9da27c308098325389c7b515f73d16..21a846146e3e1be55d40b256fdda9b540b4b6ff6 100644 (file)
@@ -72,7 +72,7 @@ hwloc_calc_SOURCES = \
        hwloc-calc.h \
        hwloc-calc.c
 
-bin_SCRIPTS = hwloc-assembler-remote
+bin_SCRIPTS = hwloc-assembler-remote hwloc-compress-dir
 
 if !HWLOC_HAVE_MINGW32
 TESTS = \
@@ -101,6 +101,7 @@ man1_pages = \
        hwloc-assembler-remote.1 \
        hwloc-bind.1 \
        hwloc-calc.1 \
+       hwloc-compress-dir.1 \
        hwloc-diff.1 \
        hwloc-distances.1 \
        hwloc-distrib.1 \
@@ -154,7 +155,8 @@ endif
        rm -f $(DESTDIR)$(man1dir)/lstopo-no-graphics.1
        cd $(DESTDIR)$(man1dir) && $(LN_S) lstopo.1 lstopo-no-graphics.1
        $(SED) -e 's/HWLOC_top_builddir\/utils/bindir/' -e '/HWLOC_top_builddir/d' $(DESTDIR)$(bindir)/hwloc-assembler-remote > $(DESTDIR)$(bindir)/hwloc-assembler-remote.tmp && mv -f $(DESTDIR)$(bindir)/hwloc-assembler-remote.tmp $(DESTDIR)$(bindir)/hwloc-assembler-remote
-       chmod +x $(DESTDIR)$(bindir)/hwloc-assembler-remote
+       $(SED) -e 's/HWLOC_top_builddir\/utils/bindir/' -e '/HWLOC_top_builddir/d' $(DESTDIR)$(bindir)/hwloc-compress-dir > $(DESTDIR)$(bindir)/hwloc-compress-dir.tmp && mv -f $(DESTDIR)$(bindir)/hwloc-compress-dir.tmp $(DESTDIR)$(bindir)/hwloc-compress-dir
+       chmod +x $(DESTDIR)$(bindir)/hwloc-assembler-remote $(DESTDIR)$(bindir)/hwloc-compress-dir
 
 uninstall-local:
        rm -f $(DESTDIR)$(bindir)/hwloc-ls$(EXEEXT)
diff --git a/utils/hwloc-compress-dir.1in b/utils/hwloc-compress-dir.1in
new file mode 100644 (file)
index 0000000..982178d
--- /dev/null
@@ -0,0 +1,82 @@
+.\" -*- nroff -*-
+.\" Copyright © 2013 inria.  All rights reserved.
+.\" See COPYING in top-level directory.
+.TH HWLOC-COMPRESS-DIR "1" "#HWLOC_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
+.SH NAME
+hwloc-compress-dir \- Compress a directory of XML topologies
+.
+.\" **************************
+.\"    Synopsis Section
+.\" **************************
+.SH SYNOPSIS
+.B hwloc-compress-dir
+[\fIoptions\fR]
+<inputdir>
+<outputdir>
+\fR
+.
+.\" **************************
+.\"    Options Section
+.\" **************************
+.SH OPTIONS
+.TP
+\fB\-v \-\-verbose\fR
+Display verbose messages.
+.
+.\" **************************
+.\"    Description Section
+.\" **************************
+.SH DESCRIPTION
+.
+hwloc-compress-dir takes an input directory containing XML exports
+and tries to compress it by computing topology diffs between them
+(with the hwloc-diff program).
+Each file is copied in the output directory either as a diff if it
+could be compressed, or as its original entire file otherwise.
+.
+.PP
+hwloc-compress-dir may recompress a directory that was previously
+compressed. All input files that are already in the output directory,
+either compressed or not, are ignored. New input files are compressed
+as much as possible as usual.
+.
+.PP
+For each file of the directory, the output filename is
+the same as the original if not compressed,
+otherwise its extension is changed to \fB.diff.xml\fR.
+.
+.PP
+Compressed files are based on another non-compressed topology.
+Its name is stored in the \fBrefname\fR topology diff attribute.
+.
+.PP
+The generated output diff files may be used with hwloc-patch
+just like any file generated by hwloc-diff.
+.
+.\" **************************
+.\"    Examples Section
+.\" **************************
+.SH EXAMPLES
+.PP
+To compress the input files from directory in into directory out:
+
+    $ hwloc-compress-dir in out
+.
+.\" **************************
+.\"    Return value section
+.\" **************************
+.SH RETURN VALUE
+Upon successful execution, hwloc-compress-dir returns 0.
+.
+.PP
+hwloc-compress-dir will return nonzero if any kind of error occurs,
+such as (but not limited to) failure to parse the command line.
+.
+.\" **************************
+.\"    See also section
+.\" **************************
+.SH SEE ALSO
+.
+.ft R
+hwloc(7), lstopo(1), hwloc-diff(1), hwloc-patch(1)
+.sp
diff --git a/utils/hwloc-compress-dir.in b/utils/hwloc-compress-dir.in
new file mode 100755 (executable)
index 0000000..5712c31
--- /dev/null
@@ -0,0 +1,110 @@
+#!/bin/sh
+#-*-sh-*-
+
+#
+# Copyright © 2013 Inria.  All rights reserved.
+# See COPYING in top-level directory.
+#
+
+HWLOC_top_builddir="@HWLOC_top_builddir@"
+prefix="@prefix@"
+exec_prefix="@exec_prefix@"
+bindir="@bindir@"
+# this will be changed into $bindir/lstopo during make install
+localhwlocdiff="$HWLOC_top_builddir/utils/hwloc-diff"
+
+verbose=0
+
+error()
+{
+       echo $@ 2>&1
+}
+
+usage()
+{
+       echo "$0 [options] <inputdir> <outputdir>"
+       echo "  Compress topologies from <inputdir> into <outputdir>"
+       echo "Options:"
+       echo "  -v --verbose            Display verbose messages"
+}
+
+while test $# -gt 2 ; do
+       case "$1" in
+       -v|--verbose)
+               verbose=1
+               ;;
+       -h)
+               usage
+               exit 0
+               ;;
+       -*)
+               error "Unrecognized option: $1"
+               usage
+               exit 1
+               ;;
+       esac
+       shift
+done
+
+if test $# -lt 2 ; then
+    usage
+    exit 1
+fi
+
+inputdir="$1"
+test x`echo $inputdir | sed -e 's/^\///'` = x$inputdir && inputdir="$PWD/$inputdir"
+outputdir="$2"
+test x`echo $outputdir | sed -e 's/^\///'` = x$outputdir && outputdir="$PWD/$outputdir"
+
+if ! cd "$outputdir" ; then
+    echo "Cannot enter output directory $outputdir"
+    exit 1
+fi
+
+alreadycompressed=0
+alreadynoncompressed=0
+newlycompressed=0
+newlynoncompressed=0
+
+inputs=`ls -1 "$inputdir"`
+for input in $inputs ; do
+    name=`echo $input | sed -e 's/.xml$//'`
+
+    if test "x${name}.xml" != "x$input"; then
+       test x$verbose = x1 && echo "Ignoring non-XML file $input"
+       continue
+    fi
+    if test -f "$outputdir/${name}.xml" ; then
+       test x$verbose = x1 && echo "$name already non-compressed, skipping"
+       alreadynoncompressed=`expr $alreadynoncompressed + 1`
+       continue
+    fi
+    if test -f  "$outputdir/${name}.diff.xml" ; then
+       test x$verbose = x1 && echo "$name already compressed, skipping"
+       alreadycompressed=`expr $alreadycompressed + 1`
+       continue
+    fi
+
+    found=
+    outputs=`ls -1 "$outputdir"`
+    for output in $outputs ; do
+       outputname=`echo $output | sed -e 's/.xml$//' | sed -e 's/.diff$//'`
+       test -f "${outputdir}/${outputname}.diff.xml" && continue
+
+       if $localhwlocdiff "$outputdir/${outputname}.xml" "$inputdir/${name}.xml" "$outputdir/${name}.diff.xml" >/dev/null 2>/dev/null; then
+           echo "Compressed $name on top of $outputname"
+           newlycompressed=`expr $newlycompressed + 1`
+           found=1
+           break
+       fi
+    done
+
+    if test x$found = x ; then
+       echo "Could not compress $name, keeping non-compressed"
+       newlynoncompressed=`expr $newlynoncompressed + 1`
+       cp "$inputdir/${name}.xml" "$outputdir/${name}.xml"
+    fi
+done
+
+echo "Compressed $newlycompressed new topologies ($alreadycompressed were already compressed)"
+echo "Kept $newlynoncompressed new topologies non-compressed ($alreadynoncompressed were already non-compressed)"
index ddc1de3f5fc5ba9508501c1e9800b4380fb743ca..8af21c481bdaab41873de1c07fba5372f1825015 100644 (file)
@@ -52,6 +52,10 @@ The output difference may later be applied to another topology
 with hwloc-patch.
 .
 .PP
+hwloc-compress-dir may be used for computing the diffs between
+all XML files in a directory.
+.
+.PP
 .B NOTE:
 It is highly recommended that you read the hwloc(7) overview page
 before reading this man page.  Most of the concepts described in
@@ -110,5 +114,5 @@ hwloc-diff also returns nonzero if any kind of error occurs, such as
 .SH SEE ALSO
 .
 .ft R
-hwloc(7), lstopo(1), hwloc-patch(1)
+hwloc(7), lstopo(1), hwloc-patch(1), hwloc-compress-dir(1)
 .sp
index 5b962bf5f05e94a0e2ce00de45f75b151fb8f2d2..98b68a9623422f85d3a4bf1aa82a5999388cbe01 100644 (file)
@@ -52,7 +52,8 @@ If - is used instead of <diff.xml>, the topology difference is read from
 the standard input.
 .
 .PP
-The XML difference may have been computed earlier with hwloc-diff.
+The XML difference may have been computed earlier with hwloc-diff
+or hwloc-compress-dir.
 .
 .PP
 .B NOTE:
@@ -99,5 +100,5 @@ hwloc-patch also returns nonzero if any kind of error occurs, such as
 .SH SEE ALSO
 .
 .ft R
-hwloc(7), lstopo(1), hwloc-diff(1)
+hwloc(7), lstopo(1), hwloc-diff(1), hwloc-compress-dir(1)
 .sp