transfer-flight-history: use \copy, copy only recent flights

diff --git a/transfer-flight-history b/transfer-flight-history
index ca1cf25431c3339752ed50d25a39c026dbd7dfed..ecff20da2d2205a2afc8aa7ef0f2d40cde495c41 100755 (executable)
--- a/transfer-flight-history
+++ b/transfer-flight-history
@@ -2,6 +2,34 @@
 
 set -ex
 
+rhost=osstest.test-lab
+cutoff=50000
+
+case "$#.$1" in
+0.)
+       
+       begin=$(
+               ssh $rhost "
+                       psql -h db -d osstestdb -q -Ptuples_only=true -c '
+                               SELECT flight FROM flights
+                                       WHERE flight < $cutoff
+                                       ORDER BY flight DESC
+                                       LIMIT 1
+                       '
+               "
+       )
+       begin=$(( begin - 2000 ))
+       ;;
+[0-9]*)
+       begin=$1
+       shift
+       ;;
+-*|*)
+       echo >&2 'bad usage'
+       exit 1
+       ;;
+esac
+
 of=cam101-history-load
 
 exec 3>$of
@@ -12,25 +40,28 @@ echo >&3 "BEGIN;"
 
 tables='flights flights_harness_touched jobs runvars steps'
 
+cond="flight >= $begin AND flight < $cutoff"
+
 df=cam101-history-deletes
 exec 4>$df
 for t in $tables
 do
-       echo >&4 "DELETE FROM $t WHERE flight < 50000;"
+       echo >&4 "DELETE FROM $t WHERE $cond;"
 done
 tac $df >&3
 
+copyhow="CSV HEADER NULL e'\\\\n'"
+
 for t in $tables;
 do
-       tf=cam101-history-$t.tmp
-       pg_dump \
-               -a -b -F plain -O \
-               --table=$t \
-               --no-security-labels \
-               --no-tablespaces \
-               --serializable-deferrable \
-               -f $tf -h osstestdb.db.cam.xci-test.com osstestdb
-       sed -e 's/^SELECT pg_catalog\.setval/-- &/g' <$tf >&3
+       echo >&3 \
+ "\\COPY $t FROM STDIN $copyhow"
+
+       psql >&3 -q -h db -d osstestdb -c \
+ "\\COPY (SELECT * FROM $t WHERE $cond) TO STDOUT $copyhow"
+
+       echo >&3 \
+ "\\."
 done
 
 echo >&3 "COMMIT;"
@@ -40,12 +71,12 @@ perl -ne '
                 $hide ||
                 m/^\-\-/ ||
                 !m/\S/;
-        $hide = 1 if m/ FROM stdin\;$/;
+        $hide = 1 if m/^\\COPY/;
         $hide = 0 if m/^\\\./;
 ' $of
 
-rsync -vP $of osstest.test-lab:.
+rsync -vP $of $rhost:.
 
-ssh osstest.test-lab psql -h db -d osstestdb -f $of
+ssh $rhost psql -eE -h db -d osstestdb -f $of
 
 echo ok.