Create ssync-reapr to clean up ssync run files

ssync-reapr will delete the run files pushed to the ssync output
dir (default ~/.local/share/ssync/runs/) which can accumulate over
time. The default is to only delete files older than a day.

We run the reap as the first step of every ssync - to ensure it clears
out the necessary files

5 files changed, 178 insertions, 33 deletions

diff --git a/docs/ssync-reapr.1 b/docs/ssync-reapr.1
new file mode 100644
index 0000000..99f36ab
--- /dev/null
+++ b/docs/ssync-reapr.1
@@ -0,0 +1,37 @@
+.TH ssync-reapr 1 {RELEASE}
+.SH NAME
+ssync-reapr \- reap temporary and run files created by other ssync commands
+.SH SYNOPSIS
+.B ssync-reapr
+.RB [option...]
+.IR DIRS...
+.YS
+.SH OPTIONS
+.TP
+.BR \-a " seconds"
+Reap files created more than the configured seconds ago. This will help clear out temporary files created by the
+.MR ssync-index 1
+and
+.MR ssync-queue 1
+commands.
+.TP
+.BR \-n " dry-run"
+Dry run the reaping process - only prints the files; doesn't delete.
+.TP
+.BR \-v " verbose logging"
+Use -vv to increase the verbose level. 2+ will print the files being reaped.
+.TP
+.BR \-h " print this message"
+.SH SEE ALSO
+.MR ssync 1
+.MR ssync 5
+.MR ssync-queue 1
+.MR ssync-fetch 1
+.SH REPOSITORY
+.UR https://git.senders.io/utils/ssync
+.UE
+.SH AUTHORS
+Steph Enders
+.MT steph@senders.io
+.ME
+ssync-reapr is open source licensed under the ISC license. See LICENSE for full copyright information.
diff --git a/docs/ssync.5 b/docs/ssync.5
index c0e3caf..e729df5 100644
--- a/docs/ssync.5
+++ b/docs/ssync.5
@@ -36,6 +36,13 @@ which will retain the outputs of each run in a subdirectory. This directory must
 .MR ssync-queue 1
 running.
 .TP
+.BI reap_age_s " seconds"
+Reap files created more than the configured seconds ago. This will help clear out temporary files created by the
+.MR ssync-index 1
+and
+.MR ssync-queue 1
+commands.
+.TP
 .BI remote_host " hostname"
 Remote Hostname to connect to when indexing and fetching remote files. This should be in the format
 .IR user@hostname
@@ -50,6 +57,7 @@ Directory containing the
 .MR ssync-index 1
 .MR ssync-queue 1
 .MR ssync-fetch 1
+.MR ssync-reapr 1
 commands.
 .SH AUTHOR
 Written by Steph Enders
@@ -58,5 +66,6 @@ Written by Steph Enders
 .MR ssync-index 1
 .MR ssync-queue 1
 .MR ssync-fetch 1
+.MR ssync-reapr 1
 
 
diff --git a/ssync b/ssync
index c135091..68f1bf4 100755
--- a/ssync
+++ b/ssync
@@ -19,16 +19,16 @@ lines() {
 }
 
 # OPTIONS
-NO_FETCH_FLAG=
+DRY_RUN_FLAG=
 VERBOSE_FLAG=
 while getopts "nvhb:" opt; do
     case "${opt}" in
         h) echo "$USAGE"
            exit 1
            ;;
-        n) NO_FETCH_FLAG=1
+        n) DRY_RUN_FLAG=1
            ;;
-        v) VERBOSE_FLAG=1
+        v) VERBOSE_FLAG=$(($VERBOSE_FLAG+1))
            ;;
         b) BACKEND_FLAG=1
            BACKEND="$OPTARG"
@@ -114,6 +114,11 @@ elif [ ! -d "$fetch_output_dir" ]; then
     exit 1
 fi
 
+reap_age_opt=
+if [ ! -z "$reap_age_s" ]; then
+    reap_age_opt="-a $reap_age_s"
+fi
+
 # SUB PROCESS OPTION GENERATION
 
 key_file_opt=""
@@ -137,8 +142,11 @@ fi
 
 verbose_opt=""
 if [ ! -z "$VERBOSE_FLAG" ]; then
-    verbose_opt="-v"
-    verbose_log "Using verbose logging for ssync subprocesses"
+    verbose_opt="-"
+    for n in $(seq 1 $VERBOSE_FLAG); do
+	verbose_opt="${verbose_opt}v"
+    done
+    verbose_log "Using verbose logging level $VERBOSE_FLAG ($verbose_opt) for ssync subprocesses"
 fi
 
 if [ ! -z "$BACKEND_FLAG" ]; then
@@ -147,9 +155,9 @@ elif [ ! -z "$backend" ]; then
     backend_opt="-b $backend"
 fi
 
-no_fetch_opt=""
-if [ ! -z "$NO_FETCH_FLAG" ]; then
-    no_fetch_opt="-n"
+dry_run_opt=""
+if [ ! -z "$DRY_RUN_FLAG" ]; then
+    dry_run_opt="-n"
 fi
 
 RUN=$(date -Is | sed 's/[:]/-/g')
@@ -163,6 +171,10 @@ remote_index_file=$output_dir/index_remote
 queue_file=$output_dir/queue
 
 ## RUNNING SUB PROCESSES
+# Reaping
+verbose_log "Reaping files"
+$ssync_dir/ssync-reapr $dry_run_opt $verbose_opt $reap_age_opt $output_files_dir
+
 # Indexing
 verbose_log "Indexing remote files at $remote_host@$remote_root_dir"
 $ssync_dir/ssync-index $verbose_opt $key_file_opt \
@@ -191,11 +203,11 @@ $ssync_dir/ssync-queue $verbose_opt \
 
 # Fetching
 verbose_log "Fetching files from queue"
-$ssync_dir/ssync-fetch $verbose_opt $key_file_opt $backend_opt $no_fetch_opt \
-		       -p 2 \
+$ssync_dir/ssync-fetch $verbose_opt $key_file_opt $backend_opt $dry_run_opt \
+                       -p 2 \
                        -r $remote_host \
-		       $queue_file \
-		       $fetch_output_dir
+                       $queue_file \
+                       $fetch_output_dir
 
 ## DONE
 verbose_log "Finished ssyncing @ $(date -Is)"
diff --git a/ssync-queue b/ssync-queue
index 25fa1cd..ddc97d3 100755
--- a/ssync-queue
+++ b/ssync-queue
@@ -2,25 +2,32 @@
 
 USAGE="ssync-queue [options] -l LOCAL_INDEX_FILE -r REMOTE_INDEX_FILE -o QUEUE_OUTPUT_FILE
     OPTIONS
+        -k keep temp files
         -l LOCAL_INDEX_FILE
-	   target local index file
-	-r REMOTE_INDEX_FILE
-	   target remote index file
-	-o QUEUE_OUTPUT_FILE
-	   queue output file
-	-v verbose logging
+           target local index file
+        -r REMOTE_INDEX_FILE
+           target remote index file
+        -o QUEUE_OUTPUT_FILE
+           queue output file
+        -v verbose logging
         -h print this message"
 
 # HELPER FUNCTIONS
 
 verbose_log() {
     if [ ! -z "$VERBOSE_FLAG" ]; then
-	echo "$@"
+        echo "$@"
     fi
 }
 lines() {
     echo $(wc -l $1 | cut -d' ' -f1)
 }
+remove_file() {
+    if [ -f "$1" ]; then
+        verbose_log "Removing $1"
+        rm $1
+    fi
+}
 
 # OPTIONS
 
@@ -32,23 +39,26 @@ CONFIG_FILE_FLAG=
 LOCAL_FILE_ARG=
 REMOTE_FILE_ARG=
 QUEUE_FILE_ARG=
+KEEP_FLAG=
 
-while getopts "hvl:r:o:" opt; do
+while getopts "hvkl:r:o:" opt; do
     case "${opt}" in
-	l) LOCAL_FILE_FLAG=1
-	   LOCAL_FILE_ARG="${OPTARG}"
-	   ;;
-	r) REMOTE_FILE_FLAG=1
-	   REMOTE_FILE_ARG="${OPTARG}"
-	   ;;
-	o) QUEUE_FILE_FLAG=1
-	   QUEUE_FILE_ARG="${OPTARG}"
-	   ;;
-	v) VERBOSE_FLAG=1
-	   ;;
-	h) echo "$USAGE"
-	   exit 1
+	k) KEEP_FLAG=1
 	   ;;
+        l) LOCAL_FILE_FLAG=1
+           LOCAL_FILE_ARG="${OPTARG}"
+           ;;
+        r) REMOTE_FILE_FLAG=1
+           REMOTE_FILE_ARG="${OPTARG}"
+           ;;
+        o) QUEUE_FILE_FLAG=1
+           QUEUE_FILE_ARG="${OPTARG}"
+           ;;
+        v) VERBOSE_FLAG=1
+           ;;
+        h) echo "$USAGE"
+           exit 1
+           ;;
     esac
 done
 
@@ -103,3 +113,9 @@ verbose_log "Found $(lines $remote_only_filenames_file) remote only files"
 cat $remote_only_filenames_file | xargs -I{} grep -F "{}" $REMOTE_FILE_ARG >> $QUEUE_FILE_ARG
 verbose_log "Added $(lines $QUEUE_FILE_ARG) to the queue"
 
+if [ -z "$KEEP_FLAG" ]; then
+    verbose_log "Clearing temp files"
+    remove_file $local_sorted
+    remove_file $remote_sorted
+    remove_file $remote_only_filenames_file
+fi
diff --git a/ssync-reapr b/ssync-reapr
new file mode 100755
index 0000000..18fa0c5
--- /dev/null
+++ b/ssync-reapr
@@ -0,0 +1,71 @@
+#!/usr/bin/env sh
+
+USAGE="ssync-reapr [options] [dirs...]
+    OPTIONS
+        -a AGE (in seconds)
+           limit the reaping of files no older than this seconds old.
+           Default is 86400 (1 day)
+        -n dry run
+        -v verbose logging (use -vv to log reaped files)
+        -h print this message"
+
+# HELPER FUNCTIONS
+verbose_log() {
+    if [ ! -z "$VERBOSE_FLAG" ]; then
+        echo "$@"
+    fi
+}
+
+# OPTIONS
+
+AGE_FLAG=
+AGE_OPT=
+DRY_RUN_FLAG=
+VERBOSE_FLAG=
+while getopts "hnva:" opt; do
+    case "${opt}" in
+        h) echo "$USAGE"
+           exit 1
+           ;;
+        a) AGE_FLAG=1
+           AGE_ARG="${OPTARG}"
+           ;;
+        n) DRY_RUN_FLAG=1
+           ;;
+        v) VERBOSE_FLAG=$(($VERBOSE_FLAG +1))
+           ;;
+    esac
+done
+
+shift $(($OPTIND -1))
+
+DIRS="$@"
+
+# Reaper
+
+rm_flag=
+if [ -z "$DRY_RUN_FLAG" ]; then
+    rm_flag="-delete"
+    if [ $VERBOSE_FLAG -gt 1 ]; then
+	rm_flag="$rm_flag -print"
+    fi
+else
+    verbose_log "Dry run!"
+    rm_flag="-print"
+fi
+
+target_date=
+if [ -z "$AGE_FLAG" ]; then
+    target_date=$(date -d "86400 seconds ago" -Is)
+else
+    target_date=$(date -d "${AGE_ARG} seconds ago" -Is)
+fi
+window_flag="-not -newermt ${target_date}"
+
+verbose_log "Reaping files older than $target_date: $window_flag"
+
+for dir in $DIRS; do
+    target=$(realpath $dir)
+    verbose_log "Clearing files in $target"
+    find $target -type f $window_flag $rm_flag
+done