From 725180192e51be7c4408cd5de45fb0688f792216 Mon Sep 17 00:00:00 2001 From: Steph Enders Date: Fri, 26 Dec 2025 21:05:24 -0500 Subject: Create ssync-reapr to clean up ssync run files ssync-reapr will delete the run files pushed to the ssync output dir (default ~/.local/share/ssync/runs/) which can accumulate over time. The default is to only delete files older than a day. We run the reap as the first step of every ssync - to ensure it clears out the necessary files --- docs/ssync-reapr.1 | 37 ++++++++++++++++++++++++++++ docs/ssync.5 | 9 +++++++ ssync | 36 ++++++++++++++++++--------- ssync-queue | 58 ++++++++++++++++++++++++++++---------------- ssync-reapr | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 33 deletions(-) create mode 100644 docs/ssync-reapr.1 create mode 100755 ssync-reapr diff --git a/docs/ssync-reapr.1 b/docs/ssync-reapr.1 new file mode 100644 index 0000000..99f36ab --- /dev/null +++ b/docs/ssync-reapr.1 @@ -0,0 +1,37 @@ +.TH ssync-reapr 1 {RELEASE} +.SH NAME +ssync-reapr \- reap temporary and run files created by other ssync commands +.SH SYNOPSIS +.B ssync-reapr +.RB [option...] +.IR DIRS... +.YS +.SH OPTIONS +.TP +.BR \-a " seconds" +Reap files created more than the configured seconds ago. This will help clear out temporary files created by the +.MR ssync-index 1 +and +.MR ssync-queue 1 +commands. +.TP +.BR \-n " dry-run" +Dry run the reaping process - only prints the files; doesn't delete. +.TP +.BR \-v " verbose logging" +Use -vv to increase the verbose level. 2+ will print the files being reaped. +.TP +.BR \-h " print this message" +.SH SEE ALSO +.MR ssync 1 +.MR ssync 5 +.MR ssync-queue 1 +.MR ssync-fetch 1 +.SH REPOSITORY +.UR https://git.senders.io/utils/ssync +.UE +.SH AUTHORS +Steph Enders +.MT steph@senders.io +.ME +ssync-reapr is open source licensed under the ISC license. See LICENSE for full copyright information. diff --git a/docs/ssync.5 b/docs/ssync.5 index c0e3caf..e729df5 100644 --- a/docs/ssync.5 +++ b/docs/ssync.5 @@ -36,6 +36,13 @@ which will retain the outputs of each run in a subdirectory. This directory must .MR ssync-queue 1 running. .TP +.BI reap_age_s " seconds" +Reap files created more than the configured seconds ago. This will help clear out temporary files created by the +.MR ssync-index 1 +and +.MR ssync-queue 1 +commands. +.TP .BI remote_host " hostname" Remote Hostname to connect to when indexing and fetching remote files. This should be in the format .IR user@hostname @@ -50,6 +57,7 @@ Directory containing the .MR ssync-index 1 .MR ssync-queue 1 .MR ssync-fetch 1 +.MR ssync-reapr 1 commands. .SH AUTHOR Written by Steph Enders @@ -58,5 +66,6 @@ Written by Steph Enders .MR ssync-index 1 .MR ssync-queue 1 .MR ssync-fetch 1 +.MR ssync-reapr 1 diff --git a/ssync b/ssync index c135091..68f1bf4 100755 --- a/ssync +++ b/ssync @@ -19,16 +19,16 @@ lines() { } # OPTIONS -NO_FETCH_FLAG= +DRY_RUN_FLAG= VERBOSE_FLAG= while getopts "nvhb:" opt; do case "${opt}" in h) echo "$USAGE" exit 1 ;; - n) NO_FETCH_FLAG=1 + n) DRY_RUN_FLAG=1 ;; - v) VERBOSE_FLAG=1 + v) VERBOSE_FLAG=$(($VERBOSE_FLAG+1)) ;; b) BACKEND_FLAG=1 BACKEND="$OPTARG" @@ -114,6 +114,11 @@ elif [ ! -d "$fetch_output_dir" ]; then exit 1 fi +reap_age_opt= +if [ ! -z "$reap_age_s" ]; then + reap_age_opt="-a $reap_age_s" +fi + # SUB PROCESS OPTION GENERATION key_file_opt="" @@ -137,8 +142,11 @@ fi verbose_opt="" if [ ! -z "$VERBOSE_FLAG" ]; then - verbose_opt="-v" - verbose_log "Using verbose logging for ssync subprocesses" + verbose_opt="-" + for n in $(seq 1 $VERBOSE_FLAG); do + verbose_opt="${verbose_opt}v" + done + verbose_log "Using verbose logging level $VERBOSE_FLAG ($verbose_opt) for ssync subprocesses" fi if [ ! -z "$BACKEND_FLAG" ]; then @@ -147,9 +155,9 @@ elif [ ! -z "$backend" ]; then backend_opt="-b $backend" fi -no_fetch_opt="" -if [ ! -z "$NO_FETCH_FLAG" ]; then - no_fetch_opt="-n" +dry_run_opt="" +if [ ! -z "$DRY_RUN_FLAG" ]; then + dry_run_opt="-n" fi RUN=$(date -Is | sed 's/[:]/-/g') @@ -163,6 +171,10 @@ remote_index_file=$output_dir/index_remote queue_file=$output_dir/queue ## RUNNING SUB PROCESSES +# Reaping +verbose_log "Reaping files" +$ssync_dir/ssync-reapr $dry_run_opt $verbose_opt $reap_age_opt $output_files_dir + # Indexing verbose_log "Indexing remote files at $remote_host@$remote_root_dir" $ssync_dir/ssync-index $verbose_opt $key_file_opt \ @@ -191,11 +203,11 @@ $ssync_dir/ssync-queue $verbose_opt \ # Fetching verbose_log "Fetching files from queue" -$ssync_dir/ssync-fetch $verbose_opt $key_file_opt $backend_opt $no_fetch_opt \ - -p 2 \ +$ssync_dir/ssync-fetch $verbose_opt $key_file_opt $backend_opt $dry_run_opt \ + -p 2 \ -r $remote_host \ - $queue_file \ - $fetch_output_dir + $queue_file \ + $fetch_output_dir ## DONE verbose_log "Finished ssyncing @ $(date -Is)" diff --git a/ssync-queue b/ssync-queue index 25fa1cd..ddc97d3 100755 --- a/ssync-queue +++ b/ssync-queue @@ -2,25 +2,32 @@ USAGE="ssync-queue [options] -l LOCAL_INDEX_FILE -r REMOTE_INDEX_FILE -o QUEUE_OUTPUT_FILE OPTIONS + -k keep temp files -l LOCAL_INDEX_FILE - target local index file - -r REMOTE_INDEX_FILE - target remote index file - -o QUEUE_OUTPUT_FILE - queue output file - -v verbose logging + target local index file + -r REMOTE_INDEX_FILE + target remote index file + -o QUEUE_OUTPUT_FILE + queue output file + -v verbose logging -h print this message" # HELPER FUNCTIONS verbose_log() { if [ ! -z "$VERBOSE_FLAG" ]; then - echo "$@" + echo "$@" fi } lines() { echo $(wc -l $1 | cut -d' ' -f1) } +remove_file() { + if [ -f "$1" ]; then + verbose_log "Removing $1" + rm $1 + fi +} # OPTIONS @@ -32,23 +39,26 @@ CONFIG_FILE_FLAG= LOCAL_FILE_ARG= REMOTE_FILE_ARG= QUEUE_FILE_ARG= +KEEP_FLAG= -while getopts "hvl:r:o:" opt; do +while getopts "hvkl:r:o:" opt; do case "${opt}" in - l) LOCAL_FILE_FLAG=1 - LOCAL_FILE_ARG="${OPTARG}" - ;; - r) REMOTE_FILE_FLAG=1 - REMOTE_FILE_ARG="${OPTARG}" - ;; - o) QUEUE_FILE_FLAG=1 - QUEUE_FILE_ARG="${OPTARG}" - ;; - v) VERBOSE_FLAG=1 - ;; - h) echo "$USAGE" - exit 1 + k) KEEP_FLAG=1 ;; + l) LOCAL_FILE_FLAG=1 + LOCAL_FILE_ARG="${OPTARG}" + ;; + r) REMOTE_FILE_FLAG=1 + REMOTE_FILE_ARG="${OPTARG}" + ;; + o) QUEUE_FILE_FLAG=1 + QUEUE_FILE_ARG="${OPTARG}" + ;; + v) VERBOSE_FLAG=1 + ;; + h) echo "$USAGE" + exit 1 + ;; esac done @@ -103,3 +113,9 @@ verbose_log "Found $(lines $remote_only_filenames_file) remote only files" cat $remote_only_filenames_file | xargs -I{} grep -F "{}" $REMOTE_FILE_ARG >> $QUEUE_FILE_ARG verbose_log "Added $(lines $QUEUE_FILE_ARG) to the queue" +if [ -z "$KEEP_FLAG" ]; then + verbose_log "Clearing temp files" + remove_file $local_sorted + remove_file $remote_sorted + remove_file $remote_only_filenames_file +fi diff --git a/ssync-reapr b/ssync-reapr new file mode 100755 index 0000000..18fa0c5 --- /dev/null +++ b/ssync-reapr @@ -0,0 +1,71 @@ +#!/usr/bin/env sh + +USAGE="ssync-reapr [options] [dirs...] + OPTIONS + -a AGE (in seconds) + limit the reaping of files no older than this seconds old. + Default is 86400 (1 day) + -n dry run + -v verbose logging (use -vv to log reaped files) + -h print this message" + +# HELPER FUNCTIONS +verbose_log() { + if [ ! -z "$VERBOSE_FLAG" ]; then + echo "$@" + fi +} + +# OPTIONS + +AGE_FLAG= +AGE_OPT= +DRY_RUN_FLAG= +VERBOSE_FLAG= +while getopts "hnva:" opt; do + case "${opt}" in + h) echo "$USAGE" + exit 1 + ;; + a) AGE_FLAG=1 + AGE_ARG="${OPTARG}" + ;; + n) DRY_RUN_FLAG=1 + ;; + v) VERBOSE_FLAG=$(($VERBOSE_FLAG +1)) + ;; + esac +done + +shift $(($OPTIND -1)) + +DIRS="$@" + +# Reaper + +rm_flag= +if [ -z "$DRY_RUN_FLAG" ]; then + rm_flag="-delete" + if [ $VERBOSE_FLAG -gt 1 ]; then + rm_flag="$rm_flag -print" + fi +else + verbose_log "Dry run!" + rm_flag="-print" +fi + +target_date= +if [ -z "$AGE_FLAG" ]; then + target_date=$(date -d "86400 seconds ago" -Is) +else + target_date=$(date -d "${AGE_ARG} seconds ago" -Is) +fi +window_flag="-not -newermt ${target_date}" + +verbose_log "Reaping files older than $target_date: $window_flag" + +for dir in $DIRS; do + target=$(realpath $dir) + verbose_log "Clearing files in $target" + find $target -type f $window_flag $rm_flag +done -- cgit v1.2.3-54-g00ecf