#!/usr/bin/env sh USAGE="ssync-queue [options] -l LOCAL_INDEX_FILE -r REMOTE_INDEX_FILE -o QUEUE_OUTPUT_FILE OPTIONS -l LOCAL_INDEX_FILE target local index file -r REMOTE_INDEX_FILE target remote index file -o QUEUE_OUTPUT_FILE queue output file -v verbose logging -h print this message" # HELPER FUNCTIONS verbose_log() { if [ ! -z "$VERBOSE_FLAG" ]; then echo "$@" fi } lines() { echo $(wc -l $1 | cut -d' ' -f1) } # OPTIONS VERBOSE_FLAG= LOCAL_FILE_FLAG= REMOTE_FILE_FLAG= QUEUE_FILE_FLAG= CONFIG_FILE_FLAG= LOCAL_FILE_ARG= REMOTE_FILE_ARG= QUEUE_FILE_ARG= while getopts "hvl:r:o:" opt; do case "${opt}" in l) LOCAL_FILE_FLAG=1 LOCAL_FILE_ARG="${OPTARG}" ;; r) REMOTE_FILE_FLAG=1 REMOTE_FILE_ARG="${OPTARG}" ;; o) QUEUE_FILE_FLAG=1 QUEUE_FILE_ARG="${OPTARG}" ;; v) VERBOSE_FLAG=1 ;; h) echo "$USAGE" exit 1 ;; esac done shift $(($OPTIND -1)) if [ -z "$LOCAL_FILE_FLAG" ]; then echo "-l LOCAL_INDEX_FILE option is required" exit 1 elif [ ! -f "$LOCAL_FILE_ARG" ]; then echo "local index file '$LOCAL_FILE_ARG' does not exist" exit 1 fi if [ -z "$REMOTE_FILE_FLAG" ]; then echo "-r REMOTE_INDEX_FILE option is required" exit 1 elif [ ! -f "$REMOTE_FILE_ARG" ]; then echo "remote index file '${REMOTE_FILE_ARG}' does not exist" exit 1 fi if [ -z "$QUEUE_FILE_FLAG" ]; then echo "-q QUEUE_OUTPUT_FILE option is required" exit 1 fi queue_tmp_dir=$(mktemp -d /tmp/ssync_queue_run.XXXXXX) verbose_log "Writing temp files to $queue_tmp_dir" # get remote filenames remote_index_filenames_file=$queue_tmp_dir/remote_filenames.idx verbose_log "Writing remote index filenames to $remote_index_filenames_file" cat $REMOTE_FILE_ARG | xargs -I{} basename {} > $remote_index_filenames_file original_line_count=$(lines $REMOTE_FILE_ARG) unique_line_count=$(sort -u $remote_index_filenames_file | lines) verbose_log "Remote index contains $unique_line_count unique filenames out of $original_line_count indexed files" if [ $original_line_count != $unique_line_count ]; then echo "Remote index contains non-unique files. Check $REMOTE_INDEX_FILE to find which files aren't unique" >&2 fi # find which filenames are unique to the remote local_sorted=$queue_tmp_dir/local_sorted.idx remote_sorted=$queue_tmp_dir/remote_sorted.idx remote_only_filenames_file=$queue_tmp_dir/remote_only_filenames.idx sort $LOCAL_FILE_ARG > $local_sorted sort $remote_index_filenames_file > $remote_sorted comm -23 $remote_sorted $local_sorted > $remote_only_filenames_file verbose_log "Found $(lines $remote_only_filenames_file) remote only files" # push matching files into queue cat $remote_only_filenames_file | xargs -I{} grep "^.*{}$" $REMOTE_FILE_ARG >> $QUEUE_FILE_ARG verbose_log "Added $(lines $QUEUE_FILE_ARG) to the queue"