Rewrite ssync main function

Use config to generate the necessary options for each sub process
running:

1) ssync-index REMOTE
2) ssync-index LOCAL
3) ssync-queue
4) ssync-fetch

1 files changed, 171 insertions, 62 deletions

diff --git a/ssync b/ssync
index 4067001..83eed3f 100755
--- a/ssync
+++ b/ssync
@@ -1,84 +1,193 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 set -e
 
-RUNID=$(openssl rand -hex 16)
-function log {
-  echo -e "${RUNID}\t$(date -Is)\t$@"
+USAGE="ssync [options] CONFIG_FILE
+    OPTIONS
+        -n no locking
+        -v verbose logging
+        -h print this message"
+
+# HELPER METHODS
+function verbose_log {
+    if [ ! -z "$VERBOSE_FLAG" ]; then
+        echo "$1"
+    fi
+}
+function lines {
+    echo $(wc -l $1 | cut -d' ' -f1)
 }
 
-if [ $# -lt 4 ]; then
-  echo "Usage: ssync KEY_FILE REMOTE REMOTE_DIR SRC_DIR"
-  exit 1
+# OPTIONS
+NO_LOCK_FLAG=
+VERBOSE_FLAG=
+while getopts "nvh" opt; do
+    case "${opt}" in
+        h) echo "$USAGE"
+           exit 1
+           ;;
+        n) NO_LOCK_FLAG=1
+           ;;
+        v) VERBOSE_FLAG=1
+           ;;
+    esac
+done
+
+shift $(($OPTIND -1))
+
+# INPUT PARAMS
+
+if [ $# -ne 1 ]; then
+    echo "$USAGE"
+    exit 1
 fi
 
-# opts
-KEY_FILE=$1
-REMOTE=$2
-REMOTE_DIR=$3
-SRC_DIR=$4
-
-# constants
-PARALLEL=5
-SSYNC_DIR=$HOME/.local/share/ssync
-FETCHED_FILE=$SSYNC_DIR/fetched
-FETCH_FILE=$SSYNC_DIR/fetch
-LASTRAN_FILE=$SSYNC_DIR/lastran
-NEXT_RUN_DATE=$(date -u -Is -d '5 minutes ago')
-RUN_DIR=$SSYNC_DIR/.runs/
-CURGET_FILE=$RUN_DIR/$RUNID
-if [ ! -f $LASTRAN_FILE ]; then
-  mkdir -p $SSYNC_DIR
-  echo $NEXT_RUN_DATE > $LASTRAN_FILE
-  log "No run existed marking next run for files newer than: $NEXT_RUN_DATE"
-  exit 0;
+CONFIG_FILE=$1
+
+# VALIDATE CONFIG
+
+if [ ! -f "$CONFIG_FILE" ]; then
+    echo "Config file '$CONFIG_FILE' does not exist"
+    exit 1
 fi
-PREV_RUN_DATE=$(cat $LASTRAN_FILE)
 
-# main
+source $CONFIG_FILE
+
+# ssync_dir=DIR
+# key_file=KEY_FILE
+# index_window_s=N
+# output_files_dir=PATH
+# remote_host=HOSTNAME
+# remote_root_dir=DIR
+# local_root_dir=DIR
+# fetch_output_dir=DIR
 
-if [[ $PREV_RUN_DATE ==  "" ]]; then
-  echo $NEXT_RUN_DATE > $LASTRAN_FILE
-  log "No run existed marking next run for files newer than: $NEXT_RUN_DATE"
-  exit 0;
+if [ -d "$ssync_dir" ]; then
+    if [ ! -f "$ssync_dir/ssync-index" ]; then
+        echo "Missing ssync-index command"
+        exit 1
+    elif [ ! -f "$ssync_dir/ssync-queue" ]; then
+        echo "Missing ssync-queue command"
+        exit 1
+    elif [ ! -f "$ssync_dir/ssync-fetch" ]; then
+        echo "Missing ssync-fetch command"
+        exit 1
+    fi
+else
+    echo "Configured ssync_dir '$ssync_dir' does not exist"
+    exit 1
 fi
 
-if [ ! -f $FETCHED_FILE ]; then
-  touch $FETCHED_FILE
+if [ ! -z "$index_window_s" ]; then
+    if [ "$index_window_s" -le 0 ]; then
+        echo "Configured index_window_s is invalid. Must be > 0"
+        exit 1
+    fi
 fi
 
+if [ -z "$output_files_dir" ]; then
+    echo "Missing configured output_files_dir"
+    exit 1
+elif [ ! -d "$output_files_dir" ]; then
+    echo "Output file directory '$output_files_dir' does not exist"
+    exit 1
+fi
 
-log "Failsafe - Running at: $NEXT_RUN_DATE - if failed to write use this timestamp in $LASTRAN_FILE"
-log "Syncing files since: $PREV_RUN_DATE"
+if [ -z "$remote_host" ]; then
+    echo "Missing configured remote_host"
+    exit 1
+fi
 
+if [ -z "$remote_root_dir" ]; then
+    echo "Missing configured remote_root_dir"
+    exit 1
+fi
 
-log "Fetching files"
-mkdir -p $RUN_DIR
-ssh -i $KEY_FILE $REMOTE "find ${REMOTE_DIR} -type f -newermt ${PREV_RUN_DATE} -exec realpath --relative-to ${REMOTE_DIR} {} \;" >> $CURGET_FILE
-GET_COUNT=$(wc -l $CURGET_FILE | cut -d' ' -f1)
+if [ -z "$local_root_dir" ]; then
+    echo "Missing configured local_root_dir"
+    exit 1
+elif [ ! -d "$local_root_dir" ]; then
+    echo "Local index directory '$local_root_dir' does not exist"
+    exit 1
+fi
 
-if [ $GET_COUNT -gt 0 ]; then
-  comm -23 <(sort -u $CURGET_FILE) <(sort -u $FETCHED_FILE) > $FETCH_FILE
-  COUNT=$(wc -l $FETCH_FILE | cut -d' ' -f1)
+if [ -z "$fetch_output_dir" ]; then
+    echo "Missing configured fetch_output_dir"
+    exit 1
+elif [ ! -d "$fetch_output_dir" ]; then
+    echo "Local fetch destination '$fetch_output_dir' does not exist"
+    exit 1
+fi
 
-  if [ $COUNT -gt 0 ]; then
-    #
-    # Syncing
-    #
-    log "Found ${COUNT} files to fetch"
+# SUB PROCESS OPTION GENERATION
 
-    cat $FETCH_FILE >> $FETCHED_FILE
-    log "Wrote files to fetched files"
-    log "Syncing now"
-    cat $FETCH_FILE | xargs -n1 -P$PARALLEL -I '{}' rsync -e "ssh -i $KEY_FILE" \
-      -av \
-      $REMOTE:"${REMOTE_DIR}/'{}'" ${SRC_DIR}
-  else
-    log "No new files to sync - found $GET_COUNT existing files" 
-  fi
+key_file_opt=""
+if [ ! -z "$KEY_FILE_FLAG" ]; then
+   if [ -f "$KEY_FILE_ARG" ]; then
+       key_file_opt="-k $KEY_FILE_ARG"
+   else
+       echo "Passed key file '$KEY_FILE_ARG' does not exist"
+       exit 1
+   fi
+elif [ ! -z "$key_file" ]; then
+    if [ -f "$key_file" ]; then
+        key_file_opt="-k $key_file"
+    else
+        echo "Configured key_file '$key_file' does not exist"
+        exit 1
+    fi
 else
-  log "No new files found - deleting $CURGET_FILE"
-  rm -f $CURGET_FILE
+    verbose_log "Using session default identity file"
+fi
+
+verbose_opt=""
+if [ ! -z "$VERBOSE_FLAG" ]; then
+    verbose_opt="-v"
+    verbose_log "Using verbose logging for ssync subprocesses"
+fi
+
+RUN=$(date -Is | sed 's/[:]/-/g')
+output_dir=$output_files_dir/$RUN
+verbose_log "Creating run output directory: $output_dir"
+mkdir -p $output_dir
+
+verbose_log "Creating index and queue files"
+local_index_file=$output_dir/index_local
+remote_index_file=$output_dir/index_remote
+queue_file=$output_dir/queue
+
+## RUNNING SUB PROCESSES
+# Indexing
+verbose_log "Indexing remote files at $remote_host@$remote_root_dir"
+$ssync_dir/ssync-index $verbose_opt $key_file_opt \
+                       -w $index_window_s \
+                       -o $remote_index_file \
+                       -r $remote_host \
+                       $remote_root_dir
+
+# quick exit if we don't have a remote index at all
+if [ $(lines $remote_index_file) -le 0 ]; then
+    echo "No remote files indexed. Quitting"
+    exit 0
 fi
 
-echo $NEXT_RUN_DATE > $LASTRAN_FILE
-log "Done syncing"
+verbose_log "Indexing local files at $local_root_dir"
+$ssync_dir/ssync-index $verbose_opt -b \
+		       -o $local_index_file \
+		       $local_root_dir
+
+# Queueing
+verbose_log "Creating queue from indexes"
+$ssync_dir/ssync-queue $verbose_opt \
+		       -l $local_index_file \
+		       -r $remote_index_file \
+		       -o $queue_file
+
+# Fetching
+verbose_log "Fetching files from queue"
+$ssync_dir/ssync-fetch $verbose_opt $key_file_opt \
+		       -r $remote_host \
+		       $queue_file \
+		       $fetch_output_dir
+
+## DONE
+verbose_log "Finished ssyncing @ $(date -Is)"