Break ssync into multiple processes

Breaking ssync into 3 sub-processes:

1) ssync-index - indexes remote and local dirs
2) ssync-queue - generates queue of yet-fetched files
3) ssync-fetch - downloads the queue

Which will ultimately be executed using ssync which will allow for
unified config files and transfer locking.

The rewrite is being done in hopes of preventing "missing files"
during large queues and ensure completeness.

The breakdown into multiple files should also help with narrowing the
logic and improving the process without interfering with the execution
and readability of the other stages.

This commit has complete subprocesses - though ssync-index needs
remediation to remove the config file - as we need predictable I/O to
be able to pass the index files into the queue process

1 files changed, 157 insertions, 0 deletions

diff --git a/ssync-index b/ssync-index
new file mode 100755
index 0000000..e19ee78
--- /dev/null
+++ b/ssync-index
@@ -0,0 +1,157 @@
+#!/usr/bin/env sh
+
+USAGE="ssync-index [options] [CONFIG_FILE]
+    OPTIONS
+        -k [KEY_FILE] override conigured key file
+        -o [OUTPUT_DIR] override configured index output dir
+        -l local only
+        -r remote only
+        -v verbose logging
+        -h print this message"
+
+# HELPER METHODS
+function verbose_log {
+    if [ ! -z "$VERBOSE_FLAG" ]; then
+	echo "$1"
+    fi
+}
+function lines {
+    wc -l $1 | cut -d' ' -f1
+}
+function get_default_index_dir {
+    local cache_path=ssync/index/
+    local cache_dir=
+    if [ ! -z "$XDG_CACHE_DIR" ]; then
+	cache_dir=$XDG_CACHE_DIR/$cache_path
+    else
+	cache_dir=/tmp/
+    fi
+
+    mkdir -p $cache_dir/$cache_path
+    echo $cache_dir/$cache_path
+}
+
+# GLOBAL VALUES
+DEFAULT_INDEX_WINDOW=86400
+LOCAL_ONLY_FLAG=
+REMOTE_ONLY_FLAG=
+VERBOSE_FLAG=
+KEY_FILE_FLAG=
+KEY_FILE_ARG=
+OUTPUT_DIR_FLAG=
+OUTPUT_DIR_ARG=
+
+# OPTIONS PARSING
+while getopts "hvlro:k:" opt; do
+    case "${opt}" in
+	h) echo "$USAGE"
+	   exit 1
+	   ;;
+	o) OUTPUT_DIR_FLAG=1
+	   OUTPUT_DIR_ARG="$OPTARG"
+	   ;;
+	k) KEY_FILE_FLAG=1
+	   KEY_FILE_ARG="$OPTARG"
+	   ;;
+	l) LOCAL_ONLY_FLAG=1
+	   ;;
+	r) REMOTE_ONLY_FLAG=1
+	   ;;
+	v) VERBOSE_FLAG=1
+	   ;;
+    esac
+done
+
+shift $(($OPTIND -1))
+
+if [ $# -eq 0 ]; then
+    echo "$USAGE"
+    exit 1
+
+fi
+
+CONFIG_FILE=$1
+
+# CONFIG PARSING
+if [ -z "$CONFIG_FILE" ]; then
+    echo "CONFIG_FILE is missing."
+    exit 1
+fi
+
+if [ ! -f $CONFIG_FILE ]; then
+    echo "$CONFIG_FILE does not exist."
+    exit 1
+fi
+
+source $CONFIG_FILE
+
+# CONFIG VALIDATION
+if [ -z "$remote_host" ]; then
+    echo "Config is missing remote_host"
+    exit 1
+fi
+
+if [ -z "$remote_root_dir" ]; then
+    echo "Config is missing remote_root_dir"
+    exit 1
+fi
+
+if [ -z "$keyfile" ]; then
+    echo "Config is missing keyfile"
+    exit 1
+elif [ ! -f "$keyfile" ]; then
+    echo "Configured keyfile '$keyfile' does not exist"
+    exit 1
+fi
+
+if [ -z "$local_root_dir" ]; then
+    echo "Config is missing local_root_dir"
+    exit 1
+elif [ ! -d "$local_root_dir" ]; then
+    echo "Configured local_root_dir '$local_root_dir' does not exist"
+    exit 1
+fi
+
+# VALUE SETTING
+
+if [ -z "$index_window_s" ]; then
+    index_window_s=$DEFAULT_INDEX_WINDOW
+fi
+
+local_index_file=
+remote_index_file=
+if [ ! -z "$OUTPUT_DIR_FLAG" ]; then
+    index_dir=$OUTPUT_DIR_ARG
+fi
+
+if [ ! -z "$index_dir" ]; then
+    if [ ! -d "$index_dir" ]; then
+	echo "Configured index_dir '$index_dir' does not exist"
+	exit 1
+    fi
+else
+  index_dir=$(get_default_index_dir)
+fi
+
+# RUN INDEXING
+
+ts=$(date -u +%s)
+local_index_file=${index_dir}/${ts}_local.idx
+remote_index_file=${index_dir}/${ts}_remote.idx
+newermt=$(date -d "$index_window_s seconds ago" -u -Is)
+verbose_log "Targetting files from $index_window_s seconds ago: $newermt"
+
+# index remote host relative to your fetch dir
+if [ -z "$LOCAL_ONLY_FLAG" ]; then
+    verbose_log "Generating remote index from ${remote_root_dir} to ${remote_index_file}"
+    ssh -i $keyfile $remote_host \
+	"find ${remote_root_dir} -type f -newermt $newermt -exec realpath {} \;" > $remote_index_file
+    verbose_log "Indexed $(lines $remote_index_file) remote files"
+fi
+
+# index local filenames ONLY
+if [ -z "$REMOTE_ONLY_FLAG" ]; then
+    verbose_log "Generate local index from ${local_root_dir} to ${local_index_file}"
+    find ${local_root_dir} -type f -exec basename {} \; > $local_index_file
+    verbose_log "Indexed $(lines $local_index_file) local files"
+fi