Add dry-run flag for testing

Should help debug potential things

1 files changed, 74 insertions, 22 deletions

diff --git a/ssync-fetch b/ssync-fetch
index 4b91a2e..b9fc806 100755
--- a/ssync-fetch
+++ b/ssync-fetch
@@ -2,16 +2,19 @@
 
 USAGE="ssync-fetch [options] QUEUE_FILE DEST_DIR
     OPTIONS
-        -r REMOTE_HOST
-           remote host to download from such as user@hostname
-           username can be omitted if identical to $USER
-           or if set in ssh_config
+        -b BACKEND
+           sftp (default) | rsync
         -k KEY_FILE
            ssh-key file to use (needs to be non-interactive)
            optional: will use default session key
            or key set in ssh_config for REMOTE_HOST
-	-b BACKEND
-	   sftp (default) | rsync
+        -n dry-run
+        -p PARALLEL
+           integer indicating how many concurrent downloads
+        -r REMOTE_HOST
+           remote host to download from such as user@hostname
+           username can be omitted if identical to $USER
+           or if set in ssh_config
         -v verbose logging
         -h print this message"
 
@@ -34,9 +37,12 @@ REMOTE_HOST_FLAG=
 REMOTE_HOST_ARG=
 BACKEND_FLAG=
 BACKEND_ARG=
+PARALLEL_FLAG=
+PARALLEL_ARG=
 VERBOSE_FLAG=
+DRY_RUN_FLAG=
 
-while getopts "hvr:k:b:" opt; do
+while getopts "hvnr:k:b:p:" opt; do
     case "${opt}" in
         h) echo "$USAGE"
            exit 1
@@ -49,9 +55,14 @@ while getopts "hvr:k:b:" opt; do
         r) REMOTE_HOST_FLAG=1
            REMOTE_HOST_ARG="${OPTARG}"
            ;;
-	b) BACKEND_FLAG=1
-	   BACKEND_ARG="${OPTARG}"
-	   ;;
+        b) BACKEND_FLAG=1
+           BACKEND_ARG="${OPTARG}"
+           ;;
+        p) PARALLEL_FLAG=1
+           PARALLEL_ARG="${OPTARG}"
+           ;;
+        n) DRY_RUN_FLAG=1
+           ;;
     esac
 done
 
@@ -96,8 +107,15 @@ fi
 
 if [ "$BACKEND" != "sftp" ]; then
     if [ "$BACKEND" != "rsync" ]; then
-	echo "Invalid backend $BACKEND"
-	exit 1
+        echo "Invalid backend $BACKEND"
+        exit 1
+    fi
+fi
+
+if [ ! -z "$PARALLEL_FLAG" ]; then
+    if [ "$PARALLEL_ARG" -le 1 ]; then
+        echo "Invalid parallel level: must be > 1"
+        exit 1
     fi
 fi
 
@@ -112,6 +130,7 @@ if [ ! -z "${KEY_FILE_FLAG}" ]; then
     ssh_id_param="-i ${KEY_FILE_ARG}"
 fi
 
+dry_run_param=""
 real_dest=$(realpath $DEST_DIR)
 tmp_dir=$(mktemp -d /tmp/ssync_fetch_run.XXXXXX)
 ts=$(date +%s)
@@ -120,21 +139,54 @@ verbose_log "Writing temp files to ${tmp_dir} with timestamp ${ts}"
 # GENERATE BATCH
 
 if [ "$BACKEND" = "sftp" ]; then
+
     batch_file=$tmp_dir/batch_${ts}
     verbose_log "Converting the queue file to sftp batch file: ${batch_file}"
-    cat $QUEUE_FILE | xargs -I{} echo -e "@reget {} ${real_dest}/" > $batch_file
+    sort $QUEUE_FILE | xargs -I{} echo -e "@reget {} ${real_dest}/" > $batch_file
+
+    if [ ! -z "$PARALLEL_FLAG" ]; then
+        split --additional-suffix=.batch -en l/${PARALLEL_ARG} $batch_file $tmp_dir/split_${ts}_
+        verbose_log "Beginning ${PARALLEL_ARG} parallel downloads for files: $(ls $tmp_dir/split_${ts}_*.batch)"
+        verbose_log "Running 'sftp -N ${ssh_id_param} -b '{}' ${REMOTE_HOST_ARG}' across ${PARALLEL_ARG} jobs"
+
+        if [ ! -z "$DRY_RUN_FLAG" ]; then
+            verbose_log "Dry-run fetch"
+            dry_run_param="echo"
+        fi
+
+        ls -1 $tmp_dir/split_${ts}_*.batch | xargs -P${PARALLEL_ARG} -I '{}' ${dry_run_param} sftp -N ${ssh_id_param} -b '{}' ${REMOTE_HOST_ARG}
+    else
+        verbose_log "Beginning download"
+        verbose_log "sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}"
+        sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}
+    fi
 
-    verbose_log "Beginning download"
-    verbose_log "sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}"
 
-    sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}
 elif [ "$BACKEND" = "rsync" ]; then
-    verbose_log "Fetching files from queue file: $QUEUE_FILE"
-    verbose_log "Beginning download"
-    verbose_log "rsync --files-from=${QUEUE_FILE} rsync://${REMOTE_HOST_ARG} ${real_dest}"
-    rsync -e "ssh ${ssh_id_param}" \
-	  -av --no-relative \
-	  --files-from=${QUEUE_FILE} ${REMOTE_HOST_ARG}:/ ${real_dest}
+
+    if [ ! -z "$DRY_RUN_FLAG" ]; then
+        verbose_log "Dry-run fetch"
+        dry_run_param="n"
+    fi
+
+    if [ ! -z "$PARALLEL_FLAG" ]; then
+        split --additional-suffix=.batch -en l/${PARALLEL_ARG} $QUEUE_FILE $tmp_dir/split_${ts}_
+        verbose_log "Beginning ${PARALLEL_ARG} parallel downloads for files: $(ls $tmp_dir/split_${ts}_*.batch)"
+        verbose_log "ls -1 $tmp_dir/split_${ts}_*.batch | xargs -P${PARALLEL_ARG} -I '{}' rsync -e \"ssh ${ssh_id_param}\" -av --no-relative --files-from='{}' ${REMOTE_HOST_ARG}:/ ${real_dest}"
+        ls -1 $tmp_dir/split_${ts}_*.batch | \
+            xargs -P${PARALLEL_ARG} -I '{}' \
+                  rsync -e "ssh ${ssh_id_param}" \
+                  -av${dry_run_param} \
+		  --no-relative \
+		  --files-from='{}' ${REMOTE_HOST_ARG}:/ ${real_dest}
+    else
+	verbose_log "Fetching files from queue file: $QUEUE_FILE"
+	verbose_log "Beginning download"
+	verbose_log "rsync -e \"ssh ${ssh_id_param}\" -av ${dry_run_param} --no-relative --files-from=${QUEUE_FILE} rsync://${REMOTE_HOST_ARG} ${real_dest}"
+	rsync -e "ssh ${ssh_id_param}" \
+	      -av${dry_run_param} --no-relative \
+	      --files-from=${QUEUE_FILE} ${REMOTE_HOST_ARG}:/ ${real_dest}
+    fi
 fi
 
 verbose_log "ssync-fetch finished"