#!/usr/bin/env sh USAGE="ssync-fetch [options] QUEUE_FILE DEST_DIR OPTIONS -r REMOTE_HOST remote host to download from such as user@hostname username can be omitted if identical to $USER or if set in ssh_config -k KEY_FILE ssh-key file to use (needs to be non-interactive) optional: will use default session key or key set in ssh_config for REMOTE_HOST -b BACKEND sftp (default) | rsync -v verbose logging -h print this message" # HELPER FUNCTIONS verbose_log() { if [ ! -z "$VERBOSE_FLAG" ]; then echo "$@" fi } lines() { echo $(wc -l $1 | cut -d' ' -f1) } # OPTIONS KEY_FILE_FLAG= KEY_FILE_ARG= REMOTE_HOST_FLAG= REMOTE_HOST_ARG= BACKEND_FLAG= BACKEND_ARG= VERBOSE_FLAG= while getopts "hvr:k:b:" opt; do case "${opt}" in h) echo "$USAGE" exit 1 ;; v) VERBOSE_FLAG=1 ;; k) KEY_FILE_FLAG=1 KEY_FILE_ARG="${OPTARG}" ;; r) REMOTE_HOST_FLAG=1 REMOTE_HOST_ARG="${OPTARG}" ;; b) BACKEND_FLAG=1 BACKEND_ARG="${OPTARG}" ;; esac done shift $(($OPTIND -1)) if [ $# -ne 2 ]; then echo "$USAGE" exit 1 fi QUEUE_FILE=$1 DEST_DIR=$2 # VALIDATION if [ ! -f "$QUEUE_FILE" ]; then echo "Queue file '$QUEUE_FILE' does not exist" exit 1 elif [ $(lines $QUEUE_FILE) -le 0 ]; then echo "Queue file empty. Exiting" exit 0 fi if [ ! -d "$DEST_DIR" ]; then echo "Destination directory '$DEST_DIR' does not exist" exit 1 fi if [ -z "$REMOTE_HOST_FLAG" ]; then echo "Remote host option -r required" exit 1 elif [ -z "$REMOTE_HOST_ARG" ]; then echo "Invalid remote host '$REMOTE_HOST_ARG'" exit 1 fi if [ -z "$BACKEND_FLAG" ]; then BACKEND="sftp" else BACKEND=$BACKEND_ARG fi if [ "$BACKEND" != "sftp" ]; then if [ "$BACKEND" != "rsync" ]; then echo "Invalid backend $BACKEND" exit 1 fi fi # CONFIGURATIONS ssh_id_param="" if [ ! -z "${KEY_FILE_FLAG}" ]; then if [ ! -f "${KEY_FILE_ARG}" ]; then echo "Identity file '${KEY_FILE_ARG}' does not exist" exit 1 fi ssh_id_param="-i ${KEY_FILE_ARG}" fi real_dest=$(realpath $DEST_DIR) tmp_dir=$(mktemp -d /tmp/ssync_fetch_run.XXXXXX) ts=$(date +%s) verbose_log "Writing temp files to ${tmp_dir} with timestamp ${ts}" # GENERATE BATCH if [ "$BACKEND" = "sftp" ]; then batch_file=$tmp_dir/batch_${ts} verbose_log "Converting the queue file to sftp batch file: ${batch_file}" cat $QUEUE_FILE | xargs -I{} echo -e "@reget {} ${real_dest}/" > $batch_file verbose_log "Beginning download" verbose_log "sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}" sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG} elif [ "$BACKEND" = "rsync" ]; then verbose_log "Fetching files from queue file: $QUEUE_FILE" verbose_log "Beginning download" verbose_log "rsync --files-from=${QUEUE_FILE} rsync://${REMOTE_HOST_ARG} ${real_dest}" rsync -av --no-relative --files-from=${QUEUE_FILE} ${REMOTE_HOST_ARG}:/ ${real_dest} fi verbose_log "ssync-fetch finished"