blob: 2a79e64d930bc51a1ae83c1df0a933c9c0fd049e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#!/usr/bin/env sh
USAGE="ssync-fetch [options] QUEUE_FILE DEST_DIR
OPTIONS
-r REMOTE_HOST
remote host to download from such as user@hostname
username can be omitted if identical to $USER
or if set in ssh_config
-k KEY_FILE
ssh-key file to use (needs to be non-interactive)
optional: will use default session key
or key set in ssh_config for REMOTE_HOST
-v verbose logging
-h print this message"
# HELPER FUNCTIONS
function verbose_log {
if [ ! -z "$VERBOSE_FLAG" ]; then
echo "$@"
fi
}
# OPTIONS
KEY_FILE_FLAG=
KEY_FILE_ARG=
REMOTE_HOST_FLAG=
REMOTE_HOST_ARG=
VERBOSE_FLAG=
while getopts "hvr:k:" opt; do
case "${opt}" in
h) echo "$USAGE"
exit 1
;;
v) VERBOSE_FLAG=1
;;
k) KEY_FILE_FLAG=1
KEY_FILE_ARG="${OPTARG}"
;;
c) CONCURRENCY_FLAG=1
CONCURRENCY_ARG="${OPTARG}"
;;
r) REMOTE_HOST_FLAG=1
REMOTE_HOST_ARG="${OPTARG}"
;;
esac
done
shift $(($OPTIND -1))
if [ $# -ne 2 ]; then
echo "$USAGE"
exit 1
fi
QUEUE_FILE=$1
DEST_DIR=$2
# VALIDATION
if [ ! -f "$QUEUE_FILE" ]; then
echo "Queue file '$QUEUE_FILE' does not exist"
exit 1
fi
if [ ! -d "$DEST_DIR" ]; then
echo "Destination directory '$DEST_DIR' does not exist"
exit 1
fi
if [ -z "$REMOTE_HOST_FLAG" ]; then
echo "Remote host option -r required"
exit 1
elif [ -z "$REMOTE_HOST_ARG" ]; then
echo "Invalid remote host '$REMOTE_HOST_ARG'"
exit 1
fi
# CONFIGURATIONS
ssh_id_param=""
if [ ! -z "${KEY_FILE_FLAG}" ]; then
if [ ! -f "${KEY_FILE_ARG}" ]; then
echo "Identity file '${KEY_FILE_ARG}' does not exist"
exit 1
fi
ssh_id_param="-i ${KEY_FILE_ARG}"
fi
concurrent_param=""
if [ ! -z "${CONCURRENCY_FLAG}" ]; then
if [ "${CONCURRENCY_ARG}" -gt 0 ]; then
concurrent_param="-X nrequests=${CONCURRENCY_ARG}"
fi
fi
real_dest=$(realpath $DEST_DIR)
tmp_dir=$(mktemp -d /tmp/ssync_fetch_run.XXXXXX)
ts=$(date +%s)
verbose_log "Writing temp files to ${tmp_dir} with timestamp ${ts}"
# GENERATE BATCH
batch_file=$tmp_dir/batch_${ts}
verbose_log "Converting the queue file to sftp batch file: ${batch_file}"
cat $QUEUE_FILE | xargs -I{} echo "@reget {} ${real_dest}/" >> $batch_file
verbose_log "Beginning download"
verbose_log "sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}"
sftp -N ${ssh_id_param} -b ${batch_file} ${REMOTE_HOST_ARG}
verbose_log "ssync-fetch finished"
|