blob: 9aacf0a7dafb181cef4a7a6ecddde499f02425b5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
#!/usr/bin/env sh
USAGE="ssync-queue [options] -l LOCAL_INDEX_FILE -r REMOTE_INDEX_FILE -o QUEUE_OUTPUT_FILE
OPTIONS
-l LOCAL_INDEX_FILE
target local index file
-r REMOTE_INDEX_FILE
target remote index file
-o QUEUE_OUTPUT_FILE
queue output file
-v verbose logging
-h print this message"
# HELPER FUNCTIONS
function verbose_log {
if [ ! -z "$VERBOSE_FLAG" ]; then
echo "$@"
fi
}
function lines {
echo $(wc -l $1 | cut -d' ' -f1)
}
# OPTIONS
VERBOSE_FLAG=
LOCAL_FILE_FLAG=
REMOTE_FILE_FLAG=
QUEUE_FILE_FLAG=
CONFIG_FILE_FLAG=
LOCAL_FILE_ARG=
REMOTE_FILE_ARG=
QUEUE_FILE_ARG=
while getopts "hvl:r:o:" opt; do
case "${opt}" in
l) LOCAL_FILE_FLAG=1
LOCAL_FILE_ARG="${OPTARG}"
;;
r) REMOTE_FILE_FLAG=1
REMOTE_FILE_ARG="${OPTARG}"
;;
o) QUEUE_FILE_FLAG=1
QUEUE_FILE_ARG="${OPTARG}"
;;
v) VERBOSE_FLAG=1
;;
h) echo "$USAGE"
exit 1
;;
esac
done
shift $(($OPTIND -1))
if [ -z "$LOCAL_FILE_FLAG" ]; then
echo "-l LOCAL_INDEX_FILE option is required"
exit 1
elif [ ! -f "$LOCAL_FILE_ARG" ]; then
echo "local index file '$LOCAL_FILE_ARG' does not exist"
exit 1
fi
if [ -z "$REMOTE_FILE_FLAG" ]; then
echo "-r REMOTE_INDEX_FILE option is required"
exit 1
elif [ ! -f "$REMOTE_FILE_ARG" ]; then
echo "remote index file '${REMOTE_FILE_ARG}' does not exist"
exit 1
fi
if [ -z "$QUEUE_FILE_FLAG" ]; then
echo "-q QUEUE_OUTPUT_FILE option is required"
exit 1
fi
queue_tmp_dir=$(mktemp -d /tmp/ssync_queue_run.XXXXXX)
verbose_log "Writing temp files to $queue_tmp_dir"
# get remote filenames
remote_index_filenames_file=$queue_tmp_dir/remote_filenames.idx
verbose_log "Writing remote index filenames to $remote_index_filenames_file"
cat $REMOTE_FILE_ARG | xargs -I{} basename {} > $remote_index_filenames_file
original_line_count=$(lines $REMOTE_FILE_ARG)
unique_line_count=$(lines <(sort -u $remote_index_filenames_file))
verbose_log "Remote index contains $unique_line_count unique filenames out of $original_line_count indexed files"
if [ $original_line_count != $unique_line_count ]; then
echo "Remote index contains non-unique files. Check $REMOTE_INDEX_FILE to find which files aren't unique" >&2
fi
# find which filenames are unique to the remote
remote_only_filenames_file=$queue_tmp_dir/remote_only_filenames.idx
comm -23 <(sort $remote_index_filenames_file) <(sort $LOCAL_FILE_ARG) \
> $remote_only_filenames_file
verbose_log "Found $(lines $remote_only_filenames_file) remote only files"
# push matching files into queue
cat $remote_only_filenames_file | xargs -I{} grep "^.*{}$" $REMOTE_FILE_ARG >> $QUEUE_FILE_ARG
verbose_log "Added $(lines $QUEUE_FILE_ARG) to the queue"
|