blob: 4067001a0230710a6861c930a3574b5e1e73884f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
#!/usr/bin/env bash
set -e
RUNID=$(openssl rand -hex 16)
function log {
echo -e "${RUNID}\t$(date -Is)\t$@"
}
if [ $# -lt 4 ]; then
echo "Usage: ssync KEY_FILE REMOTE REMOTE_DIR SRC_DIR"
exit 1
fi
# opts
KEY_FILE=$1
REMOTE=$2
REMOTE_DIR=$3
SRC_DIR=$4
# constants
PARALLEL=5
SSYNC_DIR=$HOME/.local/share/ssync
FETCHED_FILE=$SSYNC_DIR/fetched
FETCH_FILE=$SSYNC_DIR/fetch
LASTRAN_FILE=$SSYNC_DIR/lastran
NEXT_RUN_DATE=$(date -u -Is -d '5 minutes ago')
RUN_DIR=$SSYNC_DIR/.runs/
CURGET_FILE=$RUN_DIR/$RUNID
if [ ! -f $LASTRAN_FILE ]; then
mkdir -p $SSYNC_DIR
echo $NEXT_RUN_DATE > $LASTRAN_FILE
log "No run existed marking next run for files newer than: $NEXT_RUN_DATE"
exit 0;
fi
PREV_RUN_DATE=$(cat $LASTRAN_FILE)
# main
if [[ $PREV_RUN_DATE == "" ]]; then
echo $NEXT_RUN_DATE > $LASTRAN_FILE
log "No run existed marking next run for files newer than: $NEXT_RUN_DATE"
exit 0;
fi
if [ ! -f $FETCHED_FILE ]; then
touch $FETCHED_FILE
fi
log "Failsafe - Running at: $NEXT_RUN_DATE - if failed to write use this timestamp in $LASTRAN_FILE"
log "Syncing files since: $PREV_RUN_DATE"
log "Fetching files"
mkdir -p $RUN_DIR
ssh -i $KEY_FILE $REMOTE "find ${REMOTE_DIR} -type f -newermt ${PREV_RUN_DATE} -exec realpath --relative-to ${REMOTE_DIR} {} \;" >> $CURGET_FILE
GET_COUNT=$(wc -l $CURGET_FILE | cut -d' ' -f1)
if [ $GET_COUNT -gt 0 ]; then
comm -23 <(sort -u $CURGET_FILE) <(sort -u $FETCHED_FILE) > $FETCH_FILE
COUNT=$(wc -l $FETCH_FILE | cut -d' ' -f1)
if [ $COUNT -gt 0 ]; then
#
# Syncing
#
log "Found ${COUNT} files to fetch"
cat $FETCH_FILE >> $FETCHED_FILE
log "Wrote files to fetched files"
log "Syncing now"
cat $FETCH_FILE | xargs -n1 -P$PARALLEL -I '{}' rsync -e "ssh -i $KEY_FILE" \
-av \
$REMOTE:"${REMOTE_DIR}/'{}'" ${SRC_DIR}
else
log "No new files to sync - found $GET_COUNT existing files"
fi
else
log "No new files found - deleting $CURGET_FILE"
rm -f $CURGET_FILE
fi
echo $NEXT_RUN_DATE > $LASTRAN_FILE
log "Done syncing"
|