# HG changeset patch
# User casties
# Date 1489685961 -3600
# Node ID a8eac765e94dbee60a0d2c2695d0bcbea7228922

first checkin of mirror scripts.

diff -r 000000000000 -r a8eac765e94d cdli-archive-sync.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cdli-archive-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# script to synchronize the CDLI archive at UCLA to MPIWG
+#
+# V0.4.5 (23.3.2009 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/cdli-archive-sync.lock
+LOGFILE=/var/tmp/cdli-archive-sync.log
+EXCLUDEFILE=/home/mirror/scripts/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de
+
+MIRDIR=/mpiwg/mirror/CDLI/ARCHIVE
+REMOTEDIR=/Volumes
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="mpiwgsav@backup.cdli.ucla.edu"
+RSYNCOPTS="-av --delete --ignore-errors --exclude-from=$EXCLUDEFILE"
+
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC $RSYNCOPTS $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC $RSYNCOPTS $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): CDLI-ARCHIVE-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+syncdir "cdli_rawfiles"
+syncdir "cdli_archivalfiles"
+syncdir "cdli_varia"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -l $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "$( hostname ): $ERROR" <<EOF
+
+$LOG
+EOF
+fi
+
+# remove lock
+
+rm $LOCKFILE
diff -r 000000000000 -r a8eac765e94d ismi-backup-sync.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/ismi-backup-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# script to synchronize the ISMI archive at McGill to MPIWG
+#
+# V0.1 (18.4.2010 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/ismi-backup-sync.lock
+LOGFILE=/var/tmp/ismi-backup-sync.log
+EXCLUDEFILE=/home/mirror/scripts/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de
+
+MIRDIR=/mpiwg/mirror/ISMI/mcgill
+REMOTEDIR=/
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="mpiwg@dev.rasi.mcgill.ca"
+RSYNCOPTS="--delete --exclude-from=$EXCLUDEFILE"
+
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC -ave ssh $RSYNCOPTS $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC -ave ssh $RSYNCOPTS $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): ISMI-ARCHIVE-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+syncdir "data5"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -ln $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "$( hostname ): $ERROR" <<EOF
+$LOG
+EOF
+fi
+
+echo "***** finished `date`" >> $LOGFILE
+
+# remove lock
+
+rm $LOCKFILE
diff -r 000000000000 -r a8eac765e94d mpiwg-all-sync.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mpiwg-all-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# script to synchronize the MPIWG fileserver
+#
+# V0.0.3l (7.4.2015 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/mpiwg-all-sync.lock
+LOGFILE=/var/tmp/mpiwg-all-sync.log
+#EXCLUDEFILE=/usr/local/sbin/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de,jsteinhoff@mpiwg-berlin.mpg.de
+
+MIRDIR=
+REMOTEDIR=
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="foxridge2"
+#RSYNCOPTS="-e rsh --delete --exclude-from=$EXCLUDEFILE"
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC ${RSYNCOPTS[@]} $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC "${RSYNCOPTS[@]}" $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): MPIWG-ALL-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+RSYNCOPTS=(-avAX --delete -e 'ssh -i /root/.ssh/foxridge1-sync-mpiwg')
+syncdir "mpiwg"
+
+# /usr/local/mpiwg as well
+RSYNCOPTS=(-avAX --delete -e 'ssh -i /root/.ssh/foxridge1-sync-usrlocal')
+syncdir "mpiwg" "/usr/local" "/usr/local"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -l $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "foxridge1 mpiwg-all-sync: $ERROR" <<EOF
+
+$LOG
+EOF
+fi
+
+echo "***** done `date`" >> $LOGFILE
+
+# remove lock
+rm $LOCKFILE
diff -r 000000000000 -r a8eac765e94d rsync-exclude.list
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rsync-exclude.list	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,21 @@
+# files to exclude when rsync'ing
+.DS_Store
+Temporary*Items
+.TemporaryItems
+Network*Trash*Folder
+Desktop*DB
+Desktop*DF
+.Spotlight-V100
+.Trashes
+.VolumeIcon.icns
+.apdisk
+.bzvol
+.fseventsd
+# is the following still relevant?
+.FBCIndex
+.FBCLockFolder
+.HSancillary
+.HSResource
+.HSicon
+# CDLI wiki
+dokuwiki/data/cache