Mercurial > hg > mirror-tools

changeset 0:a8eac765e94d default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first checkin of mirror scripts.
author casties
date Thu, 16 Mar 2017 18:39:21 +0100
parents
children
files cdli-archive-sync.sh ismi-backup-sync.sh mpiwg-all-sync.sh rsync-exclude.list
diffstat 4 files changed, 311 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cdli-archive-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# script to synchronize the CDLI archive at UCLA to MPIWG
+#
+# V0.4.5 (23.3.2009 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/cdli-archive-sync.lock
+LOGFILE=/var/tmp/cdli-archive-sync.log
+EXCLUDEFILE=/home/mirror/scripts/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de
+
+MIRDIR=/mpiwg/mirror/CDLI/ARCHIVE
+REMOTEDIR=/Volumes
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="mpiwgsav@backup.cdli.ucla.edu"
+RSYNCOPTS="-av --delete --ignore-errors --exclude-from=$EXCLUDEFILE"
+
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC $RSYNCOPTS $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC $RSYNCOPTS $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): CDLI-ARCHIVE-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+syncdir "cdli_rawfiles"
+syncdir "cdli_archivalfiles"
+syncdir "cdli_varia"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -l $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "$( hostname ): $ERROR" <<EOF
+
+$LOG
+EOF
+fi
+
+# remove lock
+
+rm $LOCKFILE
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/ismi-backup-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# script to synchronize the ISMI archive at McGill to MPIWG
+#
+# V0.1 (18.4.2010 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/ismi-backup-sync.lock
+LOGFILE=/var/tmp/ismi-backup-sync.log
+EXCLUDEFILE=/home/mirror/scripts/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de
+
+MIRDIR=/mpiwg/mirror/ISMI/mcgill
+REMOTEDIR=/
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="mpiwg@dev.rasi.mcgill.ca"
+RSYNCOPTS="--delete --exclude-from=$EXCLUDEFILE"
+
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC -ave ssh $RSYNCOPTS $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC -ave ssh $RSYNCOPTS $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): ISMI-ARCHIVE-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+syncdir "data5"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -ln $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "$( hostname ): $ERROR" <<EOF
+$LOG
+EOF
+fi
+
+echo "***** finished `date`" >> $LOGFILE
+
+# remove lock
+
+rm $LOCKFILE
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mpiwg-all-sync.sh	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# script to synchronize the MPIWG fileserver
+#
+# V0.0.3l (7.4.2015 ROC)
+
+DEBUG=1
+
+LOCKFILE=/tmp/mpiwg-all-sync.lock
+LOGFILE=/var/tmp/mpiwg-all-sync.log
+#EXCLUDEFILE=/usr/local/sbin/rsync-exclude.list
+
+MAILTO=casties@mpiwg-berlin.mpg.de,jsteinhoff@mpiwg-berlin.mpg.de
+
+MIRDIR=
+REMOTEDIR=
+
+RSYNC=/usr/bin/rsync
+RSYNCHOST="foxridge2"
+#RSYNCOPTS="-e rsh --delete --exclude-from=$EXCLUDEFILE"
+
+function syncdir () {
+#
+# sync given directory
+#
+    DIR=$1
+    REMDIR=${2:-$REMOTEDIR}
+    if [ -n "$3" ] && [ ${3:1:1} == "-" ]
+    then
+	# if the third param starts with "-" it's the options
+	MOREOPTS=$3
+	shift
+    fi
+    LOCDIR=${3:-$MIRDIR}
+    echo "***** syncing $DIR on $(date)" >> $LOGFILE
+    if [ -n $DEBUG ] ; then echo "syncing $DIR on $(date)"; fi
+    if [ -n $DEBUG ] ; then echo "    $RSYNC ${RSYNCOPTS[@]} $MOREOPTS \"$RSYNCHOST:$REMDIR/$DIR/\" \"$LOCDIR/$DIR/\" >> $LOGFILE 2>&1" >> $LOGFILE ; fi
+    $RSYNC "${RSYNCOPTS[@]}" $MOREOPTS "$RSYNCHOST:$REMDIR/$DIR/" "$LOCDIR/$DIR/" >> $LOGFILE 2>&1 
+    if [ $? != 0 ]
+    then
+	ERROR="${ERROR}ERROR syncing $DIR "
+    fi
+}
+
+
+
+#
+# check if script is already running
+#
+if [ -f $LOCKFILE ]
+then
+    if [ -n $DEBUG ] ; then echo "lockfile present! script seems to be running!"; fi
+
+    mail -t $MAILTO -s "$( hostname ): MPIWG-ALL-MIRROR: already running!" <<EOF
+
+`ls -l $LOCKFILE `
+EOF
+    exit 1
+fi
+
+touch $LOCKFILE
+
+echo "***** started `date`" > $LOGFILE
+
+#
+# sync
+#
+
+RSYNCOPTS=(-avAX --delete -e 'ssh -i /root/.ssh/foxridge1-sync-mpiwg')
+syncdir "mpiwg"
+
+# /usr/local/mpiwg as well
+RSYNCOPTS=(-avAX --delete -e 'ssh -i /root/.ssh/foxridge1-sync-usrlocal')
+syncdir "mpiwg" "/usr/local" "/usr/local"
+
+#
+# send mail with error log if an error occurred
+#
+if [ -n "$ERROR" ]
+then
+    # don't send logs bigger than 1MB
+    LOGSIZE=`ls -l $LOGFILE |cut -d' ' -f 5`
+    if [ "$LOGSIZE" -gt 900000 ]
+    then
+	LOG="LOGFILE too big to send: $LOGSIZE"
+    else
+	LOG=`cat $LOGFILE`
+    fi
+
+    mail -t $MAILTO -s "foxridge1 mpiwg-all-sync: $ERROR" <<EOF
+
+$LOG
+EOF
+fi
+
+echo "***** done `date`" >> $LOGFILE
+
+# remove lock
+rm $LOCKFILE
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rsync-exclude.list	Thu Mar 16 18:39:21 2017 +0100
@@ -0,0 +1,21 @@
+# files to exclude when rsync'ing
+.DS_Store
+Temporary*Items
+.TemporaryItems
+Network*Trash*Folder
+Desktop*DB
+Desktop*DF
+.Spotlight-V100
+.Trashes
+.VolumeIcon.icns
+.apdisk
+.bzvol
+.fseventsd
+# is the following still relevant?
+.FBCIndex
+.FBCLockFolder
+.HSancillary
+.HSResource
+.HSicon
+# CDLI wiki
+dokuwiki/data/cache