Rsyncreposcript2

#! /bin/sh
set -e

  1. This script originates from http://www.debian.org/mirror/anonftpsync
  1. CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
  2. Version: $Id: anonftpsync,v 1.43 2008-06-15 18:16:04 spaillar Exp $
  1. Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
  2. and all newer Debian releases, or at http://rsync.samba.org/
  1. Don't forget:
  2. chmod u+x anonftpsync
  1. Set the variables below to fit your site. You can then use cron to have
  2. this script run daily to automatically update your copy of the archive.
  1. TO is the destination for the base of the Debian mirror directory
  2. (the dir that holds dists/ and ls-lR).
  3. (mandatory)

TO=

  1. RSYNC_HOST is the site you have chosen from the mirrors file.
  2. (http://www.debian.org/mirror/list-full)
  3. (mandatory)

RSYNC_HOST=

  1. RSYNC_DIR is the directory given in the "Packages over rsync:" line of
  2. the mirrors file for the site you have chosen to mirror.
  3. (mandatory)

RSYNC_DIR=debian

  1. LOGDIR is the directory where the logs will be written to
  2. (mandatory)

LOGDIR=

  1. ARCH_EXCLUDE can be used to exclude a complete architecture from
  2. mirrorring. Please use as space seperated list.
  3. Possible values are:
  4. alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc

#

  1. There is one special value: source
  2. This is not an architecture but will exclude all source code in /pool

#

  1. eg.
  2. ARCH_EXCLUDE="alpha arm armel hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
  3. With a blank ARCH_EXCLUDE you will mirror all available architectures
  4. (optional)

ARCH_EXCLUDE=

  1. EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
  2. addition to the architectures excluded by ARCH_EXCLUDE.

#

  1. Use ARCH_EXCLUDE to exclude specific architectures or all sources

#

  1. —exclude stable, testing, unstable options DON'T remove the packages of
  2. the given distribution. If you want do so, use debmirror instead.

#

  1. The following example would exclude mostly everything:

#EXCLUDE="# —exclude stable/ —exclude testing/ —exclude unstable/ # —exclude source/ # —exclude *.orig.tar.gz —exclude *.diff.gz —exclude *.dsc # —exclude /contrib/ —exclude /non-free/ # "

  1. With a blank EXCLUDE you will mirror the entire archive, except the
  2. architectures excluded by ARCH_EXCLUDE.
  3. (optional)

EXCLUDE=

  1. MAILTO is the address to send logfiles to;
  2. if it is not defined, no mail will be sent
  3. (optional)

MAILTO=

  1. LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
  2. This program creates a lock to ensure that only one copy
  3. of it is mirroring any one archive at any one time.
  4. Locks held for longer than the timeout are broken, unless
  5. a running rsync process appears to be connected to $RSYNC_HOST.

LOCK_TIMEOUT=360

  1. You may establish the connection via a web proxy by setting the environment
  2. variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
  3. that your web proxy’s configuration must support proxy connections to port 873.

#

  1. RSYNC_PROXY="IP:PORT"
  2. export RSYNC_PROXY=$RSYNC_PROXY
  1. There should be no need to edit anything below this point, unless there
  2. are problems.

#---------------—#

  1. If you are accessing a rsync server/module which is password-protected,
  2. uncomment the following lines (and edit the other file).
  1. . ftpsync.conf
  2. export RSYNC_PASSWORD
  3. RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST

#---------------—#

  1. Check for some environment variables

if [ -z "$TO" ] || [ -z "$RSYNC_HOST" ] || [ -z "$RSYNC_DIR" ] || [ -z "$LOGDIR" ]; then
echo "One of the following variables seems to be empty:"
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
exit 2
fi

  1. Note: on some non-Debian systems, hostname doesn't accept -f option.
  2. If that's the case on your system, make sure hostname prints the full
  3. hostname, and remove the -f option. If there's no hostname command,
  4. explicitly replace `hostname -f` with the hostname.

HOSTNAME=`hostname -f`

  1. The hostname must match the "Site" field written in the list of mirrors.
  2. If hostname doesn't returns the correct value, fill and uncomment below
  3. HOSTNAME=mirror.domain.tld

LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"

  1. The temp directory used by rsync —delay-updates is not
  2. world-readable remotely. It must be excluded to avoid errors.

TMP_EXCLUDE="—exclude .~tmp~/"

  1. Exclude architectures defined in $ARCH_EXCLUDE

for ARCH in $ARCH_EXCLUDE; do
EXCLUDE=$EXCLUDE" —exclude binary-$ARCH/ —exclude disks-$ARCH/ —exclude installer-$ARCH/ —exclude Contents-$ARCH.gz —exclude Contents-$ARCH.diff/ —exclude arch-$ARCH.files —exclude arch-$ARCH.list.gz —exclude *_$ARCH.deb —exclude *_$ARCH.udeb "
if [ "$ARCH" = "source" ]; then
SOURCE_EXCLUDE=" —exclude source/ —exclude *.tar.gz —exclude *.diff.gz —exclude *.dsc "
fi
done

  1. Logfile

LOGFILE=$LOGDIR/debian-mirror.log

  1. optionally, use the rsync module name in the log file name:
  2. LOGFILE=$LOGDIR/$(echo $RSYNC_DIR | tr / _)-mirror.log
  3. LOGFILE=$LOGDIR/${RSYNC_DIR/\//_}-mirror.log
  1. Get in the right directory and set the umask to be group writable

cd $HOME
umask 002

  1. If we are running mirror script for the first time, create the
  2. destination directory and the trace directory underneath it

if [ ! -d "${TO}/project/trace/" ]; then
mkdir -p ${TO}/project/trace
fi

  1. Check to see if another sync is in progress

if [ -f "$LOCK" ]; then

  1. Note: this requires the findutils find; for other finds, adjust as necessary

if [ "`find $LOCK -maxdepth 1 -cmin -$LOCK_TIMEOUT`" = "" ]; then

  1. Note: this requires the procps ps; for other ps', adjust as necessary

if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
echo "stale lock found, but a rsync is still running, aiee!"
exit 1
else
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
rm -f $LOCK
fi
else
echo "current lock file exists, unable to start rsync!"
exit 1
fi
fi

touch $LOCK

  1. Note: on some non-Debian systems, trap doesn't accept "exit" as signal
  2. specification. If that's the case on your system, try using "0".

trap "rm -f $LOCK" exit

set +e

  1. First sync /pool

rsync —recursive —links —hard-links —times —verbose —timeout=3600 $TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ » $LOGFILE 2>&1
result=$?

if [ "$result" = 0 ]; then
# Now sync the remaining stuff
rsync —recursive —links —hard-links —times —verbose —delay-updates —delete-after —timeout=3600 —exclude "Archive-Update-in-Progress-${HOSTNAME}" —exclude "project/trace/${HOSTNAME}" $TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE $RSYNC_HOST::$RSYNC_DIR/ $TO » $LOGFILE 2>&1

LANG=C date -u > "${TO}/project/trace/${HOSTNAME}"
else
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
fi

if [ -n "$MAILTO" ]; then
mail -s "debian archive synced" $MAILTO < $LOGFILE
fi

savelog $LOGFILE >/dev/null

rm $LOCK

Unless otherwise stated, the content of this page is licensed under GNU Free Documentation License.