#!/bin/bash
#
# Copyright 2016--2020 The Tor Project
# See LICENSE for licensing information.
#
# Script for creating descriptor tarballs on a CollecTor instance,
# and creating/touching the symlinks for the web-app.
#
####
# Configuration section:
# The following path should be adjusted, if the CollecTor server layout differs.
# All paths should be given absolute.
ARCHIVEDIR="/srv/collector.torproject.org/collector/indexed/archive"
WORKDIR="/srv/collector.torproject.org/collector/tarballs"
OUTDIR="/srv/collector.torproject.org/collector/out"
TARBALLTARGETDIR="/srv/collector.torproject.org/collector/data"
### end of configuration section.
#
### script start
echo `date` "Starting"
YEARONE=`date +%Y`
MONTHONE=`date +%m`
YEARTWO=`date --date='7 days ago' +%Y`
MONTHTWO=`date --date='7 days ago' +%m`
CURRENTPATH=`pwd`

mkdir -p $WORKDIR

cd $WORKDIR

if ! test -d $OUTDIR
  then echo "$OUTDIR doesn't exist.  Exiting."
  exit 1
fi

mkdir -p $TARBALLTARGETDIR

TARBALLS=(
  exit-list-$YEARONE-$MONTHONE
  exit-list-$YEARTWO-$MONTHTWO
  torperf-$YEARONE-$MONTHONE
  torperf-$YEARTWO-$MONTHTWO
  onionperf-$YEARONE-$MONTHONE
  onionperf-$YEARTWO-$MONTHTWO
  certs
  microdescs-$YEARONE-$MONTHONE
  microdescs-$YEARTWO-$MONTHTWO
  consensuses-$YEARONE-$MONTHONE
  consensuses-$YEARTWO-$MONTHTWO
  votes-$YEARONE-$MONTHONE
  votes-$YEARTWO-$MONTHTWO
  bandwidths-$YEARONE-$MONTHONE
  bandwidths-$YEARTWO-$MONTHTWO
  server-descriptors-$YEARONE-$MONTHONE
  server-descriptors-$YEARTWO-$MONTHTWO
  extra-infos-$YEARONE-$MONTHONE
  extra-infos-$YEARTWO-$MONTHTWO
  bridge-statuses-$YEARONE-$MONTHONE
  bridge-statuses-$YEARTWO-$MONTHTWO
  bridge-server-descriptors-$YEARONE-$MONTHONE
  bridge-server-descriptors-$YEARTWO-$MONTHTWO
  bridge-extra-infos-$YEARONE-$MONTHONE
  bridge-extra-infos-$YEARTWO-$MONTHTWO
  snowflakes-$YEARONE-$MONTHONE
  snowflakes-$YEARTWO-$MONTHTWO
  bridge-pool-assignments-$YEARONE-$MONTHONE
  bridge-pool-assignments-$YEARTWO-$MONTHTWO
  bridgedb-metrics-$YEARONE-$MONTHONE
  bridgedb-metrics-$YEARTWO-$MONTHTWO
  bridgestrap-$YEARONE-$MONTHONE
  bridgestrap-$YEARTWO-$MONTHTWO
)
TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))

DIRECTORIES=(
  $OUTDIR/exit-lists/$YEARONE/$MONTHONE/
  $OUTDIR/exit-lists/$YEARTWO/$MONTHTWO/
  $OUTDIR/torperf/$YEARONE/$MONTHONE/
  $OUTDIR/torperf/$YEARTWO/$MONTHTWO/
  $OUTDIR/onionperf/$YEARONE/$MONTHONE/
  $OUTDIR/onionperf/$YEARTWO/$MONTHTWO/
  $OUTDIR/relay-descriptors/certs/
  $OUTDIR/relay-descriptors/microdesc/$YEARONE/$MONTHONE
  $OUTDIR/relay-descriptors/microdesc/$YEARTWO/$MONTHTWO
  $OUTDIR/relay-descriptors/consensus/$YEARONE/$MONTHONE
  $OUTDIR/relay-descriptors/consensus/$YEARTWO/$MONTHTWO
  $OUTDIR/relay-descriptors/vote/$YEARONE/$MONTHONE/
  $OUTDIR/relay-descriptors/vote/$YEARTWO/$MONTHTWO/
  $OUTDIR/relay-descriptors/bandwidth/$YEARONE/$MONTHONE/
  $OUTDIR/relay-descriptors/bandwidth/$YEARTWO/$MONTHTWO/
  $OUTDIR/relay-descriptors/server-descriptor/$YEARONE/$MONTHONE/
  $OUTDIR/relay-descriptors/server-descriptor/$YEARTWO/$MONTHTWO/
  $OUTDIR/relay-descriptors/extra-info/$YEARONE/$MONTHONE/
  $OUTDIR/relay-descriptors/extra-info/$YEARTWO/$MONTHTWO/
  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/statuses/
  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/statuses/
  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/server-descriptors/
  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/server-descriptors/
  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/extra-infos/
  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/
  $OUTDIR/snowflakes/$YEARONE/$MONTHONE/
  $OUTDIR/snowflakes/$YEARTWO/$MONTHTWO/
  $OUTDIR/bridge-pool-assignments/$YEARONE/$MONTHONE/
  $OUTDIR/bridge-pool-assignments/$YEARTWO/$MONTHTWO/
  $OUTDIR/bridgedb-metrics/$YEARONE/$MONTHONE/
  $OUTDIR/bridgedb-metrics/$YEARTWO/$MONTHTWO/
  $OUTDIR/bridgestrap/$YEARONE/$MONTHONE/
  $OUTDIR/bridgestrap/$YEARTWO/$MONTHTWO/
)
DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))

for (( i = 0 ; i < ${#TARBALLS[@]} ; i++ )); do
  if [ ! -d ${TARBALLS[$i]} ]; then
    echo `date` "Creating symlink for" ${TARBALLS[$i]}
    ln -s ${DIRECTORIES[$i]} ${TARBALLS[$i]}
  else
    # This is a workaround for the "tar u" bug in GNU tar 1.20
    echo `date` "Touching symlink and directories for" ${TARBALLS[$i]}
    find -L ${TARBALLS[$i]} -type d | xargs touch
  fi
done

# N is a parameter that describes the number of tarballs that are created in
# parallel.

N=2
for (( i = 0 ; i < ${#TARBALLS[@]} ; i++ )); do
  ((j=j%N)); ((j++==0)) && wait
  {
    echo `date` "Creating" ${TARBALLS[$i]}'.tar.xz'
    tar -I "xz -9e" -c -h -f ${TARBALLS[$i]}.tar.xz ${TARBALLS[$i]}
  } &
done

cd $OUTDIR/webstats/
echo `date` "Creating webstats-"$YEARONE"-"$MONTHONE".tar"
find . -type f | grep "/"$YEARONE"/"$MONTHONE"/" | cut -c3- | tar cf $WORKDIR/webstats-$YEARONE-$MONTHONE.tar --transform 's,^,webstats-'$YEARONE'-'$MONTHONE'/,' -T -
echo `date` "Creating webstats-"$YEARTWO"-"$MONTHTWO".tar"
find . -type f | grep "/"$YEARTWO"/"$MONTHTWO"/" | cut -c3- | tar cf $WORKDIR/webstats-$YEARTWO-$MONTHTWO.tar --transform 's,^,webstats-'$YEARTWO'-'$MONTHTWO'/,' -T -
cd $WORKDIR

echo `date` "Moving tarballs into place"
mv *.tar* $TARBALLTARGETDIR

cd $CURRENTPATH

echo `date` "Finished tarball creation.  Starting symlink-update ..."

mkdir -p $ARCHIVEDIR/bridge-descriptors/statuses/
ln -f -s -t $ARCHIVEDIR/bridge-descriptors/statuses/ $TARBALLTARGETDIR/bridge-statuses-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridge-descriptors/server-descriptors/
ln -f -s -t $ARCHIVEDIR/bridge-descriptors/server-descriptors/ $TARBALLTARGETDIR/bridge-server-descriptors-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridge-descriptors/extra-infos/
ln -f -s -t $ARCHIVEDIR/bridge-descriptors/extra-infos/ $TARBALLTARGETDIR/bridge-extra-infos-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ $TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/exit-lists/
ln -f -s -t $ARCHIVEDIR/exit-lists/ $TARBALLTARGETDIR/exit-list-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/ $TARBALLTARGETDIR/certs.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/consensuses/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/consensuses/ $TARBALLTARGETDIR/consensuses-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/extra-infos/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/extra-infos/ $TARBALLTARGETDIR/extra-infos-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/microdescs/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/microdescs/ $TARBALLTARGETDIR/microdescs-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/server-descriptors/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/server-descriptors/ $TARBALLTARGETDIR/server-descriptors-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/statuses/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/statuses/ $TARBALLTARGETDIR/statuses-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/tor/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/tor/ $TARBALLTARGETDIR/tor-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/votes/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/votes/ $TARBALLTARGETDIR/votes-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/relay-descriptors/bandwidths/
ln -f -s -t $ARCHIVEDIR/relay-descriptors/bandwidths/ $TARBALLTARGETDIR/bandwidths-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/torperf/
ln -f -s -t $ARCHIVEDIR/torperf/ $TARBALLTARGETDIR/torperf-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/onionperf/
ln -f -s -t $ARCHIVEDIR/onionperf/ $TARBALLTARGETDIR/onionperf-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/webstats/
ln -f -s -t $ARCHIVEDIR/webstats/ $TARBALLTARGETDIR/webstats-20??-??.tar

mkdir -p $ARCHIVEDIR/snowflakes/
ln -f -s -t $ARCHIVEDIR/snowflakes/ $TARBALLTARGETDIR/snowflakes-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ $TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridgedb-metrics/
ln -f -s -t $ARCHIVEDIR/bridgedb-metrics/ $TARBALLTARGETDIR/bridgedb-metrics-20??-??.tar.xz

mkdir -p $ARCHIVEDIR/bridgestrap/
ln -f -s -t $ARCHIVEDIR/bridgestrap/ $TARBALLTARGETDIR/bridgestrap-20??-??.tar.xz

echo `date` "Finished."
