Verified Commit 95149f1e authored by Jim Newsome's avatar Jim Newsome
Browse files

Fix cache handling

parent efde457f
Loading
Loading
Loading
Loading
+10 −8
Original line number Diff line number Diff line
@@ -123,20 +123,22 @@ tornettools-stage:
    # Cache metrics downloads
    key: $CI_JOB_NAME-$DATA_MONTH
    paths:
      - jobs/network-data
      - jobs/cache/metrics
  script:
    - apt-get install -y bsdmainutils python3 python3-pip wget xz-utils
    - export DATA_DAYS=$(cal -d $DATA_MONTH | awk 'NF {DAYS = $NF}; END {print DAYS}')
    - mkdir -p jobs/network-data
    - cd jobs/network-data
    - 'if [ ! compgen -G consensuses* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi'
    - 'if [ ! compgen -G server-descriptors* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi'
    - mkdir -p jobs/cache/metrics
    - cd jobs/cache/metrics
    - 'if ! compgen -G ''consensuses*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi'
    - 'if ! compgen -G ''server-descriptors*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi'
    - 'if [ ! -f userstats-relay-country.csv ]; then wget https://metrics.torproject.org/userstats-relay-country.csv; fi'
    - 'if [ ! compgen -G onionperf-* ]; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi'
    - 'if [ ! compgen -G bandwidth-* ]; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi'
    - 'if ! compgen -G ''onionperf-*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi'
    - 'if ! compgen -G ''bandwidth-*'' > /dev/null ; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi'
    - pip3 install -r $CI_PROJECT_DIR/jobs/src/tornettools/requirements.txt
    - pip3 install -I $CI_PROJECT_DIR/jobs/src/tornettools
    - tornettools stage consensuses-$DATA_MONTH server-descriptors-$DATA_MONTH userstats-relay-country.csv --onionperf_data_path onionperf-$DATA_MONTH --bandwidth_data_path bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip
    - mkdir -p $CI_PROJECT_DIR/jobs/network-data
    - cd $CI_PROJECT_DIR/jobs/network-data
    - tornettools stage $CI_PROJECT_DIR/cache/metrics/consensuses-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/server-descriptors-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/userstats-relay-country.csv --onionperf_data_path $CI_PROJECT_DIR/cache/metrics/onionperf-$DATA_MONTH --bandwidth_data_path $CI_PROJECT_DIR/cache/metrics/bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip
  artifacts:
    paths:
      - jobs/network-data/*.json