Loading .gitlab-ci.yml +10 −8 Original line number Diff line number Diff line Loading @@ -123,20 +123,22 @@ tornettools-stage: # Cache metrics downloads key: $CI_JOB_NAME-$DATA_MONTH paths: - jobs/network-data - jobs/cache/metrics script: - apt-get install -y bsdmainutils python3 python3-pip wget xz-utils - export DATA_DAYS=$(cal -d $DATA_MONTH | awk 'NF {DAYS = $NF}; END {print DAYS}') - mkdir -p jobs/network-data - cd jobs/network-data - 'if [ ! compgen -G consensuses* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! compgen -G server-descriptors* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi' - mkdir -p jobs/cache/metrics - cd jobs/cache/metrics - 'if ! compgen -G ''consensuses*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if ! compgen -G ''server-descriptors*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! -f userstats-relay-country.csv ]; then wget https://metrics.torproject.org/userstats-relay-country.csv; fi' - 'if [ ! compgen -G onionperf-* ]; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! compgen -G bandwidth-* ]; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi' - 'if ! compgen -G ''onionperf-*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if ! compgen -G ''bandwidth-*'' > /dev/null ; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi' - pip3 install -r $CI_PROJECT_DIR/jobs/src/tornettools/requirements.txt - pip3 install -I $CI_PROJECT_DIR/jobs/src/tornettools - tornettools stage consensuses-$DATA_MONTH server-descriptors-$DATA_MONTH userstats-relay-country.csv --onionperf_data_path onionperf-$DATA_MONTH --bandwidth_data_path bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip - mkdir -p $CI_PROJECT_DIR/jobs/network-data - cd $CI_PROJECT_DIR/jobs/network-data - tornettools stage $CI_PROJECT_DIR/cache/metrics/consensuses-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/server-descriptors-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/userstats-relay-country.csv --onionperf_data_path $CI_PROJECT_DIR/cache/metrics/onionperf-$DATA_MONTH --bandwidth_data_path $CI_PROJECT_DIR/cache/metrics/bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip artifacts: paths: - jobs/network-data/*.json Loading Loading
.gitlab-ci.yml +10 −8 Original line number Diff line number Diff line Loading @@ -123,20 +123,22 @@ tornettools-stage: # Cache metrics downloads key: $CI_JOB_NAME-$DATA_MONTH paths: - jobs/network-data - jobs/cache/metrics script: - apt-get install -y bsdmainutils python3 python3-pip wget xz-utils - export DATA_DAYS=$(cal -d $DATA_MONTH | awk 'NF {DAYS = $NF}; END {print DAYS}') - mkdir -p jobs/network-data - cd jobs/network-data - 'if [ ! compgen -G consensuses* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! compgen -G server-descriptors* ]; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi' - mkdir -p jobs/cache/metrics - cd jobs/cache/metrics - 'if ! compgen -G ''consensuses*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/consensuses/consensuses-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if ! compgen -G ''server-descriptors*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/relay-descriptors/server-descriptors/server-descriptors-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! -f userstats-relay-country.csv ]; then wget https://metrics.torproject.org/userstats-relay-country.csv; fi' - 'if [ ! compgen -G onionperf-* ]; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if [ ! compgen -G bandwidth-* ]; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi' - 'if ! compgen -G ''onionperf-*'' > /dev/null ; then wget -O - https://collector.torproject.org/archive/onionperf/onionperf-$DATA_MONTH.tar.xz | tar xJ; fi' - 'if ! compgen -G ''bandwidth-*'' > /dev/null ; then wget -O bandwidth-$DATA_MONTH.csv "https://metrics.torproject.org/bandwidth.csv?start=$DATA_MONTH-01&end=$DATA_MONTH-$DATA_DAYS"; fi' - pip3 install -r $CI_PROJECT_DIR/jobs/src/tornettools/requirements.txt - pip3 install -I $CI_PROJECT_DIR/jobs/src/tornettools - tornettools stage consensuses-$DATA_MONTH server-descriptors-$DATA_MONTH userstats-relay-country.csv --onionperf_data_path onionperf-$DATA_MONTH --bandwidth_data_path bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip - mkdir -p $CI_PROJECT_DIR/jobs/network-data - cd $CI_PROJECT_DIR/jobs/network-data - tornettools stage $CI_PROJECT_DIR/cache/metrics/consensuses-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/server-descriptors-$DATA_MONTH $CI_PROJECT_DIR/cache/metrics/userstats-relay-country.csv --onionperf_data_path $CI_PROJECT_DIR/cache/metrics/onionperf-$DATA_MONTH --bandwidth_data_path $CI_PROJECT_DIR/cache/metrics/bandwidth-$DATA_MONTH.csv --geoip_path $CI_PROJECT_DIR/jobs/opt/tor/share/tor/geoip artifacts: paths: - jobs/network-data/*.json Loading