Commit ecfef769 authored by Mike Perry's avatar Mike Perry
Browse files

fixup! Scrape logs and extract important stats and bad warns

Fix paths and archiving
parent 15042bcf
Pipeline #15222 canceled with stages
......@@ -362,36 +362,30 @@ tornettools-stage:
- tornettools plot --tor_metrics_path $CI_PROJECT_DIR/jobs/network-data/tor_metrics_*.json --prefix $SIMDIR/plots -l $PL_BASELINE_LABEL experiment -- $CI_PROJECT_DIR/sim-results/$PL_BASELINE_DIR/tornet $SIMDIR
# Separate exits and relay logs into separate files, for stats aggregation
# XXX: Is this getting current run, or all runs?
- find $CI_PROJECT_DIR/jobs/tornet/*/shadow.data/hosts -path *relay*tor*stdout | grep -v exit | xargs cat > tor-nonexit-warns.log
- find $CI_PROJECT_DIR/jobs/tornet/*/shadow.data//hosts -path *exit*tor*stdout | xargs cat > tor-exit-warns.log
- find $SIMDIR/shadow.data/hosts -path *relay*tor*stdout | grep -v exit | xargs cat > $SIMDIR/tor-nonexit-warns.log
- find $SIMDIR/shadow.data/hosts -path *exit*tor*stdout | xargs cat > $SIMDIR/tor-exit-warns.log
# Process congestion control statistics
- apt-get install -y pypy3
- grep "CIRC=" tor-exit-warns.log > circ-cc.log
- $CI_PROJECT_DIR/cc_stats.py circ-cc.log > cc-stats.log
- grep "CIRC=" $SIMDIR/tor-exit-warns.log > $SIMDIR/circ-cc.log
- $CI_PROJECT_DIR/cc_stats.py $SIMDIR/circ-cc.log > $SIMDIR/cc-stats.log.txt
# Get TLS overhead
- grep "Average packaged cell fullness" tor-nonexit-warns.log > heartbeat-nonexit.log
- grep "Average packaged cell fullness" tor-exit-warns.log > heartbeat-exit.log
- $CI_PROJECT_DIR/tls_overhead.py heartbeat-nonexit.log >> cc-stats.log
- $CI_PROJECT_DIR/tls_overhead.py heartbeat-exit.log >> cc-stats.log
- grep "Average packaged cell fullness" $SIMDIR/tor-nonexit-warns.log > $SIMDIR/heartbeat-nonexit.log
- grep "Average packaged cell fullness" $SIMDIR/tor-exit-warns.log > $SIMDIR/heartbeat-exit.log
- $CI_PROJECT_DIR/tls_overhead.py $SIMDIR/heartbeat-nonexit.log >> $SIMDIR/cc-stats.log.txt
- $CI_PROJECT_DIR/tls_overhead.py $SIMDIR/heartbeat-exit.log >> $SIMDIR/cc-stats.log.txt
# Get misc bad stuff
- egrep "clock has been stalled|cells in its queue" tor-nonexit-warns.log > bad-stuff.log
- egrep "clock has been stalled|cells in its queue" tor-exit-warns.log >> bad-stuff.log
# XXX: Is this getting current run, or all runs?
- find $CI_PROJECT_DIR/jobs/tornet/*/shadow.data/hosts path *client*stdout | xargs grep -H " XO[FN]" >> bad-stuff.log
- grep " XO[FN]" tor-exit-warns.log >> bad-stuff.log
# Remove intermediate logs
- rm tor-nonexit-warns.log tor-exit-warns.log circ-cc.log
- rm heartbeat-nonexit.log heartbeat-exit.log
- egrep "clock has been stalled|cells in its queue" $SIMDIR/tor-nonexit-warns.log > $SIMDIR/bad-stuff.log.txt
- egrep "clock has been stalled|cells in its queue" $SIMDIR/tor-exit-warns.log >> $SIMDIR/bad-stuff.log.txt
- find $SIMDIR/shadow.data/hosts path *client*stdout | xargs grep -H " XO[FN]" >> $SIMDIR/bad-stuff.log.txt
- grep " XO[FN]" $SIMDIR/tor-exit-warns.log >> $SIMDIR/bad-stuff.log.txt
after_script:
# Aggressively compress raw results to save space in /cache volume.
- apt-get install -y zstd
- find $SIMDIR -type f -not -name '*.xz' -not -name '*.zst' -not -name '*.gz' -not -path '*tornet.plot.data*' -not -path '*plots*.pdf' | xargs zstd -T0 -10 --rm
- find $SIMDIR -type f -not -name '*.xz' -not -name '*.zst' -not -name '*.gz' -not -path '*tornet.plot.data*' -not -path '*plots*.pdf' -not -path "*.log.txt" | xargs zstd -T0 -10 --rm
# Copy into a separate dir for artifacts. (All artifacts must be
# subdirectory of $CI_PROJECT_DIR).
......@@ -420,6 +414,10 @@ tornettools-stage:
# Remove combined warnings log
- rm $SIMDIR/tor-warn.txt*
# Remove intermediate stats logs
- rm $ARTIFACTS_SIMDIR/tor*warns.log* $ARTIFACTS_SIMDIR/circ-cc.log*
- rm $ARTIFACTS_SIMDIR/heartbeat-*.log*
timeout: 72h
artifacts:
paths:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment