Storage and monitoring improvements

Jim Newsomerequested to merge
httpd into main

Monitoring:

  • Dump pipeline config vars to stdout, so that we can see manual overrides
  • Serve simdir over an onion service, so that we can inspect files in a running sim
  • (Temporarily) also serve cache dir over an onion service, so that we can download caches from previous runs

Save tornettools logs, since they hold some metadata about the sim. Probably redundant, but they're small.

In response to recent full sim failing due to too-large artifacts:

  • Always save simdir cache, even on job failure, for example due to too-large-artifacts.
  • Drop Non-perfclient tgen stdout files

Merge request reports

.gitlab-ci.yml

+24 −2
Changes for .gitlab-ci.yml: 24 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -136,6 +136,9 @@ default: 
    # Extra error checking

    - set -euo pipefail



    # Dump PL_* vars to stdout

    - env | grep ^PL_ | sort



    # Un-export built-in env vars. Some of these have surprisingly

    # powerful secrets.  Unexporting leaves them set in the shell, but doesn't

    # implicitly pass them to every spawned process. For a process that needs them, pass explicitly:
@@ -348,6 +351,8 @@ tornettools-stage: 
    # Roughly give each run its own cache. Note though that retries of a job

    # will still have the same cache key.

    - key: $CI_PIPELINE_ID-$CI_JOB_ID

      when: 'always'

      policy: 'push'

      paths:

        - $SIMDIR

  needs:
@@ -369,6 +374,22 @@ tornettools-stage: 
    - rm -rf $SIMDIR

    - mkdir -p `dirname $SIMDIR`



    # Serve current SIMDIR over onion service for live monitoring.

    # Serve cache volume over onion service to access previous runs's caches

    # (workaround pending access to runner host machine).

    - apt install -y tor # Use stock tor

    - echo "HiddenServiceDir /var/lib/tor/cache/" >> /etc/tor/torrc

    - echo "HiddenServicePort 80 127.0.0.1:8080" >> /etc/tor/torrc

    - (cd /cache/jnewsome/sponsor-61-sims && python3 -m http.server 8080) &

    - echo "HiddenServiceDir /var/lib/tor/simdir/" >> /etc/tor/torrc

    - echo "HiddenServicePort 80 127.0.0.1:8081" >> /etc/tor/torrc

    - (cd $SIMDIR && python3 -m http.server 8081) &

    - /usr/bin/tor &

    - while [ ! -f /var/lib/tor/cache/hostname ]; do echo "Waiting for cache service"; sleep 1; done

    - 'echo Cache service: $(cat /var/lib/tor/cache/hostname)'

    - while [ ! -f /var/lib/tor/simdir/hostname ]; do echo "Waiting for simdir service"; sleep 1; done

    - 'echo simdir service: $(cat /var/lib/tor/simdir/hostname)'



    - tornettools --seed $TRIAL_NUM generate jobs/network-data/relayinfo_staging_*.json jobs/network-data/userinfo_staging_*.json jobs/network-data/networkinfo_staging.gml jobs/network-data/tmodel --network_scale $SCALE --prefix $SIMDIR --tor jobs/opt/tor/bin/tor --torgencert jobs/opt/tor/bin/tor-gencert --process_scale $PROCESS_SCALE --load_scale $LOAD_SCALE



    # Run simulation
@@ -467,6 +488,7 @@ tornettools-stage: 
      - $SIMDIR/mappings.txt

      - $SIMDIR/tor-err.txt

      - $SIMDIR/tor-warn.txt

      - $SIMDIR/tornettools.*.log

      # Full data for a few clients

      - $SIMDIR/shadow.data/hosts/markovclient[0-3]

      # Full data for a few servers
@@ -479,8 +501,8 @@ tornettools-stage: 
      - $SIMDIR/shadow.data/hosts/*/*exitcode

      # All oniontrace raw stdout. (Redundant with $SIMDIR/oniontrace.analysis.json.xz?)

      - $SIMDIR/shadow.data/hosts/*/*oniontrace*stdout

      # All tgen raw stdout. (Redundant with $SIMDIR/tgen.analysis.json.xz?)

      - $SIMDIR/shadow.data/hosts/*/*tgen*stdout

      # All perfclient tgen raw stdout. (Redundant with $SIMDIR/tgen.analysis.json.xz?)

      - $SIMDIR/shadow.data/hosts/*perfclient*/*tgen*stdout

    exclude:

      # consensus diff caches

      - $SIMDIR/shadow.data/hosts/**/diff-cache/*