Verified Commit d2d9215a authored by Jim Newsome's avatar Jim Newsome
Browse files

Fuse tornettools-generate and run-sim stages

parent d9c3ad46
stages:
- build
- stage-sim
- generate-sim
- simulate
- post-process
- deploy
......@@ -179,63 +178,36 @@ tornettools-stage:
paths:
- jobs/network-data/*.json
tornettools-generate:
run-sim:
variables:
SCALE: '0.10'
# Needs to be 1.0 when enabling guards.
PROCESS_SCALE: '1.0'
TMODEL_REPO: 'https://github.com/tmodel-ccs2018/tmodel-ccs2018.github.io.git'
TMODEL_BRANCH: master
stage: generate-sim
SIM_TIME: '20m'
stage: simulate
needs:
- tornettools-stage
- build-tornettools
- build-oniontrace
- build-shadow
- build-tgen
- build-tor
- build-tornettools
- tornettools-stage
cache:
- key: $CI_JOB_NAME
paths:
- job-cache
# Too large to be an artifact; use cache instead.
# See https://gitlab.torproject.org/tpo/tpa/team/-/issues/40340
# TODO: should incorporate commit ID and all variables in the cache key to
# avoid potentially using stale cache.
- key: tornettools-generate-tornet
paths:
- jobs/tornet
script:
# Generate sim config
- apt-get install -y bsdmainutils libevent-dev libssl-dev python3 python3-pip wget xz-utils zlib1g-dev
- mkdir -p jobs/src
- git clone --depth=1 -b $TMODEL_BRANCH $TMODEL_REPO jobs/src/tmodel
- pip3 install -r jobs/src/tornettools/requirements.txt
- pip3 install -I jobs/src/tornettools
- tornettools generate jobs/network-data/relayinfo_staging_*.json jobs/network-data/userinfo_staging_*.json jobs/src/tmodel --network_scale $SCALE --prefix jobs/tornet --tor jobs/opt/tor/bin/tor --torgencert jobs/opt/tor/bin/tor-gencert --process_scale $PROCESS_SCALE
# Log directory sizes for debugging
- du
artifacts:
paths:
- jobs/tornet
run-sim:
variables:
SIM_TIME: '20m'
stage: simulate
needs:
- build-oniontrace
- build-shadow
- build-tgen
- build-tor
- build-tornettools
- tornettools-generate
cache:
- key: $CI_JOB_NAME
paths:
- job-cache
# Too large to be an artifact; use cache instead.
# See https://gitlab.torproject.org/tpo/tpa/team/-/issues/40340
- key: tornettools-generate-tornet
paths:
- jobs/tornet
script:
# Run simulation
- apt-get install -y libevent-2.1-7 libssl1.1 zlib1g libglib2.0-0 libigraph0v5 libprocps8 stow sysstat python3 python3-pip
- pip3 install -r jobs/src/tornettools/requirements.txt
- pip3 install -I jobs/src/tornettools
......@@ -264,6 +236,7 @@ run-sim:
- PARALLELISM=20
- tornettools simulate -s $CI_PROJECT_DIR/jobs/opt/shadow/bin/shadow -a "--use-cpu-pinning=true --interpose-method=preload -p $PARALLELISM --template-directory=shadow.data.template" $CI_PROJECT_DIR/jobs/tornet
# Parse simulation results.
# The raw results are too large to save as an artifact.
# Parse the logs and save the results of that.
# https://gitlab.torproject.org/tpo/tpa/team/-/issues/40340
......@@ -274,13 +247,9 @@ run-sim:
- tornettools parse $CI_PROJECT_DIR/jobs/tornet
- grep -E 'WARN|ERR' $CI_PROJECT_DIR/jobs/tornet/shadow.log > $CI_PROJECT_DIR/jobs/tornet/shadow.log.warn
# Log directory sizes for debugging
- du
timeout: 24h
artifacts:
paths:
- jobs/plots
- jobs/tornet/**.json.xz
- jobs/tornet/**.json
- jobs/tornet/shadow.log.warn
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment