Merge branch 'issue-40006' into develop

.gitignore

-/.onionperf/*
 onionperf-data
+onionperf-private
 venv
 *.json.xz
 *.pdf
+*.csv
 *.pyc
-onionperf/docs/_build
 .coverage

.gitlab-ci.yml

-variables:
-  GIT_STRATEGY: clone
-
-stages:
- - test
-
-test:
- stage: test
- image: debian:buster
- coverage: '/TOTAL.+ ([0-9]{1,3}%)/'
- script:
-  - apt -y update
-  - apt -y install git cmake make build-essential gcc libigraph0-dev libglib2.0-dev python-dev libxml2-dev python-lxml python-networkx python-scipy python-matplotlib python-numpy libevent-dev libssl-dev python-stem tor python-nose python-cov-core
-  - git clone https://github.com/shadow/tgen.git
-  - mkdir -p tgen/build
-  - pushd tgen/build
-  - cmake ..
-  - make
-  - ln -s `pwd`/tgen /usr/bin/
-  - popd
-  - python setup.py build
-  - python setup.py install
-  - ./run_tests.sh

CHANGELOG.md

+# Changes in version 0.6 - 2020-??-??
+
+ - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
+   update analysis results file version to 3.0. Implements #33974.
+ - Remove summaries from analysis results files, and remove the
+   `onionperf analyze -s/--do-simple-parse` switch. Implements #40005.
+ - Add JSON schema for analysis results file format 3.0. Implements
+   #40003.
+
+# Changes in version 0.5 - 2020-07-02
+
+ - Add new graph showing the cumulative distribution function of
+   throughput in Mbps. Implements #33257.
+ - Improve `README.md` to make it more useful to developers and
+   researchers. Implements #40001.
+ - Always include the `error_code` column in visualization CSV output,
+   regardless of whether data contains measurements with an error code
+   or not. Fixes #40004.
+ - Write generated torrc files to disk for debugging purposes.
+   Implements #40002.
+
+# Changes in version 0.4 - 2020-06-16
+
+ - Include all measurements when analyzing log files at midnight as
+   part of `onionperf measure`, not just the ones from the day before.
+   Also add `onionperf analyze -x/--date-prefix` switch to prepend a
+   given date string to an analysis results file. Fixes #29369.
+ - Add `size`, `last_modified`, and `sha256` fields to index.xml.
+   Implements #29365.
+ - Add support for single onion services using the switch `onionperf
+   measure -s/--single-onion`. Implements #29368.
+ - Remove unused `onionperf measure --traffic-model` switch.
+   Implements #29370.
+ - Make `onionperf measure -o/--onion-only` and `onionperf measure
+   -i/--inet-only` switches mutually exclusive. Fixes #34316.
+ - Accept one or more paths to analysis results files or directories
+   of such files per dataset in `onionperf visualize -d/--data` to
+   include all contained measurements in a dataset. Implements #34191.
+
+# Changes in version 0.3 - 2020-05-30
+
+ - Automatically compress logs when rotating them. Fixes #33396.
+ - Update to Python 3. Implements #29367.
+ - Integrate reprocessing mode into analysis mode. Implements #34142.
+ - Record download times of smaller file sizes from partial completion
+   times. Implements #26673.
+ - Stop generating .tpf files. Implements #34141.
+ - Update analysis results file version to 2.0. Implements #34224.
+ - Export visualized data to a CSV file. Implements #33258.
+ - Remove version 2 onion service support. Implements #33434.
+ - Reduce timeout and stallout values. Implements #34024.
+ - Remove 50 KiB and 1 MiB downloads. Implements #34023.
+ - Remove existing Tor control log visualizations. Implements #34214.
+ - Update to Networkx version 2.4. Fixes #34298.
+ - Update time to first/last byte definitions to include the time 
+   between starting a measurement and receiving the first/last byte. 
+   Implements #34215.
+ - Update `requirements.txt` to actual requirements, and switch from 
+   distutils to setuptools. Fixes #30586.
+ - Split visualizations into public and onion service measurements. 
+   Fixes #34216.
+
+# Changes from before 2020-04
+
+ - Changes made before 2020-04 are not listed here. See `git log` for
+   details.
+

README.md

@@ -125,6 +125,12 @@ deactivate
 
 However, in order to perform measurements or analyses, the virtual environment needs to be activated first. This will ensure all the paths are found.
 
+If needed, unit tests are run with the following command:
+
+```shell
+cd ~/onionperf/
+python3 -m nose --with-coverage --cover-package=onionperf
+```
 
 ## Measurement
 

Vagrantfile

-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-$setup_onionperf = <<SCRIPT
-export DEBIAN_FRONTEND=noninteractive
-apt-get update
-apt install -y build-essential autoconf cmake libglib2.0-dev libigraph0-dev libevent-dev libssl-dev python3 git python3-stem python3-lxml python3-networkx python3-matplotlib python3-numpy python3-scipy
-cd ~
-git clone https://git.torproject.org/tor.git
-cd tor
-./autogen.sh
-./configure --disable-asciidoc
-make
-mv src/app/tor /usr/local/bin/
-cd ~
-git clone https://github.com/shadow/shadow.git
-cd shadow/src/plugin/shadow-plugin-tgen
-mkdir build
-cd build
-cmake .. -DSKIP_SHADOW=ON -DCMAKE_MODULE_PATH=`pwd`/../../../../cmake/
-make
-mv tgen /usr/local/bin/
-cd /vagrant
-python3 setup.py build
-python3 setup.py install
-SCRIPT
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "debian/stretch64"
-
-  config.vm.define "oniondev" do |oniondev|
-    oniondev.vm.provision :shell, :inline => $setup_onionperf
-  end
-
-end

debian/changelog

-onionperf (2019052301) stretch-backports; urgency=medium
-
-  * Initial release.
-
- -- Iain R. Learmonth <irl@debian.org>  Thu, 23 May 2019 12:42:20 +0100

debian/compat

-11

debian/control

-Source: onionperf
-Maintainer: Tor Metrics <metrics-team@lists.torproject.org>
-Section: net
-Priority: optional
-Build-Depends: debhelper (>= 11),
-               dh-python,
-               cmake,
-               make,
-               build-essential,
-               gcc,
-               libigraph0-dev,
-               libglib2.0-dev,
-               python,
-               python-ipaddress,
-               python-lxml,
-               python-matplotlib,
-               python-netifaces,
-               python-networkx,
-               python-nose,
-               python-numpy,
-               python-scipy,
-               python-stem (>=1.7)
-Standards-Version: 4.1.3
-
-Package: onionperf
-Architecture: any
-Depends: ${python:Depends},
-         ${shlibs:Depends},
-         ${misc:Depends},
-         python-ipaddress,
-         python-lxml,
-         python-matplotlib,
-         python-netifaces,
-         python-networkx,
-         python-nose,
-         python-numpy,
-         python-scipy,
-         python-stem (>=1.7)
-Description: Tor Performance Measurement Tool
- Tor Performance Measurement Tool

debian/install

-tgen/build/tgen usr/bin

debian/rules

-#!/usr/bin/make -f
-
-%:
-	dh $@ --with python2 --buildsystem=pybuild
-
-override_dh_auto_test:
-	./run_tests.sh
-
-override_dh_auto_build:
-	( cd tgen; \
-	  mkdir build; \
-	  cd build; \
-	  cmake ..; \
-	  make )
-	dh_auto_build

debian/source/format

-3.0 (native)

onionperf/__init__.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/analysis.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/measurement.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/model.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/monitor.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/onionperf

@@ -3,6 +3,7 @@
 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/util.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

onionperf/visualization.py

 '''
   OnionPerf
   Authored by Rob Jansen, 2015
+  Copyright 2015-2020 The Tor Project
   See LICENSE for licensing information
 '''
 

run_tests.sh

-#!/bin/sh
-
-PYTHONPATH=. python3 -m nose --with-coverage --cover-package=onionperf