Commit dcab2143 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add new filter mode to filter analysis results.

Implements tpo/metrics/onionperf#33260.
parent e333be25
# Changes in version 0.7 - 2020-??-??
- Add a new `onionperf filter` mode that takes an OnionPerf analysis
results file as input, applies filters, and produces a new
OnionPerf analysis results file as output.
# Changes in version 0.6 - 2020-08-08 # Changes in version 0.6 - 2020-08-08
- Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and
......
...@@ -62,6 +62,11 @@ class OPAnalysis(Analysis): ...@@ -62,6 +62,11 @@ class OPAnalysis(Analysis):
self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") self.json_db['data'][self.nickname]["tgen"].pop("stream_summary")
self.did_analysis = True self.did_analysis = True
def set_tgen_transfers(self, node, tgen_transfers):
self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers
def set_tgen_streams(self, node, tgen_streams):
self.json_db['data'][node]['tgen']['streams'] = tgen_streams
def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None): def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None):
if filename is None: if filename is None:
...@@ -98,6 +103,12 @@ class OPAnalysis(Analysis): ...@@ -98,6 +103,12 @@ class OPAnalysis(Analysis):
except: except:
return None return None
def get_tor_circuits(self, node):
try:
return self.json_db['data'][node]['tor']['circuits']
except:
return None
def get_tor_streams(self, node): def get_tor_streams(self, node):
try: try:
return self.json_db['data'][node]['tor']['streams'] return self.json_db['data'][node]['tor']['streams']
......
'''
OnionPerf
Authored by Rob Jansen, 2015
Copyright 2015-2020 The Tor Project
See LICENSE for licensing information
'''
import re
from onionperf.analysis import OPAnalysis
class Filtering(object):
def __init__(self):
self.fingerprints_to_include = None
self.fingerprints_to_exclude = None
self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})")
def read_input(self, path):
self.analysis = OPAnalysis.load(filename=path)
def include_fingerprints(self, path):
self.fingerprints_to_include = []
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
if fingerprint_match:
fingerprint = fingerprint_match.group(1).upper()
self.fingerprints_to_include.append(fingerprint)
def exclude_fingerprints(self, path):
self.exclude_fingerprints = []
with open(path, 'rt') as f:
for line in f:
fingerprint_match = self.fingerprint_pattern.match(line)
if fingerprint_match:
fingerprint = fingerprint_match.group(1).upper()
self.exclude_fingerprints.append(fingerprint)
def apply_filters(self):
if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None:
return
for source in self.analysis.get_nodes():
tor_streams_by_source_port = {}
tor_streams = self.analysis.get_tor_streams(source)
for tor_stream in tor_streams.values():
if "source" in tor_stream and ":" in tor_stream["source"]:
source_port = tor_stream["source"].split(":")[1]
tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream)
tor_circuits = self.analysis.get_tor_circuits(source)
tgen_streams = self.analysis.get_tgen_streams(source)
tgen_transfers = self.analysis.get_tgen_transfers(source)
retained_tgen_streams = {}
retained_tgen_transfers = {}
while tgen_streams or tgen_transfers:
stream_id = None
transfer_id = None
source_port = None
unix_ts_end = None
keep = False
if tgen_streams:
stream_id, stream_data = tgen_streams.popitem()
if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2:
source_port = stream_data["transport_info"]["local"].split(":")[2]
if "unix_ts_end" in stream_data:
unix_ts_end = stream_data["unix_ts_end"]
elif tgen_transfers:
transfer_id, transfer_data = tgen_transfers.popitem()
if "endpoint_local" in transfer_data and len(transfer_data["endpoint_local"].split(":")) > 2:
source_port = transfer_data["endpoint_local"].split(":")[2]
if "unix_ts_end" in transfer_data:
unix_ts_end = transfer_data["unix_ts_end"]
if source_port and unix_ts_end:
for tor_stream in tor_streams_by_source_port[source_port]:
if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0:
circuit_id = tor_stream["circuit_id"]
if circuit_id and circuit_id in tor_circuits:
tor_circuit = tor_circuits[circuit_id]
path = tor_circuit["path"]
keep = True
for long_name, _ in path:
fingerprint_match = self.fingerprint_pattern.match(long_name)
if fingerprint_match:
fingerprint = fingerprint_match.group(1).upper()
if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include:
keep = False
break
if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude:
keep = False
break
if keep:
if stream_id:
retained_tgen_streams[stream_id] = stream_data
if transfer_id:
retained_tgen_transfers[transfer_id] = transfer_data
self.analysis.set_tgen_streams(source, retained_tgen_streams)
self.analysis.set_tgen_transfers(source, retained_tgen_transfers)
def write_output(self, path):
self.analysis.save(filename=path)
...@@ -74,6 +74,15 @@ Stats files in the default Torperf format can also be exported. ...@@ -74,6 +74,15 @@ Stats files in the default Torperf format can also be exported.
HELP_ANALYZE = """ HELP_ANALYZE = """
Analyze Tor and TGen output Analyze Tor and TGen output
""" """
DESC_FILTER = """
Takes an OnionPerf analysis results file as input, applies filters,
and produces a new OnionPerf analysis results file as output.
"""
HELP_FILTER = """
Filter OnionPerf analysis results
"""
DESC_VISUALIZE = """ DESC_VISUALIZE = """
Loads an OnionPerf json file, e.g., one produced with the `analyze` subcommand, Loads an OnionPerf json file, e.g., one produced with the `analyze` subcommand,
and plots various interesting performance metrics to PDF files. and plots various interesting performance metrics to PDF files.
...@@ -280,6 +289,36 @@ files generated by this script will be written""", ...@@ -280,6 +289,36 @@ files generated by this script will be written""",
action="store", dest="date_prefix", action="store", dest="date_prefix",
default=None) default=None)
# filter
filter_parser = sub_parser.add_parser('filter', description=DESC_FILTER, help=HELP_FILTER,
formatter_class=my_formatter_class)
filter_parser.set_defaults(func=filter, formatter_class=my_formatter_class)
filter_parser.add_argument('-i', '--input',
help="""read the OnionPerf analysis results at PATH as input""",
metavar="PATH", required="True",
action="store", dest="input")
filter_parser.add_argument('--include-fingerprints',
help="""include only measurements with known circuit path and with all
relays being contained in the fingerprints file located at
PATH""",
metavar="PATH", action="store", dest="include_fingerprints",
default=None)
filter_parser.add_argument('--exclude-fingerprints',
help="""exclude measurements without known circuit path or with any
relays being contained in the fingerprints file located at
PATH""",
metavar="PATH", action="store", dest="exclude_fingerprints",
default=None)
filter_parser.add_argument('-o', '--output',
help="""write the filtered output OnionPerf analysis results file to
PATH""",
metavar="PATH", required="True",
action="store", dest="output")
# visualize # visualize
visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE, visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE,
formatter_class=my_formatter_class) formatter_class=my_formatter_class)
...@@ -397,6 +436,18 @@ def analyze(args): ...@@ -397,6 +436,18 @@ def analyze(args):
else: else:
logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed") logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed")
def filter(args):
from onionperf.filtering import Filtering
filtering = Filtering()
filtering.read_input(args.input)
if args.include_fingerprints is not None:
filtering.include_fingerprints(args.include_fingerprints)
if args.exclude_fingerprints is not None:
filtering.exclude_fingerprints(args.exclude_fingerprints)
filtering.apply_filters()
filtering.write_output(args.output)
def visualize(args): def visualize(args):
from onionperf.visualization import TGenVisualization from onionperf.visualization import TGenVisualization
from onionperf.analysis import OPAnalysis from onionperf.analysis import OPAnalysis
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment