Commit 14d6a4c2 authored by Hiro's avatar Hiro 🏄
Browse files

Moved exitscanner files from metrics cloud repository to its own project

parents
import collections
import datetime
import glob
import ipaddress
import json
import os
import os.path
import re
import subprocess
import threading
import stem.descriptor
fortyeighthoursago = datetime.datetime.utcnow() - datetime.timedelta(hours=48)
Measurement = collections.namedtuple("Measurement", ["address", "date"])
exits = dict()
def merge_addresses(fp, new):
addresses = exits[fp].exit_addresses
addresses.extend(new)
addresses.sort(key=lambda x: x[1], reverse=True)
uniq_addresses = []
while len(uniq_addresses) < len(addresses):
if addresses[len(uniq_addresses)][0] in uniq_addresses:
addresses.remove(addresses[len(uniq_addresses)])
continue
uniq_addresses.append(addresses[len(uniq_addresses)][0])
return [
a for a in addresses
if a[1] > fortyeighthoursago
]
def merge(desc):
if desc.fingerprint not in exits:
exits[desc.fingerprint] = desc
return
fp = desc.fingerprint
exits[fp].published = max(exits[fp].published, desc.published)
exits[fp].last_status = max(exits[fp].last_status, desc.last_status)
exits[fp].exit_addresses = merge_addresses(fp, desc.exit_addresses)
def run():
exit_lists = list(glob.iglob('lists/2*')) # fix this glob before 23:59 on 31st Dec 2999
# Import latest exit list from disc
if exit_lists:
latest_exit_list = max(exit_lists, key=os.path.getctime)
for desc in stem.descriptor.parse_file(latest_exit_list,
descriptor_type="tordnsel 1.0"):
merge(desc)
# Import new measurements
kill = lambda process: process.kill()
with subprocess.Popen(["./bin/exitmap", "ipscan", "-o", "/dev/stdout"],
cwd="/srv/tordnsel.torproject.org/exitscanner/exitmap",
stdout=subprocess.PIPE,
encoding='utf-8') as p:
scantimer = threading.Timer(7200, kill, [p])
scantimer.start()
for line in p.stdout:
result = re.match(
r"^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}),[0-9]{3} modules\.ipscan \[INFO\] (\{.*\})$",
line)
if result:
check_result = json.loads(result.group(2))
desc = stem.descriptor.tordnsel.TorDNSEL("", False)
desc.fingerprint = check_result["Fingerprint"]
desc.last_status = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0)
desc.published = datetime.datetime.strptime(
check_result["DescPublished"], "%Y-%m-%dT%H:%M:%S")
desc.exit_addresses = [
(check_result["IP"],
datetime.datetime.strptime(result.group(1),
"%Y-%m-%d %H:%M:%S"))
]
merge(desc)
scantimer.cancel()
# Format exit list filename
now = datetime.datetime.utcnow()
filename = (f"{now.year}-{now.month:02d}-"
f"{now.day:02d}-{now.hour:02d}-"
f"{now.minute:02d}-{now.second:02d}")
# Format an exit list
with open(f"lists/{filename}", "w") as out:
for desc in exits.values():
if desc.exit_addresses:
out.write(f"ExitNode {desc.fingerprint}\n")
out.write(f"Published {desc.published}\n")
out.write(f"LastStatus {desc.last_status}\n")
for a in desc.exit_addresses:
out.write(f"ExitAddress {a[0]} {a[1]}\n")
# Provide the snapshot emulation
try:
os.unlink("lists/latest")
except FileNotFoundError:
pass # ok maybe this is the first time we run
os.symlink(os.path.abspath(f"lists/{filename}"), "lists/latest")
# Format a DNS zone
with open(f"dnsel.torproject.org", "w") as out:
out.write("""$TTL 1200 ; seconds
$ORIGIN dnsel.torproject.org.
@ 1D IN SOA check-01.torproject.org. metrics-team.lists.torproject.org. (
""" + datetime.datetime.utcnow().strftime("%y%m%d%H%M") + """
1H ; refresh
15 ; retry
1H ; expire
15 ; nxdomain ttl
)
IN NS check-01.torproject.org.
""") # fix the serial number generation before 2042 because 2^32 overflow
exit_addresses = collections.defaultdict(list)
for desc in exits.values():
for exit_address in desc.exit_addresses:
exit_addresses[exit_address[0]].append(desc.fingerprint)
for exit_address in exit_addresses:
reverse = ipaddress.ip_address(exit_address).reverse_pointer.split(".i")[0]
out.write(reverse + " IN A 127.0.0.2\n")
for fingerprint in exit_addresses[exit_address]:
out.write(reverse + " IN TXT \"" + fingerprint + "\"\n")
os.system("sudo /usr/sbin/rndc reload dnsel.torproject.org")
# Format a bulk exit list
with open("lists/bulk", "w") as out:
for exit_address in exit_addresses:
out.write(exit_address + "\n")
if __name__ == "__main__":
while True:
start = datetime.datetime.utcnow()
run()
while datetime.datetime.utcnow() < start + datetime.timedelta(minutes=40):
pass
#!/usr/bin/env python2
# Copyright 2013-2017 Philipp Winter <phw@nymity.ch>
#
# This file is part of exitmap.
#
# exitmap is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# exitmap is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with exitmap. If not, see <http://www.gnu.org/licenses/>.
"""
Module to detect false negatives for <https://check.torproject.org>.
"""
import sys
import json
import logging
try:
import urllib2
except ImportError:
import urllib.request as urllib2
from util import exiturl
import stem.descriptor.server_descriptor as descriptor
log = logging.getLogger(__name__)
# exitmap needs this variable to figure out which relays can exit to the given
# destination(s).
destinations = [("check.torproject.org", 443)]
def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
url = exiturl(exit_desc.fingerprint)
try:
data = urllib2.urlopen("https://check.torproject.org/api/ip",
timeout=10).read()
except Exception as err:
log.debug("urllib2.urlopen says: %s" % err)
return
if not data:
return
try:
check_answer = json.loads(data)
except ValueError as err:
log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
return
check_answer["DescPublished"] = exit_desc.published.isoformat()
check_answer["Fingerprint"] = exit_desc.fingerprint
log.info(json.dumps(check_answer))
def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs):
"""
Probe the given exit relay and look for check.tp.o false negatives.
"""
run_python_over_tor(fetch_page, exit_desc)
def main():
"""
Entry point when invoked over the command line.
"""
desc = descriptor.ServerDescriptor("")
desc.fingerprint = "bogus"
desc.address = "0.0.0.0"
fetch_page(desc)
return 0
if __name__ == "__main__":
sys.exit(main())
[Unit]
Description=Exit Scanner
[Service]
Type=simple
WorkingDirectory=/srv/tordnsel.torproject.org
ExecStart=/usr/bin/python3 /srv/tordnsel.torproject.org/exitscanner/exitscan.py
StandardOutput=file:/srv/tordnsel.torproject.org/exitscanner.log
[Install]
WantedBy=default.target
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment