#!/usr/bin/env python3

# Usage: scrape-geoip ~/src/tor > geoip-events.txt
#
# Scrapes geoip database changes from the tor source code history and formats
# them in the format expected by the Metrics Timeline.
#
# ~/src/tor (or whatever the path is) must be a tor source repo; i.e. a clone of
# https://git.torproject.org/tor.git.

import datetime
import difflib
import getopt
import hashlib
import os.path
import re
import socket
import subprocess
import sys
import urllib.parse

import parser

def usage(file=sys.stdout):
    print("""\
Usage: {sys.argv[0]} /path/to/tor

  --output-country-codes  include countries in the "places" column and
                            relative increase/decrease in "description"
  --since=YYYY-MM-DD      only include commits since YYYY-MM-DD
  -h, --help show this help\
""", file=file)

class Options:
    output_country_codes = False
    since_date = None

# A file-like object that copies whatever is read into a hash object.
class HashFile:
    def __init__(self, f, h):
        self.f = f
        self.h = h

    def readline(self):
        line = self.f.readline()
        self.h.update(line)
        return line

    def __iter__(self):
        return self

    def __next__(self):
        line = self.readline()
        if not line:
            raise StopIteration
        return line

class DB:
    def __init__(self):
        self.version = None
        self.digest = None
        self.entries = []
        self.prev = None

    def parse(self, f, parse_ip):
        self.digest = hashlib.sha1()
        maxmind_version = None
        ipfire_version = None
        ipfire_metadata = {}
        for line in HashFile(f, self.digest):
            line = line.decode("utf-8").strip()

            # Maxmind style comment header.
            m = re.match(r'^# Last updated based on (.*?)\.?$', line)
            if m:
                assert maxmind_version is None, maxmind_version
                maxmind_version, = m.groups()

            # IPFire style comment header.
            m = re.match(r'^# (Generated|Vendor): *(.*)$', line)
            if m:
                key, value = m.groups()
                assert key not in ipfire_metadata, (key, value, ipfire_metadata)
                ipfire_metadata[key] = value

            if line.startswith("#"):
                continue

            start, end, country = line.strip().split(",")
            start = parse_ip(start)
            end = parse_ip(end)
            country = country.lower()
            self.entries.append((country, start, end))

        ipfire_generated = ipfire_metadata.get("Generated")
        ipfire_vendor = ipfire_metadata.get("Vendor")
        # Ensure that if one key is set, the other is set.
        assert (ipfire_generated is None) == (ipfire_vendor is None), ipfire_metadata
        if ipfire_generated is not None and ipfire_vendor is not None:
            ipfire_version = f"{ipfire_vendor} {ipfire_generated}"

        assert not (maxmind_version is not None and ipfire_version is not None), (maxmind_version, ipfire_version)
        self.version = maxmind_version or ipfire_version

    # Parsing geoip and geoip6 databases is the same except for the
    # representation of addresses.

    def parse_geoip(self, f):
        return self.parse(f, int)

    def parse_geoip6(self, f):
        return self.parse(f, lambda s: int(socket.inet_pton(socket.AF_INET6, s).hex(), 16))

# Yield (date, commit_hash) tuples.
def git_log(dirname, filename):
    cmdline = ["git", "log", "--date=unix", "--pretty=%ad %H"]
    if Options.since_date is not None:
        cmdline.append(f"--since={Options.since_date}")
    cmdline.append(filename)
    proc = subprocess.Popen(cmdline, cwd=dirname, stdout=subprocess.PIPE, encoding="utf-8")
    for line in proc.stdout:
        date_str, commit_hash = line.strip().split()
        date = datetime.datetime.utcfromtimestamp(int(date_str))
        yield (date, commit_hash)

def git_show(dirname, filename, commit_hash):
    proc = subprocess.Popen(["git", "show", commit_hash+":"+filename],
        cwd=dirname, stdout=subprocess.PIPE)
    return proc.stdout

def changed_countries(a, b):
    result = set()
    if a is None or b is None:
        return result
    sm = difflib.SequenceMatcher(None, a.entries, b.entries, autojunk=False)
    for tag, i1, i2, j1, j2 in sm.get_opcodes():
        if tag == "equal":
            continue
        # Add countries from all lines added or deleted.
        for i in range(i1, i2):
            country, _, _ = a.entries[i]
            result.add(country)
        for j in range(j1, j2):
            country, _, _ = b.entries[j]
            result.add(country)
    return result

def process_history(path, parse_fn):
    prev = None
    for date, commit_hash in sorted(git_log(TOR_PATH, path)):
        db = DB()
        db.prev = prev
        prev = db
        try:
            parse_fn(db, git_show(TOR_PATH, path, commit_hash))
        except Exception as err:
            print(f"Skipping {path} {commit_hash}: {err}", file=sys.stderr)
            continue
        yield date, commit_hash, db


opts, args = getopt.gnu_getopt(sys.argv[1:], "h", ["help", "output-country-codes", "since="])
for o, a in opts:
    if o == "-h" or o == "--help":
        usage()
        sys.exit()
    elif o == "--output-country-codes":
        Options.output_country_codes = True
    elif o == "--since":
        Options.since_date = a

try:
    TOR_PATH, = args
except ValueError:
    usage(sys.stderr)
    sys.exit(1)

class Entry:
    def __init__(self, date, commit_hash):
        self.date = date
        self.commit_hash = commit_hash
        self.geoip_db = None
        self.geoip6_db = None

    def __lt__(self, other):
        return (self.date, self.commit_hash) < (other.date, other.commit_hash)

# parse histories and merge them by commit hash
entries = {}
for date, commit_hash, db in process_history("src/config/geoip", DB.parse_geoip):
    entries.setdefault(commit_hash, Entry(date, commit_hash))
    assert entries[commit_hash].geoip_db is None
    entries[commit_hash].geoip_db = db
for date, commit_hash, db in process_history("src/config/geoip6", DB.parse_geoip6):
    entries.setdefault(commit_hash, Entry(date, commit_hash))
    assert entries[commit_hash].geoip6_db is None
    entries[commit_hash].geoip6_db = db

for entry in sorted(entries.values()):
    places = []
    protocols = []

    if Options.output_country_codes:
        places = set()
        if entry.geoip_db is not None:
            places.update(changed_countries(entry.geoip_db, entry.geoip_db.prev))
        if entry.geoip6_db is not None:
            places.update(changed_countries(entry.geoip6_db, entry.geoip6_db.prev))
        places = sorted(places)

    if entry.geoip_db is not None:
        protocols.append("ipv4")
    if entry.geoip6_db is not None:
        protocols.append("ipv6")

    if entry.geoip_db is not None and entry.geoip6_db is not None and entry.geoip_db.version == entry.geoip6_db.version:
            description = "geoip and geoip6 databases updated "
            if entry.geoip_db.version is not None:
                description += f"to \"{entry.geoip_db.version}\" "
            description += "(`geoip-db-digest {}`, `geoip6-db-digest {}`).".format(
                entry.geoip_db.digest.hexdigest().upper(),
                entry.geoip6_db.digest.hexdigest().upper(),
            )
    else:
        parts = []
        if entry.geoip_db is not None:
            description = "geoip database updated "
            if entry.geoip_db.version is not None:
                description += f"to \"{entry.geoip_db.version}\" "
            description += "(`geoip-db-digest {}`).".format(entry.geoip_db.digest.hexdigest().upper())
            parts.append(description)
        if entry.geoip6_db is not None:
            description = "geoip6 database updated "
            if entry.geoip6_db.version is not None:
                description += f"to \"{entry.geoip6_db.version}\" "
            description += "(`geoip6-db-digest {}`).".format(entry.geoip6_db.digest.hexdigest().upper())
            parts.append(description)
        description = " ".join(parts)

    start_date = entry.date.strftime("%Y-%m-%d")
    end_date = ""
    places = " ".join(places)
    protocols = " ".join(protocols)
    links = parser.MarkdownLink(
        parser.MarkdownLiteral("commit"),
        urllib.parse.urlunparse(("https", "gitweb.torproject.org", "/tor.git/commit/", None, urllib.parse.urlencode({"id": entry.commit_hash}), None)),
    ).to_markdown()
    unexplained = ""

    row = (start_date, end_date, places, protocols, description, links, unexplained)
    print("|" + "|".join(parser.backslash_escape(cell, "|") for cell in row) + "|")
