Commit a61e3361 authored by Arturo Filastò's avatar Arturo Filastò
Browse files

Add support for downloading GeoIP files via the ooniresource tool. 

* Add better exception handling to ooniresource.
* Add gunzip to utils.
parent 491a78e3

ooni/resources/__init__.py

+20 −1
Changes for ooni/resources/__init__.py: 20 added lines, 1 removed line.
Original line number Diff line number Diff line 
from ooni.settings import config

from ooni.utils import unzip

from ooni.utils import unzip, gunzip



from ooni.deckgen.processors import citizenlab_test_lists

from ooni.deckgen.processors import namebench_dns_servers



config.read_config_file()



__version__ = "0.0.1"



inputs = {
@@ -20,3 +22,20 @@ inputs = { 
        "processor": citizenlab_test_lists

    }

}



geoip = {

    "GeoIPASNum.dat.gz": {

        "url": "http://www.maxmind.com/download/"

               "geoip/database/asnum/GeoIPASNum.dat.gz",

        "action": gunzip,

        "action_args": [config.advanced.geoip_data_dir],

        "processor": None

    },

    "GeoIP.dat.gz": {

        "url": "http://geolite.maxmind.com/"

               "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",

        "action": gunzip,

        "action_args": [config.advanced.geoip_data_dir],

        "processor": None

    }

}

ooni/resources/cli.py

+25 −7
Changes for ooni/resources/cli.py: 25 added lines, 7 removed lines.
Original line number Diff line number Diff line 
import sys



from twisted.internet import defer

from twisted.python import usage



from ooni.utils import log

from ooni.settings import config



from ooni.resources import __version__
@@ -9,10 +11,11 @@ from ooni.resources import update 




class Options(usage.Options):

    synopsis = """%s"""

    synopsis = """%s""" % sys.argv[0]



    optFlags = [

        ["update-inputs", None, "Update the resources needed for inputs"]

        ["update-inputs", None, "Update the resources needed for inputs."],

        ["update-geoip", None, "Update the geoip related resources."]

    ]

    optParameters = []
@@ -21,9 +24,10 @@ class Options(usage.Options): 
        sys.exit(0)





@defer.inlineCallbacks

def run():

    options = Options()

    config.read_config_file()

    options = Options()

    try:

        options.parseOptions()

    except usage.UsageError as error_message:
@@ -31,9 +35,23 @@ def run(): 
        print "%s: Try --help for usage details." % (sys.argv[0])

        sys.exit(1)



    if not any(options.values()):

        print("%s: no command specified" % sys.argv[0])

        print options

        sys.exit(1)



    if options['update-inputs']:

        return update.download_inputs()

        print "Downloading inputs"

        try:

            yield update.download_inputs()

        except Exception as exc:

            log.err("failed to download geoip files")

            log.exception(exc)



    print "%s: no command specified" % sys.argv[0]

    print "%s: Try --help for usage details." % (sys.argv[0])

    sys.exit(1)
 
    if options['update-geoip']:

        print "Downloading geoip files"

        try:

            yield update.download_geoip()

        except Exception as exc:

            log.err("failed to download geoip files")

            log.exception(exc)

ooni/resources/update.py

+11 −4
Changes for ooni/resources/update.py: 11 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ from twisted.internet import reactor, defer, protocol 
from twisted.web.client import RedirectAgent, Agent



from ooni.settings import config

from ooni.resources import inputs

from ooni.resources import inputs, geoip



agent = RedirectAgent(Agent(reactor))
@@ -29,12 +29,11 @@ class SaveToFile(protocol.Protocol): 




@defer.inlineCallbacks

def download_inputs():

    for filename, resource in inputs.items():

def download_resource(resources):

    for filename, resource in resources.items():

        print "Downloading %s" % filename



        filename = os.path.join(config.resources_directory, filename)



        response = yield agent.request("GET", resource['url'])

        finished = defer.Deferred()

        response.deliverBody(SaveToFile(finished, response.length, filename))
@@ -45,3 +44,11 @@ def download_inputs(): 
                                      filename,

                                      *resource['action_args'])

        print "%s written." % filename





def download_inputs():

    return download_resource(inputs)





def download_geoip():

    return download_resource(geoip)

ooni/utils/__init__.py

+13 −0
Changes for ooni/utils/__init__.py: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line 
import shutil

import string

import random

import glob

import os



import gzip

from zipfile import ZipFile



from ooni import otime
@@ -156,3 +158,14 @@ def unzip(filename, dst): 
        zip_file = ZipFile(zfp)

        zip_file.extractall(dst_path)

    return dst_path



def gunzip(filename, dst):

    assert filename.endswith(".gz")

    dst_path = os.path.join(

        dst,

        os.path.basename(filename).replace(".gz", "")

    )

    with open(dst_path, "w+") as fw:

        gzip_file = gzip.open(filename)

        shutil.copyfileobj(gzip_file, fw)

        gzip_file.close()