Merge branch 'state_file_02'

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Stop deleting the latest.v3bw symlink. Instead, do an atomic rename.
   (#26740)
+- State file for storing the last time `sbws scanner` was started, and able to
+  be used for storing many other types of state in the future. (GH#166)
 
 ### Changed
 

README.md

@@ -137,6 +137,8 @@ In this directory you will find
 - `log/` If configured, this directory stores logs generated by all the sbws
   commands in rotating log files.
 - `v3bw/` This directory stores the v3bw files created with `sbws generate`.
+- `state.dat` A file for storing state needed between sbws commands. See its
+  documentation for more information.
 
 ## Running tests
 

docs/source/index.rst

@@ -27,6 +27,7 @@ Contents
    faq
    glossary
    diagrams
+   state
    sbws
 
 Proposals

docs/source/sbws.util.rst

@@ -28,6 +28,14 @@ sbws.util.parser module
     :undoc-members:
     :show-inheritance:
 
+sbws.util.state module
+~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: sbws.util.state
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 sbws.util.stem module
 ~~~~~~~~~~~~~~~~~~~~~
 

docs/source/state.rst

+The ``state.dat`` file
+======================
+
+This file contains state that multiple sbws commands may want access to and
+that needs to persist across processes. Both read and write access to this file
+is wrapped in the ``State`` class, allowing for safe concurrent access: the
+file is locked before reading or writing, and (for now) only simple data types
+are allowed so we can be sure to update the state file on disk every time the
+state is modified in memory.
+
+At the time of writing, the following fields can exist in the state file.
+
+``scanner_started``
+-------------------
+
+The last time ``sbws scanner`` was started.
+
+- **Producer**: ``sbws scanner``, once at startup.
+
+- **Consumer**: ``sbws generate``, once each time it is ran.

sbws/config.default.ini

@@ -4,7 +4,7 @@ v3bw_dname = ${sbws_home}/v3bw
 # The latest bandwidth file is atomically symlinked to
 # V3BandwidthsFile ${v3bw_dname}/latest.v3bw
 v3bw_fname = ${v3bw_dname}/{}.v3bw
-started_filepath = ${sbws_home}/started_at
+state_fname = ${sbws_home}/state.dat
 log_dname = ${sbws_home}/log
 
 [destinations]

sbws/core/scanner.py

@@ -7,10 +7,8 @@ from ..lib.resultdump import ResultErrorStream
 from ..lib.relaylist import RelayList
 from ..lib.relayprioritizer import RelayPrioritizer
 from ..lib.destination import DestinationList
-from ..util.filelock import FileLock
 from ..util.timestamp import now_isodt_str
-# from ..util.simpleauth import authenticate_to_server
-# from ..util.sockio import (make_socket, close_socket)
+from ..util.state import State
 from sbws.globals import (fail_hard, is_initted)
 import sbws.util.stem as stem_utils
 import sbws.util.requests as requests_utils
@@ -316,22 +314,7 @@ def result_putter_error(target):
     return closure
 
 
-def write_start_ts(conf):
-    """Write ISO formated timestamp which represents the date and time
-    when scanner started.
-
-    :param ConfigParser conf: configuration
-    """
-    generator_started = now_isodt_str()
-    log.info('Scanner started at {}'.format(generator_started))
-    filepath = conf['paths']['started_filepath']
-    with FileLock(filepath):
-        with open(filepath, 'w') as fd:
-            fd.write(generator_started)
-
-
 def run_speedtest(args, conf):
-    write_start_ts(conf)
     controller, _ = stem_utils.init_controller(
         path=conf['tor']['control_socket'])
     if not controller:
@@ -398,6 +381,9 @@ def main(args, conf):
 
     os.makedirs(conf['paths']['datadir'], exist_ok=True)
 
+    state = State(conf['paths']['state_fname'])
+    state['scanner_started'] = now_isodt_str()
+
     try:
         run_speedtest(args, conf)
     except KeyboardInterrupt as e:

sbws/lib/v3bwfile.py

@@ -9,8 +9,9 @@ from statistics import median
 from sbws import __version__
 from sbws.globals import SPEC_VERSION, BW_LINE_SIZE
 from sbws.lib.resultdump import ResultSuccess, _ResultType
-from sbws.util.filelock import FileLock, DirectoryLock
+from sbws.util.filelock import DirectoryLock
 from sbws.util.timestamp import now_isodt_str, unixts_to_isodt_str
+from sbws.util.state import State
 
 log = logging.getLogger(__name__)
 
@@ -66,27 +67,6 @@ def warn_if_not_accurate_enough(bw_lines, scale_constant):
                     'allowed', (1 - accuracy_ratio) * 100, margin * 100)
 
 
-def read_started_ts(conf):
-    """Read ISO formated timestamp which represents the date and time
-    when scanner started.
-
-    :param ConfigParser conf: configuration
-    :returns: str, ISO formated timestamp
-    """
-    try:
-        filepath = conf['paths']['started_filepath']
-    except TypeError:
-        return None
-    try:
-        with FileLock(filepath):
-            with open(filepath, 'r') as fd:
-                generator_started = fd.read()
-    except FileNotFoundError as e:
-        log.warn('File %s not found.%s', filepath, e)
-        return None
-    return generator_started
-
-
 def num_results_of_type(results, type_str):
     return len([r for r in results if r.type == type_str])
 
@@ -210,7 +190,15 @@ class V3BWHeader(object):
 
     @staticmethod
     def generator_started_from_file(conf):
-        return read_started_ts(conf)
+        '''
+        ISO formatted timestamp for the time when the scanner process most
+        recently started.
+        '''
+        state = State(conf['paths']['state_fname'])
+        if 'scanner_started' in state:
+            return state['scanner_started']
+        else:
+            return None
 
     @staticmethod
     def latest_bandwidth_from_results(results):

sbws/util/config.py

@@ -210,8 +210,8 @@ def _validate_paths(conf):
     sec = 'paths'
     err_tmpl = Template('$sec/$key ($val): $e')
     unvalidated_keys = [
-        'datadir', 'sbws_home', 'v3bw_fname', 'v3bw_dname',
-        'started_filepath', 'log_dname']
+        'datadir', 'sbws_home', 'v3bw_fname', 'v3bw_dname', 'state_fname',
+        'log_dname']
     all_valid_keys = unvalidated_keys
     allow_missing = ['sbws_home']
     errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl,

sbws/util/state.py

+from sbws.util.filelock import FileLock
+import os
+import json
+
+
+class State:
+    '''
+    State allows one to atomically access and update a simple state file on
+    disk across threads and across processes.
+
+    To put it blunty, due to limited developer time and his inability to
+    quickly find a way to safely access and update more complex data types
+    (namely, collections like list, set, and dict), you may only store simple
+    types of data as enumerated in _ALLOWED_TYPES. Keys must be strings.
+
+    Data is stored as JSON on disk in the provided file file.
+
+    >>> state = State('foo.state')
+    >>> # state == {}
+
+    >>> state['linux'] = True
+    >>> # 'foo.state' now exists on disk with the JSON for {'linux': True}
+
+    >>> # We read 'foo.state' from disk in order to get the most up-to-date
+    >>> #     state info. Pretend another process has updated 'linux' to be
+    >>> #     False
+    >>> state['linux']
+    >>> # returns False
+
+    >>> # Pretend another process has added the user's age to the state file.
+    >>> #     As before, we read the state file from disk for the most
+    >>> #     up-to-date info.
+    >>> state['age']
+    >>> # Returns 14
+
+    >>> # We now set their name. We read the state file first, set the option,
+    >>> #     and then write it out.
+    >>> state['name'] = 'John'
+
+    >>> # We can do many of the same things with a State object as with a dict
+    >>> for key in state: print(key)
+    >>> # Prints 'linux', 'age', and 'name'
+    '''
+    _ALLOWED_TYPES = (int, float, str, bool, type(None))
+
+    def __init__(self, fname):
+        self._fname = fname
+        self._state = self._read()
+
+    def _read(self):
+        with FileLock(self._fname):
+            if not os.path.exists(self._fname):
+                return {}
+            with open(self._fname, 'rt') as fd:
+                return json.load(fd)
+
+    def _write(self):
+        with FileLock(self._fname):
+            with open(self._fname, 'wt') as fd:
+                return json.dump(self._state, fd, indent=4)
+
+    def __len__(self):
+        self._state = self._read()
+        return self._state.__len__()
+
+    def __getitem__(self, key):
+        if not isinstance(key, str):
+            raise TypeError(
+                'Keys must be strings. %s is a %s' % (key, type(key)))
+        self._state = self._read()
+        return self._state.__getitem__(key)
+
+    def __delitem__(self, key):
+        if not isinstance(key, str):
+            raise TypeError(
+                'Keys must be strings. %s is a %s' % (key, type(key)))
+        self._state = self._read()
+        self._state.__delitem__(key)
+        self._write()
+
+    def __setitem__(self, key, value):
+        if not isinstance(key, str):
+            raise TypeError(
+                'Keys must be strings. %s is a %s' % (key, type(key)))
+        if type(value) not in State._ALLOWED_TYPES:
+            raise TypeError(
+                'May only store value with type in %s, not %s' %
+                (State._ALLOWED_TYPES, type(value)))
+        self._state = self._read()
+        self._state.__setitem__(key, value)
+        self._write()
+
+    def __iter__(self):
+        self._state = self._read()
+        return self._state.__iter__()
+
+    def __contains__(self, item):
+        self._state = self._read()
+        return self._state.__contains__(item)

tests/unit/util/test_state.py

+from sbws.util.state import State
+import os
+# from tempfile import NamedTemporaryFile as NTF
+
+
+def test_state_set_allowed_key_types(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    attempt_keys = ('k')
+    for key in attempt_keys:
+        state[key] = 4
+        assert state[key] == 4
+
+
+def test_state_set_bad_key_types(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    attempt_keys = (15983, None, True, -1.2, [], {}, set())
+    for key in attempt_keys:
+        try:
+            state[key] = 4
+        except TypeError:
+            pass
+        else:
+            assert None, 'Should not have been able to use %s %s as a key' %\
+                (key, type(key))
+    try:
+        state[key]
+    except TypeError:
+        pass
+    else:
+        assert None, '%s %s is not a valid key type, so should have got '\
+            'TypeError when giving it' % (key, type(key))
+
+
+def test_state_set_allowed_value_types(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    attempt_vals = (15983, None, True, -1.2, 'loooooool')
+    for val in attempt_vals:
+        state['foo'] = val
+        assert state['foo'] == val
+
+
+def test_state_set_bad_value_types(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    attempt_vals = ([], {}, set())
+    for val in attempt_vals:
+        try:
+            state['foo'] = val
+        except TypeError:
+            pass
+        else:
+            assert None, 'Should not have been able to use %s %s as a value' %\
+                (val, type(val))
+
+
+def test_state_del(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    d = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
+    for key in d:
+        state[key] = d[key]
+    assert len(state) == len(d)
+
+    del d['a']
+    del state['a']
+    assert len(state) == len(d)
+    for key in d:
+        assert d[key] == state[key]
+
+    attempt_keys = (15983, None, True, -1.2, [], {}, set())
+    for key in attempt_keys:
+        try:
+            del state[key]
+        except TypeError:
+            pass
+        else:
+            assert None, 'Should not have been allowed to delete %s %s '\
+                'because it is not a valid key type' % (key, type(key))
+
+    d['e'] = 5
+    state['e'] = 5
+    d['e'] = 5.5
+    state['e'] = 5.5
+    assert len(state) == len(d)
+
+
+def test_state_get_len(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    d = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
+    for key in d:
+        state[key] = d[key]
+    assert len(state) == len(d)
+
+    del d['a']
+    del state['a']
+    assert len(state) == len(d)
+
+    d['e'] = 5
+    state['e'] = 5
+    d['e'] = 5.5
+    state['e'] = 5.5
+    assert len(state) == len(d)
+
+
+def test_state_contains(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    d = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
+    for key in d:
+        state[key] = d[key]
+    assert 'a' in state
+    assert 'e' not in state
+
+
+def test_state_iter(tmpdir):
+    state = State(os.path.join(str(tmpdir), 'statefoo'))
+    for key in state:
+        pass
+    d = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
+    for key in d:
+        state[key] = d[key]
+    assert set([key for key in state]) == set(d)