Commit f05a15fa authored by Nick Mathewson's avatar Nick Mathewson 🎨
Browse files

Merge branch 'ticket-40002-plus-squashed'

parents 99bd06c7 67f83d77
......@@ -2,7 +2,7 @@ language: python
# The default python version on Travis is 2.7
# But we add this line to show the python version in the Travis UI
python: "2.7"
python: "3.8"
os:
- linux
......@@ -30,7 +30,7 @@ env:
- TOR="master-nightly" NETWORK_FLAVOUR="basic-min"
matrix:
# include creates Linux, python 2.7, tor master builds by default
# include creates Linux, python 3.8, tor master builds by default
# we use tor master to catch tor issues before stable releases
# the key(s) in each item override these defaults
include:
......@@ -39,9 +39,6 @@ matrix:
##
## We need to use macOS to test IPv6 networks, because Travis Linux doesn't
## support IPv6. But macOS is tricky:
## - We use the default python version on macOS, which is currently 2.7.
## (But we don't show the version, because Travis might change it
## without us noticing.)
## - We use language: c, because language: python fails on Travis macOS.
## - We get the tor version in the homebrew cache on the macOS image.
## The latest tor version in homebrew is on this page:
......@@ -63,19 +60,19 @@ matrix:
##
## IPv6 networks (on macOS)
## "make test-network-all" on all supported tor versions
- env: TOR="stable-release" NETWORK_FLAVOUR="bridges+ipv6-min"
- env: PYTHON="python3" TOR="stable-release" NETWORK_FLAVOUR="bridges+ipv6-min"
os: osx
language: c
python:
## The IPv6 exit test doesn't actually require IPv6, see #30182.
## But we'll keep this test, because it does test IPv6 exit config.
- env: TOR="master-nightly" NETWORK_FLAVOUR="ipv6-exit-min"
- env: TOR="stable-release" NETWORK_FLAVOUR="hs-v23-ipv6-md"
- env: PYTHON="python3" TOR="stable-release" NETWORK_FLAVOUR="hs-v23-ipv6-md"
os: osx
language: c
python:
## v3 onion service IPv6 tests
- env: TOR="stable-release" NETWORK_FLAVOUR="single-onion-v23-ipv6-md"
- env: PYTHON="python3" TOR="stable-release" NETWORK_FLAVOUR="single-onion-v23-ipv6-md"
os: osx
language: c
python:
......@@ -140,10 +137,6 @@ matrix:
## Pre-installed in Travis Bionic:
## https://docs.travis-ci.com/user/reference/bionic/#python-support
## End of Life: 1 January 2020
## https://www.python.org/dev/peps/pep-0373/#update
- python: "2.7"
## End of Life: December 2021
## https://www.python.org/dev/peps/pep-0494/#lifespan
- python: "3.6"
......@@ -174,22 +167,6 @@ matrix:
## Tor master doesn't work on Travis Xenial, because it only has
## OpenSSL 1.1.0
## Pypy 2
## End of Life: "forever"
## http://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2
## But chutney can decide not to support python 2 after 1 Jan 2020.
- python: "pypy"
dist: xenial
addons:
apt:
sources:
- sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.4.2.x-xenial main'
key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc'
packages:
- shellcheck
- tor
env: TOR="0.4.2-nightly" NETWORK_FLAVOUR="basic-min"
## PyPy does not have documented end of life dates
- python: "pypy3"
dist: xenial
......
......@@ -10,7 +10,7 @@ It is supposed to be a good tool for:
Right now it only sorta does these things.
You will need:
- Python 2.7, or a supported Python 3,
- A supported version of Python 3
- (we support Python versions that are still getting updates), and
- Tor binaries.
......
......@@ -12,7 +12,15 @@ then
export CHUTNEY_PATH
fi
binaries="python3 python python2"
python_ok ()
{
checkpy="$1"
test -f "${checkpy}" && \
test -x "${checkpy}" && \
"${checkpy}" -c 'import sys;sys.exit(sys.version_info[0]<3)'
}
binaries="python3 python"
if ! test "${PYTHON+y}"
then
......@@ -31,7 +39,7 @@ then
;;
esac
abs_path="${directory}${binary}"
if test -f "${abs_path}" && test -x "${abs_path}"
if python_ok "${abs_path}"
then
PYTHON="${abs_path}"
break
......@@ -44,12 +52,18 @@ then
fi
done
IFS="${saved_IFS}"
else
if ! python_ok "$(command -v "${PYTHON}")"
then
echo "Python in \$PYTHON envvar (\"$PYTHON\") is not present, or is not python 3." >&2
exit 1
fi
fi
if ! test "${PYTHON+y}"
then
printf "No compatible Python version found.\n" >&2
printf "Is Python installed and in your PATH?\n" >&2
printf "Is python 3 installed and in your PATH?\n" >&2
exit 1
fi
......
......@@ -81,6 +81,8 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from pathlib import Path
import string
import os
......@@ -258,18 +260,18 @@ class IncluderDict(_DictWrapper):
if not key.startswith("include:"):
raise KeyError(key)
filename = key[len("include:"):]
if os.path.isabs(filename):
with open(filename, 'r') as f:
filename = Path(key[len("include:"):])
if filename.is_absolute():
with filename.open(mode='r') as f:
stat = os.fstat(f.fileno())
if stat.st_mtime > self._st_mtime:
self._st_mtime = stat.st_mtime
return f.read()
for elt in self._includePath:
fullname = os.path.join(elt, filename)
if os.path.exists(fullname):
with open(fullname, 'r') as f:
fullname = Path(elt, filename)
if fullname.exists():
with fullname.open(mode='r') as f:
stat = os.fstat(f.fileno())
if stat.st_mtime > self._st_mtime:
self._st_mtime = stat.st_mtime
......@@ -298,9 +300,9 @@ class PathDict(_DictWrapper):
key = key[len("path:"):]
for location in self._path:
p = os.path.join(location, key)
p = Path(location, key)
try:
s = os.stat(p)
s = p.stat()
if s and s.st_mode & 0x111:
return p
except OSError:
......
......@@ -12,6 +12,8 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from pathlib import Path
import cgitb
import errno
import importlib
......@@ -98,7 +100,7 @@ def getenv_bool(env_var, default):
else:
return getenv_type(env_var, default, bool, type_name='a bool')
def mkdir_p(d, mode=448):
def mkdir_p(*d, mode=448):
"""Create directory 'd' and all of its parents as needed. Unlike
os.makedirs, does not give an error if d already exists.
......@@ -110,12 +112,7 @@ def mkdir_p(d, mode=448):
permissions for the intermediate directories. In python3, 'mode'
only sets the mode for the last directory created.
"""
try:
os.makedirs(d, mode=mode)
except OSError as e:
if e.errno == errno.EEXIST:
return
raise
Path(*d).mkdir(mode=mode, parents=True, exist_ok=True)
def make_datadir_subdirectory(datadir, subdir):
"""
......@@ -123,7 +120,7 @@ def make_datadir_subdirectory(datadir, subdir):
that datadirectory. Ensure that both are mode 700.
"""
mkdir_p(datadir)
mkdir_p(os.path.join(datadir, subdir))
mkdir_p(datadir, subdir)
def get_absolute_chutney_path():
"""
......@@ -134,8 +131,8 @@ def get_absolute_chutney_path():
# (./chutney already sets CHUTNEY_PATH using the path to the script)
# use tools/test-network.sh if you want chutney to try really hard to find
# itself
relative_chutney_path = os.environ.get('CHUTNEY_PATH', os.getcwd())
return os.path.abspath(relative_chutney_path)
relative_chutney_path = Path(os.environ.get('CHUTNEY_PATH', os.getcwd()))
return relative_chutney_path.resolve()
def get_absolute_net_path():
"""
......@@ -153,23 +150,23 @@ def get_absolute_net_path():
Finally, return the path relative to the current working directory,
regardless of whether the path actually exists.
"""
data_dir = os.environ.get('CHUTNEY_DATA_DIR', 'net')
if os.path.isabs(data_dir):
data_dir = Path(os.environ.get('CHUTNEY_DATA_DIR', 'net'))
if data_dir.is_absolute():
# if we are given an absolute path, we should use it
# regardless of whether the directory exists
return data_dir
# use the chutney path as the default
absolute_chutney_path = get_absolute_chutney_path()
relative_net_path = data_dir
relative_net_path = Path(data_dir)
# but what is it relative to?
# let's check if there's an existing directory with this name in
# CHUTNEY_PATH first, to preserve backwards-compatible behaviour
chutney_net_path = os.path.join(absolute_chutney_path, relative_net_path)
if os.path.isdir(chutney_net_path):
chutney_net_path = Path(absolute_chutney_path, relative_net_path)
if chutney_net_path.is_dir():
return chutney_net_path
# ok, it's relative to the current directory, whatever that is, and whether
# or not the path actually exists
return os.path.abspath(relative_net_path)
return relative_net_path.resolve()
def get_absolute_nodes_path():
"""
......@@ -182,7 +179,7 @@ def get_absolute_nodes_path():
See get_new_absolute_nodes_path() for more details.
"""
return os.path.join(get_absolute_net_path(), 'nodes')
return Path(get_absolute_net_path(), 'nodes')
def get_new_absolute_nodes_path(now=time.time()):
"""
......@@ -202,12 +199,12 @@ def get_new_absolute_nodes_path(now=time.time()):
# should only be called by 'chutney configure', all other chutney commands
# should use get_absolute_nodes_path()
nodesdir = get_absolute_nodes_path()
newdir = newdirbase = "%s.%d" % (nodesdir, now)
newdir = newdirbase = Path("%s.%d" % (nodesdir, now))
# if the time is the same, fall back to a simple integer count
# (this is very unlikely to happen unless the clock changes: it's not
# possible to run multiple chutney networks at the same time)
i = 0
while os.path.exists(newdir):
while newdir.exists():
i += 1
newdir = "%s.%d" % (newdirbase, i)
return newdir
......@@ -735,20 +732,20 @@ class LocalNodeBuilder(NodeBuilder):
datadir = self._env['dir']
tor_gencert = self._env['tor_gencert']
lifetime = self._env['auth_cert_lifetime']
idfile = os.path.join(datadir, 'keys', "authority_identity_key")
skfile = os.path.join(datadir, 'keys', "authority_signing_key")
certfile = os.path.join(datadir, 'keys', "authority_certificate")
idfile = Path(datadir, 'keys', "authority_identity_key")
skfile = Path(datadir, 'keys', "authority_signing_key")
certfile = Path(datadir, 'keys', "authority_certificate")
addr = self.expand("${ip}:${dirport}")
passphrase = self._env['auth_passphrase']
if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
if all(f.exists() for f in [idfile, skfile, certfile]):
return
cmdline = [
tor_gencert,
'--create-identity-key',
'--passphrase-fd', '0',
'-i', idfile,
'-s', skfile,
'-c', certfile,
'-i', str(idfile),
'-s', str(skfile),
'-c', str(certfile),
'-m', str(lifetime),
'-a', addr,
]
......@@ -790,9 +787,9 @@ class LocalNodeBuilder(NodeBuilder):
return ""
datadir = self._env['dir']
certfile = os.path.join(datadir, 'keys', "authority_certificate")
certfile = Path(datadir, 'keys', "authority_certificate")
v3id = None
with open(certfile, 'r') as f:
with certfile.open(mode='r') as f:
for line in f:
if line.startswith("fingerprint"):
v3id = line.split()[1].strip()
......@@ -882,24 +879,23 @@ class LocalNodeController(NodeController):
Raises a ValueError if the file appears to be corrupt.
"""
datadir = self._env['dir']
key_file = os.path.join(datadir, 'keys',
"ed25519_master_id_public_key")
key_file = Path(datadir, 'keys', 'ed25519_master_id_public_key')
# If we're called early during bootstrap, the file won't have been
# created yet. (And some very old tor versions don't have ed25519.)
if not os.path.exists(key_file):
if not key_file.exists():
debug(("File {} does not exist. Are you running a very old tor "
"version?").format(key_file))
return None
EXPECTED_ED25519_FILE_SIZE = 64
key_file_size = os.stat(key_file).st_size
key_file_size = key_file.stat().st_size
if key_file_size != EXPECTED_ED25519_FILE_SIZE:
raise ValueError(
("The current size of the file is {} bytes, which is not"
"matching the expected value of {} bytes")
.format(key_file_size, EXPECTED_ED25519_FILE_SIZE))
with open(key_file, 'rb') as f:
with key_file.open(mode='rb') as f:
ED25519_KEY_POSITION = 32
f.seek(ED25519_KEY_POSITION)
rest_file = f.read()
......@@ -1063,11 +1059,11 @@ class LocalNodeController(NodeController):
"""Read the pidfile, and return the pid of the running process.
Returns None if there is no pid in the file.
"""
pidfile = self._env['pidfile']
if not os.path.exists(pidfile):
pidfile = Path(self._env['pidfile'])
if not pidfile.exists():
return None
with open(pidfile, 'r') as f:
with pidfile.open(mode='r') as f:
try:
return int(f.read())
except ValueError:
......@@ -1115,7 +1111,7 @@ class LocalNodeController(NodeController):
print("{:12} is running with PID {:5}: {}"
.format(nick, pid, tor_version))
return True
elif corefile and os.path.exists(os.path.join(datadir, corefile)):
elif corefile and Path(datadir, corefile).exists():
if listNonRunning:
print("{:12} seems to have crashed, and left core file {}: {}"
.format(nick, corefile, tor_version))
......@@ -1199,8 +1195,8 @@ class LocalNodeController(NodeController):
def cleanup_lockfile(self):
"""Remove lock file if this node is no longer running."""
lf = self._env['lockfile']
if not self.isRunning() and os.path.exists(lf):
lf = Path(self._env['lockfile'])
if not self.isRunning() and lf.exists():
debug("Removing stale lock file for {} ..."
.format(self._env['nick']))
os.remove(lf)
......@@ -1209,8 +1205,8 @@ class LocalNodeController(NodeController):
"""Move PID file to pidfile.old if this node is no longer running
so that we don't try to stop the node again.
"""
pidfile = self._env['pidfile']
if not self.isRunning() and os.path.exists(pidfile):
pidfile = Path(self._env['pidfile'])
if not self.isRunning() and pidfile.exists():
debug("Renaming stale pid file for {} ..."
.format(self._env['nick']))
os.rename(pidfile, pidfile + ".old")
......@@ -1251,7 +1247,7 @@ class LocalNodeController(NodeController):
logname = "info.log"
else:
logname = "notice.log"
return os.path.join(datadir, logname)
return Path(datadir, logname)
INTERNAL_ERROR_CODE = -500
MISSING_FILE_CODE = -400
......@@ -1309,13 +1305,13 @@ class LocalNodeController(NodeController):
received.
"""
logfname = self.getLogfile()
if not os.path.exists(logfname):
if not logfname.exists():
return (LocalNodeController.MISSING_FILE_CODE,
"no_logfile", "There is no logfile yet.")
percent = LocalNodeController.NO_RECORDS_CODE
keyword = "no_message"
message = "No bootstrap messages yet."
with open(logfname, 'r') as f:
with logfname.open(mode='r') as f:
for line in f:
m = re.search(r'Bootstrapped (\d+)%(?: \(([^\)]*)\))?: (.*)',
line)
......@@ -1397,16 +1393,15 @@ class LocalNodeController(NodeController):
datadir = self._env['dir']
to_dir_server = self.getDirServer()
desc = os.path.join(datadir, "cached-descriptors")
desc_new = os.path.join(datadir, "cached-descriptors.new")
desc = Path(datadir, "cached-descriptors")
desc_new = Path(datadir, "cached-descriptors.new")
paths = None
if v2_dir_paths:
ns_cons = os.path.join(datadir, "cached-consensus")
md_cons = os.path.join(datadir,
"cached-microdesc-consensus")
md = os.path.join(datadir, "cached-microdescs")
md_new = os.path.join(datadir, "cached-microdescs.new")
ns_cons = Path(datadir, "cached-consensus")
md_cons = Path(datadir, "cached-microdesc-consensus")
md = Path(datadir, "cached-microdescs")
md_new = Path(datadir, "cached-microdescs.new")
paths = { 'ns_cons': ns_cons,
'desc': desc,
......@@ -1425,7 +1420,7 @@ class LocalNodeController(NodeController):
# bootstrapping, and trying to publish their descriptors too early
if to_bridge_auth:
paths = None
# br_status = os.path.join(datadir, "networkstatus-bridges")
# br_status = Path(datadir, "networkstatus-bridges")
# paths['br_status'] = br_status
else:
# We're looking for bridges, but other nodes don't use bridges
......@@ -1515,14 +1510,15 @@ class LocalNodeController(NodeController):
* a set containing dir_format; and
* a status message string.
"""
if not os.path.exists(dir_path):
dir_path = Path(dir_path)
if not dir_path.exists():
return (LocalNodeController.MISSING_FILE_CODE,
{ dir_format }, "No dir file")
dir_pattern = self.getNodeDirInfoStatusPattern(dir_format)
line_count = 0
with open(dir_path, 'r') as f:
with dir_path.open(mode='r') as f:
for line in f:
line_count = line_count + 1
if dir_pattern:
......@@ -2106,10 +2102,9 @@ class TorEnviron(chutney.Templating.Environ):
return my['ptport_base'] + my['nodenum']
def _get_dir(self, my):
return os.path.abspath(os.path.join(my['net_base_dir'],
"nodes",
"%03d%s" % (
my['nodenum'], my['tag'])))
return Path(my['net_base_dir'],
"nodes",
"%03d%s" % (my['nodenum'], my['tag'])).resolve()
def _get_nick(self, my):
return "test%03d%s" % (my['nodenum'], my['tag'])
......@@ -2121,13 +2116,13 @@ class TorEnviron(chutney.Templating.Environ):
return self['nick'] # OMG TEH SECURE!
def _get_torrc_template_path(self, my):
return [os.path.join(my['chutney_dir'], 'torrc_templates')]
return [Path(my['chutney_dir'], 'torrc_templates')]
def _get_lockfile(self, my):
return os.path.join(self['dir'], 'lock')
return Path(self['dir'], 'lock')
def _get_pidfile(self, my):
return os.path.join(self['dir'], 'pid')
return Path(self['dir'], 'pid')
# A hs generates its key on first run,
# so check for it at the last possible moment,
......@@ -2138,8 +2133,7 @@ class TorEnviron(chutney.Templating.Environ):
if my['hs-hostname'] is None:
datadir = my['dir']
# a file containing a single line with the hs' .onion address
hs_hostname_file = os.path.join(datadir, my['hs_directory'],
'hostname')
hs_hostname_file = Path(datadir, my['hs_directory'], 'hostname')
try:
with open(hs_hostname_file, 'r') as hostnamefp:
hostname = hostnamefp.read()
......@@ -2181,11 +2175,11 @@ class TorEnviron(chutney.Templating.Environ):
dns_conf = TorEnviron.DEFAULT_DNS_RESOLV_CONF
else:
dns_conf = my['dns_conf']
dns_conf = os.path.abspath(dns_conf)
dns_conf = Path(dns_conf).resolve()
# work around Tor bug #21900, where exits fail when the DNS conf
# file does not exist, or is a broken symlink
# (os.path.exists returns False for broken symbolic links)
if not os.path.exists(dns_conf):
# (Path.exists returns False for broken symbolic links)
if not dns_conf.exists():
# Issue a warning so the user notices
print("CHUTNEY_DNS_CONF '{}' does not exist, using '{}'."
.format(dns_conf, TorEnviron.OFFLINE_DNS_RESOLV_CONF))
......@@ -2225,17 +2219,17 @@ class Network(object):
nodesdir = get_absolute_nodes_path()
# only move the directory if it exists
if not os.path.exists(nodesdir):
if not nodesdir.exists():
return
# and if it's not a link
if os.path.islink(nodesdir):
if nodesdir.is_symlink():
return
# subtract 1 second to avoid collisions and get the correct ordering
newdir = get_new_absolute_nodes_path(time.time() - 1)
print("NOTE: renaming %r to %r" % (nodesdir, newdir))
os.rename(nodesdir, newdir)
nodesdir.rename(newdir)
def create_new_nodes_dir(self):
"""Create a new directory with a unique name, and symlink it to nodes
......@@ -2250,12 +2244,12 @@ class Network(object):
nodeslink = get_absolute_nodes_path()
# this path should be unique and should not exist
if os.path.exists(newnodesdir):
if newnodesdir.exists():
raise RuntimeError(
'get_new_absolute_nodes_path returned a path that exists')
# if this path exists, it must be a link
if os.path.exists(nodeslink) and not os.path.islink(nodeslink):
if nodeslink.exists() and not nodeslink.is_symlink():
raise RuntimeError(
'get_absolute_nodes_path returned a path that exists and '
'is not a link')
......@@ -2265,14 +2259,14 @@ class Network(object):
# this gets created with mode 0700, that's probably ok
mkdir_p(newnodesdir)
try:
os.unlink(nodeslink)
nodeslink.unlink()
except OSError as e:
# it's ok if the link doesn't exist, we're just about to make it
if e.errno == errno.ENOENT:
pass
else:
raise
os.symlink(newnodesdir, nodeslink)
nodeslink.symlink_to(newnodesdir)
def _checkConfig(self):
for n in self._nodes:
......@@ -2654,14 +2648,11 @@ def ConfigureNodes(nodelist):
network._dfltEnv['hasbridgeauth'] = True
def getTests():
tests = []
chutney_path = get_absolute_chutney_path()
if len(chutney_path) > 0 and chutney_path[-1] != '/':
chutney_path += "/"
for x in os.listdir(chutney_path + "scripts/chutney_tests/"):
if not x.startswith("_") and os.path.splitext(x)[1] == ".py":
tests.append(os.path.splitext(x)[0])
return tests
chutney_tests_path = chutney_path / "scripts" / "chutney_tests"
return [test.name for test in chutney_tests_path.glob("*.py")
if not test.name.startswith("_")]
def usage(network):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment