Merge branch 'bug30649_040' into bug30649_maint_041

.gitignore

@@ -168,6 +168,8 @@ uptime-*.json
 /src/lib/libtor-crypt-ops-testing.a
 /src/lib/libtor-ctime.a
 /src/lib/libtor-ctime-testing.a
+/src/lib/libtor-dispatch.a
+/src/lib/libtor-dispatch-testing.a
 /src/lib/libtor-encoding.a
 /src/lib/libtor-encoding-testing.a
 /src/lib/libtor-evloop.a
@@ -200,6 +202,8 @@ uptime-*.json
 /src/lib/libtor-osinfo-testing.a
 /src/lib/libtor-process.a
 /src/lib/libtor-process-testing.a
+/src/lib/libtor-pubsub.a
+/src/lib/libtor-pubsub-testing.a
 /src/lib/libtor-sandbox.a
 /src/lib/libtor-sandbox-testing.a
 /src/lib/libtor-string.a

.travis.yml

@@ -42,15 +42,18 @@ matrix:
   ## include creates builds with gcc, linux
   include:
     ## We include a single coverage build with the best options for coverage
-    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
     ## We only want to check these build option combinations once
     ## (they shouldn't vary by compiler or OS)
     ## We run rust and coverage with hardening off, which seems like enough
     # - env: HARDENING_OPTIONS=""
     ## We check asciidoc with distcheck, to make sure we remove doc products
-    - env: DISTCHECK="yes" ASCIIDOC_OPTIONS=""
+    - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes"
+    # We also try running a hardened clang build with chutney on Linux.
+    - env: CHUTNEY="yes" SKIP_MAKE_CHECK="yes" CHUTNEY_ALLOW_FAILURES="2"
+      compiler: clang
     # We clone our stem repo and run `make test-stem`
-    - env: TEST_STEM="yes"
+    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
     ## Check rust online with distcheck, to make sure we remove rust products
     - env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     ## Check disable module dirauth with and without rust
@@ -72,7 +75,7 @@ matrix:
   ## https://docs.travis-ci.com/user/customizing-the-build#matching-jobs-with-allow_failures
   allow_failures:
     ## test-stem sometimes hangs on Travis
-    - env: TEST_STEM="yes"
+    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
 
   exclude:
     ## gcc on OSX is less useful, because the default compiler is clang.
@@ -88,7 +91,7 @@ matrix:
     - compiler: gcc
       os: linux
       ## TOR_RUST_DEPENDENCIES is spelt RUST_DEPENDENCIES in 0.3.2
-      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
+      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
 
 ## (Linux only) Use the latest Linux image (Ubuntu Trusty)
 dist: trusty
@@ -175,6 +178,9 @@ install:
   - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
   ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
   - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
+  ## If we're running chutney, install it.
+  - if [[ "$CHUTNEY" != "" ]]; then git clone --depth 1 https://github.com/torproject/chutney.git ; export CHUTNEY_PATH="$(pwd)/chutney"; fi
+  ## If we're running stem, install it.
   - if [[ "$TEST_STEM" != "" ]]; then git clone --depth 1 https://github.com/torproject/stem.git ; export STEM_SOURCE_DIR=`pwd`/stem; fi
   ##
   ## Finally, list installed package versions
@@ -189,7 +195,9 @@ install:
   - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
   ## Get python version
   - python --version
-  ## run stem tests if they are enabled.
+  ## If we're running chutney, show the chutney commit
+  - if [[ "$CHUTNEY" != "" ]]; then pushd "$CHUTNEY_PATH"; git log -1 ; popd ; fi
+  ## If we're running stem, show the stem version and commit
   - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi
 
 script:
@@ -200,10 +208,12 @@ script:
   - echo "Configure flags are $CONFIGURE_FLAGS"
   - ./configure $CONFIGURE_FLAGS
   ## We run `make check` because that's what https://jenkins.torproject.org does.
-  - if [[ "$DISTCHECK" == "" && "$TEST_STEM" == "" ]]; then make check; fi
-  ## Diagnostic for bug 29437: kill stem if it hangs for 15 minutes
-  - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --log notice --target RUN_ALL; fi
-  - if [[ "$DISTCHECK" != "" && "$TEST_STEM" == "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
+  - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then make check; fi
+  - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
+  - if [[ "$CHUTNEY" != "" ]]; then make test-network-all; fi
+  ## Diagnostic for bug 29437: kill stem if it hangs for 9.5 minutes
+  ## Travis will kill the job after 10 minutes with no output
+  - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --test control.controller --test control.base_controller --test process --log TRACE --log-file stem.log; fi
   ## If this build was one that produced coverage, upload it.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p' || echo "Coverage failed"; fi
 
@@ -212,11 +222,13 @@ after_failure:
   ## But the log is too long for travis' rendered view, so tail it.
   - tail -1000 config.log || echo "tail failed"
   ## `make check` will leave a log file with more details of test failures.
-  - if [[ "$DISTCHECK" == "" ]]; then cat test-suite.log || echo "cat failed"; fi
+  - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then cat test-suite.log || echo "cat failed"; fi
   ## `make distcheck` puts it somewhere different.
   - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-testlog || echo "make failed"; fi
   - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-core || echo "make failed"; fi
-  - if [[ "$TEST_STEM" != "" ]]; then cat "$STEM_SOURCE_DIR"/test/data/tor_log || echo "cat failed"; fi
+  - if [[ "$CHUTNEY" != "" ]]; then ls test_network_log || echo "ls failed"; cat test_network_log/* || echo "cat failed"; fi
+  - if [[ "$TEST_STEM" != "" ]]; then tail -1000 "$STEM_SOURCE_DIR"/test/data/tor_log || echo "tail failed"; fi
+  - if [[ "$TEST_STEM" != "" ]]; then grep -v "SocketClosed" stem.log | tail -1000 || echo "grep | tail failed"; fi
 
 before_cache:
   ## Delete all gcov files.

Makefile.am

@@ -41,6 +41,8 @@ TOR_UTIL_LIBS = \
         src/lib/libtor-geoip.a \
 	src/lib/libtor-process.a \
         src/lib/libtor-buf.a \
+	src/lib/libtor-pubsub.a \
+	src/lib/libtor-dispatch.a \
 	src/lib/libtor-time.a \
 	src/lib/libtor-fs.a \
 	src/lib/libtor-encoding.a \
@@ -72,6 +74,8 @@ TOR_UTIL_TESTING_LIBS = \
         src/lib/libtor-geoip-testing.a \
 	src/lib/libtor-process-testing.a \
         src/lib/libtor-buf-testing.a \
+	src/lib/libtor-pubsub-testing.a \
+	src/lib/libtor-dispatch-testing.a \
 	src/lib/libtor-time-testing.a \
 	src/lib/libtor-fs-testing.a \
 	src/lib/libtor-encoding-testing.a \
@@ -161,7 +165,12 @@ EXTRA_DIST+= \
 	README						\
 	ReleaseNotes					\
 	scripts/maint/checkIncludes.py                  \
-	scripts/maint/checkSpace.pl
+	scripts/maint/checkSpace.pl 			\
+	scripts/maint/practracker/exceptions.txt	\
+	scripts/maint/practracker/metrics.py		\
+	scripts/maint/practracker/practracker.py	\
+	scripts/maint/practracker/problem.py		\
+	scripts/maint/practracker/util.py
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -328,11 +337,8 @@ coverage-html-full: all
 	lcov --remove "$(HTML_COVER_DIR)/lcov.tmp" --rc lcov_branch_coverage=1 'test/*' 'ext/tinytest*' '/usr/*' --output-file "$(HTML_COVER_DIR)/lcov.info"
 	genhtml --branch-coverage -o "$(HTML_COVER_DIR)" "$(HTML_COVER_DIR)/lcov.info"
 
-# Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
-# tinytest*.[ch]
-check-spaces:
-if USE_PERL
-	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+# For scripts: avoid src/ext and src/trunnel.
+OWNED_TOR_C_FILES=\
 		$(top_srcdir)/src/lib/*/*.[ch] \
 		$(top_srcdir)/src/core/*/*.[ch] \
 		$(top_srcdir)/src/feature/*/*.[ch] \
@@ -340,6 +346,11 @@ if USE_PERL
 		$(top_srcdir)/src/test/*.[ch] \
 		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+
+check-spaces:
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+		$(OWNED_TOR_C_FILES)
 endif
 
 check-includes:
@@ -347,6 +358,14 @@ if USEPYTHON
 	$(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py
 endif
 
+check-best-practices:
+if USEPYTHON
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir)
+endif
+
+practracker-regen:
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py --regen $(top_srcdir)
+
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
 
@@ -442,6 +461,25 @@ version:
 	   (cd "$(top_srcdir)" && git rev-parse --short=16 HEAD); \
 	fi
 
+.PHONY: autostyle-ifdefs
+autostyle-ifdefs:
+	$(PYTHON) scripts/maint/annotate_ifdef_directives $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle-ifdefs
+autostyle-operators:
+	$(PERL) scripts/coccinelle/test-operator-cleanup $(OWNED_TOR_C_FILES)
+
+.PHONY: rectify-includes
+rectify-includes:
+	$(PYTHON) scripts/maint/rectify_include_paths.py
+
+.PHONY: update-copyright
+update-copyright:
+	$(PERL) scripts/maint/updateCopyright.pl $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle
+autostyle: update-versions rustfmt autostyle-ifdefs rectify-includes
+
 mostlyclean-local:
 	rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
 	rm -rf $(HTML_COVER_DIR)

autogen.sh

 #!/bin/sh
 
-if [ -x "`which autoreconf 2>/dev/null`" ] ; then
+if command -v autoreconf; then
   opt="-i -f -W all,error"
 
-  for i in $@; do
+  for i in "$@"; do
     case "$i" in
       -v)
         opt="${opt} -v"
@@ -11,6 +11,7 @@ if [ -x "`which autoreconf 2>/dev/null`" ] ; then
     esac
   done
 
+  # shellcheck disable=SC2086
   exec autoreconf $opt
 fi
 

changes/29241_diagnostic

-  o Minor features (NSS, diagnostic):
-    - Try to log an error from NSS (if there is any) and a more useful
-      description of our situation if we are using NSS and a call to
-      SSL_ExportKeyingMaterial() fails.  Diagnostic for ticket 29241.

changes/bug13221

-  o Minor bugfixes (logging):
-    - Correct a misleading error message when IPv4Only or IPv6Only
-      is used but the resolved address can not be interpreted as an
-      address of the specified IP version.  Fixes bug 13221; bugfix
-      on 0.2.3.9-alpha.  Patch from Kris Katterjohn.

changes/bug22619

+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.

changes/bug23507

+  o Minor bugfixes (v3 single onion services):
+    - Make v3 single onion services fall back to a 3-hop intro, when there
+      all intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.

changes/bug23818_v2

+  o Minor bugfixes (v2 single onion services):
+    - Always retry v2 single onion service intro and rend circuits with a
+      3-hop path. Previously, v2 single onion services used a 3-hop path
+      when rend circuits were retried after a remote or delayed failure,
+      but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.2.9.3-alpha.

changes/bug23818_v3

+  o Minor bugfixes (v3 single onion services):
+    - Always retry v3 single onion service intro and rend circuits with a
+      3-hop path. Previously, v3 single onion services used a 3-hop path
+      when rend circuits were retried after a remote or delayed failure,
+      but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.

changes/bug27199

-  o Minor bugfixes (rust):
-    - Abort on panic in all build profiles, instead of potentially unwinding
-      into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.

changes/bug28525

-  o Minor features (address selection):
-    - Make Tor aware of the RFC 6598 (Carrier Grade NAT) IP range, which is the
-      subnet 100.64.0.0/10. This is deployed by many ISPs as an alternative to
-      RFC 1918 that does not break existing internal networks. This patch fixes
-      security issues caused by RFC 6518 by blocking control ports on these
-      addresses and warns users if client ports or ExtORPorts are listening on
-      a RFC 6598 address. Closes ticket 28525. Patch by Neel Chauhan.

changes/bug28614_better_logging

-  o Minor bugfixes (logging):
-    - On Windows, when errors cause us to reload a consensus from disk, tell
-      the user that we are retrying at log level "notice". Previously we only
-      logged this information at "info", which was confusing because the
-      errors themselves were logged at "warning". Improves previous fix for
-      28614.  Fixes bug 30004; bugfix on 0.4.0.2-alpha.

changes/bug28656

-  o Minor bugfixes (logging):
-    - Stop logging a BUG() warning when tor is waiting for exit descriptors.
-      Fixes bug 28656; bugfix on 0.3.5.1-alpha.

changes/bug28698

-  o Minor bugfix (logging):
-    - Avoid logging about relaxing circuits when their time is fixed.
-      Fixes bug 28698; bugfix on 0.2.4.7-alpha

changes/bug28925

-  o Minor bugfixes (bootstrap reporting):
-    - During bootstrap reporting, correctly distinguish pluggable
-      transports from plain proxies. Fixes bug 28925; bugfix on
-      0.4.0.1-alpha.

changes/bug28979

-  o Minor bugfixes (documentation):
-    - Describe the contents of the v3 onion service client authorization
-      files correctly: They hold public keys, not private keys. Fixes bug
-      28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".

changes/bug28981

-  o Minor bugfixes (misc):
-    - The amount of total available physical memory is now determined
-      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
-      when it is defined and a 64-bit variant is not available.  Fixes
-      bug 28981; bugfix on 0.2.5.4-alpha.  Patch from Kris Katterjohn.