From a9733a32d35205274d6e808cd846688576ab0b6a Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Sat, 24 Feb 2018 02:16:28 +0000 Subject: [PATCH 0001/1724] build: Revert "Use -lresolv in LIBS with rust on OSX." This reverts commit 2ac9734bd21f3eecb20560cde5c4037230029157. * FIXES #25341: https://bugs.torproject.org/25341 Signed-off-by: Isis Lovecruft --- Makefile.am | 3 +-- configure.ac | 11 ----------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Makefile.am b/Makefile.am index 04ca88a236..02ccbd91b5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,8 +26,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif if USE_RUST -rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ \ - @TOR_RUST_EXTRA_LIBS@ +rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_UTIL_STATIC_NAME@ else rust_ldadd= endif diff --git a/configure.ac b/configure.ac index 22c8a56137..d00e1f74be 100644 --- a/configure.ac +++ b/configure.ac @@ -468,15 +468,6 @@ if test "x$enable_rust" = "xyes"; then fi fi - dnl This is a workaround for #46797 - dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ). Once the - dnl upstream bug is fixed, we can remove this workaround. - case "$host_os" in - darwin*) - TOR_RUST_EXTRA_LIBS="-lresolv" - ;; - esac - dnl For now both MSVC and MinGW rust libraries will output static libs with dnl the MSVC naming convention. if test "$bwin32" = "true"; then @@ -503,8 +494,6 @@ if test "x$enable_rust" = "xyes"; then AC_MSG_RESULT([$RUSTC_VERSION]) fi -AC_SUBST(TOR_RUST_EXTRA_LIBS) - AC_SEARCH_LIBS(socket, [socket network]) AC_SEARCH_LIBS(gethostbyname, [nsl]) AC_SEARCH_LIBS(dlopen, [dl]) -- GitLab From af3e5bed1084fa321e90eaf87e5dc9a7256dd920 Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Sat, 24 Feb 2018 02:17:36 +0000 Subject: [PATCH 0002/1724] build: Fix Makefile.am TOR_RUST_STATIC_NAME variable name after revert. * FIXES #25341: https://bugs.torproject.org/25341 --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 02ccbd91b5..2c6de35743 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,7 +26,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif if USE_RUST -rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_UTIL_STATIC_NAME@ +rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ else rust_ldadd= endif -- GitLab From 306ab8ae17b8f39a6b83bcb24fe02afd8389dc13 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 25 Jun 2018 14:52:40 -0500 Subject: [PATCH 0003/1724] Update fallback lists to match #24805 --- scripts/maint/fallback.blacklist | 3 +++ scripts/maint/fallback.whitelist | 40 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist index a118cb5919..52ef4f0ec5 100644 --- a/scripts/maint/fallback.blacklist +++ b/scripts/maint/fallback.blacklist @@ -230,3 +230,6 @@ # Email sent directly to teor 212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110 + +# Was moved from blacklist to whitelist by mistake (https://trac.torproject.org/projects/tor/ticket/24805#comment:6) +85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index e9158e1280..819fa48f46 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -56,7 +56,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional emails to teor with updated relays @@ -295,9 +294,6 @@ # Email sent directly to teor, verified using relay contact info 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - # Email sent directly to teor, verified using relay contact info 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 @@ -377,10 +373,6 @@ # Email sent directly to teor, verified using relay contact info 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - # Email sent directly to teor, verified using relay contact info 178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 @@ -405,9 +397,6 @@ # Email sent directly to teor, verified using relay contact info 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - # Email sent directly to teor, verified using relay contact info 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D @@ -563,7 +552,6 @@ 185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B 78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 # Email sent directly to teor, verified using relay contact info @@ -684,6 +672,7 @@ # Email sent directly to teor, verified using relay contact info # Assume details update is permanent 188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik +129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee # Email sent directly to teor, verified using relay contact info 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 @@ -695,8 +684,8 @@ # Email sent directly to teor, verified using relay contact info 176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C 176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F +94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 # Email sent directly to teor, verified using relay contact info 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA @@ -743,7 +732,8 @@ 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF # Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 +# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 +94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 # Email sent directly to teor, verified using relay contact info 173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 @@ -942,10 +932,13 @@ 199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html 5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 +54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 +54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html 104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor @@ -959,16 +952,10 @@ # Email sent directly to teor 62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - # https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html 51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - # https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html 24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia @@ -994,3 +981,16 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html 82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html +37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html +87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html +185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html +51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay + -- GitLab From 23d9ffcdf4877f574278b0f0b39ce511072898af Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 28 Jun 2018 15:22:11 -0500 Subject: [PATCH 0004/1724] Update whitelist to match feedback from operators --- scripts/maint/fallback.whitelist | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 819fa48f46..0cca5f29cb 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -319,10 +319,14 @@ 185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 +185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 +185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 + # Email sent directly to teor, verified using relay contact info 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 @@ -752,6 +756,7 @@ # Email sent directly to teor, verified using relay contact info 5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 +5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 # Email sent directly to teor, verified using relay contact info 46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 @@ -823,9 +828,12 @@ 195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F +31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 + # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 @@ -994,3 +1002,26 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html 51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay +# Email sent directly to Phoul +185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn + +# Email sent directly to Phoul +78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 + +# Email sent directly to Phoul +5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel +64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html +132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html +96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 + +# Email sent directly to Phoul +163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 +163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 + +# Email sent directly to Phoul +158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 + -- GitLab From d7e7c0d20d15d90882bc0b963f897a99b5d7585a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:14:20 -0500 Subject: [PATCH 0005/1724] Updating whitelist to match operator feedback --- scripts/maint/fallback.whitelist | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 0cca5f29cb..1ea3874451 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1025,3 +1025,17 @@ # Email sent directly to Phoul 158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 +# Email sent directly to Phoul +45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 + +# Email sent directly to Phoul +51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E +217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html +195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 +51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 +163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 +51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 +54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 +51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 -- GitLab From cbdf6a04b10a27721b51df45daf4e3d8cd084287 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:17:22 -0500 Subject: [PATCH 0006/1724] Fixing spacing issue in whitelist --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1ea3874451..7097d8911a 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1038,4 +1038,4 @@ 163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 -51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 +51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 -- GitLab From 855feab70d35dd5093aae1f27d6cf19ee6cb3338 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 13 Jul 2018 13:17:42 -0500 Subject: [PATCH 0007/1724] Adding 3 more fallback mirrors --- scripts/maint/fallback.whitelist | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 7097d8911a..e60b29304c 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1039,3 +1039,12 @@ 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 + +# Email sent directly to Phoul +54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC + +# Email sent directly to Phoul +67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 + +# Email sent directly to Phoul +195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F -- GitLab From 54cf36400669c555a3af84b167fd79f19b9ebbe3 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 16 Jul 2018 12:21:46 -0500 Subject: [PATCH 0008/1724] Add mirrors from EmeraldOnion --- scripts/maint/fallback.whitelist | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index e60b29304c..7d9d136780 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -12,7 +12,8 @@ # To replace this list with the hard-coded fallback list (for testing), use # a command similar to: -# cat src/or/fallback_dirs.inc | grep \" | grep -v weight | tr -d '\n' | \ +# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ +# tr -d '\n' | \ # sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ # > scripts/maint/fallback.whitelist # @@ -1048,3 +1049,8 @@ # Email sent directly to Phoul 195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F + +# Email sent directly to Phoul +23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 +23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 +23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 -- GitLab From 04de9443fef50a27036dd2e8550fde87731d2adc Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 25 Jun 2018 14:52:40 -0500 Subject: [PATCH 0009/1724] Update fallback lists to match #24805 --- scripts/maint/fallback.whitelist | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 79551948c6..ff7eadda88 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -57,7 +57,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional emails to teor with updated relays @@ -296,9 +295,6 @@ # Email sent directly to teor, verified using relay contact info 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - # Email sent directly to teor, verified using relay contact info 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 @@ -378,10 +374,6 @@ # Email sent directly to teor, verified using relay contact info 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - # Email sent directly to teor, verified using relay contact info 178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 @@ -406,9 +398,6 @@ # Email sent directly to teor, verified using relay contact info 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - # Email sent directly to teor, verified using relay contact info 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D @@ -564,7 +553,6 @@ 185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B 78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 # Email sent directly to teor, verified using relay contact info @@ -685,6 +673,7 @@ # Email sent directly to teor, verified using relay contact info # Assume details update is permanent 188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik +129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee # Email sent directly to teor, verified using relay contact info 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 @@ -696,8 +685,8 @@ # Email sent directly to teor, verified using relay contact info 176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C 176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F +94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 # Email sent directly to teor, verified using relay contact info 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA @@ -744,7 +733,8 @@ 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF # Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 +# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 +94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 # Email sent directly to teor, verified using relay contact info 173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 @@ -943,10 +933,13 @@ 199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html 5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 +54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 +54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html 104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor @@ -960,16 +953,10 @@ # Email sent directly to teor 62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - # https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html 51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - # https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html 24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia @@ -995,3 +982,16 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html 82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html +37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html +87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html +185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html +51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay + -- GitLab From 7479f75b95eaee2003a8bc9bb8ba6465cd8f9754 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 28 Jun 2018 15:22:11 -0500 Subject: [PATCH 0010/1724] Update whitelist to match feedback from operators --- scripts/maint/fallback.whitelist | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index ff7eadda88..1077a0054b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -320,10 +320,14 @@ 185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 +185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 +185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 + # Email sent directly to teor, verified using relay contact info 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 @@ -753,6 +757,7 @@ # Email sent directly to teor, verified using relay contact info 5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 +5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 # Email sent directly to teor, verified using relay contact info 46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 @@ -824,9 +829,12 @@ 195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F +31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 + # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 @@ -995,3 +1003,26 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html 51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay +# Email sent directly to Phoul +185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn + +# Email sent directly to Phoul +78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 + +# Email sent directly to Phoul +5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel +64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html +132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html +96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 + +# Email sent directly to Phoul +163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 +163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 + +# Email sent directly to Phoul +158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 + -- GitLab From eca58aec76ce87ea8eb56a8f7689945f79e0f319 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:14:20 -0500 Subject: [PATCH 0011/1724] Updating whitelist to match operator feedback --- scripts/maint/fallback.whitelist | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1077a0054b..0560d49108 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1026,3 +1026,17 @@ # Email sent directly to Phoul 158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 +# Email sent directly to Phoul +45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 + +# Email sent directly to Phoul +51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E +217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html +195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 +51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 +163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 +51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 +54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 +51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 -- GitLab From 226816df558ccfc92b106bfefe129604448cd43a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:17:22 -0500 Subject: [PATCH 0012/1724] Fixing spacing issue in whitelist --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 0560d49108..30c5b3f707 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1039,4 +1039,4 @@ 163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 -51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 +51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 -- GitLab From 831abc0e10ba19d7e33cdee8e328d39a1d849496 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 13 Jul 2018 13:17:42 -0500 Subject: [PATCH 0013/1724] Adding 3 more fallback mirrors --- scripts/maint/fallback.whitelist | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 30c5b3f707..53458ef6c0 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1040,3 +1040,12 @@ 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 + +# Email sent directly to Phoul +54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC + +# Email sent directly to Phoul +67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 + +# Email sent directly to Phoul +195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F -- GitLab From 5c2431a3436506862b2148a43821bf22f33c7269 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 16 Jul 2018 12:21:46 -0500 Subject: [PATCH 0014/1724] Add mirrors from EmeraldOnion --- scripts/maint/fallback.whitelist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 53458ef6c0..7d9d136780 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1049,3 +1049,8 @@ # Email sent directly to Phoul 195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F + +# Email sent directly to Phoul +23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 +23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 +23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 -- GitLab From 0260b720053717cdc348a51ede1d7e51e7ea2741 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 3 Aug 2018 09:49:03 -0500 Subject: [PATCH 0015/1724] Adding relay from Edmond --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 7d9d136780..20751b6e47 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1054,3 +1054,7 @@ 23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 + +# Email sent directly to Phoul +37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 + -- GitLab From 9aceb928392ef8f3e2f737c14702a9cc330bd109 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 10 Aug 2018 15:19:03 -0500 Subject: [PATCH 0016/1724] Adding 2 new fallback mirrors --- scripts/maint/fallback.whitelist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 20751b6e47..1b27cf7d29 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1058,3 +1058,8 @@ # Email sent directly to Phoul 37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 +# Email sent directly to Phoul +178.254.7.88:9030 orpport=9001 id=85A885433E50B1874F11CEC9BE98451E24660976 + +# https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html +5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 -- GitLab From 51f8db1d92d64331d000d2847edb8b66e9fb4c0c Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 16 Aug 2018 14:32:23 -0500 Subject: [PATCH 0017/1724] Drop relays from fallback list that were terminated by provider --- scripts/maint/fallback.whitelist | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1b27cf7d29..cd38e7f1fa 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -832,9 +832,6 @@ # Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 -31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F -31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 - # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 -- GitLab From f0792537e9cf25e76b5cab16635de7b78bfbf8b7 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 28 Aug 2018 13:05:59 -0500 Subject: [PATCH 0018/1724] Adding new fallback --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index cd38e7f1fa..375ac8f748 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1060,3 +1060,7 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html 5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 + +# https://trac.torproject.org/projects/tor/ticket/27297 +37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 + -- GitLab From 23ed863da4f66d9e07f578118f328b9f1898fe52 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 18:55:36 +0300 Subject: [PATCH 0019/1724] Improve bracket handling in tor_addr_parse() * Actually check for second bracket * Only attempt parsing IPv4 address when no brackets found --- src/lib/net/address.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 619fa13e9b..14c086a5b0 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1187,14 +1187,19 @@ tor_addr_parse(tor_addr_t *addr, const char *src) int result; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + if (src[0] == '[' && src[1] && src[strlen(src)-1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + } else if (!brackets_detected && + tor_inet_pton(AF_INET, src, &in_tmp) > 0) { result = AF_INET; tor_addr_from_in(addr, &in_tmp); } else { -- GitLab From feae813e1bf32e7fc35a14d88ece4a9457cfd172 Mon Sep 17 00:00:00 2001 From: David Fifield Date: Wed, 2 Aug 2017 02:04:35 -0700 Subject: [PATCH 0020/1724] Add tests for tor_addr_parse, separate from tor_addr_port_parse. --- src/test/test_addr.c | 56 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index a9004048a5..1d97db52a6 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -723,7 +723,7 @@ test_addr_ip6_helpers(void *arg) ; } -/** Test tor_addr_port_parse(). */ +/** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { @@ -734,6 +734,60 @@ test_addr_parse(void *arg) /* Correct call. */ (void)arg; + r= tor_addr_parse(&addr, "192.0.2.1"); + tt_int_op(r,OP_EQ, AF_INET); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "192.0.2.1"); + + r= tor_addr_parse(&addr, "11:22::33:44"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "[11:22::33:44]"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + + r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + + /* Empty string. */ + r= tor_addr_parse(&addr, ""); + tt_int_op(r,OP_EQ, -1); + + /* Square brackets around IPv4 address. */ + r= tor_addr_parse(&addr, "[192.0.2.1]"); + tt_int_op(r,OP_EQ, -1); + + /* Only left square bracket. */ + r= tor_addr_parse(&addr, "[11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Only right square bracket. */ + r= tor_addr_parse(&addr, "11:22::33:44]"); + tt_int_op(r,OP_EQ, -1); + + /* Leading colon. */ + r= tor_addr_parse(&addr, ":11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Trailing colon. */ + r= tor_addr_parse(&addr, "11:22::33:44:"); + tt_int_op(r,OP_EQ, -1); + + /* Too many hex words in IPv4-mapped IPv6 address. */ + r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); + tt_int_op(r,OP_EQ, -1); + + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", &addr, &port, -1); -- GitLab From 01eb164574bdcc938509810634dfb0348c484662 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 19:34:14 +0300 Subject: [PATCH 0021/1724] Reject addresses with needless trailing colon --- src/lib/net/address.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 14c086a5b0..336693b464 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1189,13 +1189,19 @@ tor_addr_parse(tor_addr_t *addr, const char *src) struct in6_addr in6_tmp; int brackets_detected = 0; + size_t len = strlen(src); + tor_assert(addr && src); - if (src[0] == '[' && src[1] && src[strlen(src)-1] == ']') { + if (src[0] == '[' && src[1] && src[len - 1] == ']') { brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + len -= 2; } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + result = -1; + } else if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); } else if (!brackets_detected && -- GitLab From 1baa2703c36cf964d5fc1f07433196e8ccc0e55a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 20:05:54 +0300 Subject: [PATCH 0022/1724] Add changes file --- changes/bug23082 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug23082 diff --git a/changes/bug23082 b/changes/bug23082 new file mode 100644 index 0000000000..fc4b52c364 --- /dev/null +++ b/changes/bug23082 @@ -0,0 +1,4 @@ + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to + reject certain incorrect inputs that previously were + not detected. Fixes bug 23082; bugfix on 0.2.0.10-alpha. -- GitLab From 5595b212270215eaa020603cabbe2c7b3b34d624 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 09:38:20 -0400 Subject: [PATCH 0023/1724] Consdiff: use lengths on inputs so they don't need NUL at the end This is part of #27244, so that we can safely mmap consensus documents. --- src/feature/dircache/consdiffmgr.c | 9 ++- src/feature/dircache/directory.c | 10 +++- src/feature/dircommon/consdiff.c | 42 +++++++------ src/feature/dircommon/consdiff.h | 15 ++--- src/feature/nodelist/routerparse.c | 13 +++-- src/feature/nodelist/routerparse.h | 4 +- src/test/bench.c | 6 +- src/test/fuzz/fuzz_diff.c | 15 +++-- src/test/fuzz/fuzz_diff_apply.c | 13 ++--- src/test/test_consdiff.c | 94 ++++++++++++++++++++---------- src/test/test_consdiffmgr.c | 25 ++++++-- 11 files changed, 156 insertions(+), 90 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 304b64f3b6..7999df08d5 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1496,7 +1496,10 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? - consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); + consensus_diff = consensus_diff_generate(diff_from_nt, + strlen(diff_from_nt), + diff_to_nt, + strlen(diff_to_nt)); tor_free(diff_from_nt); tor_free(diff_to_nt); } @@ -1746,8 +1749,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) (const uint8_t *)consensus, bodylen); { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen, + &start, &end) < 0) { start = consensus; end = consensus+bodylen; } diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c index de0bcdbfa7..8e5fc86836 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dircache/directory.c @@ -2607,12 +2607,17 @@ handle_response_fetch_consensus(dir_connection_t *conn, /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); const char *consensus_body; + size_t consensus_body_len; char *owned_consensus = NULL; if (cd) { consensus_body = cd->dir; + consensus_body_len = cd->dir_len; } else { owned_consensus = networkstatus_read_cached_consensus(flavname); - consensus_body = owned_consensus; + if (owned_consensus) { + consensus_body = owned_consensus; + consensus_body_len = strlen(consensus_body); + } } if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " @@ -2622,7 +2627,8 @@ handle_response_fetch_consensus(dir_connection_t *conn, return -1; } - new_consensus = consensus_diff_apply(consensus_body, body); + new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, + body, body_len); tor_free(owned_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index 1823dc07fb..c67a118125 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest,(const char *cons, +consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)) { int r = crypto_digest256((char*)digest_out->sha3_256, - cons, strlen(cons), DIGEST_SHA3_256); + cons, len, DIGEST_SHA3_256); return r; } @@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons, /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest_as_signed,(const char *cons, +consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)) { return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, - cons); + cons, len); } /** Return true iff d1 and d2 contain the same digest */ @@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1, cons2_str = consensus_join_lines(cons2); consensus_digest_t cons2_digests; - if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + if (consensus_compute_digest(cons2_str, strlen(cons2_str), + &cons2_digests) < 0) { /* LCOV_EXCL_START -- digest can't fail */ log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " "resulting from applying a consensus diff."); @@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1, * generated cdlines will become invalid. */ STATIC int -consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +consensus_split_lines(smartlist_t *out, + const char *s, size_t len, + memarea_t *area) { - const char *end_of_str = s + strlen(s); - tor_assert(*end_of_str == '\0'); + const char *end_of_str = s + len; - while (*s) { + while (s < end_of_str) { const char *eol = memchr(s, '\n', end_of_str - s); if (!eol) { /* File doesn't end with newline. */ @@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp) * success, retun a newly allocated string containing that diff. On failure, * return NULL. */ char * -consensus_diff_generate(const char *cons1, - const char *cons2) +consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len) { consensus_digest_t d1, d2; smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; int r1, r2; char *result = NULL; - r1 = consensus_compute_digest_as_signed(cons1, &d1); - r2 = consensus_compute_digest(cons2, &d2); + r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1); + r2 = consensus_compute_digest(cons2, cons2len, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE memarea_t *area = memarea_new(); lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, cons1, area) < 0) + if (consensus_split_lines(lines1, cons1, cons1len, area) < 0) goto done; - if (consensus_split_lines(lines2, cons2, area) < 0) + if (consensus_split_lines(lines2, cons2, cons2len, area) < 0) goto done; result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); @@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1, * consensus. On failure, return NULL. */ char * consensus_diff_apply(const char *consensus, - const char *diff) + size_t consensus_len, + const char *diff, + size_t diff_len) { consensus_digest_t d1; smartlist_t *lines1 = NULL, *lines2 = NULL; @@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest_as_signed(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1); if (BUG(r1 < 0)) return NULL; // LCOV_EXCL_LINE lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, consensus, area) < 0) + if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0) goto done; - if (consensus_split_lines(lines2, diff, area) < 0) + if (consensus_split_lines(lines2, diff, diff_len, area) < 0) goto done; result = consdiff_apply_diff(lines1, lines2, &d1); diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index a5e4ba5cbf..eb7c9f9fe0 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -7,10 +7,10 @@ #include "core/or/or.h" -char *consensus_diff_generate(const char *cons1, - const char *cons2); -char *consensus_diff_apply(const char *consensus, - const char *diff); +char *consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len); +char *consensus_diff_apply(const char *consensus, size_t consensus_len, + const char *diff, size_t diff_len); int looks_like_a_consensus_diff(const char *document, size_t len); @@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, const smartlist_slice_t *slice1, const smartlist_slice_t *slice2); -STATIC int consensus_split_lines(smartlist_t *out, const char *s, +STATIC int consensus_split_lines(smartlist_t *out, + const char *s, size_t len, struct memarea_t *area); STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, const char *s); @@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b); STATIC int line_str_eq(const cdline_t *a, const char *b); MOCK_DECL(STATIC int, - consensus_compute_digest,(const char *cons, + consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, - consensus_compute_digest_as_signed,(const char *cons, + consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 73d320de40..9c51799d9b 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -1024,10 +1024,11 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) * -1. */ int router_get_networkstatus_v3_signed_boundaries(const char *s, + size_t len, const char **start_out, const char **end_out) { - return router_get_hash_impl_helper(s, strlen(s), + return router_get_hash_impl_helper(s, len, "network-status-version", "\ndirectory-signature", ' ', LOG_INFO, @@ -1039,12 +1040,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s, * signed portion can be identified. Return 0 on success, -1 on failure. */ int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s) + const char *s, size_t len) { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(s, len, + &start, &end) < 0) { start = s; - end = s + strlen(s); + end = s + len; } tor_assert(start); tor_assert(end); @@ -3415,7 +3417,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, *eos_out = NULL; if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, + s, strlen(s))<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index 87c2a75aa5..be455984d1 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -32,11 +32,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests); -int router_get_networkstatus_v3_signed_boundaries(const char *s, +int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out); int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s); + const char *s, size_t len); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 char *router_get_dirobj_signature(const char *digest, diff --git a/src/test/bench.c b/src/test/bench.c index 959d4374b1..9da1b46a1b 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -702,11 +702,13 @@ main(int argc, const char **argv) perror("X"); return 1; } + size_t f1len = strlen(f1); + size_t f2len = strlen(f2); for (i = 0; i < N; ++i) { - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); tor_free(diff); } - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); printf("%s", diff); tor_free(f1); tor_free(f2); diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 1079856fdb..8966856be2 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -42,14 +44,14 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_generate(c1, c2); + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { - char *c4 = consensus_diff_apply(c1, c3); + char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); if (strcmp(c2, c4)) { printf("%s\n", escaped(c1)); @@ -61,9 +63,6 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) tor_free(c3); tor_free(c4); } - tor_free(c1); - tor_free(c2); return 0; } - diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 165d0e6126..9b25185225 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_apply(c1, c2); + char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len); - tor_free(c1); - tor_free(c2); tor_free(c3); return 0; } - diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index b836befd22..23e8f7167e 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -14,6 +14,39 @@ #define tt_str_eq_line(a,b) \ tt_assert(line_str_eq((b),(a))) +static int +consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area) +{ + size_t len = strlen(s); + return consensus_split_lines(out, s, len, area); +} + +static int +consensus_compute_digest_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest(tmp, len, digest_out); + tor_free(tmp); + return r; +} + +static int +consensus_compute_digest_as_signed_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest_as_signed(tmp, len, digest_out); + tor_free(tmp); + return r; +} + static void test_consdiff_smartlist_slice(void *arg) { @@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg) /* Create a regular smartlist. */ (void)arg; - consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area); /* See that smartlist_slice_string_pos respects the bounds of the slice. */ sls = smartlist_slice(sl, 2, 5); @@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg) int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; (void)arg; - consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); - consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); - consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); - consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); - consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); sls3 = smartlist_slice(sl3, 0, -1); @@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\nb\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); /* Length of sls1 is 0. */ sls1 = smartlist_slice(sl1, 0, 0); @@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\na\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\na\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg) tt_assert(!bitarray_is_set(changed2, 3)); smartlist_clear(sl2); - consensus_split_lines(sl2, "a\nb\na\nb\n", area); + consensus_split_lines_(sl2, "a\nb\na\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg) bitarray_clear(changed1, 3); smartlist_clear(sl2); - consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + consensus_split_lines_(sl2, "b\nb\nb\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg) /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); - consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(7, OP_EQ, smartlist_len(diff)); @@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg) smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "B\n", area); - consensus_split_lines(cons2, "A\nB\n", area); + consensus_split_lines_(cons1, "B\n", area); + consensus_split_lines_(cons2, "A\nB\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(3, OP_EQ, smartlist_len(diff)); @@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg) diff = smartlist_new(); setup_capture_of_logs(LOG_WARN); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); /* Command without range. */ smartlist_add_linecpy(diff, area, "a"); @@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); /* Test appending text, 'a'. */ - consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(8, OP_EQ, smartlist_len(cons2)); @@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test deleting text, 'd'. */ - consensus_split_lines(diff, "4d\n1,2d\n", area); + consensus_split_lines_(diff, "4d\n1,2d\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(2, OP_EQ, smartlist_len(cons2)); @@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); @@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test 'a', 'd' and 'c' together. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(6, OP_EQ, smartlist_len(cons2)); @@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg) ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons2_str, &digests2)); + consensus_compute_digest_(cons2_str, &digests2)); - consensus_split_lines(cons1, cons1_str, area); - consensus_split_lines(cons2, cons2_str, area); + consensus_split_lines_(cons1, cons1_str, area); + consensus_split_lines_(cons2, cons2_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_EQ, diff); @@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); smartlist_clear(cons1); - consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines_(cons1, cons1_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(11, OP_EQ, smartlist_len(diff)); @@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons1_str, &digests1)); - consensus_split_lines(cons1, cons1_str, area); + consensus_compute_digest_(cons1_str, &digests1)); + consensus_split_lines_(cons1, cons1_str, area); /* diff doesn't have enough lines. */ cons2 = consdiff_apply_diff(cons1, diff, &digests1); @@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = { CONSDIFF_LEGACY(apply_diff), END_OF_TESTCASES }; - diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 6c0601b504..dc4fea7f6f 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,21 @@ #include "test/test.h" #include "test/log_test_helpers.h" +static char * +consensus_diff_apply_(const char *c, const char *d) +{ + size_t c_len = strlen(c); + size_t d_len = strlen(d); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + char *c_tmp = tor_memdup(c, c_len); + char *d_tmp = tor_memdup(d, d_len); + char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len); + tor_free(c_tmp); + tor_free(d_tmp); + return result; +} + // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module @@ -153,7 +168,8 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + if (router_get_networkstatus_v3_sha3_as_signed(digest, + str1, strlen(str1))<0) { TT_FAIL(("Unable to compute sha3-as-signed")); return CONSDIFF_NOT_FOUND; } @@ -181,7 +197,7 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply(str1, diff_string); + char *applied = consensus_diff_apply_(str1, diff_string); tor_free(diff_string); if (applied == NULL) return -1; @@ -370,7 +386,8 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body, + strlen(md_ns_body)); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); @@ -414,7 +431,7 @@ test_consdiffmgr_make_diffs(void *arg) r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); - applied = consensus_diff_apply(md_ns_body, diff_text); + applied = consensus_diff_apply_(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); -- GitLab From e014b72b73b2a299068f1ca5b7a22f2bea2f58f8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 10:09:12 -0400 Subject: [PATCH 0024/1724] Stop memcpy'ing uncompressed consensuses when making diffs --- src/feature/dircache/consdiffmgr.c | 50 ++++++++++++++++++------------ src/feature/dircache/consdiffmgr.h | 5 +-- src/test/fuzz/fuzz_diff.c | 17 +++++++--- src/test/test_consdiffmgr.c | 16 +++++----- 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 7999df08d5..bf3a0ef3cf 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1387,19 +1387,21 @@ typedef struct consensus_diff_worker_job_t { } consensus_diff_worker_job_t; /** Given a consensus_cache_entry_t, check whether it has a label claiming - * that it was compressed. If so, uncompress its contents into out and - * set outlen to hold their size. If not, just copy the body into - * out and set outlen to its length. Return 0 on success, - * -1 on failure. - * - * In all cases, the output is nul-terminated. */ + * that it was compressed. If so, uncompress its contents into *out and + * set outlen to hold their size, and set *owned_out to a pointer + * that the caller will need to free. If not, just set *out and + * outlen to its extent in memory. Return 0 on success, -1 on failure. + **/ STATIC int -uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent) +uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent) { const uint8_t *body; size_t bodylen; + *owned_out = NULL; + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) return -1; @@ -1410,8 +1412,17 @@ uncompress_or_copy(char **out, size_t *outlen, if (lv_compression) method = compression_method_get_by_name(lv_compression); - return tor_uncompress(out, outlen, (const char *)body, bodylen, + int rv; + if (method == NO_METHOD) { + *out = (const char *)body; + *outlen = bodylen; + rv = 0; + } else { + rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen, method, 1, LOG_WARN); + *out = *owned_out; + } + return rv; } /** @@ -1478,16 +1489,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) char *consensus_diff; { - char *diff_from_nt = NULL, *diff_to_nt = NULL; + const char *diff_from_nt = NULL, *diff_to_nt = NULL; + char *owned1 = NULL, *owned2 = NULL; size_t diff_from_nt_len, diff_to_nt_len; - if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, - job->diff_from) < 0) { + if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1, + job->diff_from) < 0) { return WQ_RPL_REPLY; } - if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, - job->diff_to) < 0) { - tor_free(diff_from_nt); + if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2, + job->diff_to) < 0) { + tor_free(owned1); return WQ_RPL_REPLY; } tor_assert(diff_from_nt); @@ -1497,11 +1509,11 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? consensus_diff = consensus_diff_generate(diff_from_nt, - strlen(diff_from_nt), + diff_from_nt_len, diff_to_nt, - strlen(diff_to_nt)); - tor_free(diff_from_nt); - tor_free(diff_to_nt); + diff_to_nt_len); + tor_free(owned1); + tor_free(owned2); } if (!consensus_diff) { /* Couldn't generate consensus; we'll leave the reply blank. */ diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 66c3d65002..d6f273cc4e 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -68,8 +68,9 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); -STATIC int uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent); +STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 8966856be2..64aecc8a64 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -48,18 +48,27 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) size_t c2_len = data_size - c1_len - SEPLEN; const char *c2 = (const char *)separator + SEPLEN; + const char *cp = memchr(c1, 0, c1_len); + if (cp) + c1_len = cp - c1; + + cp = memchr(c2, 0, c2_len); + if (cp) + c2_len = cp - c2; + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); - if (strcmp(c2, c4)) { - printf("%s\n", escaped(c1)); - printf("%s\n", escaped(c2)); + int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len); + if (! equal) { + //printf("%s\n", escaped(c1)); + //printf("%s\n", escaped(c2)); printf("%s\n", escaped(c3)); printf("%s\n", escaped(c4)); } - tor_assert(! strcmp(c2, c4)); + tor_assert(equal); tor_free(c3); tor_free(c4); } diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index dc4fea7f6f..4b49fdb6aa 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -191,14 +191,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, consensus_cache_entry_incref(ent); size_t size; - char *diff_string = NULL; - int r = uncompress_or_copy(&diff_string, &size, ent); + const char *diff_string = NULL; + char *diff_owned = NULL; + int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent); consensus_cache_entry_decref(ent); if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply_(str1, diff_string); - tor_free(diff_string); + char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size); + tor_free(diff_owned); if (applied == NULL) return -1; @@ -298,7 +299,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time(); - char *body = NULL; + const char *body = NULL; + char *body_owned = NULL; consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); @@ -340,7 +342,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - r = uncompress_or_copy(&body, &s, ent); + r = uncompress_or_set_ptr(&body, &s, &body_owned, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -353,7 +355,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); - tor_free(body); + tor_free(body_owned); } static void -- GitLab From abaca3fc8c6bc54408084e7514468fa2cd7b3edf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 10:32:17 -0400 Subject: [PATCH 0025/1724] Revise networkstatus parsing code to use lengths This way the networkstatus can be parsed without being NUL-terminated, so we can implement 27244 and mmap our consensus objects. --- src/feature/dirauth/dirvote.c | 15 ++++++---- src/feature/nodelist/networkstatus.c | 4 ++- src/feature/nodelist/routerparse.c | 44 ++++++++++++++++------------ src/feature/nodelist/routerparse.h | 6 ++-- src/test/fuzz/fuzz_consensus.c | 6 ++-- src/test/fuzz/fuzz_vrs.c | 16 +++++----- src/test/test_dir.c | 32 +++++++++++++++----- src/test/test_dir_common.c | 5 ++-- src/test/test_routerlist.c | 4 ++- 9 files changed, 85 insertions(+), 47 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 14357c770e..42821760c5 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -401,7 +401,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, { networkstatus_t *v; - if (!(v = networkstatus_parse_vote_from_string(status, NULL, + if (!(v = networkstatus_parse_vote_from_string(status, strlen(status), + NULL, v3_ns->type))) { log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: " "<<%s>>", @@ -2398,7 +2399,8 @@ networkstatus_compute_consensus(smartlist_t *votes, { networkstatus_t *c; - if (!(c = networkstatus_parse_vote_from_string(result, NULL, + if (!(c = networkstatus_parse_vote_from_string(result, strlen(result), + NULL, NS_TYPE_CONSENSUS))) { log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); @@ -3121,7 +3123,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *msg_out = NULL; again: - vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote, + vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body), + &end_of_vote, NS_TYPE_VOTE); if (!end_of_vote) end_of_vote = vote_body + strlen(vote_body); @@ -3379,7 +3382,9 @@ dirvote_compute_consensuses(void) flavor_name); continue; } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + consensus = networkstatus_parse_vote_from_string(consensus_body, + strlen(consensus_body), + NULL, NS_TYPE_CONSENSUS); if (!consensus) { log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", @@ -3518,7 +3523,7 @@ dirvote_add_signatures_to_pending_consensus( * just in case we break detached signature processing at some point. */ { networkstatus_t *v = networkstatus_parse_vote_from_string( - pc->body, NULL, + pc->body, strlen(pc->body), NULL, NS_TYPE_CONSENSUS); tor_assert(v); networkstatus_vote_free(v); diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 6492b828b1..4338fde599 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1861,7 +1861,9 @@ networkstatus_set_current_consensus(const char *consensus, } /* Make sure it's parseable. */ - c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); + c = networkstatus_parse_vote_from_string(consensus, + strlen(consensus), + NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); result = -2; diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 9c51799d9b..9abdfb614c 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -1057,9 +1057,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, /** Set digests to all the digests of the consensus document in * s */ int -router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, size_t len, + common_digests_t *digests) { - return router_get_hashes_impl(s,strlen(s),digests, + return router_get_hashes_impl(s, len, digests, "network-status-version", "\ndirectory-signature", ' '); @@ -2489,18 +2490,19 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) return NULL; } -/** Helper: given a string s, return the start of the next router-status +/** Helper: given a string s ending at s_eos, return the + * start of the next router-status * object (starting with "r " at the start of a line). If none is found, * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ static inline const char * -find_start_of_next_routerstatus(const char *s) +find_start_of_next_routerstatus(const char *s, const char *s_eos) { const char *eos, *footer, *sig; - if ((eos = strstr(s, "\nr "))) + if ((eos = tor_memstr(s, s_eos - s, "\nr "))) ++eos; else - eos = s + strlen(s); + eos = s_eos; footer = tor_memstr(s, eos-s, "\ndirectory-footer"); sig = tor_memstr(s, eos-s, "\ndirectory-signature"); @@ -2632,7 +2634,8 @@ summarize_protover_flags(protover_summary_flags_t *out, **/ STATIC routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *s_eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, @@ -2651,7 +2654,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, flav = FLAV_NS; tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); - eos = find_start_of_next_routerstatus(*s); + eos = find_start_of_next_routerstatus(*s, s_eos); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { log_warn(LD_DIR, "Error tokenizing router status"); @@ -3394,7 +3397,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from s, and return the result. Return NULL on failure. */ networkstatus_t * -networkstatus_parse_vote_from_string(const char *s, const char **eos_out, +networkstatus_parse_vote_from_string(const char *s, + size_t s_len, + const char **eos_out, networkstatus_type_t ns_type) { smartlist_t *tokens = smartlist_new(); @@ -3410,21 +3415,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, memarea_t *area = NULL, *rs_area = NULL; consensus_flavor_t flav = FLAV_NS; char *last_kwd=NULL; + const char *eos = s + s_len; tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests) || + if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) || router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, - s, strlen(s))<0) { + s, s_len)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } area = memarea_new(); - end_of_header = find_start_of_next_routerstatus(s); + end_of_header = find_start_of_next_routerstatus(s, eos); if (tokenize_string(area, s, end_of_header, tokens, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : @@ -3455,7 +3461,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; - if (!(cert = strstr(s, "\ndir-key-certificate-version"))) + if (!(cert = tor_memstr(s, end_of_header - s, + "\ndir-key-certificate-version"))) goto err; ++cert; ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); @@ -3768,10 +3775,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, s = end_of_header; ns->routerstatus_list = smartlist_new(); - while (!strcmpstart(s, "r ")) { + while (eos-s >= 2 && fast_memeq(s, "r ", 2)) { if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, + if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns, rs, 0, 0)) { smartlist_add(ns->routerstatus_list, rs); } else { @@ -3779,7 +3786,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } else { routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, + if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos, + rs_tokens, NULL, NULL, ns->consensus_method, flav))) { @@ -3824,10 +3832,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* Parse footer; check signature. */ footer_tokens = smartlist_new(); - if ((end_of_footer = strstr(s, "\nnetwork-status-version "))) + if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version "))) ++end_of_footer; else - end_of_footer = s + strlen(s); + end_of_footer = eos; if (tokenize_string(area,s, end_of_footer, footer_tokens, networkstatus_vote_footer_token_table, 0)) { log_warn(LD_DIR, "Error tokenizing network-status vote footer."); diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index be455984d1..390088c948 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -30,7 +30,7 @@ enum networkstatus_type_t; int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hashes(const char *s, +int router_get_networkstatus_v3_hashes(const char *s, size_t len, common_digests_t *digests); int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, @@ -81,6 +81,7 @@ void dump_distinct_digest_count(int severity); int compare_vote_routerstatus_entries(const void **_a, const void **_b); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + size_t len, const char **eos_out, enum networkstatus_type_t ns_type); ns_detached_signatures_t *networkstatus_parse_detached_signatures( @@ -139,7 +140,8 @@ STATIC void dump_desc_fifo_cleanup(void); struct memarea_t; STATIC routerstatus_t *routerstatus_parse_entry_from_string( struct memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index b170fd33d8..5a04683a11 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -59,13 +59,13 @@ int fuzz_main(const uint8_t *data, size_t sz) { networkstatus_t *ns; - char *str = tor_memdup_nulterm(data, sz); const char *eos = NULL; networkstatus_type_t tp = NS_TYPE_CONSENSUS; if (tor_memstr(data, MIN(sz, 1024), "tus vote")) tp = NS_TYPE_VOTE; const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote"; - ns = networkstatus_parse_vote_from_string(str, + ns = networkstatus_parse_vote_from_string((const char *)data, + sz, &eos, tp); if (ns) { @@ -74,6 +74,6 @@ fuzz_main(const uint8_t *data, size_t sz) } else { log_debug(LD_GENERAL, "Parsing as %s failed", what); } - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 8c96851b1f..5665ffeaca 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -52,24 +52,24 @@ fuzz_cleanup(void) int fuzz_main(const uint8_t *data, size_t sz) { - char *str = tor_memdup_nulterm(data, sz); const char *s; routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL; vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs)); smartlist_t *tokens = smartlist_new(); + const char *eos = (const char *)data + sz; - s = str; - rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_md = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_MICRODESC); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens, dummy_vote, vrs, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); @@ -81,6 +81,6 @@ fuzz_main(const uint8_t *data, size_t sz) vote_routerstatus_free(vrs); memarea_clear(area); smartlist_free(tokens); - tor_free(str); + return 0; } diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 723799ee8a..0fa5c31039 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -70,6 +70,23 @@ #define NS_MODULE dir +static networkstatus_t * +networkstatus_parse_vote_from_string_(const char *s, + const char **eos_out, + enum networkstatus_type_t ns_type) +{ + size_t len = strlen(s); + // memdup so that it won't be nul-terminated. + char *tmp = tor_memdup(s, len); + networkstatus_t *result = + networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type); + if (eos_out && *eos_out) { + *eos_out = s + (*eos_out - tmp); + } + tor_free(tmp); + return result; +} + static void test_dir_nicknames(void *arg) { @@ -2888,7 +2905,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_NS); tt_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, + con = networkstatus_parse_vote_from_string_(consensus_text, NULL, NS_TYPE_CONSENSUS); tt_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", @@ -2900,7 +2917,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_MICRODESC); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -2999,13 +3016,13 @@ test_a_networkstatus( tt_assert(consensus_text3); tt_assert(consensus_text_md2); tt_assert(consensus_text_md3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL, NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL, NS_TYPE_CONSENSUS); - con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL, NS_TYPE_CONSENSUS); - con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL, NS_TYPE_CONSENSUS); tt_assert(con2); tt_assert(con3); @@ -6020,9 +6037,10 @@ test_dir_assumed_flags(void *arg) "192.168.0.1 9001 0\n" "m thisoneislongerbecauseitisa256bitmddigest33\n" "s Fast Guard Stable\n"; + const char *eos = str1 + strlen(str1); const char *cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, + rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index e65e2b0111..6e3bb9945d 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -264,7 +264,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out, /* dump the vote and try to parse it. */ v_text = format_networkstatus_vote(sign_skey, vote); tt_assert(v_text); - *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE); + *vote_out = networkstatus_parse_vote_from_string(v_text, + strlen(v_text), + NULL, NS_TYPE_VOTE); done: if (v_text) @@ -422,4 +424,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert, return 0; } - diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 89d1f4f90f..7fe4c15b18 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -270,7 +270,9 @@ test_router_pick_directory_server_impl(void *arg) construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string(consensus_text_md, + strlen(consensus_text_md), + NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); -- GitLab From 7e3005af30b94fd1925b0be475d72875272b9044 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 7 Sep 2018 19:38:21 -0400 Subject: [PATCH 0026/1724] Replace "read consensus from disk" with "map consensus from disk". Implements 27244, and should save a bunch of RAM on clients. --- src/feature/dirauth/dirvote.c | 4 +- src/feature/dircache/directory.c | 18 +++--- src/feature/nodelist/networkstatus.c | 89 +++++++++++++++------------- src/feature/nodelist/networkstatus.h | 4 +- src/test/test_routerlist.c | 3 +- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 42821760c5..771e0363aa 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -3648,7 +3648,9 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) + if (networkstatus_set_current_consensus(pending->body, + strlen(pending->body), + name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c index 8e5fc86836..1473299029 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dircache/directory.c @@ -2606,17 +2606,17 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (looks_like_a_consensus_diff(body, body_len)) { /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); - const char *consensus_body; + const char *consensus_body = NULL; size_t consensus_body_len; - char *owned_consensus = NULL; + tor_mmap_t *mapped_consensus = NULL; if (cd) { consensus_body = cd->dir; consensus_body_len = cd->dir_len; } else { - owned_consensus = networkstatus_read_cached_consensus(flavname); - if (owned_consensus) { - consensus_body = owned_consensus; - consensus_body_len = strlen(consensus_body); + mapped_consensus = networkstatus_map_cached_consensus(flavname); + if (mapped_consensus) { + consensus_body = mapped_consensus->data; + consensus_body_len = mapped_consensus->size; } } if (!consensus_body) { @@ -2629,7 +2629,7 @@ handle_response_fetch_consensus(dir_connection_t *conn, new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, body, body_len); - tor_free(owned_consensus); + tor_munmap_file(mapped_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " "'%s:%d'", conn->base_.address, conn->base_.port); @@ -2651,7 +2651,9 @@ handle_response_fetch_consensus(dir_connection_t *conn, sourcename = "downloaded"; } - if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + if ((r=networkstatus_set_current_consensus(consensus, + strlen(consensus), + flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory %s from " diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 4338fde599..847ca0cdfc 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -105,8 +105,6 @@ STATIC networkstatus_t *current_md_consensus = NULL; typedef struct consensus_waiting_for_certs_t { /** The consensus itself. */ networkstatus_t *consensus; - /** The encoded version of the consensus, nul-terminated. */ - char *body; /** When did we set the current value of consensus_waiting_for_certs? If * this is too recent, we shouldn't try to fetch a new consensus for a * little while, to give ourselves time to get certificates for this one. */ @@ -199,14 +197,11 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } -/** - * Read and and return the cached consensus of type flavorname. If - * unverified is false, get the one we haven't verified. Return NULL if - * the file isn't there. */ +/** Return the filename used to cache the consensus of a given flavor */ static char * -networkstatus_read_cached_consensus_impl(int flav, - const char *flavorname, - int unverified_consensus) +networkstatus_get_cache_fname(int flav, + const char *flavorname, + int unverified_consensus) { char buf[128]; const char *prefix; @@ -221,21 +216,35 @@ networkstatus_read_cached_consensus_impl(int flav, tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); } - char *filename = get_cachedir_fname(buf); - char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + return get_cachedir_fname(buf); +} + +/** + * Read and and return the cached consensus of type flavorname. If + * unverified is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static tor_mmap_t * +networkstatus_map_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char *filename = networkstatus_get_cache_fname(flav, + flavorname, + unverified_consensus); + tor_mmap_t *result = tor_mmap_file(filename); tor_free(filename); return result; } -/** Return a new string containing the current cached consensus of flavor - * flavorname. */ -char * -networkstatus_read_cached_consensus(const char *flavorname) - { +/** Map the file containing the current cached consensus of flavor + * flavorname */ +tor_mmap_t * +networkstatus_map_cached_consensus(const char *flavorname) +{ int flav = networkstatus_parse_flavor_name(flavorname); if (flav < 0) return NULL; - return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); + return networkstatus_map_cached_consensus_impl(flav, flavorname, 0); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -248,24 +257,26 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { + tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags, NULL) < -1) { log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } - s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, + m = networkstatus_map_cached_consensus_impl(flav, flavor, 1); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags | NSSET_WAS_WAITING_FOR_CERTS, NULL)) { log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " "from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } } @@ -1833,6 +1844,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor, */ int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir) @@ -1862,7 +1874,7 @@ networkstatus_set_current_consensus(const char *consensus, /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, - strlen(consensus), + consensus_len, NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); @@ -1951,14 +1963,12 @@ networkstatus_set_current_consensus(const char *consensus, c->valid_after > current_valid_after) { waiting = &consensus_waiting_for_certs[flav]; networkstatus_vote_free(waiting->consensus); - tor_free(waiting->body); waiting->consensus = c; free_consensus = 0; - waiting->body = tor_strdup(consensus); waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_str_to_file(unverified_fname, consensus, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 0); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -2049,10 +2059,6 @@ networkstatus_set_current_consensus(const char *consensus, waiting->consensus->valid_after <= c->valid_after) { networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; - if (consensus != waiting->body) - tor_free(waiting->body); - else - waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { @@ -2147,14 +2153,16 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - char *waiting_body = waiting->body; - if (!networkstatus_set_current_consensus( - waiting_body, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir)) { - tor_free(waiting_body); + tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl( + i, flavor_name, 1); + if (mapping) { + networkstatus_set_current_consensus(mapping->data, + mapping->size, + flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, + source_dir); } + tor_munmap_file(mapping); } } } @@ -2730,6 +2738,5 @@ networkstatus_free_all(void) networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; } - tor_free(waiting->body); } } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index cc6badf0b2..997d7cfa39 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -16,7 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -char *networkstatus_read_cached_consensus(const char *flavorname); +tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free_(routerstatus_t *rs); #define routerstatus_free(rs) \ @@ -108,6 +108,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options); #define NSSET_ACCEPT_OBSOLETE 8 #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir); @@ -159,4 +160,3 @@ extern networkstatus_t *current_md_consensus; #endif /* defined(NETWORKSTATUS_PRIVATE) */ #endif /* !defined(TOR_NETWORKSTATUS_H) */ - diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 7fe4c15b18..bf76570b3f 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -659,7 +659,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) MOCK(clock_skew_warning, mock_clock_skew_warning); /* Caller will call teardown_capture_of_logs() */ setup_capture_of_logs(LOG_WARN); - retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + retval = networkstatus_set_current_consensus(consensus, strlen(consensus), + "microdesc", 0, NULL); done: -- GitLab From 04bb70199be924804708bd4ace18b28b5acdbf19 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 11:37:55 -0400 Subject: [PATCH 0027/1724] Followup: Make authority_cert_parse_from_string() take length too --- src/feature/nodelist/routerlist.c | 3 ++- src/feature/nodelist/routerparse.c | 19 +++++++++++-------- src/feature/nodelist/routerparse.h | 1 + src/feature/relay/router.c | 2 +- src/test/test_dir.c | 12 +++++++++--- src/test/test_dir_common.c | 12 +++++++++--- src/test/test_dir_handle_get.c | 16 ++++++++++++---- src/test/test_routerlist.c | 12 +++++++++--- src/test/test_shared_random.c | 12 +++++++++--- 9 files changed, 63 insertions(+), 26 deletions(-) diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index bcc5c1f074..c3c72e9c78 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -555,7 +555,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, int added_trusted_cert = 0; for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s), + &eos); cert_list_t *cl; if (!cert) { failure_code = -1; diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 9abdfb614c..8a5efc007e 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -2308,7 +2308,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, /** Parse a key certificate from s; point end-of-string to * the first character after the certificate. */ authority_cert_t * -authority_cert_parse_from_string(const char *s, const char **end_of_string) +authority_cert_parse_from_string(const char *s, size_t maxlen, + const char **end_of_string) { /** Reject any certificate at least this big; it is probably an overflow, an * attack, a bug, or some other nonsense. */ @@ -2319,24 +2320,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) char digest[DIGEST_LEN]; directory_token_t *tok; char fp_declared[DIGEST_LEN]; - char *eos; + const char *eos; size_t len; int found; memarea_t *area = NULL; + const char *end_of_s = s + maxlen; const char *s_dup = s; - s = eat_whitespace(s); - eos = strstr(s, "\ndir-key-certification"); + s = eat_whitespace_eos(s, end_of_s); + eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification"); if (! eos) { log_warn(LD_DIR, "No signature found on key certificate"); return NULL; } - eos = strstr(eos, "\n-----END SIGNATURE-----\n"); + eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n"); if (! eos) { log_warn(LD_DIR, "No end-of-signature found on key certificate"); return NULL; } - eos = strchr(eos+2, '\n'); + eos = memchr(eos+2, '\n', end_of_s - (eos+2)); tor_assert(eos); ++eos; len = eos - s; @@ -2353,7 +2355,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_warn(LD_DIR, "Error tokenizing key certificate"); goto err; } - if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", + if (router_get_hash_impl(s, eos-s, digest, "dir-key-certificate-version", "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); @@ -3465,7 +3467,8 @@ networkstatus_parse_vote_from_string(const char *s, "\ndir-key-certificate-version"))) goto err; ++cert; - ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); + ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert, + &end_of_cert); if (!ns->cert || !end_of_cert || end_of_cert > end_of_header) goto err; } diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index 390088c948..c60046e8e8 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -93,6 +93,7 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, smartlist_t *invalid_digests_out); authority_cert_t *authority_cert_parse_from_string(const char *s, + size_t maxlen, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char *desc_id_out, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 1f316ebf08..17f63e9983 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -717,7 +717,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname); goto done; } - parsed = authority_cert_parse_from_string(cert, &eos); + parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos); if (!parsed) { log_warn(LD_DIR, "Unable to parse certificate in %s", fname); goto done; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0fa5c31039..078a383dd8 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -2799,11 +2799,17 @@ test_a_networkstatus( MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); /* Parse certificates and keys. */ - cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert1); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(cert3); sign_skey_1 = crypto_pk_new(); sign_skey_2 = crypto_pk_new(); diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 6e3bb9945d..63bd082eea 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -40,14 +40,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1, { /* Parse certificates and keys. */ authority_cert_t *cert; - cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert); tt_assert(cert->identity_key); *cert1 = cert; tt_assert(*cert1); - *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(*cert2); - *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(*cert3); *sign_skey_1 = crypto_pk_new(); *sign_skey_2 = crypto_pk_new(); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 09799a0e5f..ecb7c1b5fc 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1270,7 +1270,9 @@ test_dir_handle_get_server_keys_authority(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1420,7 +1422,9 @@ test_dir_handle_get_server_keys_sk(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2388,7 +2392,9 @@ test_dir_handle_get_status_vote_next_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2466,7 +2472,9 @@ test_dir_handle_get_status_vote_current_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index bf76570b3f..e7c577cf66 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -260,7 +260,9 @@ test_router_pick_directory_server_impl(void *arg) /* Init SR subsystem. */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -472,7 +474,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -645,7 +649,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 70adf580ab..2043613641 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -64,7 +64,9 @@ init_authority_state(void) MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); or_options_t *options = get_options_mutable(); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(mock_cert); options->AuthoritativeDir = 1; tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0); @@ -420,7 +422,9 @@ test_sr_commit(void *arg) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -823,7 +827,9 @@ test_sr_setup_commits(void) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; -- GitLab From 81a5448c187458ea3edcc0a72044a4cfdf0273bf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 11:54:37 -0400 Subject: [PATCH 0028/1724] Changes file for feature27244 --- changes/feature27244 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/feature27244 diff --git a/changes/feature27244 b/changes/feature27244 new file mode 100644 index 0000000000..a4debbbe53 --- /dev/null +++ b/changes/feature27244 @@ -0,0 +1,5 @@ + o Minor features (memory usage): + - Tor clients no longer need to keep the full text of a consensus in + memory in order to parse it, or apply a diff to it. Instead, they + use mmap() to read the consensus files from disk. Closes ticket + 27244. -- GitLab From cb1891b412b40ad943306ad0894ad2fc726cf562 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 12 Sep 2018 13:39:49 -0500 Subject: [PATCH 0029/1724] Adding 1 new fallback --- scripts/maint/fallback.whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 375ac8f748..088545471b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1064,3 +1064,5 @@ # https://trac.torproject.org/projects/tor/ticket/27297 37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 +# Email sent directly to Phoul +139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 -- GitLab From 7e862c3ec029d3cdfdd07483e31843c8f3b48a68 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 15 Sep 2018 01:23:02 +1000 Subject: [PATCH 0030/1724] check-changes: Check bugfix version formatting Check that bugfix versions in changes files look like Tor versions from the versions spec. Part of ticket 27761. --- changes/ticket27761 | 3 +++ scripts/maint/lintChanges.py | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 changes/ticket27761 diff --git a/changes/ticket27761 b/changes/ticket27761 new file mode 100644 index 0000000000..ef4686bdca --- /dev/null +++ b/changes/ticket27761 @@ -0,0 +1,3 @@ + o Minor features (changelogs): + - Check that bugfix versions in changes files look like Tor versions + from the versions spec. Closes ticket 27761. diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index d5b8fcae5c..5c5d45da1d 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -87,6 +87,12 @@ def lintfile(fname): warn("Bugfix does not say 'Fixes bug X; bugfix on Y'") elif re.search('tor-([0-9]+)', contents): warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)") + else: + bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents) + if bugfix_match is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + elif bugfix_match.group(0) is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") return have_warned != [] -- GitLab From 3eafa61f6324c91b4bb8c02a6e54db0f12ef42c2 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 18 Sep 2018 20:39:03 +1000 Subject: [PATCH 0031/1724] check-changes: Warn about bugfixes on future releases Warn when bugfix changes files say that the bug is in a future release. Closes ticket 27761. --- Makefile.am | 2 +- changes/ticket27761 | 3 ++- scripts/maint/lintChanges.py | 50 ++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 36a5dd2e9e..9a65e083a5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -408,7 +408,7 @@ endif check-changes: if USEPYTHON @if test -d "$(top_srcdir)/changes"; then \ - $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ + PACKAGE_VERSION=$(PACKAGE_VERSION) $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ fi endif diff --git a/changes/ticket27761 b/changes/ticket27761 index ef4686bdca..35106ee9c6 100644 --- a/changes/ticket27761 +++ b/changes/ticket27761 @@ -1,3 +1,4 @@ o Minor features (changelogs): - Check that bugfix versions in changes files look like Tor versions - from the versions spec. Closes ticket 27761. + from the versions spec. Warn when bugfixes claim to be on a future + release. Closes ticket 27761. diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 5c5d45da1d..39fa08bb4a 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -35,6 +35,36 @@ NEEDS_SUBCATEGORIES = set([ "Major features", ]) +def split_tor_version(version): + ''' + Return the initial numeric components of the Tor version as a list of ints. + For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO. + For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present. + + If the version is malformed, returns None. + ''' + version_match = re.search('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) + if version_match is None: + return None + + version_groups = version_match.groups() + if version_groups is None: + return None + if len(version_groups) < 3: + return None + + if len(version_groups) != 5: + return None + version_components = version_groups[0:3] + version_components += version_groups[4:5] + + try: + version_list = [int(v) for v in version_components if v is not None] + except ValueError: + return None + + return version_list + def lintfile(fname): have_warned = [] @@ -93,6 +123,26 @@ def lintfile(fname): warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") elif bugfix_match.group(0) is None: warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + else: + bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents) + bugfix_group = bugfix_match.groups(0) if bugfix_match is not None else None + bugfix_version = bugfix_group[0] if bugfix_group is not None else None + package_version = os.environ.get('PACKAGE_VERSION', None) + if bugfix_version is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version.") + elif package_version is not None: + # If $PACKAGE_VERSION isn't set, skip this check + bugfix_split = split_tor_version(bugfix_version) + package_split = split_tor_version(package_version) + if bugfix_split is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version: '{}'.".format(bugfix_version)) + elif package_split is None: + # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed + warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version)) + elif bugfix_split > package_split: + warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version)) return have_warned != [] -- GitLab From 1663a1dd633213b750fe72b72b8c1bca7e15cd0a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 19 Sep 2018 15:39:39 -0500 Subject: [PATCH 0032/1724] Adjusting IP of fallback --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 088545471b..81e6785333 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -731,7 +731,7 @@ 185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat # Email sent directly to teor, verified using relay contact info -64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 +198.232.165.2:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 # Emails sent directly to teor, verified using relay contact info 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF -- GitLab From bc68e80e0a801d498ce215de4a689c09927181df Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 20 Sep 2018 13:48:45 -0500 Subject: [PATCH 0033/1724] Adding fallback relay --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 81e6785333..1fd231abf2 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1066,3 +1066,7 @@ # Email sent directly to Phoul 139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 + +# Email sent directly to Phoul +104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 + -- GitLab From b0c41e3ec2bca832466ea7709827518159af0c08 Mon Sep 17 00:00:00 2001 From: Jay Bitron Date: Wed, 10 Oct 2018 14:12:53 -0700 Subject: [PATCH 0034/1724] Fix the missing unpack function in mmdb-convert.py --- src/config/mmdb-convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 3a454a3fc1..706a8b03cc 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -77,7 +77,7 @@ def to_int32(s): def to_int28(s): "Parse a pair of big-endian 28-bit integers from bytestring s." - a, b = unpack("!LL", s + b'\x00') + a, b = struct.unpack("!LL", s + b'\x00') return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) class Tree(object): -- GitLab From 8b5ad246e81a8346b79d37052320753f87e22a04 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 11 Oct 2018 20:28:11 +0300 Subject: [PATCH 0035/1724] Fix issues that shellcheck found in chutney-git-bisect.sh --- changes/ticket28006 | 3 +++ scripts/test/chutney-git-bisect.sh | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ticket28006 diff --git a/changes/ticket28006 b/changes/ticket28006 new file mode 100644 index 0000000000..95a4b2cae4 --- /dev/null +++ b/changes/ticket28006 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh index 8a3f2c70c8..dc1319a27a 100755 --- a/scripts/test/chutney-git-bisect.sh +++ b/scripts/test/chutney-git-bisect.sh @@ -20,7 +20,7 @@ if [ ! -z "$1" ]; then fi if [ ! -z "$2" ]; then - cd "$2" + cd "$2" || exit fi CHUTNEY_TEST_CMD="make test-network-all" @@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good" exit 0 fi - i=$[$i+1] + i=$((i+1)) done -i=$[$i-1] +i=$((i-1)) echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad" exit 1 -- GitLab From 57bba19bbeb74d97f3e429259a19b5b6b1f8be53 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 12 Oct 2018 15:21:32 -0500 Subject: [PATCH 0036/1724] Remove aurora and chulak from fallback list --- scripts/maint/fallback.whitelist | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1fd231abf2..1b06299cca 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -172,8 +172,6 @@ # Email sent directly to teor, verified using relay contact info 94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 -176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 -- GitLab From 5033e950ccd0e0f37134f5c4124e1f79d4d784b3 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 12 Oct 2018 15:25:17 -0500 Subject: [PATCH 0037/1724] Adding Quake to fallback list --- scripts/maint/fallback.whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1b06299cca..128cbc8613 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1068,3 +1068,5 @@ # Email sent directly to Phoul 104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 +# Email sent directly to Phoul / Teor +178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 -- GitLab From 9b946c13a2e825311eff40713574ba1608137010 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 15 Oct 2018 11:43:02 -0500 Subject: [PATCH 0038/1724] Adding hviv104 to fallback list --- scripts/maint/fallback.whitelist | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 128cbc8613..3696da8e18 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1070,3 +1070,6 @@ # Email sent directly to Phoul / Teor 178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 + +# Email sent directly to Phoul +192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 -- GitLab From 17f4388b0fa377f7bbdf6f9df23ff429a717d4f8 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 15 Oct 2018 13:47:31 -0500 Subject: [PATCH 0039/1724] Remove mullbinde9 from fallback list --- scripts/maint/fallback.whitelist | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 3696da8e18..94db9272fd 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -747,9 +747,6 @@ # Email sent directly to teor, verified using relay contact info 51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63 -# Email sent directly to teor, verified using relay contact info -163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001 - # Email sent directly to teor, verified using relay contact info 5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001 -- GitLab From 55412c4f3d3486d28fe337b919e7fddc2f93e1b4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 11 Oct 2018 15:22:12 +0300 Subject: [PATCH 0040/1724] Add new source file to test target --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_parsecommon.c | 44 +++++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 src/test/test_parsecommon.c diff --git a/src/test/include.am b/src/test/include.am index 1055cd0a81..dd2986c67c 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -182,6 +182,7 @@ src_test_test_SOURCES += \ src/test/test_x509.c \ src/test/test_helpers.c \ src/test/test_dns.c \ + src/test/test_parsecommon.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test.c b/src/test/test.c index 70d91e3967..56eb153289 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -926,5 +926,6 @@ struct testgroup_t testgroups[] = { { "util/thread/", thread_tests }, { "util/handle/", handle_tests }, { "dns/", dns_tests }, + { "parsecommon/", parsecommon_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index a46fedf3e0..281551aa69 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -266,6 +266,7 @@ extern struct testcase_t dns_tests[]; extern struct testcase_t handle_tests[]; extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_util_tests[]; diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c new file mode 100644 index 0000000000..f152450f5a --- /dev/null +++ b/src/test/test_parsecommon.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "test/test.h" +#include "lib/memarea/memarea.h" +#include "lib/encoding/binascii.h" +#include "feature/dirparse/parsecommon.h" +#include "test/log_test_helpers.h" + +static void +test_parsecommon_tokenize_string_null(void *arg) +{ + + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + const char *str_with_null = "a\0bccccccccc"; + + int retval = + tokenize_string(area, str_with_null, + str_with_null + 3, + tokens, NULL, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +#define PARSECOMMON_TEST(name) \ + { #name, test_parsecommon_ ## name, 0, NULL, NULL } + +struct testcase_t parsecommon_tests[] = { + PARSECOMMON_TEST(tokenize_string_null), + END_OF_TESTCASES +}; + -- GitLab From 7829e3a86875da16e3d7ac55be85145bd672fc12 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 14:19:40 +0300 Subject: [PATCH 0041/1724] First testcase for get_next_token() --- src/test/test_parsecommon.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index f152450f5a..ba778d9969 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -34,11 +34,38 @@ test_parsecommon_tokenize_string_null(void *arg) return; } +static void +test_parsecommon_get_next_token_success(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "1024"); + tt_assert(!token->object_type); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + + tt_ptr_op(*s, OP_EQ, end); + + done: + memarea_drop_all(area); + return; +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(get_next_token_success), END_OF_TESTCASES }; -- GitLab From 5c891dba770b752d16f7e7cea8f61f15abd09ef5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 15:48:38 +0300 Subject: [PATCH 0042/1724] Test argument concatenation in get_next_token() --- src/test/test_parsecommon.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index ba778d9969..b63327ecd1 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -60,12 +60,33 @@ test_parsecommon_get_next_token_success(void *arg) return; } +static void +test_parsecommon_get_next_token_concat_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "proto A=1 B=2"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_PROTO); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "A=1 B=2"); + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(get_next_token_success), + PARSECOMMON_TEST(get_next_token_concat_args), END_OF_TESTCASES }; -- GitLab From 569d8d8cd73987cd0ca126a6f98a75dd34b21137 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 18:33:27 +0300 Subject: [PATCH 0043/1724] Test-case for public key parsing using get_next_token() --- src/test/test_parsecommon.c | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index b63327ecd1..519bbf79ea 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -80,6 +80,47 @@ test_parsecommon_get_next_token_concat_args(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_parse_keys(void *arg) +{ + (void)arg; + + memarea_t *area = memarea_new(); + const char *base64_key = + "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" + "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"; + char *str; + tor_asprintf(&str, "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "%s" + "-----END RSA PUBLIC KEY-----\n", base64_key); + const char *end = str + strlen(str); + const char **s = (const char **)&str; + const char decoded[128]; + + base64_decode((char *)decoded, sizeof(decoded), base64_key, + strlen(base64_key)); + + token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); + tt_int_op(token->n_args, OP_EQ, 0); + tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + tt_assert(token->key); + tt_assert(!token->error); + + // TODO: same with secret key + + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -87,6 +128,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), + PARSECOMMON_TEST(get_next_token_parse_keys), END_OF_TESTCASES }; -- GitLab From 7764d6dfc914d9a29c706cbcb2b399b09680c9b7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 19:19:40 +0300 Subject: [PATCH 0044/1724] Test RSA private key parsing with get_next_token() --- src/test/test_parsecommon.c | 41 +++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 519bbf79ea..ffd3902a8d 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -114,8 +114,45 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(token->key); tt_assert(!token->error); - // TODO: same with secret key - + const char *base64_skey = + "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" + "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" + "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" + "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n" + "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n" + "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n" + "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n" + "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n" + "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n" + "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" + "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" + "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"; + + const char decoded2[128]; + base64_decode((char *)decoded2, sizeof(decoded2), base64_skey, + strlen(base64_skey)); + + char *str2; + tor_asprintf(&str2, "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" + "%s" + "-----END RSA PRIVATE KEY-----\n", base64_skey); + const char *end2 = str2 + strlen(str2); + const char **s2 = (const char **)&str2; + + token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, + NEED_SKEY_1024); + + directory_token_t *token2 = get_next_token(area, s2, end2, &rule2); + + tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); + tt_int_op(token2->n_args, OP_EQ, 0); + tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); + tt_int_op(token2->object_size, OP_EQ, 0); + tt_assert(!token2->object_body); + tt_assert(token2->key); + tt_assert(!token->error); done: memarea_drop_all(area); -- GitLab From 38a7033d3378a7b0cdc68258ebcd0e53371e8f24 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 17:44:40 +0300 Subject: [PATCH 0045/1724] Fix memory management in test_parsecommon_get_next_token_parse_keys --- src/test/test_parsecommon.c | 41 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index ffd3902a8d..6b5307619a 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -86,25 +86,22 @@ test_parsecommon_get_next_token_parse_keys(void *arg) (void)arg; memarea_t *area = memarea_new(); - const char *base64_key = + const char *str = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" - "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"; - char *str; - tor_asprintf(&str, "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "%s" - "-----END RSA PUBLIC KEY-----\n", base64_key); + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + const char *end = str + strlen(str); const char **s = (const char **)&str; - const char decoded[128]; - - base64_decode((char *)decoded, sizeof(decoded), base64_key, - strlen(base64_key)); + directory_token_t *token = NULL; + directory_token_t *token2 = NULL; token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); - directory_token_t *token = get_next_token(area, s, end, &rule); + token = get_next_token(area, s, end, &rule); tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); tt_int_op(token->n_args, OP_EQ, 0); @@ -114,7 +111,9 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(token->key); tt_assert(!token->error); - const char *base64_skey = + const char *str2 = + "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" @@ -127,24 +126,16 @@ test_parsecommon_get_next_token_parse_keys(void *arg) "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" - "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"; - - const char decoded2[128]; - base64_decode((char *)decoded2, sizeof(decoded2), base64_skey, - strlen(base64_skey)); + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n" + "-----END RSA PRIVATE KEY-----\n"; - char *str2; - tor_asprintf(&str2, "client-key\n" - "-----BEGIN RSA PRIVATE KEY-----\n" - "%s" - "-----END RSA PRIVATE KEY-----\n", base64_skey); const char *end2 = str2 + strlen(str2); const char **s2 = (const char **)&str2; token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024); - directory_token_t *token2 = get_next_token(area, s2, end2, &rule2); + token2 = get_next_token(area, s2, end2, &rule2); tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); tt_int_op(token2->n_args, OP_EQ, 0); @@ -155,6 +146,8 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(!token->error); done: + if (token) token_clear(token); + if (token2) token_clear(token2); memarea_drop_all(area); } -- GitLab From 6c5ba2662af08c8094e024a2c04141e776d966a6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 18:51:08 +0300 Subject: [PATCH 0046/1724] Test object parsing in get_next_token() --- src/test/test_parsecommon.c | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 6b5307619a..59e366bc4e 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -151,6 +151,62 @@ test_parsecommon_get_next_token_parse_keys(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_object(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE); + tt_int_op(token->n_args, OP_EQ, 2); + tt_str_op(token->args[0], OP_EQ, + "0232AF901C31A04EE9848595AF9BB7620D4C5B2E"); + tt_str_op(token->args[1], OP_EQ, + "CD1FD971855430880D3C31E0331C5C55800C2F79"); + + tt_assert(!token->error); + + char decoded[256]; + const char *signature = + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + tt_assert(signature); + size_t signature_len = strlen(signature); + base64_decode(decoded, sizeof(decoded), signature, signature_len); + + tt_str_op(token->object_type, OP_EQ, "SIGNATURE"); + tt_int_op(token->object_size, OP_EQ, 256); + tt_mem_op(token->object_body, OP_EQ, decoded, 256); + + tt_assert(!token->key); + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -159,6 +215,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), + PARSECOMMON_TEST(get_next_token_object), END_OF_TESTCASES }; -- GitLab From 81731a290d2a806047901c1835afb557533a0350 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 20:01:42 +0300 Subject: [PATCH 0047/1724] Unit-test some error conditions in get_next_token() --- src/test/test_parsecommon.c | 173 ++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 59e366bc4e..d8719747e2 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -207,6 +207,173 @@ test_parsecommon_get_next_token_object(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_err_too_many_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024 1024 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too many arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_too_few_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too few arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_obj_missing_endline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_beginline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-Z---\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_tag_mismatch(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SOMETHINGELSE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, + "Malformed object: mismatched end tag SIGNATURE"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_base64(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "%%@%%%%%%%!!!'\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data"); + + done: + memarea_drop_all(area); + return; +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -216,6 +383,12 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), PARSECOMMON_TEST(get_next_token_object), + PARSECOMMON_TEST(get_next_token_err_too_many_args), + PARSECOMMON_TEST(get_next_token_err_too_few_args), + PARSECOMMON_TEST(get_next_token_err_obj_missing_endline), + PARSECOMMON_TEST(get_next_token_err_bad_beginline), + PARSECOMMON_TEST(get_next_token_err_tag_mismatch), + PARSECOMMON_TEST(get_next_token_err_bad_base64), END_OF_TESTCASES }; -- GitLab From 78c446af7ef9900c31e84496a4547550d4c191e2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 14:53:04 +0300 Subject: [PATCH 0048/1724] Unit-test multiple line parsing with tokenize_string() --- src/test/test_parsecommon.c | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index d8719747e2..0c754a8fee 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -34,6 +34,51 @@ test_parsecommon_tokenize_string_null(void *arg) return; } +static void +test_parsecommon_tokenize_string_multiple_lines(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, GE(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup( + "hibernating 0\nuptime 1024\n" + "published 2018-10-15 10:00:00\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(smartlist_len(tokens), OP_EQ, 3); + directory_token_t *token = smartlist_get(tokens, 0); + + tt_int_op(token->tp, OP_EQ, K_HIBERNATING); + + token = smartlist_get(tokens, 1); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + + token = smartlist_get(tokens, 2); + + tt_int_op(token->tp, OP_EQ, K_PUBLISHED); + + tt_int_op(retval, OP_EQ, 0); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -379,6 +424,7 @@ test_parsecommon_get_next_token_err_bad_base64(void *arg) struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), -- GitLab From 7c8bf2f7c7adbae3b0f56631c2a2c036076b322d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:22:48 +0300 Subject: [PATCH 0049/1724] Add testcase for too few elements in tokenize_string() input --- src/test/test_parsecommon.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 0c754a8fee..e68d0fb884 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -79,6 +79,36 @@ test_parsecommon_tokenize_string_multiple_lines(void *arg) return; } +static void +test_parsecommon_tokenize_string_min_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(2), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // Missing "uptime" + char *str = tor_strdup("uptime 1024\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -425,6 +455,7 @@ test_parsecommon_get_next_token_err_bad_base64(void *arg) struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(tokenize_string_multiple_lines), + PARSECOMMON_TEST(tokenize_string_min_cnt), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), -- GitLab From 1a4edceee9049d19a97b08dc28e87fa346d02536 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:31:47 +0300 Subject: [PATCH 0050/1724] Add testcase for too many elements in tokenize_string() input --- src/test/test_parsecommon.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index e68d0fb884..182f6fba5d 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -109,6 +109,37 @@ test_parsecommon_tokenize_string_min_cnt(void *arg) return; } +static void +test_parsecommon_tokenize_string_max_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // "uptime" expected once, but occurs twice in input. + char *str = tor_strdup( + "uptime 1024\nuptime 2048\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -456,6 +487,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(tokenize_string_min_cnt), + PARSECOMMON_TEST(tokenize_string_max_cnt), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), -- GitLab From f10d664fd14b318fba23c7290c800590d0a474b4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:54:11 +0300 Subject: [PATCH 0051/1724] Test AT_END checking in tokenize_string() --- src/test/test_parsecommon.c | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 182f6fba5d..13e4ac8c7f 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -140,6 +140,66 @@ test_parsecommon_tokenize_string_max_cnt(void *arg) return; } +static void +test_parsecommon_tokenize_string_at_start(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the first line. + char *str = tor_strdup( + "uptime 1024\nclient-name Alice\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_end(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the last line. + char *str = tor_strdup( + "client-name Alice\nuptime 1024\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -488,6 +548,8 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(tokenize_string_min_cnt), PARSECOMMON_TEST(tokenize_string_max_cnt), + PARSECOMMON_TEST(tokenize_string_at_start), + PARSECOMMON_TEST(tokenize_string_at_end), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), -- GitLab From 7fd82a4570c4c3ea887c02190a54e424eb5f4c5f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 17:39:52 +0300 Subject: [PATCH 0052/1724] One testcase for annotation handling in tokenize_string() --- src/test/test_parsecommon.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 13e4ac8c7f..7ce4b71b00 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -200,6 +200,33 @@ test_parsecommon_tokenize_string_at_end(void *arg) return; } +static void +test_parsecommon_tokenize_string_no_annotations(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -550,6 +577,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_max_cnt), PARSECOMMON_TEST(tokenize_string_at_start), PARSECOMMON_TEST(tokenize_string_at_end), + PARSECOMMON_TEST(tokenize_string_no_annotations), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), -- GitLab From 65864be9bc59fb3743d2cebf497d0a4268596f95 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 17:53:17 +0300 Subject: [PATCH 0053/1724] Add changes file --- changes/ticket27625 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket27625 diff --git a/changes/ticket27625 b/changes/ticket27625 new file mode 100644 index 0000000000..33d40adf34 --- /dev/null +++ b/changes/ticket27625 @@ -0,0 +1,4 @@ + o Testing: + - Write some unit tests for tokenize_string() and + get_next_token() functions. Resolves ticket 27625. + -- GitLab From 2f0744b3e6f579f25db1ed6e048d0418ac2ab570 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 17 Oct 2018 00:16:21 +0000 Subject: [PATCH 0054/1724] rust/tor_util: drop unsafe block in cstr! This is unnecessary just to get an empty string, there's Default::default(). Fix on 8fff331bb095dc6f5e2fe2ecfc9ab08ea9e2fe97. --- changes/ticket28077 | 3 +++ src/rust/tor_util/strings.rs | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changes/ticket28077 diff --git a/changes/ticket28077 b/changes/ticket28077 new file mode 100644 index 0000000000..2b5afb1678 --- /dev/null +++ b/changes/ticket28077 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Remove unnecessarily unsafe code from the rust macro cstr!. Closes + ticket 28077. diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index d64275e06b..71a908a58c 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -105,11 +105,7 @@ macro_rules! cstr { ($($bytes:expr),*) => ( ::std::ffi::CStr::from_bytes_with_nul( concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or( - unsafe{ - ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") - } - ) + ).unwrap_or_default() ) } -- GitLab From fd2e0ac1c39088a315ccce648e52536a43ba3c27 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 17 Oct 2018 14:46:38 -0400 Subject: [PATCH 0055/1724] Bump to 0.3.6.0-alpha-dev. --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index d6e0c84904..292244f56f 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.5.3-alpha]) +AC_INIT([tor],[0.3.6.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 8a82b013e4..8b9b709698 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.5.3-alpha" +!define VERSION "0.3.6.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 7a4a1d72c0..de3bf09282 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.5.3-alpha" +#define VERSION "0.3.6.0-alpha-dev" -- GitLab From f07ab5b95c3d38cf3fb85f9c1ea1f08aed003971 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Sun, 30 Sep 2018 14:05:48 +0000 Subject: [PATCH 0056/1724] evloop: fix docs alert_sockets_t was moved in 544ab27a949406628809869111b7288017a5bcb1. --- src/lib/evloop/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..5e59b48fe6 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. -- GitLab From a56451af4269235df45416a37a5e37be6e63c827 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Sun, 30 Sep 2018 14:37:11 +0000 Subject: [PATCH 0057/1724] evloop: fix docs for threadpool_register_reply_event Commit 6a5f62f68f2c73dbbbbddb4fa3759586f4c2b0dc ultimately didn't include the base argument, and the callback is named cb. --- src/lib/evloop/workqueue.c | 4 ++-- src/lib/evloop/workqueue.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 5e59b48fe6..5471f87b04 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -622,8 +622,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool tp's reply queue with the libevent - * mainloop of base. If tp is provided, it is run after +/** Register the threadpool tp's reply queue with Tor's global + * libevent mainloop. If cb is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index da292d1f05..10d5d47464 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); -- GitLab From 3a8f32067d1cb4d5aee320fbd3d1c02541f4e112 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Sep 2018 13:50:12 -0400 Subject: [PATCH 0058/1724] hs-v3: Consolidate descriptor cookie computation code Both client and service had their own code for this. Consolidate into one place so we avoid duplication. Closes #27549 Signed-off-by: David Goulet --- changes/ticket27549 | 3 ++ src/feature/hs/hs_descriptor.c | 89 ++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 changes/ticket27549 diff --git a/changes/ticket27549 b/changes/ticket27549 new file mode 100644 index 0000000000..51d0f24757 --- /dev/null +++ b/changes/ticket27549 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (hidden service v3): + - Consolidate the authorized client descriptor cookie computation code + from client and service into one function. Closes ticket 27549. diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index d0cdffdf10..1232182ea5 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1400,6 +1400,49 @@ encrypted_data_length_is_valid(size_t len) return 0; } +/* Build the KEYS component for the authorized client computation. The format + * of the construction is: + * + * SECRET_SEED = x25519(sk, pk) + * KEYS = KDF(subcredential | SECRET_SEED, 40) + * + * The keys_out parameter will points to the buffer containing the KEYS. The + * caller should wipe and free its content once done with it. This function + * can't fail. */ +static void +build_descriptor_cookie_keys(const uint8_t *subcredential, + size_t subcredential_len, + const curve25519_secret_key_t *sk, + const curve25519_public_key_t *pk, + uint8_t **keys_out) +{ + uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; + uint8_t *keystream; + size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN; + crypto_xof_t *xof; + + tor_assert(subcredential); + tor_assert(sk); + tor_assert(pk); + tor_assert(keys_out); + + keystream = tor_malloc_zero(keystream_len); + + /* Calculate x25519(sk, pk) to get the secret seed. */ + curve25519_handshake(secret_seed, sk, pk); + + /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, subcredential, subcredential_len); + crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); + crypto_xof_squeeze_bytes(xof, keystream, keystream_len); + crypto_xof_free(xof); + + memwipe(secret_seed, 0, sizeof(secret_seed)); + + *keys_out = keystream; +} + /* Decrypt the descriptor cookie given the descriptor, the auth client, * and the client secret key. On sucess, return 0 and a newly allocated * descriptor cookie descriptor_cookie_out. On error or if the client id @@ -1412,12 +1455,10 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, uint8_t **descriptor_cookie_out) { int ret = -1; - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key = NULL; + uint8_t *keystream = NULL; uint8_t *descriptor_cookie = NULL; + const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; - crypto_xof_t *xof = NULL; tor_assert(desc); tor_assert(client); @@ -1429,16 +1470,11 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, sizeof(*client_auth_sk))); tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); - /* Calculate x25519(client_x, hs_Y) */ - curve25519_handshake(secret_seed, client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1464,8 +1500,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(secret_seed, 0, sizeof(secret_seed)); memwipe(keystream, 0, sizeof(keystream)); + tor_free(keystream); return ret; } @@ -2864,11 +2900,9 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, const uint8_t *descriptor_cookie, hs_desc_authorized_client_t *client_out) { - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key; + uint8_t *keystream = NULL; + const uint8_t *cookie_key; crypto_cipher_t *cipher; - crypto_xof_t *xof; tor_assert(client_auth_pk); tor_assert(auth_ephemeral_sk); @@ -2884,18 +2918,11 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(!tor_mem_is_zero((char *) subcredential, DIGEST256_LEN)); - /* Calculate x25519(hs_y, client_X) */ - curve25519_handshake(secret_seed, - auth_ephemeral_sk, - client_auth_pk); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, &keystream); + /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); cookie_key = keystream + HS_DESC_CLIENT_ID_LEN; @@ -2910,8 +2937,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(secret_seed, 0, sizeof(secret_seed)); memwipe(keystream, 0, sizeof(keystream)); + tor_free(keystream); crypto_cipher_free(cipher); } -- GitLab From 91fa12aedc5f30e0f3703bed252e5b30ce374c5c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 18:31:27 +0300 Subject: [PATCH 0059/1724] Fallback to local DNS when no other nameservers are known --- src/feature/relay/dns.c | 55 ++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index bc507d47f6..f056629b8d 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1388,16 +1388,23 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { log_debug(LD_FS, "stat()ing %s", conf_fname); - if (stat(sandbox_intern_string(conf_fname), &st)) { + int missing_resolv_conf = 0; + int stat_res = stat(sandbox_intern_string(conf_fname), &st); + + if (stat_res) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); - goto err; - } - if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname) + missing_resolv_conf = 1; + } else if (!force && resolv_conf_fname && + !strcmp(conf_fname,resolv_conf_fname) && st.st_mtime == resolv_conf_mtime) { log_info(LD_EXIT, "No change to '%s'", conf_fname); return 0; } + + if (stat_res == 0 && st.st_size == 0) + missing_resolv_conf = 1; + if (nameservers_configured) { evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); @@ -1410,20 +1417,34 @@ configure_nameservers(int force) sandbox_intern_string("/etc/hosts")); } #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */ - log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, - sandbox_intern_string(conf_fname)))) { - log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", - conf_fname, conf_fname, r); - goto err; - } - if (evdns_base_count_nameservers(the_evdns_base) == 0) { - log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); - goto err; + + if (!missing_resolv_conf) { + log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { + log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers " + "in '%s' (%d)", conf_fname, conf_fname, r); + + if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" - + goto err; // in which case we expect libevent to add 127.0.0.1 as + // fallback. + } + if (evdns_base_count_nameservers(the_evdns_base) == 0) { + log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", + conf_fname); + } + + tor_free(resolv_conf_fname); + resolv_conf_fname = tor_strdup(conf_fname); + resolv_conf_mtime = st.st_mtime; + } else { + log_warn(LD_EXIT, "Could not read your DNS config from '%s' - " + "please investigate your DNS configuration. " + "This is possibly a problem. Meanwhile, falling" + " back to local DNS at 127.0.0.1.", conf_fname); + evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1"); } - tor_free(resolv_conf_fname); - resolv_conf_fname = tor_strdup(conf_fname); - resolv_conf_mtime = st.st_mtime; + if (nameservers_configured) evdns_base_resume(the_evdns_base); } -- GitLab From 98fd3b410463a25be0864cfaa8ef2e56420125fe Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 18:48:32 +0300 Subject: [PATCH 0060/1724] Mention DNS fallback in manpage --- doc/tor.1.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 406372433f..8729b8ed50 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2211,7 +2211,8 @@ is non-zero): __filename__. The file format is the same as the standard Unix "**resolv.conf**" file (7). This option, like all other ServerDNS options, only affects name lookups that your server does on behalf of clients. - (Defaults to use the system DNS configuration.) + (Defaults to use the system DNS configuration or a localhost DNS service + in case no nameservers are found in a given configuration.) [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**:: If this option is false, Tor exits immediately if there are problems -- GitLab From 1a1b088f8cab73ca258c35e83d21df900f8e2e32 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 19:05:40 +0300 Subject: [PATCH 0061/1724] Add changes file --- changes/bug21900 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug21900 diff --git a/changes/bug21900 b/changes/bug21900 new file mode 100644 index 0000000000..686cb6c584 --- /dev/null +++ b/changes/bug21900 @@ -0,0 +1,4 @@ + o Minor bugfixes (DNS): + - Gracefully handle empty or absent resolve.conf file by falling + back to using localhost DNS service and hoping it works. Fixes + bug 21900; bugfix on 0.2.1.10-alpha. -- GitLab From d827902cb1107593473552974be5fcbb1fbb4abe Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:15:06 +0300 Subject: [PATCH 0062/1724] Unit test for DNS fallback in configure_nameservers --- src/feature/relay/dns.c | 30 ++++++++++++++++++++ src/feature/relay/dns.h | 3 ++ src/test/test_dns.c | 61 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index f056629b8d..1b4914b49a 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1357,6 +1357,36 @@ evdns_err_is_transient(int err) } } +/** + * Return number of configured nameservers in the_evdns_base. + */ +size_t +number_of_configured_nameservers(void) +{ + return evdns_base_count_nameservers(the_evdns_base); +} + +/** + * Return address of configured nameserver in the_evdns_base + * at index idx. + */ +tor_addr_t * +configured_nameserver_address(const size_t idx) +{ + struct sockaddr_storage sa; + ev_socklen_t sa_len = sizeof(sa); + + if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx, + (struct sockaddr *)&sa, + sa_len) > 0) { + tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); + tor_addr_from_sockaddr(tor_addr, (const struct sockaddr *)&sa, NULL); + return tor_addr; + } + + return NULL; +} + /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on * our last attempt. On Unix, this reads from /etc/resolv.conf or diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 1dd6f903d1..3a17ea7b36 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -45,6 +45,9 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #ifdef DNS_PRIVATE #include "feature/relay/dns_structs.h" +size_t number_of_configured_nameservers(void); +tor_addr_t *configured_nameserver_address(const size_t idx); + MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, cached_resolve_t **resolve_out)); diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 8369f844f6..ea59a49b5b 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -13,9 +13,69 @@ #include "core/or/edge_connection_st.h" #include "core/or/or_circuit_st.h" +#include "app/config/or_options_st.h" +#include "app/config/config.h" + +#include +#include #define NS_MODULE dns +#define NS_SUBMODULE configure_nameservers_fallback + +static or_options_t options = { + .ORPort_set = 1, +}; + +static const or_options_t * +mock_get_options(void) +{ + return &options; +} + +static void +NS(test_main)(void *arg) +{ + (void)arg; + tor_addr_t *nameserver_addr = NULL; + + MOCK(get_options, mock_get_options); + + options.ServerDNSResolvConfFile = (char *)"no_such_file!!!"; + + dns_init(); // calls configure_nameservers() + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); + +#ifndef _WIN32 + tor_free(nameserver_addr); + + options.ServerDNSResolvConfFile = (char *)"/dev/null"; + + dns_init(); + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); +#endif + + UNMOCK(get_options); + + done: + tor_free(nameserver_addr); + return; +} + +#undef NS_SUBMODULE + #define NS_SUBMODULE clip_ttl static void @@ -736,6 +796,7 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { + TEST_CASE(configure_nameservers_fallback), TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), -- GitLab From 92f0e04f8d08574be60da7e6ebad90a2883a488a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:18:54 +0300 Subject: [PATCH 0063/1724] Check if libevent comes with evdns_base_get_nameserver_addr() --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index 292244f56f..af3d64b7f2 100644 --- a/configure.ac +++ b/configure.ac @@ -812,6 +812,8 @@ fi dnl Now check for particular libevent functions. AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \ evutil_secure_rng_add_bytes \ + evdns_base_get_nameserver_addr \ + ]) LIBS="$save_LIBS" -- GitLab From 98cef6807eb70e7c459f6f80a06f894fac63100a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:32:26 +0300 Subject: [PATCH 0064/1724] Exclude test and a supporting function when evdns_base_get_nameserver_addr() is not available --- src/feature/relay/dns.c | 2 ++ src/feature/relay/dns.h | 2 ++ src/test/test_dns.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 1b4914b49a..dfd84652ad 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1366,6 +1366,7 @@ number_of_configured_nameservers(void) return evdns_base_count_nameservers(the_evdns_base); } +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR /** * Return address of configured nameserver in the_evdns_base * at index idx. @@ -1386,6 +1387,7 @@ configured_nameserver_address(const size_t idx) return NULL; } +#endif /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 3a17ea7b36..5758ea4363 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -46,7 +46,9 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #include "feature/relay/dns_structs.h" size_t number_of_configured_nameservers(void); +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR tor_addr_t *configured_nameserver_address(const size_t idx); +#endif MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, diff --git a/src/test/test_dns.c b/src/test/test_dns.c index ea59a49b5b..ea0fcf8c5e 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,6 +1,7 @@ /* Copyright (c) 2015-2018, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "orconfig.h" #include "core/or/or.h" #include "test/test.h" @@ -21,6 +22,7 @@ #define NS_MODULE dns +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR #define NS_SUBMODULE configure_nameservers_fallback static or_options_t options = { @@ -75,6 +77,7 @@ NS(test_main)(void *arg) } #undef NS_SUBMODULE +#endif #define NS_SUBMODULE clip_ttl @@ -796,7 +799,9 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR TEST_CASE(configure_nameservers_fallback), +#endif TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), -- GitLab From 7aa9fc1637aeb876c0eebceec3dcdcbcba603302 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 21 Oct 2018 23:46:09 -0400 Subject: [PATCH 0065/1724] clean up a tor2web comment --- src/feature/rend/rendmid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 8ca19a2522..67a8b5f8f4 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -236,8 +236,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Check if we are configured to accept established rendezvous cells from - * client or in other words Tor2Web clients. */ + /* Check if we are configured to defend ourselves from clients that + * attempt to establish rendezvous points directly to us. */ if (channel_is_client(circ->p_chan) && dos_should_refuse_single_hop_client()) { /* Note it down for the heartbeat log purposes. */ -- GitLab From 3704c4a01219cca7273fa5fa0bc6751d98ad05ac Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 29 Aug 2018 13:32:52 +0000 Subject: [PATCH 0066/1724] string: add BOM helper --- src/lib/string/util_string.c | 13 +++++++++++++ src/lib/string/util_string.h | 1 + 2 files changed, 14 insertions(+) diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index b2b85d151d..e76e73046f 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -541,3 +541,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 746ece0d33..99467a27c3 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -53,5 +53,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ -- GitLab From f874ab26401ca269074963697ddcad879b3b4e3a Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 29 Aug 2018 08:49:10 +0000 Subject: [PATCH 0067/1724] dircache: make dirauths reject non UTF-8 descriptors and extrainfo Ticket #27367. --- changes/feature27367 | 4 ++++ src/feature/dirauth/process_descs.c | 12 +++++++++--- src/feature/dirauth/process_descs.h | 3 ++- src/feature/dircache/dircache.c | 4 ++-- 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 changes/feature27367 diff --git a/changes/feature27367 b/changes/feature27367 new file mode 100644 index 0000000000..99c0839621 --- /dev/null +++ b/changes/feature27367 @@ -0,0 +1,4 @@ + o Minor features (parsing): + - Directory authorities now validate that router descriptors and ExtraInfo + documents are in a valid subset of UTF-8, and reject them if not. + Closes ticket 27367. diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index c379f25bdd..dca87b3eaf 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -519,7 +519,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) /** As for dirserv_add_descriptor(), but accepts multiple documents, and * returns the most severe error that occurred for any one of them. */ was_router_added_t -dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, +dirserv_add_multiple_descriptors(const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg) { @@ -536,6 +537,11 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + if (!string_is_utf8_no_bom(desc, desclen)) { + *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; + return ROUTER_AUTHDIR_REJECTS; + } + format_iso_time(time_buf, now); if (tor_snprintf(annotation_buf, sizeof(annotation_buf), "@uploaded-at %s\n" @@ -552,7 +558,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0, annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; @@ -568,7 +574,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0, NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index ad9d5c3d4c..5a0914acd8 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -17,7 +17,8 @@ void dirserv_free_fingerprint_list(void); int dirserv_add_own_fingerprint(crypto_pk_t *pk); enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, + const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg); enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..930a8b87ef 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1608,8 +1608,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); + was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len, + purpose, conn->base_.address, &msg); tor_assert(msg); if (r == ROUTER_ADDED_SUCCESSFULLY) { -- GitLab From c0bd800d267285d7372094b77121c63a4ba720d2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Oct 2018 09:09:40 -0400 Subject: [PATCH 0068/1724] Re-alphabetize the list of tests in tests.[ch] --- src/test/test.c | 20 ++++++++++---------- src/test/test.h | 34 +++++++++++++++++----------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/test/test.c b/src/test/test.c index 56eb153289..17b736d305 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -847,8 +847,8 @@ struct testgroup_t testgroups[] = { { "circuitbuild/", circuitbuild_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "circuituse/", circuituse_tests }, { "circuitstats/", circuitstats_tests }, + { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, @@ -865,34 +865,36 @@ struct testgroup_t testgroups[] = { #endif { "crypto/pem/", pem_tests }, { "dir/", dir_tests }, - { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, { "dir/voting-schedule/", voting_schedule_tests }, + { "dir_handle_get/", dir_handle_get_tests }, + { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, - { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, { "geoip/", geoip_tests }, - { "legacy_hs/", hs_tests }, + { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, - { "hs_client/", hs_client_tests }, - { "hs_intropoint/", hs_intropoint_tests }, { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, + { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, @@ -910,8 +912,8 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, - { "socks/", socks_tests }, { "shared-random/", sr_tests }, + { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, @@ -921,11 +923,9 @@ struct testgroup_t testgroups[] = { { "tortls/x509/", x509_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, + { "util/handle/", handle_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, { "util/thread/", thread_tests }, - { "util/handle/", handle_tests }, - { "dns/", dns_tests }, - { "parsecommon/", parsecommon_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index 281551aa69..092356f0fb 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -177,11 +177,11 @@ extern const struct testcase_setup_t ed25519_test_setup; extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; -extern struct testcase_t bwmgt_tests[]; extern struct testcase_t buffer_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; @@ -191,8 +191,8 @@ extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t circuituse_tests[]; extern struct testcase_t circuitstats_tests[]; +extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; @@ -200,30 +200,32 @@ extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; +extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; -extern struct testcase_t dir_tests[]; +extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; extern struct testcase_t geoip_tests[]; -extern struct testcase_t hs_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; -extern struct testcase_t hs_client_tests[]; -extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; @@ -234,6 +236,7 @@ extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; @@ -252,21 +255,18 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; -extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; +extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; extern struct testcase_t tortls_openssl_tests[]; -extern struct testcase_t util_tests[]; +extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t util_tests[]; extern struct testcase_t voting_schedule_tests[]; -extern struct testcase_t dns_tests[]; -extern struct testcase_t handle_tests[]; -extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; -extern struct testcase_t parsecommon_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_util_tests[]; -- GitLab From 594140574e7366efac693d440a636a1e1cce82ff Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Oct 2018 11:06:34 -0400 Subject: [PATCH 0069/1724] Fix remaining cases of using consensus without a len parameter. (Thanks to cyberpunks for noting two of them!) --- src/feature/dircache/consdiffmgr.c | 28 ++++++++++++++++++++++++---- src/feature/dircache/consdiffmgr.h | 6 ++++++ src/feature/dircache/dirserv.c | 5 ++++- src/feature/dircache/dirserv.h | 1 + src/feature/nodelist/networkstatus.c | 5 +++-- src/test/test_consdiffmgr.c | 2 ++ src/test/test_dir_handle_get.c | 2 ++ 7 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index bf3a0ef3cf..2cddf68b69 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); @@ -509,8 +510,25 @@ get_max_age_to_cache(void) MAX_MAX_AGE_TO_CACHE); } +#ifdef TOR_UNIT_TESTS +/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For + * testing. */ +int +consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed) +{ + size_t len = strlen(consensus); + /* make a non-nul-terminated copy so that we can have a better chance + * of catching errors. */ + char *ctmp = tor_memdup(consensus, len); + int r = consdiffmgr_add_consensus(ctmp, len, as_parsed); + tor_free(ctmp); + return r; +} +#endif + /** - * Given a string containing a networkstatus consensus, and the results of + * Given a buffer containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not * already present and not too old. Create new consensus diffs from or to * that consensus as appropriate. @@ -519,6 +537,7 @@ get_max_age_to_cache(void) */ int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) @@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, as_parsed); + return consensus_queue_compression_work(consensus, consensus_len, as_parsed); } /** @@ -1826,14 +1845,15 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { tor_assert(consensus); tor_assert(as_parsed); consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); - job->consensus = tor_strdup(consensus); - job->consensus_len = strlen(consensus); + job->consensus = tor_memdup_nulterm(consensus, consensus_len); + job->consensus_len = strlen(job->consensus); job->flavor = as_parsed->flavor; char va_str[ISO_TIME_LEN+1]; diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index d6f273cc4e..011c8799d6 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t { struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); consdiff_status_t consdiffmgr_find_consensus( @@ -73,4 +74,9 @@ STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ +#ifdef TOR_UNIT_TESTS +int consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed); +#endif + #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index b85db8324f..433d3f4ce4 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -1272,6 +1272,7 @@ free_cached_dir_(void *_d) * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, @@ -1282,7 +1283,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, if (!cached_consensuses) cached_consensuses = strmap_new(); - new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + new_networkstatus = + new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len), + published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index 9be4bf9db2..c430987aa7 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -148,6 +148,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options, cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, + size_t consensus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 847ca0cdfc..97a8779bea 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2107,17 +2107,18 @@ networkstatus_set_current_consensus(const char *consensus, if (we_want_to_fetch_flavor(options, flav)) { dirserv_set_cached_consensus_networkstatus(consensus, + consensus_len, flavor, &c->digests, c->digest_sha3_as_signed, c->valid_after); if (dir_server_mode(get_options())) { - consdiffmgr_add_consensus(consensus, c); + consdiffmgr_add_consensus(consensus, consensus_len, c); } } if (!from_cache) { - write_str_to_file(consensus_fname, consensus, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 0); } warn_early_consensus(c, flavor, now); diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 4b49fdb6aa..966314be13 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,8 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static char * consensus_diff_apply_(const char *c, const char *d) { diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ecb7c1b5fc..b591396913 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -67,6 +67,8 @@ ENABLE_GCC_WARNING(overlength-strings) #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \ "Consensus not signed by sufficient number of requested authorities\r\n\r\n" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static dir_connection_t * new_dir_conn(void) { -- GitLab From 52a82bc53c82c82f754c1267aaa68b48737ba07c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Oct 2018 09:24:15 -0400 Subject: [PATCH 0070/1724] Add a couple more checks to test_parsecommon.c These checks should make coverity stop giving us a "dereference before null check" warning here. --- src/test/test_parsecommon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 7ce4b71b00..6da125dd0a 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -295,6 +295,7 @@ test_parsecommon_get_next_token_parse_keys(void *arg) token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); token = get_next_token(area, s, end, &rule); + tt_assert(token); tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); tt_int_op(token->n_args, OP_EQ, 0); @@ -329,6 +330,7 @@ test_parsecommon_get_next_token_parse_keys(void *arg) NEED_SKEY_1024); token2 = get_next_token(area, s2, end2, &rule2); + tt_assert(token2); tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); tt_int_op(token2->n_args, OP_EQ, 0); @@ -590,4 +592,3 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_err_bad_base64), END_OF_TESTCASES }; - -- GitLab From b7edfcbf6bb3a27d914ad883ae75413a4d25739a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 26 Oct 2018 10:26:47 +0300 Subject: [PATCH 0071/1724] In configured_nameserver_address, check if tor_addr_from_sockaddr succeeded --- src/feature/relay/dns.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index dfd84652ad..371c2f5069 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1381,8 +1381,11 @@ configured_nameserver_address(const size_t idx) (struct sockaddr *)&sa, sa_len) > 0) { tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); - tor_addr_from_sockaddr(tor_addr, (const struct sockaddr *)&sa, NULL); - return tor_addr; + if (tor_addr_from_sockaddr(tor_addr, + (const struct sockaddr *)&sa, + NULL) == 0) { + return tor_addr; + } } return NULL; -- GitLab From b59eedc25992f8c460d4c99968a1fbaa28896610 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 12:39:11 +0300 Subject: [PATCH 0072/1724] Add trunnel spec and impl for NETINFO cells --- src/trunnel/include.am | 6 +- src/trunnel/netinfo.c | 721 ++++++++++++++++++++++++++++++++++++ src/trunnel/netinfo.h | 226 +++++++++++ src/trunnel/netinfo.trunnel | 21 ++ 4 files changed, 972 insertions(+), 2 deletions(-) create mode 100644 src/trunnel/netinfo.c create mode 100644 src/trunnel/netinfo.h create mode 100644 src/trunnel/netinfo.trunnel diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 03c1753e96..b5db0609a8 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -23,7 +23,8 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -37,7 +38,8 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/socks5.h + src/trunnel/socks5.h \ + src/trunnel/netinfo.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c new file mode 100644 index 0000000000..170e9bf034 --- /dev/null +++ b/src/trunnel/netinfo.c @@ -0,0 +1,721 @@ +/* netinfo.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include +#include "trunnel-impl.h" + +#include "netinfo.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int netinfo_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +netinfo_addr_t * +netinfo_addr_new(void) +{ + netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); + if (NULL == val) + return NULL; + val->addr_type = NETINFO_ADDR_TYPE_IPV4; + val->len = 4; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_addr_clear(netinfo_addr_t *obj) +{ + (void) obj; +} + +void +netinfo_addr_free(netinfo_addr_t *obj) +{ + if (obj == NULL) + return; + netinfo_addr_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_addr_t)); + trunnel_free_(obj); +} + +uint8_t +netinfo_addr_get_addr_type(const netinfo_addr_t *inp) +{ + return inp->addr_type; +} +int +netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) +{ + if (! ((val == NETINFO_ADDR_TYPE_IPV4 || val == NETINFO_ADDR_TYPE_IPV6))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->addr_type = val; + return 0; +} +uint8_t +netinfo_addr_get_len(const netinfo_addr_t *inp) +{ + return inp->len; +} +int +netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) +{ + if (! ((val == 4 || val == 16))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->len = val; + return 0; +} +uint32_t +netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp) +{ + return inp->addr_ipv4; +} +int +netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val) +{ + inp->addr_ipv4 = val; + return 0; +} +size_t +netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp) +{ + (void)inp; return 16; +} + +uint8_t +netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx) +{ + trunnel_assert(idx < 16); + return inp->addr_ipv6[idx]; +} + +uint8_t +netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx) +{ + return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx); +} +int +netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 16); + inp->addr_ipv6[idx] = elt; + return 0; +} + +uint8_t * +netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp) +{ + return inp->addr_ipv6; +} +const uint8_t * +netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp) +{ + return (const uint8_t *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp); +} +const char * +netinfo_addr_check(const netinfo_addr_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) + return "Integer out of bounds"; + if (! (obj->len == 4 || obj->len == 16)) + return "Integer out of bounds"; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + break; + + case NETINFO_ADDR_TYPE_IPV6: + break; + + default: + return "Bad tag for union"; + break; + } + return NULL; +} + +ssize_t +netinfo_addr_encoded_len(const netinfo_addr_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_addr_check(obj)) + return -1; + + + /* Length of u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + result += 1; + + /* Length of u8 len IN [4, 16] */ + result += 1; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Length of u32 addr_ipv4 */ + result += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Length of u8 addr_ipv6[16] */ + result += 16; + break; + + default: + trunnel_assert(0); + break; + } + return result; +} +int +netinfo_addr_clear_errors(netinfo_addr_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_addr_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_addr_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->addr_type)); + written += 1; ptr += 1; + + /* Encode u8 len IN [4, 16] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->len)); + written += 1; ptr += 1; + + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; + + default: + trunnel_assert(0); + break; + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_addr_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + CHECK_REMAINING(1, truncated); + obj->addr_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) + goto fail; + + /* Parse u8 len IN [4, 16] */ + CHECK_REMAINING(1, truncated); + obj->len = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->len == 4 || obj->len == 16)) + goto fail; + + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, truncated); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, truncated); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; + + default: + goto fail; + break; + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_addr_new(); + if (NULL == *output) + return -1; + result = netinfo_addr_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_addr_free(*output); + *output = NULL; + } + return result; +} +netinfo_cell_t * +netinfo_cell_new(void) +{ + netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_cell_clear(netinfo_cell_t *obj) +{ + (void) obj; + netinfo_addr_free(obj->other_addr); + obj->other_addr = NULL; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs); + TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs); +} + +void +netinfo_cell_free(netinfo_cell_t *obj) +{ + if (obj == NULL) + return; + netinfo_cell_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_cell_t)); + trunnel_free_(obj); +} + +uint32_t +netinfo_cell_get_timestamp(const netinfo_cell_t *inp) +{ + return inp->timestamp; +} +int +netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val) +{ + inp->timestamp = val; + return 0; +} +struct netinfo_addr_st * +netinfo_cell_get_other_addr(netinfo_cell_t *inp) +{ + return inp->other_addr; +} +const struct netinfo_addr_st * +netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp) +{ + return netinfo_cell_get_other_addr((netinfo_cell_t*) inp); +} +int +netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + if (inp->other_addr && inp->other_addr != val) + netinfo_addr_free(inp->other_addr); + return netinfo_cell_set0_other_addr(inp, val); +} +int +netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + inp->other_addr = val; + return 0; +} +uint8_t +netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp) +{ + return inp->n_my_addrs; +} +int +netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val) +{ + inp->n_my_addrs = val; + return 0; +} +size_t +netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs); +} + +struct netinfo_addr_st * +netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); +} + + const struct netinfo_addr_st * +netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx) +{ + return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx); +} +int +netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); + if (oldval && oldval != elt) + netinfo_addr_free(oldval); + return netinfo_cell_set0_my_addrs(inp, idx, elt); +} +int +netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt); + return 0; +} +int +netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->my_addrs.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct netinfo_addr_st * * +netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp) +{ + return inp->my_addrs.elts_; +} +const struct netinfo_addr_st * const * +netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp) +{ + return (const struct netinfo_addr_st * const *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp); +} +int +netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen) +{ + struct netinfo_addr_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_, + &inp->my_addrs.n_, inp->my_addrs.elts_, newlen, + sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->my_addrs.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +netinfo_cell_check(const netinfo_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + if (NULL != (msg = netinfo_addr_check(obj->other_addr))) + return msg; + } + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs) + return "Length mismatch for my_addrs"; + return NULL; +} + +ssize_t +netinfo_cell_encoded_len(const netinfo_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_cell_check(obj)) + return -1; + + + /* Length of u32 timestamp */ + result += 4; + + /* Length of struct netinfo_addr other_addr */ + result += netinfo_addr_encoded_len(obj->other_addr); + + /* Length of u8 n_my_addrs */ + result += 1; + + /* Length of struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + return result; +} +int +netinfo_cell_clear_errors(netinfo_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_cell_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u32 timestamp */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp)); + written += 4; ptr += 4; + + /* Encode struct netinfo_addr other_addr */ + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, obj->other_addr); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + + /* Encode u8 n_my_addrs */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_my_addrs)); + written += 1; ptr += 1; + + /* Encode struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_cell_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u32 timestamp */ + CHECK_REMAINING(4, truncated); + obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + + /* Parse struct netinfo_addr other_addr */ + result = netinfo_addr_parse(&obj->other_addr, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + + /* Parse u8 n_my_addrs */ + CHECK_REMAINING(1, truncated); + obj->n_my_addrs = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct netinfo_addr my_addrs[n_my_addrs] */ + TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {}); + { + netinfo_addr_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_my_addrs; ++idx) { + result = netinfo_addr_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_cell_new(); + if (NULL == *output) + return -1; + result = netinfo_cell_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h new file mode 100644 index 0000000000..ac46e603ba --- /dev/null +++ b/src/trunnel/netinfo.h @@ -0,0 +1,226 @@ +/* netinfo.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_NETINFO_H +#define TRUNNEL_NETINFO_H + +#include +#include "trunnel.h" + +#define NETINFO_ADDR_TYPE_IPV4 4 +#define NETINFO_ADDR_TYPE_IPV6 6 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR) +struct netinfo_addr_st { + uint8_t addr_type; + uint8_t len; + uint32_t addr_ipv4; + uint8_t addr_ipv6[16]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_addr_st netinfo_addr_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL) +struct netinfo_cell_st { + uint32_t timestamp; + struct netinfo_addr_st *other_addr; + uint8_t n_my_addrs; + TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_cell_st netinfo_cell_t; +/** Return a newly allocated netinfo_addr with all elements set to + * zero. + */ +netinfo_addr_t *netinfo_addr_new(void); +/** Release all storage held by the netinfo_addr in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_addr_free(netinfo_addr_t *victim); +/** Try to parse a netinfo_addr from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_addr_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_addr in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj); +/** Try to encode the netinfo_addr from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input); +/** Check whether the internal state of the netinfo_addr in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_addr_check(const netinfo_addr_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_addr_clear_errors(netinfo_addr_t *obj); +/** Return the value of the addr_type field of the netinfo_addr_t in + * 'inp' + */ +uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp); +/** Set the value of the addr_type field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the len field of the netinfo_addr_t in 'inp' + */ +uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp); +/** Set the value of the len field of the netinfo_addr_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' + */ +uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp); +/** Set the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val); +/** Return the (constant) length of the array holding the addr_ipv6 + * field of the netinfo_addr_t in 'inp'. + */ +size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp); +/** Return the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp'. + */ +uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the + * value 'elt'. + */ +int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'. + */ +uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +const uint8_t * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp); +/** Return a newly allocated netinfo_cell with all elements set to + * zero. + */ +netinfo_cell_t *netinfo_cell_new(void); +/** Release all storage held by the netinfo_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_cell_free(netinfo_cell_t *victim); +/** Try to parse a netinfo_cell from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj); +/** Try to encode the netinfo_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input); +/** Check whether the internal state of the netinfo_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_cell_check(const netinfo_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_cell_clear_errors(netinfo_cell_t *obj); +/** Return the value of the timestamp field of the netinfo_cell_t in + * 'inp' + */ +uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp); +/** Set the value of the timestamp field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val); +/** Return the value of the other_addr field of the netinfo_cell_t in + * 'inp' + */ +struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp); +/** As netinfo_cell_get_other_addr, but take and return a const + * pointer + */ +const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp); +/** Set the value of the other_addr field of the netinfo_cell_t in + * 'inp' to 'val'. Free the old value if any. Steals the referenceto + * 'val'.Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** As netinfo_cell_set_other_addr, but does not free the previous + * value. + */ +int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** Return the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' + */ +uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp); +/** Set the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the my_addrs field + * of the netinfo_cell_t in 'inp'. + */ +size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp'. + */ +struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ + const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the + * value 'elt'. Free the previous value, if any. + */ +int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** As netinfo_cell_set_my_addrs, but does not free the previous + * value. + */ +int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** Append a new element 'elt' to the dynamic array field my_addrs of + * the netinfo_cell_t in 'inp'. + */ +int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt); +/** Return a pointer to the variable-length array field my_addrs of + * 'inp'. + */ +struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ +const struct netinfo_addr_st * const * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp); +/** Change the length of the variable-length array field my_addrs of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen); + + +#endif diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel new file mode 100644 index 0000000000..9c80fcc533 --- /dev/null +++ b/src/trunnel/netinfo.trunnel @@ -0,0 +1,21 @@ +const NETINFO_ADDR_TYPE_IPV4 = 4; +const NETINFO_ADDR_TYPE_IPV6 = 6; + +struct netinfo_addr { + u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6]; + u8 len IN [4, 16]; + union addr[addr_type] { + NETINFO_ADDR_TYPE_IPV4: u32 ipv4; + NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; + default: fail; + }; + +} + +struct netinfo_cell { + u32 timestamp; + struct netinfo_addr other_addr; + u8 n_my_addrs; + struct netinfo_addr my_addrs[n_my_addrs]; +} + -- GitLab From d3e6112bb2536f0437cabc3e675177a95d3e374c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 14:58:19 +0300 Subject: [PATCH 0073/1724] Use trunnel for NETINFO cell parsing --- src/core/or/channeltls.c | 94 ++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 883acacf37..392e78506e 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -59,6 +59,7 @@ #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" #include "trunnel/channelpadding_negotiation.h" +#include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/cell_st.h" @@ -1636,6 +1637,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) channelpadding_negotiate_free(negotiation); } +/** + * Convert netinfo_addr into corresponding tor_addr. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + /** * Process a 'netinfo' cell. * @@ -1648,8 +1678,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) time_t timestamp; uint8_t my_addr_type; uint8_t my_addr_len; - const uint8_t *my_addr_ptr; - const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); const routerinfo_t *me = router_get_my_routerinfo(); @@ -1720,34 +1748,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); - if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { - apparent_skew = now - timestamp; + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; } - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + + if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { + apparent_skew = now - timestamp; + } /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { * * This is actually never going to happen, since my_addr_len is at most 255, * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ - if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { - tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) { + connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); + return; + } + if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) { if (!get_options()->BridgeRelay && me && - get_uint32(my_addr_ptr) == htonl(me->addr)) { + tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) { TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; } - - } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { - tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); - + } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 && + my_addr_len == 16) { if (!get_options()->BridgeRelay && me && !tor_addr_is_null(&me->ipv6_addr) && tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { @@ -1755,17 +1797,21 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } } - n_other_addrs = (uint8_t) *cp++; - while (n_other_addrs && cp < end-2) { + n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell); + for (uint8_t i = 0; i < n_other_addrs; i++) { /* Consider all the other addresses; if any matches, this connection is * "canonical." */ + + const netinfo_addr_t *netinfo_addr = + netinfo_cell_getconst_my_addrs(netinfo_cell, i); + tor_addr_t addr; - const uint8_t *next = - decode_address_from_payload(&addr, cp, (int)(end-cp)); - if (next == NULL) { + + if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Bad address in netinfo cell; closing connection."); connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); return; } /* A relay can connect from anywhere and be canonical, so @@ -1779,10 +1825,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_set_canonical(chan->conn, 1); break; } - cp = next; - --n_other_addrs; } + netinfo_cell_free(netinfo_cell); + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { const char *descr = -- GitLab From 5cc86e364f4efbaa39b599e007765fc1770c8010 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 17:56:39 +0300 Subject: [PATCH 0074/1724] Generate NETINFO cell using trunnel --- src/core/or/connection_or.c | 82 +++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 65f4e28c92..274cf85904 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -46,6 +46,7 @@ #include "lib/geoip/geoip.h" #include "core/mainloop/mainloop.h" #include "trunnel/link_handshake.h" +#include "trunnel/netinfo.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -2428,6 +2429,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) return 0; } +static netinfo_addr_t * +netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr) +{ + sa_family_t addr_family = tor_addr_family(tor_addr); + + if (BUG(addr_family != AF_INET && addr_family != AF_INET6)) + return NULL; + + netinfo_addr_t *netinfo_addr = netinfo_addr_new(); + + if (addr_family == AF_INET) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4); + netinfo_addr_set_len(netinfo_addr, 4); + netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr)); + } else if (addr_family == AF_INET6) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6); + netinfo_addr_set_len(netinfo_addr, 16); + uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr); + memcpy(ipv6_buf, in6_addr, 16); + } + + return netinfo_addr; +} + /** Send a NETINFO cell on conn, telling the other server what we know * about their address, our address, and the current time. */ MOCK_IMPL(int, @@ -2436,8 +2462,7 @@ connection_or_send_netinfo,(or_connection_t *conn)) cell_t cell; time_t now = time(NULL); const routerinfo_t *me; - int len; - uint8_t *out; + int r = -1; tor_assert(conn->handshake_state); @@ -2450,20 +2475,21 @@ connection_or_send_netinfo,(or_connection_t *conn)) memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; + netinfo_cell_t *netinfo_cell = netinfo_cell_new(); + /* Timestamp, if we're a relay. */ if (public_server_mode(get_options()) || ! conn->is_outgoing) - set_uint32(cell.payload, htonl((uint32_t)now)); + netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now); /* Their address. */ - out = cell.payload + 4; + const tor_addr_t *remote_tor_addr = + !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr; /* We use &conn->real_addr below, unless it hasn't yet been set. If it * hasn't yet been set, we know that base_.addr hasn't been tampered with * yet either. */ - len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr) - ? &conn->real_addr : &conn->base_.addr); - if (len<0) - return -1; - out += len; + netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr); + + netinfo_cell_set_other_addr(netinfo_cell, their_addr); /* My address -- only include it if I'm a public relay, or if I'm a * bridge and this is an incoming connection. If I'm a bridge and this @@ -2471,28 +2497,42 @@ connection_or_send_netinfo,(or_connection_t *conn)) if ((public_server_mode(get_options()) || !conn->is_outgoing) && (me = router_get_my_routerinfo())) { tor_addr_t my_addr; - *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr); - tor_addr_from_ipv4h(&my_addr, me->addr); - len = append_address_to_payload(out, &my_addr); - if (len < 0) - return -1; - out += len; + + uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr); + netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs); + + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&my_addr)); if (!tor_addr_is_null(&me->ipv6_addr)) { - len = append_address_to_payload(out, &me->ipv6_addr); - if (len < 0) - return -1; + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&me->ipv6_addr)); } - } else { - *out = 0; + } + + const char *errmsg = NULL; + if ((errmsg = netinfo_cell_check(netinfo_cell))) { + log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s", + errmsg); + goto cleanup; + } + + if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE, + netinfo_cell) < 0) { + log_warn(LD_OR, "Failed generating NETINFO cell"); + goto cleanup; } conn->handshake_state->digest_sent_data = 0; conn->handshake_state->sent_netinfo = 1; connection_or_write_cell_to_buf(&cell, conn); - return 0; + r = 0; + cleanup: + netinfo_cell_free(netinfo_cell); + + return r; } /** Helper used to add an encoded certs to a cert cell */ -- GitLab From b64e1e602bac5e108e40a284616e3fe7f7597b17 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 22 Sep 2018 20:01:15 +0300 Subject: [PATCH 0075/1724] Add changes file --- changes/ticket27325 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket27325 diff --git a/changes/ticket27325 b/changes/ticket27325 new file mode 100644 index 0000000000..2cf2bb7d69 --- /dev/null +++ b/changes/ticket27325 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket + 27325. -- GitLab From bdf6540edf5b9d3d45214393412b8fe7e42da5ca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 26 Oct 2018 11:30:26 +0300 Subject: [PATCH 0076/1724] Add a comment about address type field to netinfo.trunnel --- src/trunnel/netinfo.trunnel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel index 9c80fcc533..21afcd136e 100644 --- a/src/trunnel/netinfo.trunnel +++ b/src/trunnel/netinfo.trunnel @@ -1,3 +1,6 @@ +// Warning: make sure these values are consistent with RESOLVED_TYPE_* +// constants in Tor code and numbers in Section 6.4 of tor-spec.txt. + const NETINFO_ADDR_TYPE_IPV4 = 4; const NETINFO_ADDR_TYPE_IPV6 = 6; -- GitLab From 5c2212c7346c5368a9d5e35ac78b143999682297 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Oct 2018 14:55:17 +0300 Subject: [PATCH 0077/1724] HSv3: Correctly memwipe client auth keystream. Wipe the whole thing, not just the size of the pointer. --- src/feature/hs/hs_descriptor.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 0e3c761bf6..1b2008c804 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1406,10 +1406,10 @@ encrypted_data_length_is_valid(size_t len) * SECRET_SEED = x25519(sk, pk) * KEYS = KDF(subcredential | SECRET_SEED, 40) * - * The keys_out parameter will points to the buffer containing the KEYS. The - * caller should wipe and free its content once done with it. This function - * can't fail. */ -static void + * Set the keys_out argument to point to the buffer containing the KEYS, + * and return the buffer's length. The caller should wipe and free its content + * once done with it. This function can't fail. */ +static size_t build_descriptor_cookie_keys(const uint8_t *subcredential, size_t subcredential_len, const curve25519_secret_key_t *sk, @@ -1441,6 +1441,7 @@ build_descriptor_cookie_keys(const uint8_t *subcredential, memwipe(secret_seed, 0, sizeof(secret_seed)); *keys_out = keystream; + return keystream_len; } /* Decrypt the descriptor cookie given the descriptor, the auth client, @@ -1456,6 +1457,7 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, { int ret = -1; uint8_t *keystream = NULL; + size_t keystream_length = 0; uint8_t *descriptor_cookie = NULL; const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; @@ -1471,10 +1473,12 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ - build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, - client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey, - &keystream); + keystream_length = + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); + tor_assert(keystream_length > 0); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1500,7 +1504,7 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); tor_free(keystream); return ret; } @@ -2915,6 +2919,7 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, hs_desc_authorized_client_t *client_out) { uint8_t *keystream = NULL; + size_t keystream_length = 0; const uint8_t *cookie_key; crypto_cipher_t *cipher; @@ -2933,8 +2938,11 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, DIGEST256_LEN)); /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ - build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, - auth_ephemeral_sk, client_auth_pk, &keystream); + keystream_length = + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, + &keystream); + tor_assert(keystream_length > 0); /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); @@ -2951,7 +2959,7 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); tor_free(keystream); crypto_cipher_free(cipher); -- GitLab From fe89d9df6935e3a61f8014ef9033b1ca2cfff5a1 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 29 Oct 2018 13:47:43 +1000 Subject: [PATCH 0078/1724] doc: Spell make test-network-all correctly in ReleasingTor.md Closes ticket 28821. --- doc/HACKING/ReleasingTor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 55a40fc89b..29e1ae1668 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -39,7 +39,7 @@ new Tor release: Does 'make distcheck' complain? How about 'make test-stem' and 'make test-network' and - `make test-network-full`? + `make test-network-all`? - Are all those tests still happy with --enable-expensive-hardening ? -- GitLab From b0c456e578229420ed833ba919c181f7dd5f5e2b Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 29 Oct 2018 13:49:47 +1000 Subject: [PATCH 0079/1724] doc: Use `` for commands and "" for quotes in ReleasingTor.md --- doc/HACKING/ReleasingTor.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 29e1ae1668..b5444afa96 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -36,9 +36,9 @@ new Tor release: What about clang scan-build? - Does 'make distcheck' complain? + Does `make distcheck` complain? - How about 'make test-stem' and 'make test-network' and + How about `make test-stem` and `make test-network` and `make test-network-all`? - Are all those tests still happy with --enable-expensive-hardening ? @@ -79,7 +79,7 @@ new Tor release: Present and imperative tense: not past. - 'Relays', not 'servers' or 'nodes' or 'Tor relays'. + "Relays", not "servers" or "nodes" or "Tor relays". "Stop FOOing", not "Fix a bug where we would FOO". @@ -100,7 +100,7 @@ new Tor release: For stable releases that backport things from later, we try to compose their releases, we try to make sure that we keep the changelog entries - identical to their original versions, with a 'backport from 0.x.y.z' + identical to their original versions, with a "backport from 0.x.y.z" note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged -- GitLab From 635312fc2a8fe96aa282ed242279629f814b9333 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 10:45:14 +0200 Subject: [PATCH 0080/1724] Silence shellcheck SC2086 in run_trunnel.sh --- scripts/codegen/run_trunnel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 428804342a..9df402cf59 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -10,7 +10,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} $file + python -m trunnel ${OPTIONS} "$file" done python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ -- GitLab From a0dd6bfdb03f53a52778e65161b9f3c68234bb82 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 10:54:31 +0200 Subject: [PATCH 0081/1724] run_trunnel.sh: Use 'find -exec' instead of a 'for' loop This fixes shellcheck warnings SC2044 and SC2006. --- scripts/codegen/run_trunnel.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 9df402cf59..645b3c2158 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. -for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} "$file" -done +find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \; python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ -- GitLab From 4af27e016814f4817174ed87e7b660d65c6eaf9c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:05:55 +0200 Subject: [PATCH 0082/1724] Add changes file --- changes/ticket28010 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28010 diff --git a/changes/ticket28010 b/changes/ticket28010 new file mode 100644 index 0000000000..4fca17d022 --- /dev/null +++ b/changes/ticket28010 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue + 28010. -- GitLab From eab81b12e92ea3ae53d14337a6fafc9d6c32b39a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:31:03 +0200 Subject: [PATCH 0083/1724] Fix shellcheck warning SC2086 in run_calltool.sh --- scripts/maint/run_calltool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index efb8706fea..f36810f5b6 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -15,7 +15,7 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr for calculation in $SUBITEMS; do echo "======== $calculation" - python -m calltool $calculation > callgraph/$calculation + python -m calltool "$calculation" > callgraph/"$calculation" done echo < callgraph/README -- GitLab From 6aef0ce94e9bfe5d3bd06f0a3349f9e8930218f5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:37:16 +0200 Subject: [PATCH 0084/1724] Fix shellcheck issue SC2217 in run_calltool.sh --- scripts/maint/run_calltool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index f36810f5b6..b0268322f4 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -18,7 +18,7 @@ for calculation in $SUBITEMS; do python -m calltool "$calculation" > callgraph/"$calculation" done -echo < callgraph/README +cat < callgraph/README This directory holds output from calltool, as run on Tor. For more information about each of these files, see the NOTES and README files in the calltool distribution. -- GitLab From 5a3cb495ce5bf010440fc0288c1ca00fff6ec8e5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:41:36 +0200 Subject: [PATCH 0085/1724] Add changes file --- changes/ticket28011 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28011 diff --git a/changes/ticket28011 b/changes/ticket28011 new file mode 100644 index 0000000000..5efc3c917b --- /dev/null +++ b/changes/ticket28011 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_calltool.sh. Resolves + ticket 28011. -- GitLab From 91748cd17c1c1c2d5590d3b5533b74fcc1cd2140 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Oct 2018 08:58:19 -0400 Subject: [PATCH 0086/1724] doc: Add Maintaining.md documentation Closes #28225 Signed-off-by: David Goulet --- doc/HACKING/Maintaining.md | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 doc/HACKING/Maintaining.md diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md new file mode 100644 index 0000000000..22d62b5471 --- /dev/null +++ b/doc/HACKING/Maintaining.md @@ -0,0 +1,113 @@ +# Maintaining Tor + +This document details the duties and processes on maintaining the Tor code +base. + +The first section describes who is the current Tor maintainer and what are the +responsabilities. Tor has one main single maintainer but does have many +committers and subsystem maintainers. + +The second third section describes how the **alpha and master** branches are +maintained and by whom. + +Finally, the last section describes how the **stable** branches are maintained +and by whom. + +This document does not cover how Tor is released, please see +[ReleasingTor.md](ReleasingTor.md) for that information. + +## Tor Maintainer + +The current maintainer is Nick Mathewson . + +The maintainer takes final decisions in terms of engineering, architecture and +protocol design. Releasing Tor falls under their responsability. + +## Alpha and Master Branches + +The Tor repository always has at all time a **master** branch which contains +the upstream ongoing development. + +It may also contains a branch for a released feature freezed version which is +called the **alpha** branch. The git tag and version number is always +postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or +`tor-0.3.5.3-alpha`. + +Tor is separated into subsystems and some of those are maintained by other +developers than the main maintainer. Those people have commit access to the +code base but only commit (in most cases) into the subsystem they maintain. + +Upstream merges are restricted to the alpha and master branches. Subsystem +maintainers should never push a patch into a stable branch which is the +responsability of the [stable branch maintainer](#stable-branches). + +### Who + +In alphabetical order, the following people have upstream commit access and +maintain the following subsystems: + +- David Goulet + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Channels, Circuitmux, Connection, Scheduler. + ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]* + * Cell Logic (Handling/Parsing). + ***keywords:*** *[tor-cell]* + * Threading backend. + ***keywords:*** *[tor-thread]* + +- George Kadianakis + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Guard. + ***keywords:*** *[tor-guard]* + * Pluggable Transport (excluding Bridge networking). + ***keywords:*** *[tor-pt]* + +### Tasks + +These are the tasks of a subsystem maintainer: + +1. Regurlarly go over `merge_ready` tickets relevant to the related subsystem + and for the current alpha or development (master branch) Milestone. + +2. A subsystem maintainer is expected to contribute to any design changes + (including proposals) or large patch set about the subsystem. + +3. Leave their ego at the door. Mistakes will be made but they have to be + taking care of seriously. Learn and move on quickly. + +### Merging Policy + +These are few important items to follow when merging code upstream: + +1. To merge code upstream, the patch must have passed our CI (currently + github.com/torproject), have a corresponding ticket and reviewed by + **at least** one person that is not the original coder. + + Example A: If Alice writes a patch then Bob, a Tor network team member, + reviews it and flags it `merge_ready`. Then, the maintainter is required + to look at the patch and makes a decision. + + Example B: If the maintainer writes a patch then Bob, a Tor network + team member, reviews it and flags it `merge_ready`, then the maintainer + can merge the code upstream. + +2. Maintainer makes sure the commit message should describe what was fixed + and, if it applies, how was it fixed. It should also always refer to + the ticket number. + +3. Trivial patches such as comment change, documentation, syntax issues or + typos can be merged without a ticket or reviewers. + +4. Tor uses the "merge forward" method that is if a patch applies to the + alpha branch, it has to be merged there first and then merged forward + into master. + +5. Maintainer should always consult with the network team about any doubts, + mis-understandings or unknowns of a patch. Final word will always go to the + main Tor maintainer. + +## Stable Branches + +(Currently being drafted and reviewed by the network team.) -- GitLab From 742cd1564993faefded2d33b6839428a1fe4412a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 1 Sep 2018 20:56:33 +0300 Subject: [PATCH 0087/1724] Move a check for trailing colon to tor_inet_pton() That way, string_is_valid_ipv6_address() can benefit from it --- src/lib/net/address.c | 5 +---- src/lib/net/inaddr.c | 8 +++++++- src/test/test_util.c | 13 +++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 336693b464..17f4b1cf7a 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1198,10 +1198,7 @@ tor_addr_parse(tor_addr_t *addr, const char *src) len -= 2; } - /* Reject if src has needless trailing ':'. */ - if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { - result = -1; - } else if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); } else if (!brackets_detected && diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index dcd8fcdd65..0960d323c5 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/test/test_util.c b/src/test/test_util.c index 6cbd504e34..0921bae109 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -5749,6 +5749,18 @@ test_util_ipv4_validation(void *arg) return; } +static void +test_util_ipv6_validation(void *arg) +{ + (void)arg; + + tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e")); + tt_assert(!string_is_valid_ipv6_address("11:22::33:44:")); + + done: + return; +} + static void test_util_writepid(void *arg) { @@ -6439,6 +6451,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(hostname_validation, 0), UTIL_TEST(dest_validation_edgecase, 0), UTIL_TEST(ipv4_validation, 0), + UTIL_TEST(ipv6_validation, 0), UTIL_TEST(writepid, 0), UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), -- GitLab From 1425549ca61cab8aa9476a25be0a31f726672bcc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 1 Sep 2018 21:22:47 +0300 Subject: [PATCH 0088/1724] Code cleanups for tor_addr_parse() --- src/lib/net/address.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 17f4b1cf7a..a87d4a36a3 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1189,13 +1189,13 @@ tor_addr_parse(tor_addr_t *addr, const char *src) struct in6_addr in6_tmp; int brackets_detected = 0; + tor_assert(addr && src); + size_t len = strlen(src); - tor_assert(addr && src); - if (src[0] == '[' && src[1] && src[len - 1] == ']') { + if (len && src[0] == '[' && src[len - 1] == ']') { brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); - len -= 2; } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { -- GitLab From 067b16eae2b7d37c7ec1595226bc7bf26aac1ff5 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 5 Sep 2018 10:12:35 -0400 Subject: [PATCH 0089/1724] Check IPv6 subnets as well as IPv4 subnets where possible when choosing client paths --- changes/bug24393 | 6 ++++++ src/feature/nodelist/nodelist.c | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 changes/bug24393 diff --git a/changes/bug24393 b/changes/bug24393 new file mode 100644 index 0000000000..e190192319 --- /dev/null +++ b/changes/bug24393 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - When using addrs_in_same_network_family(), check IPv6 subnets as well as + IPv4 ones where possible when a client chooses circuit paths. Previously, + we used this function only for IPv4 subnets. Closes ticket 24393. Patch + by Neel Chauhan. + diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index a98a5c8655..a1a1b0ea37 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1867,6 +1867,9 @@ int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { + if (tor_addr_is_null(a1) || tor_addr_is_null(a2)) + return 0; + switch (tor_addr_family(a1)) { case AF_INET: return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC); @@ -1917,7 +1920,13 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) tor_addr_t a1, a2; node_get_addr(node1, &a1); node_get_addr(node2, &a2); - if (addrs_in_same_network_family(&a1, &a2)) + + tor_addr_port_t ap6_1, ap6_2; + node_get_pref_ipv6_orport(node1, &ap6_1); + node_get_pref_ipv6_orport(node2, &ap6_2); + + if (addrs_in_same_network_family(&a1, &a2) || + addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr)) return 1; } @@ -1974,12 +1983,17 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) /* First, add any nodes with similar network addresses. */ if (options->EnforceDistinctSubnets) { tor_addr_t node_addr; + tor_addr_port_t node_ap6; node_get_addr(node, &node_addr); + node_get_pref_ipv6_orport(node, &node_ap6); SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) { tor_addr_t a; + tor_addr_port_t ap6; node_get_addr(node2, &a); - if (addrs_in_same_network_family(&a, &node_addr)) + node_get_pref_ipv6_orport(node2, &ap6); + if (addrs_in_same_network_family(&a, &node_addr) || + addrs_in_same_network_family(&ap6.addr, &node_ap6.addr)) smartlist_add(sl, (void*)node2); } SMARTLIST_FOREACH_END(node2); } -- GitLab From a182301152afe9cd066516ae02f588840b2efc43 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 31 Oct 2018 08:30:48 -0400 Subject: [PATCH 0090/1724] Fix memory leak (#28257, CID 1440805). --- src/feature/relay/dns.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 371c2f5069..701719af95 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1386,6 +1386,7 @@ configured_nameserver_address(const size_t idx) NULL) == 0) { return tor_addr; } + tor_free(tor_addr); } return NULL; -- GitLab From e9adc200aab8bf2068dc7d7fb0cf2e2d43149182 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 12 Sep 2018 15:50:11 -0400 Subject: [PATCH 0091/1724] Add test for nodes_in_same_family() --- src/test/test_address.c | 75 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/test/test_address.c b/src/test/test_address.c index e99220f838..69de072cfa 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" #include "lib/net/address.h" #include "test/test.h" @@ -1170,6 +1172,78 @@ test_address_tor_addr_in_same_network_family(void *ignored) return; } +static node_t * +helper_create_mock_node(char id_char) +{ + node_t *node = tor_malloc_zero(sizeof(node_t)); + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + tor_addr_make_null(&ri->ipv6_addr, AF_INET6); + node->ri = ri; + memset(node->identity, id_char, sizeof(node->identity)); + return node; +} + +static void +helper_free_mock_node(node_t *node) +{ + tor_free(node->ri); + tor_free(node); +} + +#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \ + tor_addr_t addr; \ + tor_addr_parse(&addr, ipv4_addr); \ + node->ri->addr = tor_addr_to_ipv4h(&addr); \ + node->ri->or_port = ipv4_port; \ + } + +#define NODE_CLEAR_IPV4(node) { \ + node->ri->addr = 0; \ + node->ri->or_port = 0; \ + } + +#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \ + tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \ + node->ri->ipv6_orport = ipv6_port; \ + } + +static void +test_address_tor_node_in_same_network_family(void *ignored) +{ + (void)ignored; + node_t *node_a = helper_create_mock_node('a'); + node_t *node_b = helper_create_mock_node('b'); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "8.8.4.4", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "1.1.1.1", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_a); + NODE_SET_IPV6(node_a, "2001:470:20::2", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_b); + NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + helper_free_mock_node(node_a); + helper_free_mock_node(node_b); + + done: + return; +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } @@ -1202,6 +1276,7 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0), ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), + ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; -- GitLab From f60607ee96bf85286c7ef58f7fcf5ab00f0c3ad0 Mon Sep 17 00:00:00 2001 From: Fernando Fernandez Mancera Date: Mon, 29 Oct 2018 17:07:41 +0100 Subject: [PATCH 0092/1724] Improve log message in hs_service.c Signed-off-by: Fernando Fernandez Mancera --- src/feature/hs/hs_service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 7d56c9e2ad..c288e28e80 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2931,8 +2931,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now, /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */ tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR); - log_info(LD_REND, "Encrypted revision counter %d to %ld", - (int) seconds_since_start_of_srv, (long int) rev_counter); + log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64, + (int) seconds_since_start_of_srv, rev_counter); hs_desc->desc->plaintext_data.revision_counter = rev_counter; } -- GitLab From a0402c6f33206468f57c381c0022e547520d14c2 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 1 Nov 2018 12:37:17 +0200 Subject: [PATCH 0093/1724] Add changes file for #27707. --- changes/bug27707 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/bug27707 diff --git a/changes/bug27707 b/changes/bug27707 new file mode 100644 index 0000000000..e114222741 --- /dev/null +++ b/changes/bug27707 @@ -0,0 +1,3 @@ + o Minor features (log messages): + - Improve log message in HSv3 service that could print out negative + revision counters. Closes ticket 27707. Patch by "ffmancera". \ No newline at end of file -- GitLab From da716fdfbb08952b971882eba1dabca2fef9c7f3 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 1 Nov 2018 12:55:04 +0200 Subject: [PATCH 0094/1724] Add tests for the string_is_utf8_no_bom() function. --- src/test/test_util.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/test_util.c b/src/test/test_util.c index 7bc1b7921a..0678251137 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4024,6 +4024,13 @@ test_util_string_is_utf8(void *ptr) tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7)); tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9)); + /* Test the utf8_no_bom function */ + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7)); + tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect", + strlen("loveandrespect"))); + // Validate exactly 'len' bytes. tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2)); tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6)); -- GitLab From 0ce1f2d46646fd73abee56888650288055f16a53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 08:18:29 -0400 Subject: [PATCH 0095/1724] Declare the subsystem structure. --- src/include.am | 1 + src/lib/subsys/.may_include | 1 + src/lib/subsys/include.am | 3 ++ src/lib/subsys/subsys.h | 63 +++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 src/lib/subsys/.may_include create mode 100644 src/lib/subsys/include.am create mode 100644 src/lib/subsys/subsys.h diff --git a/src/include.am b/src/include.am index d2f83da814..247b0db8da 100644 --- a/src/include.am +++ b/src/include.am @@ -25,6 +25,7 @@ include src/lib/osinfo/include.am include src/lib/process/include.am include src/lib/sandbox/include.am include src/lib/string/include.am +include src/lib/subsys/include.am include src/lib/smartlist_core/include.am include src/lib/term/include.am include src/lib/testsupport/include.am diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..4741126b14 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,3 @@ + +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..7e4fe53636 --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include + +struct dispatch_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * Subsystems + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct dispatch_connector_t *); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#endif -- GitLab From 1b75de85b3cbc7706078a9899e483d18579a6fd1 Mon Sep 17 00:00:00 2001 From: "Alex Xu (Hello71)" Date: Fri, 19 Oct 2018 09:53:23 -0400 Subject: [PATCH 0096/1724] Don't overwrite the Content-Type when compressing --- changes/ticket28100 | 3 +++ src/feature/dircache/dircache.c | 14 ++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 changes/ticket28100 diff --git a/changes/ticket28100 b/changes/ticket28100 new file mode 100644 index 0000000000..b8e3271013 --- /dev/null +++ b/changes/ticket28100 @@ -0,0 +1,3 @@ + o Minor features (HTTP standards compliance): + - Don't send Content-Type: application/octet-stream for transparently + compressed documents, which confused browsers. Closes ticket 28100. diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..dff4b85caa 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -166,22 +166,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, buf_free(buf); } -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be compressed or not. */ +/** As write_http_response_header_impl, but translates method into + * encoding */ static void write_http_response_headers(dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime) { - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - doctype, - methodname, + "text/plain", + compression_method_get_name(method), extra_headers, cache_lifetime); } -- GitLab From 3e2423d19b78ab71cb1ac8205406c7575296ecd3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:29:24 -0400 Subject: [PATCH 0097/1724] Update address tests to avoid offending coverity. --- src/test/test_address.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/test_address.c b/src/test/test_address.c index 69de072cfa..a823fd9cd5 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1186,6 +1186,8 @@ helper_create_mock_node(char id_char) static void helper_free_mock_node(node_t *node) { + if (!node) + return; tor_free(node->ri); tor_free(node); } @@ -1237,11 +1239,9 @@ test_address_tor_node_in_same_network_family(void *ignored) NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + done: helper_free_mock_node(node_a); helper_free_mock_node(node_b); - - done: - return; } #define ADDRESS_TEST(name, flags) \ @@ -1279,4 +1279,3 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; - -- GitLab From 18a4eaf5c142bae55780716464d43c2f8a9e2e49 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:30:55 -0400 Subject: [PATCH 0098/1724] Avoid mmap leak if we get a consensus diff we can't use. Fixes CID 1440819; bug not in any released Tor. --- src/feature/dirclient/dirclient.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 705bf75e5c..cd88fa5ebf 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2221,6 +2221,7 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " "any %s-flavored consensus in our current cache.",flavname); + tor_munmap_file(mapped_consensus); networkstatus_consensus_download_failed(0, flavname); // XXXX if this happens too much, see below return -1; -- GitLab From 674ef53a7e953a724b4f3bfe2f1e06ba2897bba2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:32:43 -0400 Subject: [PATCH 0099/1724] Add a warning if we can't write networkstatus-bridges Fixes CID 1440818. --- src/feature/nodelist/networkstatus.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ec1a69b9e2..f1def9afb1 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2403,7 +2403,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) published, thresholds, fingerprint_line ? fingerprint_line : "", status); fname = get_datadir_fname("networkstatus-bridges"); - write_str_to_file(fname,published_thresholds_and_status,0); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } tor_free(thresholds); tor_free(published_thresholds_and_status); tor_free(fname); -- GitLab From 5b48af9c4cc591d4db00cc1060de4d519a80023b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 4 Nov 2018 19:34:46 +0200 Subject: [PATCH 0100/1724] Fix all instances of shellcheck warning SC2006 --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index a5a92579d3..d2680856b8 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,11 +20,11 @@ set -e -HERE=`dirname $(realpath $0)` -TOPLEVEL=`dirname $(dirname $HERE)` +HERE=$(dirname $(realpath $0)) +TOPLEVEL=$(dirname $(dirname $HERE)) TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=`which cargo` +CARGO=$(which cargo) if ! test -f "$TOML" ; then printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" @@ -38,7 +38,7 @@ if test -z "$CARGO" ; then printf "Error: cargo must be installed and in your \$PATH\n" fi -if test -z `cargo --list | grep vendor` ; then +if test -z $(cargo --list | grep vendor) ; then printf "Error: cargo-vendor not installed\n" fi -- GitLab From 45b28167d7e2b1d5afb26db6f76ca2329a9bbc04 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 17 Oct 2018 21:43:59 -0400 Subject: [PATCH 0101/1724] In count_acceptable_nodes(), count direct and indirect nodes with node_has_preferred_descriptor() --- changes/bug25885 | 7 +++++++ src/core/or/circuitbuild.c | 21 ++++++++++++--------- src/core/or/circuitbuild.h | 3 ++- src/test/test_circuitbuild.c | 4 ++-- 4 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 changes/bug25885 diff --git a/changes/bug25885 b/changes/bug25885 new file mode 100644 index 0000000000..1b89acfe06 --- /dev/null +++ b/changes/bug25885 @@ -0,0 +1,7 @@ + o Minor bugfixes (guards): + - In count_acceptable_nodes(), check if we have at least one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we have added up the sum of all nodes with a descriptor, but that + could cause us to build circuits that fail if we had either too + many bridges, or not enough guard nodes. Fixes bug 25885; bugfix + on 0.3.6.1-alpha. Patch by Neel Chauhan. diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index a69457571e..4f9f09bc8f 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1658,22 +1658,25 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { - int num_acceptable_routers; int routelen; tor_assert(nodes); routelen = route_len_for_purpose(purpose, exit_ei); - num_acceptable_routers = count_acceptable_nodes(nodes); + int num_acceptable_direct = count_acceptable_nodes(nodes, 1); + int num_acceptable_indirect = count_acceptable_nodes(nodes, 0); - log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", - routelen, num_acceptable_routers, smartlist_len(nodes)); + log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect " + "routers suitable).", routelen, num_acceptable_direct, + num_acceptable_indirect); - if (num_acceptable_routers < routelen) { + if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) { log_info(LD_CIRC, - "Not enough acceptable routers (%d/%d). Discarding this circuit.", - num_acceptable_routers, routelen); + "Not enough acceptable routers (%d/%d direct and %d/%d " + "indirect routers suitable). Discarding this circuit.", + num_acceptable_direct, routelen, + num_acceptable_indirect, routelen); return -1; } @@ -2315,7 +2318,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes)) +count_acceptable_nodes, (smartlist_t *nodes, int direct)) { int num=0; @@ -2329,7 +2332,7 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; - if (! node_has_any_descriptor(node)) + if (! node_has_preferred_descriptor(node, direct)) continue; /* The node has a descriptor, so we can just check the ntor key directly */ if (!node_has_curve25519_onion_key(node)) diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index cee71b297b..93f903f060 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -84,7 +84,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 02eadecd98..dd47ad7689 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -21,11 +21,11 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes) +mock_count_acceptable_nodes(smartlist_t *nodes, int direct) { (void)nodes; - return DEFAULT_ROUTE_LEN + 1; + return direct ? 1 : DEFAULT_ROUTE_LEN + 1; } /* Test route lengths when the caller of new_route_len() doesn't -- GitLab From 7bb76b24cf755799b7950ef078ac5ccf4d6e3a8a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 11:51:33 -0400 Subject: [PATCH 0102/1724] Code to manage the list of subsystems. --- src/app/main/main.c | 6 ++ src/app/main/subsysmgr.c | 130 ++++++++++++++++++++++++++++++++++ src/app/main/subsysmgr.h | 20 ++++++ src/app/main/subsystem_list.c | 20 ++++++ src/core/include.am | 3 + src/lib/subsys/subsys.h | 5 ++ 6 files changed, 184 insertions(+) create mode 100644 src/app/main/subsysmgr.c create mode 100644 src/app/main/subsysmgr.h create mode 100644 src/app/main/subsystem_list.c diff --git a/src/app/main/main.c b/src/app/main/main.c index ae87add67d..444d6ea7ec 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,6 +15,7 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -813,6 +814,9 @@ tor_free_all(int postfork) release_lockfile(); } tor_libevent_free_all(); + + subsystems_shutdown(); + /* Stuff in util.c and address.c*/ if (!postfork) { escaped(NULL); @@ -1426,6 +1430,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif + subsystems_init(); + init_protocol_warning_severity_level(); update_approx_time(time(NULL)); diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c new file mode 100644 index 0000000000..7974f2d238 --- /dev/null +++ b/src/app/main/subsysmgr.c @@ -0,0 +1,130 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/err/torerr.h" + +#include +#include +#include + +/** + * True iff we have checked tor_subsystems for consistency. + **/ +static bool subsystem_array_validated = false; + +/** + * True if a given subsystem is initialized. Expand this array if there + * are more than this number of subsystems. (We'd rather not + * dynamically allocate in this module.) + **/ +static bool sys_initialized[128]; + +/** + * Exit with a raw assertion if the subsystems list is inconsistent; + * initialize the subsystem_initialized array. + **/ +static void +check_and_setup(void) +{ + if (subsystem_array_validated) + return; + + raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems); + memset(sys_initialized, 0, sizeof(sys_initialized)); + + int last_level = MIN_SUBSYS_LEVEL; + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) { + fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. " + "It is supposed to be between %d and %d (inclusive).\n", + sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + if (sys->level < last_level) { + fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. " + "Its level is %d; but the previous subsystem's level was %d.\n", + sys->name, i, sys->level, last_level); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + last_level = sys->level; + } + + subsystem_array_validated = true; +} + +/** + * Initialize all the subsystems; exit on failure. + **/ +int +subsystems_init(void) +{ + return subsystems_init_upto(MAX_SUBSYS_LEVEL); +} + +/** + * Initialize all the subsystems whose level is less than or equal to + * target_level; exit on failure. + **/ +int +subsystems_init_upto(int target_level) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (sys_initialized[i]) + continue; + int r = 0; + if (sys->initialize) + r = sys->initialize(); + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", + sys->name, i); + raw_assert_unreached_msg("A subsystem couldn't be initialized."); + } + sys_initialized[i] = true; + } + + return 0; +} + +/** + * Shut down all the subsystems. + **/ +void +subsystems_shutdown(void) +{ + subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1); +} + +/** + * Shut down all the subsystems whose level is above target_level. + **/ +void +subsystems_shutdown_downto(int target_level) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level <= target_level) + break; + if (! sys_initialized[i]) + continue; + if (sys->shutdown) + sys->shutdown(); + sys_initialized[i] = false; + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h new file mode 100644 index 0000000000..c9b892eee4 --- /dev/null +++ b/src/app/main/subsysmgr.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYSMGR_T +#define TOR_SUBSYSMGR_T + +#include "lib/subsys/subsys.h" + +extern const struct subsys_fns_t *tor_subsystems[]; +extern const unsigned n_tor_subsystems; + +int subsystems_init(void); +int subsystems_init_upto(int level); + +void subsystems_shutdown(void); +void subsystems_shutdown_downto(int level); + +#endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c new file mode 100644 index 0000000000..fc1249e1c6 --- /dev/null +++ b/src/app/main/subsystem_list.c @@ -0,0 +1,20 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include + +/** + * Global list of the subsystems in Tor, in the order of their initialization. + **/ +const subsys_fns_t *tor_subsystems[] = { + NULL // placeholder. +}; + +const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..d3fce54285 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,8 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/subsystem_list.c \ + src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ @@ -191,6 +193,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 7e4fe53636..012b218da7 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -43,6 +43,8 @@ typedef struct subsys_fns_t { * * This function MUST NOT rely on any runtime configuration information; * it is only for global state or pre-configuration state. + * + * This function MUST NOT have any parts that can fail. **/ int (*initialize)(void); @@ -60,4 +62,7 @@ typedef struct subsys_fns_t { } subsys_fns_t; +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + #endif -- GitLab From 6e7ff8cba0efaf803e3ef5b5aba4123633fe0658 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:33:22 -0400 Subject: [PATCH 0103/1724] Move the code that knows our tor version into a lowest-level lib --- .gitignore | 2 + Makefile.am | 2 + src/app/config/config.c | 41 +------------------ src/app/config/config.h | 2 - src/app/config/statefile.c | 1 + src/app/main/main.c | 1 + src/feature/control/control.c | 1 + src/feature/dirauth/shared_random_state.c | 1 + src/feature/relay/router.c | 1 + src/include.am | 1 + src/lib/log/.may_include | 3 +- src/lib/log/include.am | 8 ---- src/lib/log/log.c | 2 +- src/lib/version/.may_include | 3 ++ src/lib/{log => version}/git_revision.c | 2 +- src/lib/{log => version}/git_revision.h | 0 src/lib/version/include.am | 25 ++++++++++++ src/lib/version/torversion.h | 12 ++++++ src/lib/version/version.c | 50 +++++++++++++++++++++++ src/rust/build.rs | 1 + src/test/fuzz/fuzzing_common.c | 1 + src/test/testing_common.c | 1 + 22 files changed, 107 insertions(+), 54 deletions(-) create mode 100644 src/lib/version/.may_include rename src/lib/{log => version}/git_revision.c (94%) rename src/lib/{log => version}/git_revision.h (100%) create mode 100644 src/lib/version/include.am create mode 100644 src/lib/version/torversion.h create mode 100644 src/lib/version/version.c diff --git a/.gitignore b/.gitignore index cedff8fb37..ee2de376a6 100644 --- a/.gitignore +++ b/.gitignore @@ -210,6 +210,8 @@ uptime-*.json /src/lib/libtor-tls.a /src/lib/libtor-tls-testing.a /src/lib/libtor-trace.a +/src/lib/libtor-version.a +/src/lib/libtor-version-testing.a /src/lib/libtor-wallclock.a /src/lib/libtor-wallclock-testing.a diff --git a/Makefile.am b/Makefile.am index e5c1be31b5..cb76edfa2f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -62,6 +62,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-malloc.a \ src/lib/libtor-wallclock.a \ src/lib/libtor-err.a \ + src/lib/libtor-version.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime.a @@ -91,6 +92,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-malloc-testing.a \ src/lib/libtor-wallclock-testing.a \ src/lib/libtor-err-testing.a \ + src/lib/libtor-version-testing.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime-testing.a endif diff --git a/src/app/config/config.c b/src/app/config/config.c index 6e7e131055..7b49387bcf 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -112,9 +112,9 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" -#include "lib/log/git_revision.h" #include "lib/net/resolve.h" #include "lib/sandbox/sandbox.h" +#include "lib/version/torversion.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -972,42 +972,6 @@ set_options(or_options_t *new_val, char **msg) return 0; } -/** The version of this Tor process, as parsed. */ -static char *the_tor_version = NULL; -/** A shorter version of this Tor process's version, for export in our router - * descriptor. (Does not include the git version, if any.) */ -static char *the_short_tor_version = NULL; - -/** Return the current Tor version. */ -const char * -get_version(void) -{ - if (the_tor_version == NULL) { - if (strlen(tor_git_revision)) { - tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(), - tor_git_revision); - } else { - the_tor_version = tor_strdup(get_short_version()); - } - } - return the_tor_version; -} - -/** Return the current Tor version, without any git tag. */ -const char * -get_short_version(void) -{ - - if (the_short_tor_version == NULL) { -#ifdef TOR_BUILD_TAG - tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG); -#else - the_short_tor_version = tor_strdup(VERSION); -#endif - } - return the_short_tor_version; -} - /** Release additional memory allocated in options */ STATIC void @@ -1067,9 +1031,6 @@ config_free_all(void) tor_free(torrc_defaults_fname); tor_free(global_dirfrontpagecontents); - tor_free(the_short_tor_version); - tor_free(the_tor_version); - cleanup_protocol_warning_severity_level(); have_parsed_cmdline = 0; diff --git a/src/app/config/config.h b/src/app/config/config.h index a169cfd451..4c497b83a6 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); void init_protocol_warning_severity_level(void); int get_protocol_warning_severity_level(void); -const char *get_version(void); -const char *get_short_version(void); /** An error from options_trial_assign() or options_init_from_string(). */ typedef enum setopt_err_t { diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 8a8b7ced01..4ba7be1519 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -45,6 +45,7 @@ #include "app/config/statefile.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index 444d6ea7ec..031f570097 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -84,6 +84,7 @@ #include "lib/encoding/confline.h" #include "lib/evloop/timers.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 3fa47747eb..b31b448e96 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -92,6 +92,7 @@ #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" #include "lib/evloop/compat_libevent.h" +#include "lib/version/torversion.h" #include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 38c7fd76d0..1ce06744d4 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -22,6 +22,7 @@ #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" #include "lib/encoding/confline.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 3a819f592c..9d61ced11c 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -49,6 +49,7 @@ #include "lib/encoding/confline.h" #include "lib/osinfo/uname.h" #include "lib/tls/tortls.h" +#include "lib/version/torversion.h" #include "feature/dirauth/authmode.h" diff --git a/src/include.am b/src/include.am index 247b0db8da..8279499936 100644 --- a/src/include.am +++ b/src/include.am @@ -33,6 +33,7 @@ include src/lib/thread/include.am include src/lib/time/include.am include src/lib/tls/include.am include src/lib/trace/include.am +include src/lib/version/include.am include src/lib/wallclock/include.am include src/trunnel/include.am diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..7ca1863a52 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -10,6 +10,5 @@ lib/log/*.h lib/malloc/*.h lib/string/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i \ No newline at end of file diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..c6f404e269 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ src/lib/log/util_bug.c @@ -21,15 +20,8 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ src/lib/log/util_bug.h \ diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d60ce6308a..bc7b36dcb9 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,7 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h \ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c similarity index 94% rename from src/lib/log/git_revision.c rename to src/lib/version/git_revision.c index 9d29ecd2a2..e5b2ff534e 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h similarity index 100% rename from src/lib/log/git_revision.h rename to src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..6944eb05e3 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..761d6f25ab --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..29ada39c9d --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include +#include + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/rust/build.rs b/src/rust/build.rs index 123d5c0682..bf566c56bf 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -162,6 +162,7 @@ pub fn main() { cfg.component("tor-malloc"); cfg.component("tor-wallclock"); cfg.component("tor-err-testing"); + cfg.component("tor-version-testing"); cfg.component("tor-intmath-testing"); cfg.component("tor-ctime-testing"); cfg.component("curve25519_donna"); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 1401e4c28d..879f9e74dc 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -9,6 +9,7 @@ #include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" static or_options_t *mock_options = NULL; static const or_options_t * diff --git a/src/test/testing_common.c b/src/test/testing_common.c index c52683afca..8d648ee175 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -25,6 +25,7 @@ #include "lib/compress/compress.h" #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include #ifdef HAVE_FCNTL_H -- GitLab From 175153a3290b3987faacac9d5390e87e1ad4a457 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:40:55 -0400 Subject: [PATCH 0104/1724] Make initialization for the "err" library into a subsystem. --- src/app/main/main.c | 10 --------- src/app/main/subsystem_list.c | 4 +++- src/lib/err/.may_include | 2 ++ src/lib/err/include.am | 8 ++++--- src/lib/err/torerr.c | 10 +++++++++ src/lib/err/torerr.h | 1 + src/lib/err/torerr_sys.c | 39 +++++++++++++++++++++++++++++++++++ src/lib/err/torerr_sys.h | 14 +++++++++++++ 8 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 src/lib/err/torerr_sys.c create mode 100644 src/lib/err/torerr_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 031f570097..e3d7610c82 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -69,7 +69,6 @@ #include "lib/container/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/err/backtrace.h" #include "lib/geoip/geoip.h" #include "lib/process/waitpid.h" @@ -822,7 +821,6 @@ tor_free_all(int postfork) if (!postfork) { escaped(NULL); esc_router_info(NULL); - clean_up_backtrace_handler(); logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -1419,14 +1417,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) #endif /* !defined(_WIN64) */ #endif /* defined(_WIN32) */ - { - int bt_err = configure_backtrace_handler(get_version()); - if (bt_err < 0) { - log_warn(LD_BUG, "Unable to install backtrace handler: %s", - strerror(-bt_err)); - } - } - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index fc1249e1c6..244dbadbd9 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,13 +8,15 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/err/torerr_sys.h" + #include /** * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { - NULL // placeholder. + &sys_torerr, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..daa1b6e4ca 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,5 @@ orconfig.h lib/cc/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h \ No newline at end of file diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..43adcd2694 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -6,8 +6,9 @@ noinst_LIBRARIES += src/lib/libtor-err-testing.a endif src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) @@ -16,4 +17,5 @@ src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index f9e139f967..e9de86837f 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -122,6 +122,16 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) n_sigsafe_log_fds = n; } +/** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + /** * Set the granularity (in ms) to use when reporting fatal errors outside * the logging system. diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index d4bba6916f..b415ef73ef 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -39,6 +39,7 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..54666f4106 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include + +static int +torerr_subsys_init(void) +{ + configure_backtrace_handler(get_version()); + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +torerr_subsys_shutdown(void) +{ + tor_log_reset_sigsafe_err_fds(); + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + .level = -100, + .supported = true, + .initialize = torerr_subsys_init, + .shutdown = torerr_subsys_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..b56270d538 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ -- GitLab From 178c1821b2115972ce3c3f194d1fcbd0d75ca364 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:55:10 -0400 Subject: [PATCH 0105/1724] Make the windows process parameter initialization a subsystem Also, move it from "main" into lib/process --- src/app/main/main.c | 29 --------------- src/app/main/subsystem_list.c | 2 + src/lib/process/.may_include | 1 + src/lib/process/include.am | 6 ++- src/lib/process/winprocess_sys.c | 64 ++++++++++++++++++++++++++++++++ src/lib/process/winprocess_sys.h | 14 +++++++ 6 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 src/lib/process/winprocess_sys.c create mode 100644 src/lib/process/winprocess_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index e3d7610c82..1e4cd37feb 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1388,35 +1388,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) { int result = 0; -#ifdef _WIN32 -#ifndef HeapEnableTerminationOnCorruption -#define HeapEnableTerminationOnCorruption 1 -#endif - /* On heap corruption, just give up; don't try to play along. */ - HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); - - /* SetProcessDEPPolicy is only supported on 32-bit Windows. - * (On 64-bit Windows it always fails, and some compilers don't like the - * PSETDEP cast.) - * 32-bit Windows defines _WIN32. - * 64-bit Windows defines _WIN32 and _WIN64. */ -#ifndef _WIN64 - /* Call SetProcessDEPPolicy to permanently enable DEP. - The function will not resolve on earlier versions of Windows, - and failure is not dangerous. */ - HMODULE hMod = GetModuleHandleA("Kernel32.dll"); - if (hMod) { - typedef BOOL (WINAPI *PSETDEP)(DWORD); - PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, - "SetProcessDEPPolicy"); - if (setdeppolicy) { - /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ - setdeppolicy(3); - } - } -#endif /* !defined(_WIN64) */ -#endif /* defined(_WIN32) */ - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 244dbadbd9..0f7d5d2ccc 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -9,6 +9,7 @@ #include "lib/cc/torint.h" #include "lib/err/torerr_sys.h" +#include "lib/process/winprocess_sys.h" #include @@ -16,6 +17,7 @@ * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { + &sys_winprocess, &sys_torerr, }; diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..a2d57a52f3 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -11,6 +11,7 @@ lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..2aa30cc3c1 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -12,7 +12,8 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) @@ -26,4 +27,5 @@ noinst_HEADERS += \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..e00f94c915 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include +#include + +#ifdef _WIN32 +#include + +#define WINPROCESS_SYS_ENABLED true + +static int +init_windows_process_params(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define init_windows_process_params NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = init_windows_process_params, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..cb096e0c92 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ -- GitLab From b8c50eabfee1bd9f5ed03f8ec569cc53b980f1d1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:14:43 -0400 Subject: [PATCH 0106/1724] Add a subsystem for our threading support --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 2 ++ src/lib/thread/.may_include | 1 + src/lib/thread/compat_threads.c | 16 ++++++++++++++++ src/lib/thread/include.am | 5 +++-- src/lib/thread/thread_sys.h | 14 ++++++++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/lib/thread/thread_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 1e4cd37feb..21a2832781 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1397,7 +1397,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); update_approx_time(time(NULL)); - tor_threads_init(); tor_compress_init(); init_logging(0); monotime_init(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 0f7d5d2ccc..c3b731ca39 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,6 +10,7 @@ #include "lib/err/torerr_sys.h" #include "lib/process/winprocess_sys.h" +#include "lib/thread/thread_sys.h" #include @@ -19,6 +20,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, + &sys_threads, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index 93ad0cd734..c26a426923 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,5 +2,6 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 7f1970af45..3d41faa8ce 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include @@ -109,3 +111,17 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +sys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + .level = -95, + .initialize = sys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..695795a2c8 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -23,5 +23,6 @@ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..984abe88e8 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ -- GitLab From d3e4afcc9b835e0f862207ef16d7e706ceea9ce1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:26:33 -0400 Subject: [PATCH 0107/1724] Turn the logging code into a subsystem --- src/app/main/main.c | 3 --- src/app/main/subsystem_list.c | 2 ++ src/lib/log/.may_include | 1 + src/lib/log/include.am | 2 ++ src/lib/log/log.c | 1 + src/lib/log/log_sys.c | 35 +++++++++++++++++++++++++++++++++++ src/lib/log/log_sys.h | 14 ++++++++++++++ 7 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/lib/log/log_sys.c create mode 100644 src/lib/log/log_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 21a2832781..f44f3475dd 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -819,9 +819,7 @@ tor_free_all(int postfork) /* Stuff in util.c and address.c*/ if (!postfork) { - escaped(NULL); esc_router_info(NULL); - logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -1398,7 +1396,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) update_approx_time(time(NULL)); tor_compress_init(); - init_logging(0); monotime_init(); int argc = tor_cfg->argc + tor_cfg->argc_owned; diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index c3b731ca39..4a2994ec49 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -11,6 +11,7 @@ #include "lib/err/torerr_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" +#include "lib/log/log_sys.h" #include @@ -21,6 +22,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, &sys_threads, + &sys_logging, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 7ca1863a52..11c87f0a0d 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -9,6 +9,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/version/*.h lib/wallclock/*.h diff --git a/src/lib/log/include.am b/src/lib/log/include.am index c6f404e269..9d3dbe3104 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -9,6 +9,7 @@ src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -24,5 +25,6 @@ noinst_HEADERS += \ src/lib/log/escape.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index bc7b36dcb9..46107fe848 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,6 +32,7 @@ #define LOG_PRIVATE #include "lib/log/log.h" +#include "lib/log/log_sys.h" #include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..94ec97fdc1 --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +init_logging_subsys(void) +{ + init_logging(0); + return 0; +} + +static void +shutdown_logging_subsys(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + .level = -90, + .initialize = init_logging_subsys, + .shutdown = shutdown_logging_subsys, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..f7afbb279d --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ -- GitLab From a0ee54549fec3ae710ab5e3623d707bd08adcafe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:34:07 -0400 Subject: [PATCH 0108/1724] Turn the wallclock module into a subsystem. (This may be slightly gratuitous.) --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 4 +++- src/lib/wallclock/.may_include | 1 + src/lib/wallclock/approx_time.c | 16 ++++++++++++++++ src/lib/wallclock/include.am | 3 ++- src/lib/wallclock/wallclock_sys.h | 14 ++++++++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/lib/wallclock/wallclock_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index f44f3475dd..5740efb0b6 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1394,7 +1394,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); - update_approx_time(time(NULL)); tor_compress_init(); monotime_init(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 4a2994ec49..3d03a9a4df 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -9,9 +9,10 @@ #include "lib/cc/torint.h" #include "lib/err/torerr_sys.h" +#include "lib/log/log_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" -#include "lib/log/log_sys.h" +#include "lib/wallclock/wallclock_sys.h" #include @@ -21,6 +22,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, + &sys_wallclock, &sys_threads, &sys_logging, }; diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index bb9a292369..c7a7ae9bd7 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,17 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +init_wallclock_subsys(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + .level = -99, + .initialize = init_wallclock_subsys, +}; diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2351252e0c 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -19,4 +19,5 @@ noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..e009578a83 --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ -- GitLab From 05b54f6a6a24ebdb47de4f7e41cf94f2f6be93bd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 14:13:36 -0400 Subject: [PATCH 0109/1724] Use subsystems manager for subsystems used in tests. --- src/lib/subsys/subsys.h | 4 ++++ src/test/bench.c | 7 ++++--- src/test/fuzz/fuzzing_common.c | 9 ++++----- src/test/testing_common.c | 8 +++----- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 012b218da7..25451bc450 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -65,4 +65,8 @@ typedef struct subsys_fns_t { #define MIN_SUBSYS_LEVEL -100 #define MAX_SUBSYS_LEVEL 100 +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + #endif diff --git a/src/test/bench.c b/src/test/bench.c index 9da1b46a1b..ff8707d41c 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -24,6 +24,7 @@ #include "core/or/circuitlist.h" #include "app/config/config.h" +#include "app/main/subsysmgr.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/crypto/onion_ntor.h" @@ -690,9 +691,10 @@ main(int argc, const char **argv) char *errmsg; or_options_t *options; - tor_threads_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); + tor_compress_init(); - init_logging(1); if (argc == 4 && !strcmp(argv[1], "diff")) { const int N = 200; @@ -739,7 +741,6 @@ main(int argc, const char **argv) init_protocol_warning_severity_level(); options = options_new(); - init_logging(1); options->command = CMD_RUN_UNITTESTS; options->DataDirectory = tor_strdup(""); options->KeyDirectory = tor_strdup(""); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 879f9e74dc..21aa07cfe2 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -3,6 +3,7 @@ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #include "core/or/or.h" +#include "app/main/subsysmgr.h" #include "lib/err/backtrace.h" #include "app/config/config.h" #include "test/fuzz/fuzzing.h" @@ -95,12 +96,10 @@ disable_signature_checking(void) static void global_init(void) { - tor_threads_init(); - tor_compress_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); - /* Initialise logging first */ - init_logging(1); - configure_backtrace_handler(get_version()); + tor_compress_init(); if (crypto_global_init(0, NULL, NULL) < 0) abort(); diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 8d648ee175..eef393d3a8 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -26,6 +26,7 @@ #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" #include "lib/version/torversion.h" +#include "app/main/subsysmgr.h" #include #ifdef HAVE_FCNTL_H @@ -251,12 +252,9 @@ main(int c, const char **v) int loglevel = LOG_ERR; int accel_crypto = 0; - /* We must initialise logs before we call tor_assert() */ - init_logging(1); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); - update_approx_time(time(NULL)); options = options_new(); - tor_threads_init(); tor_compress_init(); network_init(); @@ -268,7 +266,6 @@ main(int c, const char **v) tor_libevent_initialize(&cfg); control_initialize_event_queue(); - configure_backtrace_handler(get_version()); for (i_out = i = 1; i < c; ++i) { if (!strcmp(v[i], "--warn")) { @@ -295,6 +292,7 @@ main(int c, const char **v) s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } + flush_log_messages_from_startup(); init_protocol_warning_severity_level(); options->command = CMD_RUN_UNITTESTS; -- GitLab From cfe5b35edb38cef6312ef0b4ae44fb0e20342706 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 11:11:21 -0400 Subject: [PATCH 0110/1724] Move networking startup/cleanup logic into a subsystem. --- src/app/main/main.c | 19 --------------- src/app/main/subsystem_list.c | 2 ++ src/lib/net/.may_include | 1 + src/lib/net/include.am | 2 ++ src/lib/net/network_sys.c | 44 +++++++++++++++++++++++++++++++++++ src/lib/net/network_sys.h | 14 +++++++++++ src/test/testing_common.c | 2 -- 7 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 src/lib/net/network_sys.c create mode 100644 src/lib/net/network_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 5740efb0b6..3e80725b9a 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -427,18 +427,6 @@ dumpstats(int severity) rend_service_dump_stats(severity); } -/** Called by exit() as we shut down the process. - */ -static void -exit_function(void) -{ - /* NOTE: If we ever daemonize, this gets called immediately. That's - * okay for now, because we only use this on Windows. */ -#ifdef _WIN32 - WSACleanup(); -#endif -} - #ifdef _WIN32 #define UNIX_ONLY 0 #else @@ -632,12 +620,6 @@ tor_init(int argc, char *argv[]) rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ - if (network_init()<0) { - log_err(LD_BUG,"Error initializing network; exiting."); - return -1; - } - atexit(exit_function); - int init_rv = options_init_from_torrc(argc,argv); if (init_rv < 0) { log_err(LD_CONFIG,"Reading config failed--see warnings above."); @@ -784,7 +766,6 @@ tor_free_all(int postfork) routerparse_free_all(); ext_orport_free_all(); control_free_all(); - tor_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 3d03a9a4df..cb186c14d9 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,6 +10,7 @@ #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" +#include "lib/net/network_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" #include "lib/wallclock/wallclock_sys.h" @@ -25,6 +26,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_wallclock, &sys_threads, &sys_logging, + &sys_network, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..f93f0e1552 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -11,5 +11,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h \ No newline at end of file diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..8a88f0f2ae 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -11,6 +11,7 @@ src_lib_libtor_net_a_SOURCES = \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..c9d33a94d3 --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include +#include +#endif + +static int +init_network_sys(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +shutdown_network_sys(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + .level = -90, + .supported = true, + .initialize = init_network_sys, + .shutdown = shutdown_network_sys, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..62b778bb66 --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/test/testing_common.c b/src/test/testing_common.c index eef393d3a8..818bb58c9a 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -257,8 +257,6 @@ main(int c, const char **v) options = options_new(); tor_compress_init(); - network_init(); - monotime_init(); struct tor_libevent_cfg cfg; -- GitLab From 50436ccea4bd200e45196ccce7acff28f293a4de Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 11:21:06 -0400 Subject: [PATCH 0111/1724] Add crypto module as a subsystem. --- src/app/main/main.c | 7 ------- src/app/main/subsystem_list.c | 2 ++ src/lib/crypt_ops/.may_include | 1 + src/lib/crypt_ops/crypto_init.c | 26 ++++++++++++++++++++++++++ src/lib/crypt_ops/crypto_sys.h | 14 ++++++++++++++ src/lib/crypt_ops/include.am | 1 + src/test/testing_common.c | 2 -- 7 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 src/lib/crypt_ops/crypto_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 3e80725b9a..74c3c41e5b 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -535,12 +535,6 @@ tor_init(int argc, char *argv[]) tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); log_set_application_name(progname); - /* Set up the crypto nice and early */ - if (crypto_early_init() < 0) { - log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); - return -1; - } - /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ @@ -859,7 +853,6 @@ tor_cleanup(void) later, if it makes shutdown unacceptably slow. But for now, leave it here: it's helped us catch bugs in the past. */ - crypto_global_cleanup(); } /** Read/create keys as needed, and echo our fingerprint to stdout. */ diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index cb186c14d9..dd64568226 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" #include "lib/net/network_sys.h" @@ -27,6 +28,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_threads, &sys_logging, &sys_network, + &sys_crypto, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..352fde858c 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,6 +12,7 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/testsupport.h lib/thread/*.h lib/log/*.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 9d6e2da0d0..cc7865ef72 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -20,6 +20,9 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" + +#include "lib/subsys/subsys.h" #include "siphash.h" @@ -202,3 +205,26 @@ tor_is_using_nss(void) return 0; #endif } + +static int +init_crypto_sys(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +shutdown_crypto_sys(void) +{ + crypto_global_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = init_crypto_sys, + .shutdown = shutdown_crypto_sys, +}; diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..31644d088b --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..d0ccc13bff 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -66,5 +66,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 818bb58c9a..d4c5632334 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -331,8 +331,6 @@ main(int c, const char **v) free_pregenerated_keys(); - crypto_global_cleanup(); - if (have_failed) return 1; else -- GitLab From cad61f0f6de48c6eab6e811a081f154b03de57b8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:00:56 -0400 Subject: [PATCH 0112/1724] Move prefork, postfork, and thread-exit hooks into subsys So far, crypto is the only module that uses them, but others are likely to do so in the future. --- src/app/config/config.c | 5 +-- src/app/main/subsysmgr.c | 57 +++++++++++++++++++++++++++++++++ src/app/main/subsysmgr.h | 4 +++ src/lib/crypt_ops/crypto_init.c | 3 ++ src/lib/subsys/subsys.h | 16 +++++++++ src/test/testing_common.c | 4 +-- 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 7b49387bcf..76df7ec67e 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -64,6 +64,7 @@ #include "app/config/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -1393,10 +1394,10 @@ options_act_reversible(const or_options_t *old_options, char **msg) * processes. */ if (running_tor && options->RunAsDaemon) { if (! start_daemon_has_been_called()) - crypto_prefork(); + subsystems_prefork(); /* No need to roll back, since you can't change the value. */ if (start_daemon()) - crypto_postfork(); + subsystems_postfork(); } #ifdef HAVE_SYSTEMD diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index 7974f2d238..05803ee946 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -128,3 +128,60 @@ subsystems_shutdown_downto(int target_level) sys_initialized[i] = false; } } + +/** + * Run pre-fork code on all subsystems that declare any + **/ +void +subsystems_prefork(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->prefork) + sys->prefork(); + } +} + +/** + * Run post-fork code on all subsystems that declare any + **/ +void +subsystems_postfork(void) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->postfork) + sys->postfork(); + } +} + +/** + * Run thread-clanup code on all subsystems that declare any + **/ +void +subsystems_thread_cleanup(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->thread_cleanup) + sys->thread_cleanup(); + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h index c9b892eee4..4b3cad62ad 100644 --- a/src/app/main/subsysmgr.h +++ b/src/app/main/subsysmgr.h @@ -17,4 +17,8 @@ int subsystems_init_upto(int level); void subsystems_shutdown(void); void subsystems_shutdown_downto(int level); +void subsystems_prefork(void); +void subsystems_postfork(void); +void subsystems_thread_cleanup(void); + #endif diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index cc7865ef72..a03f5eff7c 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -227,4 +227,7 @@ const struct subsys_fns_t sys_crypto = { .level = -60, .initialize = init_crypto_sys, .shutdown = shutdown_crypto_sys, + .prefork = crypto_prefork, + .postfork = crypto_postfork, + .thread_cleanup = crypto_thread_cleanup, }; diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 25451bc450..b06d67e624 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -53,6 +53,22 @@ typedef struct subsys_fns_t { **/ int (*add_pubsub)(struct dispatch_connector_t *); + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + /** * Free all resources held by this subsystem. * diff --git a/src/test/testing_common.c b/src/test/testing_common.c index d4c5632334..1362f29711 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -232,12 +232,12 @@ void tinytest_prefork(void) { free_pregenerated_keys(); - crypto_prefork(); + subsystems_prefork(); } void tinytest_postfork(void) { - crypto_postfork(); + subsystems_postfork(); init_pregenerated_keys(); } -- GitLab From 207253df8d7c040840c7f4305534e6979bfc7bf7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:09:44 -0400 Subject: [PATCH 0113/1724] Move monotonic time setup into a subsystem --- src/app/main/main.c | 2 -- src/app/main/subsystem_list.c | 2 ++ src/lib/time/.may_include | 1 + src/lib/time/include.am | 2 ++ src/lib/time/time_sys.c | 26 ++++++++++++++++++++++++++ src/lib/time/time_sys.h | 14 ++++++++++++++ src/test/testing_common.c | 2 -- 7 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 src/lib/time/time_sys.c create mode 100644 src/lib/time/time_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 74c3c41e5b..bb2e9f5cdb 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1248,7 +1248,6 @@ static int run_tor_main_loop(void) { handle_signals(); - monotime_init(); timers_initialize(); initialize_mainloop_events(); @@ -1369,7 +1368,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); tor_compress_init(); - monotime_init(); int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index dd64568226..a9189b9941 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -14,6 +14,7 @@ #include "lib/net/network_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" +#include "lib/time/time_sys.h" #include "lib/wallclock/wallclock_sys.h" #include @@ -27,6 +28,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_wallclock, &sys_threads, &sys_logging, + &sys_time, &sys_network, &sys_crypto, }; diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..40a18805ac 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,6 +4,7 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dae16f49ac 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -16,4 +17,5 @@ src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..2303874f29 --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +init_time_sys(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + .level = -90, + .supported = true, + .initialize = init_time_sys, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..0f1aebc268 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_time.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 1362f29711..333dbc436f 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -257,8 +257,6 @@ main(int c, const char **v) options = options_new(); tor_compress_init(); - monotime_init(); - struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); tor_libevent_initialize(&cfg); -- GitLab From 019a044e5e6586fb42a171cb98aea15a72bd5a13 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:34:56 -0400 Subject: [PATCH 0114/1724] Turn "compress" into a subsystem. --- src/app/main/main.c | 2 -- src/app/main/subsystem_list.c | 2 ++ src/lib/compress/.may_include | 1 + src/lib/compress/compress.c | 15 ++++++++++++++- src/lib/compress/compress.h | 2 +- src/lib/compress/compress_sys.h | 14 ++++++++++++++ src/lib/compress/include.am | 1 + src/test/testing_common.c | 1 - 8 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 src/lib/compress/compress_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index bb2e9f5cdb..6240609ee6 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1367,8 +1367,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); - tor_compress_init(); - int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); memcpy(argv, tor_cfg->argv, tor_cfg->argc*sizeof(char*)); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index a9189b9941..e47b05da15 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" @@ -30,6 +31,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_logging, &sys_time, &sys_network, + &sys_compress, &sys_crypto, }; diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..4870259ec9 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -8,5 +8,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 2ad9b15b2e..0d134fd1be 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,14 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = tor_compress_init, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 4466e27c4d..4dd6506238 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..a162140cfb --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..b952779578 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -22,5 +22,6 @@ noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 333dbc436f..6d2db28f15 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -255,7 +255,6 @@ main(int c, const char **v) subsystems_init_upto(SUBSYS_LEVEL_LIBS); options = options_new(); - tor_compress_init(); struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); -- GitLab From 32b23a4c40880591ecadab59f932f4a4c1e7560a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:46:35 -0400 Subject: [PATCH 0115/1724] Make tortls use the subsystems interface This one only needs a shutdown right now. --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 2 ++ src/lib/tls/.may_include | 1 + src/lib/tls/include.am | 1 + src/lib/tls/tortls.c | 8 ++++++++ src/lib/tls/tortls_sys.h | 14 ++++++++++++++ 6 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/lib/tls/tortls_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 6240609ee6..4dedae9c0c 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -776,7 +776,6 @@ tor_free_all(int postfork) policies_free_all(); } if (!postfork) { - tor_tls_free_all(); #ifndef _WIN32 tor_getpwnam(NULL); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index e47b05da15..62c87005c6 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -16,6 +16,7 @@ #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" #include "lib/time/time_sys.h" +#include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" #include @@ -33,6 +34,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_network, &sys_compress, &sys_crypto, + &sys_tortls, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..79301bc318 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -11,6 +11,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/testsupport.h lib/tls/*.h diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..1817739eef 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -36,5 +36,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 56f70bc371..fdeea9e0d4 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include @@ -440,3 +442,9 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = tor_tls_free_all +}; diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..fd909f6019 --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ -- GitLab From adecda753996611e9a5b82c5fa87ea78ec683806 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 09:42:16 -0500 Subject: [PATCH 0116/1724] changes file for subsystems api (28330) --- changes/subsystems | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/subsystems diff --git a/changes/subsystems b/changes/subsystems new file mode 100644 index 0000000000..a51fb8e2b1 --- /dev/null +++ b/changes/subsystems @@ -0,0 +1,6 @@ + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when initializing + and shutting down. Previously, these systems were managed implicitly + though various places throughout the codebase. (There still some + subsystems using the old system.) + Closes ticket 28330. -- GitLab From 1a6060fa4263d20d3c1ccf29163165126a413957 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 23 Oct 2018 23:53:39 +0000 Subject: [PATCH 0117/1724] New macro CTASSERT(condition) to assert condition at compile-time. To get it, use: #include "lib/cc/ctassert.h" --- src/lib/cc/ctassert.h | 75 +++++++++++++++++++++++++++++++++++++++++++ src/lib/cc/include.am | 1 + 2 files changed, 76 insertions(+) create mode 100644 src/lib/cc/ctassert.h diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..7307bca53a --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,75 @@ +/*- + * Copyright (c) 2018 Taylor R. Campbell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert(x, #x) + +#else + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN(x, c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN(x, __INCLUDE_LEVEL__, __LINE__) +#else +#define CTASSERT(x) CTASSERT_EXPN(x, l, __LINE__) /* hope it's unique enough */ +#endif + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..52cf8a9f72 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h -- GitLab From e69a4ad6b321dbdb63236687ac8924c301c60f9d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:39:58 -0500 Subject: [PATCH 0118/1724] Add a user of CTASSERT(). --- src/lib/net/address.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index c97a17037a..a351b9df28 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in a, with port in port, into a * sockaddr object in *sa_out of object size len. If not enough -- GitLab From 3c9dd9ef2d86463e535952528bc5151ce499a74f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:41:14 -0500 Subject: [PATCH 0119/1724] Add parentheses to the ctassert macro expansions --- src/lib/cc/ctassert.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 7307bca53a..5a1b137cba 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -43,7 +43,7 @@ #if __STDC_VERSION__ >= 201112L /* If C11 is available, just use _Static_assert. */ -#define CTASSERT(x) _Static_assert(x, #x) +#define CTASSERT(x) _Static_assert((x), #x) #else @@ -59,11 +59,12 @@ * error. */ #if defined(__COUNTER__) -#define CTASSERT(x) CTASSERT_EXPN(x, c, __COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) #elif defined(__INCLUDE_LEVEL__) -#define CTASSERT(x) CTASSERT_EXPN(x, __INCLUDE_LEVEL__, __LINE__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) #else -#define CTASSERT(x) CTASSERT_EXPN(x, l, __LINE__) /* hope it's unique enough */ +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) #endif #define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) -- GitLab From 6b706bcf199b82c6b1a87c839e8d13bf9d8e60b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:41:32 -0500 Subject: [PATCH 0120/1724] Remove a tab. --- src/lib/cc/ctassert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 5a1b137cba..674c0c2e10 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -38,7 +38,7 @@ /** * CTASSERT(expression) * - * Trigger a compiler error if expression is false. + * Trigger a compiler error if expression is false. */ #if __STDC_VERSION__ >= 201112L -- GitLab From d9508d8ede7ecbdfa35ce8a12c6e1d66d1bbb4b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:42:18 -0500 Subject: [PATCH 0121/1724] Change copyright statement (with permission) --- src/lib/cc/ctassert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 674c0c2e10..c6de3bf1ec 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2018 Taylor R. Campbell + * Copyright (c) 2018 The Tor Project, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without -- GitLab From 770653ff456c9ae9ecf29008f7abf367a776e557 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 16:59:39 -0500 Subject: [PATCH 0122/1724] Allow lib/cc to include its own files. --- src/lib/cc/.may_include | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h \ No newline at end of file -- GitLab From a7a060a637ac714abdb3f944df1bbbb30a7c9b14 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:37:02 -0500 Subject: [PATCH 0123/1724] Switch ctassert.h to 3bsd (with permission) --- src/lib/cc/ctassert.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index c6de3bf1ec..ea3a3e119e 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -10,6 +10,9 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. + * 3. Neither the names of the copyright owners nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -- GitLab From 6d93820499a8bfb19128759893b18c1437f99c6b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 19 Oct 2018 15:04:45 -0400 Subject: [PATCH 0124/1724] Memoize summarize_protover_flags() Our tests showed that this function is responsible for a huge number of our malloc/free() calls. It's a prime candidate for being memoized. Closes ticket 27225. --- changes/ticket27225 | 5 +++ src/app/main/main.c | 2 ++ src/core/or/versions.c | 82 ++++++++++++++++++++++++++++++++++-------- src/core/or/versions.h | 2 ++ 4 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 changes/ticket27225 diff --git a/changes/ticket27225 b/changes/ticket27225 new file mode 100644 index 0000000000..4c05a269d6 --- /dev/null +++ b/changes/ticket27225 @@ -0,0 +1,5 @@ + o Minor features (performance): + - Avoid parsing the same protocol-versions string over and over + in summarize_protover_flags(). This should save us a huge number + of malloc calls on startup, and may reduce memory fragmentation with + some allocators. Closes ticket 27225. diff --git a/src/app/main/main.c b/src/app/main/main.c index ae87add67d..04bbfadcb7 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -33,6 +33,7 @@ #include "core/or/relay.h" #include "core/or/scheduler.h" #include "core/or/status.h" +#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" @@ -791,6 +792,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + protover_summary_cache_free_all(); if (!postfork) { config_free_all(); diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 06274996a7..6f8eea7a67 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -377,6 +377,62 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_); } +/** If there are more than this many entries, we're probably under + * some kind of weird DoS. */ +static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024; + +/** + * Map from protover string to protover_summary_flags_t. + */ +static strmap_t *protover_summary_map = NULL; + +/** + * Helper. Given a non-NULL protover string protocols, set out + * to its summary, and memoize the result in protover_summary_map. + */ +static void +memoize_protover_summary(protover_summary_flags_t *out, + const char *protocols) +{ + if (!protover_summary_map) + protover_summary_map = strmap_new(); + + if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { + protover_summary_cache_free_all(); + } + + const protover_summary_flags_t *cached = + strmap_get(protover_summary_map, protocols); + + if (cached != NULL) { + /* We found a cached entry; no need to parse this one. */ + memcpy(out, cached, sizeof(protover_summary_flags_t)); + tor_assert(out->protocols_known); + return; + } + + memset(out, 0, sizeof(*out)); + out->protocols_known = 1; + out->supports_extend2_cells = + protocol_list_supports_protocol(protocols, PRT_RELAY, 2); + out->supports_ed25519_link_handshake_compat = + protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_link_handshake_any = + protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_hs_intro = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); + out->supports_v3_hsdir = + protocol_list_supports_protocol(protocols, PRT_HSDIR, + PROTOVER_HSDIR_V3); + out->supports_v3_rendezvous_point = + protocol_list_supports_protocol(protocols, PRT_HSREND, + PROTOVER_HS_RENDEZVOUS_POINT_V3); + + protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); + cached = strmap_set(protover_summary_map, protocols, new_cached); + tor_assert(!cached); +} + /** Summarize the protocols listed in protocols into out, * falling back or correcting them based on version as appropriate. */ @@ -388,21 +444,7 @@ summarize_protover_flags(protover_summary_flags_t *out, tor_assert(out); memset(out, 0, sizeof(*out)); if (protocols) { - out->protocols_known = 1; - out->supports_extend2_cells = - protocol_list_supports_protocol(protocols, PRT_RELAY, 2); - out->supports_ed25519_link_handshake_compat = - protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_link_handshake_any = - protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_hs_intro = - protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); - out->supports_v3_hsdir = - protocol_list_supports_protocol(protocols, PRT_HSDIR, - PROTOVER_HSDIR_V3); - out->supports_v3_rendezvous_point = - protocol_list_supports_protocol(protocols, PRT_HSREND, - PROTOVER_HS_RENDEZVOUS_POINT_V3); + memoize_protover_summary(out, protocols); } if (version && !strcmpstart(version, "Tor ")) { if (!out->protocols_known) { @@ -420,3 +462,13 @@ summarize_protover_flags(protover_summary_flags_t *out, } } } + +/** + * Free all space held in the protover_summary_map. + */ +void +protover_summary_cache_free_all(void) +{ + strmap_free(protover_summary_map, tor_free_); + protover_summary_map = NULL; +} diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 0c773f3f4c..4fc50a0018 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -41,4 +41,6 @@ void summarize_protover_flags(protover_summary_flags_t *out, const char *protocols, const char *version); +void protover_summary_cache_free_all(void); + #endif /* !defined(TOR_VERSIONS_H) */ -- GitLab From 9e48d9a920c1e1f8c6fc551363b28905f5580f8e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:54:03 -0500 Subject: [PATCH 0125/1724] Change version on master to 0.4.0.0-alpha-dev. --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index b80aa821ad..4524c6b467 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.6.0-alpha-dev]) +AC_INIT([tor],[0.4.0.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 8b9b709698..af01a2b499 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.6.0-alpha-dev" +!define VERSION "0.4.0.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index de3bf09282..cfc3bc9e9e 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.6.0-alpha-dev" +#define VERSION "0.4.0.0-alpha-dev" -- GitLab From 6e828ced56c04aa38fa6e2d595528bf0d046ca8c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 7 Nov 2018 11:02:26 -0500 Subject: [PATCH 0126/1724] simplify now that it uses tor's copyright and license --- src/lib/cc/ctassert.h | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index ea3a3e119e..e42976360f 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -1,31 +1,5 @@ -/*- - * Copyright (c) 2018 The Tor Project, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the names of the copyright owners nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ /** * \file ctassert.h -- GitLab From 4fe4bcf8a10967a668895e962099f50635ba9e4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 10:55:18 -0500 Subject: [PATCH 0127/1724] Explain that configuration should happen elsewhere, but not init. --- src/lib/subsys/subsys.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index b06d67e624..462314567e 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -44,6 +44,9 @@ typedef struct subsys_fns_t { * This function MUST NOT rely on any runtime configuration information; * it is only for global state or pre-configuration state. * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * * This function MUST NOT have any parts that can fail. **/ int (*initialize)(void); -- GitLab From 61695e3d622dfcc196b8b829842f2b12fecebeab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 10:58:20 -0500 Subject: [PATCH 0128/1724] Document that subsystem callbacks are optional. --- src/lib/subsys/subsys.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 462314567e..2452ec6e2f 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -14,7 +14,11 @@ struct dispatch_connector_t; * A subsystem is a part of Tor that is initialized, shut down, configured, * and connected to other parts of Tor. * - * Subsystems + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. **/ typedef struct subsys_fns_t { /** -- GitLab From e80595f562e199049a41fdf1f3e12baced7e74d5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:00:31 -0500 Subject: [PATCH 0129/1724] fixup! Make initialization for the "err" library into a subsystem. Check for failure to install backtrace handler. --- src/lib/err/torerr_sys.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index 54666f4106..2f9e33e233 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -18,7 +18,8 @@ static int torerr_subsys_init(void) { - configure_backtrace_handler(get_version()); + if (configure_backtrace_handler(get_version()) < 0) + return -1; tor_log_reset_sigsafe_err_fds(); return 0; -- GitLab From c6336727cac937b4b5ca38c9b49ed3a66ce0b579 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:12:12 -0500 Subject: [PATCH 0130/1724] Rename subsystem callback functions to make them consistent --- src/lib/compress/compress.c | 8 +++++++- src/lib/crypt_ops/crypto_init.c | 32 +++++++++++++++++++++++++------- src/lib/err/torerr_sys.c | 8 ++++---- src/lib/log/log_sys.c | 8 ++++---- src/lib/net/network_sys.c | 8 ++++---- src/lib/process/winprocess_sys.c | 6 +++--- src/lib/thread/compat_threads.c | 4 ++-- src/lib/time/time_sys.c | 4 ++-- src/lib/tls/tortls.c | 8 +++++++- src/lib/wallclock/approx_time.c | 4 ++-- 10 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 0d134fd1be..6cb9bd492b 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -686,9 +686,15 @@ tor_compress_log_init_warnings(void) tor_zstd_warn_if_version_mismatched(); } +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + const subsys_fns_t sys_compress = { .name = "compress", .supported = true, .level = -70, - .initialize = tor_compress_init, + .initialize = subsys_compress_initialize, }; diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index a03f5eff7c..4c4cc3e43b 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -207,7 +207,7 @@ tor_is_using_nss(void) } static int -init_crypto_sys(void) +subsys_crypto_initialize(void) { if (crypto_early_init() < 0) return -1; @@ -216,18 +216,36 @@ init_crypto_sys(void) } static void -shutdown_crypto_sys(void) +subsys_crypto_shutdown(void) { crypto_global_cleanup(); } +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + const struct subsys_fns_t sys_crypto = { .name = "crypto", .supported = true, .level = -60, - .initialize = init_crypto_sys, - .shutdown = shutdown_crypto_sys, - .prefork = crypto_prefork, - .postfork = crypto_postfork, - .thread_cleanup = crypto_thread_cleanup, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, }; diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index 2f9e33e233..96bb1308a4 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -16,7 +16,7 @@ #include static int -torerr_subsys_init(void) +subsys_torerr_initialize(void) { if (configure_backtrace_handler(get_version()) < 0) return -1; @@ -25,7 +25,7 @@ torerr_subsys_init(void) return 0; } static void -torerr_subsys_shutdown(void) +subsys_torerr_shutdown(void) { tor_log_reset_sigsafe_err_fds(); clean_up_backtrace_handler(); @@ -35,6 +35,6 @@ const subsys_fns_t sys_torerr = { .name = "err", .level = -100, .supported = true, - .initialize = torerr_subsys_init, - .shutdown = torerr_subsys_shutdown + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown }; diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c index 94ec97fdc1..e20f3156ca 100644 --- a/src/lib/log/log_sys.c +++ b/src/lib/log/log_sys.c @@ -13,14 +13,14 @@ #include "lib/log/log_sys.h" static int -init_logging_subsys(void) +subsys_logging_initialize(void) { init_logging(0); return 0; } static void -shutdown_logging_subsys(void) +subsys_logging_shutdown(void) { logs_free_all(); escaped(NULL); @@ -30,6 +30,6 @@ const subsys_fns_t sys_logging = { .name = "log", .supported = true, .level = -90, - .initialize = init_logging_subsys, - .shutdown = shutdown_logging_subsys, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, }; diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c index c9d33a94d3..ac49288ee6 100644 --- a/src/lib/net/network_sys.c +++ b/src/lib/net/network_sys.c @@ -18,7 +18,7 @@ #endif static int -init_network_sys(void) +subsys_network_initialize(void) { if (network_init() < 0) return -1; @@ -27,7 +27,7 @@ init_network_sys(void) } static void -shutdown_network_sys(void) +subsys_network_shutdown(void) { #ifdef _WIN32 WSACleanup(); @@ -39,6 +39,6 @@ const subsys_fns_t sys_network = { .name = "network", .level = -90, .supported = true, - .initialize = init_network_sys, - .shutdown = shutdown_network_sys, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, }; diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index e00f94c915..ef66f8bfb1 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -19,7 +19,7 @@ #define WINPROCESS_SYS_ENABLED true static int -init_windows_process_params(void) +subsys_winprocess_initialize(void) { #ifndef HeapEnableTerminationOnCorruption #define HeapEnableTerminationOnCorruption 1 @@ -53,12 +53,12 @@ init_windows_process_params(void) } #else /* !defined(_WIN32) */ #define WINPROCESS_SYS_ENABLED false -#define init_windows_process_params NULL +#define subsys_winprocess_initialize NULL #endif /* defined(_WIN32) */ const subsys_fns_t sys_winprocess = { .name = "winprocess", .level = -100, .supported = WINPROCESS_SYS_ENABLED, - .initialize = init_windows_process_params, + .initialize = subsys_winprocess_initialize, }; diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 3d41faa8ce..0b466da212 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -113,7 +113,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) #endif /* !defined(HAVE_WORKING_STDATOMIC) */ static int -sys_threads_initialize(void) +subsys_threads_initialize(void) { tor_threads_init(); return 0; @@ -123,5 +123,5 @@ const subsys_fns_t sys_threads = { .name = "threads", .supported = true, .level = -95, - .initialize = sys_threads_initialize, + .initialize = subsys_threads_initialize, }; diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c index 2303874f29..b29ca35e69 100644 --- a/src/lib/time/time_sys.c +++ b/src/lib/time/time_sys.c @@ -12,7 +12,7 @@ #include "lib/time/compat_time.h" static int -init_time_sys(void) +subsys_time_initialize(void) { monotime_init(); return 0; @@ -22,5 +22,5 @@ const subsys_fns_t sys_time = { .name = "time", .level = -90, .supported = true, - .initialize = init_time_sys, + .initialize = subsys_time_initialize, }; diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index fdeea9e0d4..654cacacf7 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -443,8 +443,14 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + const subsys_fns_t sys_tortls = { .name = "tortls", .level = -50, - .shutdown = tor_tls_free_all + .shutdown = subsys_tortls_shutdown }; diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index c7a7ae9bd7..0b0ef382c2 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -45,7 +45,7 @@ update_approx_time(time_t now) #endif /* !defined(TIME_IS_FAST) */ static int -init_wallclock_subsys(void) +subsys_wallclock_initialize(void) { update_approx_time(time(NULL)); return 0; @@ -55,5 +55,5 @@ const subsys_fns_t sys_wallclock = { .name = "wallclock", .supported = true, .level = -99, - .initialize = init_wallclock_subsys, + .initialize = subsys_wallclock_initialize, }; -- GitLab From ba722e47995e106b46d848263638fa3009687cd9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:15:27 -0500 Subject: [PATCH 0131/1724] Add list of levels in subsystem_list.c --- src/app/main/subsystem_list.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 62c87005c6..190e6579d8 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -25,7 +25,7 @@ * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { - &sys_winprocess, + &sys_winprocess, /* -100 */ &sys_torerr, &sys_wallclock, &sys_threads, -- GitLab From 60d10812368458cb88aa9f9d628c49766d4bb490 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:56:26 -0500 Subject: [PATCH 0132/1724] Log before performing a subsystem operation --- src/app/main/subsysmgr.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index 05803ee946..abd2edd10b 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -7,6 +7,8 @@ #include "app/main/subsysmgr.h" #include "lib/err/torerr.h" +#include "lib/log/log.h" + #include #include #include @@ -85,8 +87,13 @@ subsystems_init_upto(int target_level) if (sys_initialized[i]) continue; int r = 0; - if (sys->initialize) + if (sys->initialize) { + // Note that the logging subsystem is designed so that it does no harm + // to log a message in an uninitialized state. These messages will be + // discarded for now, however. + log_debug(LD_GENERAL, "Initializing %s", sys->name); r = sys->initialize(); + } if (r < 0) { fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", sys->name, i); @@ -123,8 +130,10 @@ subsystems_shutdown_downto(int target_level) break; if (! sys_initialized[i]) continue; - if (sys->shutdown) + if (sys->shutdown) { + log_debug(LD_GENERAL, "Shutting down %s", sys->name); sys->shutdown(); + } sys_initialized[i] = false; } } @@ -143,8 +152,10 @@ subsystems_prefork(void) continue; if (! sys_initialized[i]) continue; - if (sys->prefork) + if (sys->prefork) { + log_debug(LD_GENERAL, "Pre-fork: %s", sys->name); sys->prefork(); + } } } @@ -162,13 +173,15 @@ subsystems_postfork(void) continue; if (! sys_initialized[i]) continue; - if (sys->postfork) + if (sys->postfork) { + log_debug(LD_GENERAL, "Post-fork: %s", sys->name); sys->postfork(); + } } } /** - * Run thread-clanup code on all subsystems that declare any + * Run thread-cleanup code on all subsystems that declare any **/ void subsystems_thread_cleanup(void) @@ -181,7 +194,9 @@ subsystems_thread_cleanup(void) continue; if (! sys_initialized[i]) continue; - if (sys->thread_cleanup) + if (sys->thread_cleanup) { + log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name); sys->thread_cleanup(); + } } } -- GitLab From 91355c0fac0f90ba286edb3d3f12d71d250be16d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 22:17:18 -0500 Subject: [PATCH 0133/1724] Annotate subsystem list with their levels. --- src/app/main/subsystem_list.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 190e6579d8..8640329e92 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -26,15 +26,15 @@ **/ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, /* -100 */ - &sys_torerr, - &sys_wallclock, - &sys_threads, - &sys_logging, - &sys_time, - &sys_network, - &sys_compress, - &sys_crypto, - &sys_tortls, + &sys_torerr, /* -100 */ + &sys_wallclock, /* -99 */ + &sys_threads, /* -95 */ + &sys_logging, /* -90 */ + &sys_time, /* -90 */ + &sys_network, /* -90 */ + &sys_compress, /* -70 */ + &sys_crypto, /* -60 */ + &sys_tortls, /* -50 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); -- GitLab From 100136ca8624151605601d80b63746cfaeb6df47 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 20:24:07 +0200 Subject: [PATCH 0134/1724] Create new periodic event for pruning old info about Tor routers --- changes/bug27929 | 5 +++++ src/core/mainloop/mainloop.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 changes/bug27929 diff --git a/changes/bug27929 b/changes/bug27929 new file mode 100644 index 0000000000..a97d18f202 --- /dev/null +++ b/changes/bug27929 @@ -0,0 +1,5 @@ + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new periodic + event that will run every minute even if Tor is not configured + as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index a9f1429787..be19136130 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1359,6 +1359,7 @@ CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(launch_reachability_tests); +CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); @@ -1391,6 +1392,8 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, PERIODIC_EVENT_FLAG_NEED_NET), CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(prune_old_routers, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), @@ -1454,6 +1457,7 @@ static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; static periodic_event_item_t *save_state_event=NULL; +static periodic_event_item_t *prune_old_routers_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1556,6 +1560,7 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -2220,6 +2225,27 @@ retry_dns_callback(time_t now, const or_options_t *options) return RETRY_DNS_INTERVAL; } +/** + * Periodic callback: prune routerlist of old information about Tor network. + */ +static int +prune_old_routers_callback(time_t now, const or_options_t *options) +{ +#define ROUTERLIST_PRUNING_INTERVAL (60) // 1 minute. + (void)now; + (void)options; + + if (!net_is_disabled()) { + /* If any networkstatus documents are no longer recent, we need to + * update all the descriptors' running status. */ + /* Remove dead routers. */ + log_debug(LD_GENERAL, "Pruning routerlist..."); + routerlist_remove_old_routers(); + } + + return ROUTERLIST_PRUNING_INTERVAL; +} + /** Periodic callback: consider rebuilding or and re-uploading our descriptor * (if we've passed our internal checks). */ static int @@ -2239,12 +2265,6 @@ check_descriptor_callback(time_t now, const or_options_t *options) check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* If any networkstatus documents are no longer recent, we need to - * update all the descriptors' running status. */ - /* Remove dead routers. */ - /* XXXX This doesn't belong here, but it was here in the pre- - * XXXX refactoring code. */ - routerlist_remove_old_routers(); } return CHECK_DESCRIPTOR_INTERVAL; -- GitLab From ec93385cb235a9aafc7bd3bd83a440b3f35ff6fd Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 13 Nov 2018 10:33:51 -0500 Subject: [PATCH 0135/1724] Comment for rend_cache_failure in feature/rend/rendcache.c: "usuable" should be "usable" --- src/feature/rend/rendcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 848386b97d..b851e71959 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -45,7 +45,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL; * looked up in this cache and if present, it is discarded from the fetched * descriptor. At the end, all IP(s) in the cache, for a specific service * ID, that were NOT present in the descriptor are removed from this cache. - * Which means that if at least one IP was not in this cache, thus usuable, + * Which means that if at least one IP was not in this cache, thus usable, * it's considered a new descriptor so we keep it. Else, if all IPs were in * this cache, we discard the descriptor as it's considered unusable. * -- GitLab From c9f9c9bc497bbfdd5e1acdc37f84da5f3f7396c2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 17:52:44 -0500 Subject: [PATCH 0136/1724] Make memarea use smartlist_core, not container. --- src/lib/memarea/.may_include | 2 +- src/lib/memarea/memarea.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h \ No newline at end of file diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index dd7c48c079..96d94c89d9 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -- GitLab From f6b8c7da66bf93a9505b397661616cc4af2a34f6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 18:05:14 -0500 Subject: [PATCH 0137/1724] Move buffers.c out of lib/containers to resolve a circularity. --- .gitignore | 2 ++ Makefile.am | 2 ++ src/app/main/main.c | 2 +- src/core/mainloop/connection.c | 2 +- src/core/mainloop/mainloop.c | 2 +- src/core/or/circuitlist.c | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/connection_or.c | 2 +- src/core/or/or.h | 2 +- src/core/or/relay.c | 2 +- src/core/or/scheduler.c | 2 +- src/core/or/scheduler_kist.c | 2 +- src/core/proto/proto_cell.c | 2 +- src/core/proto/proto_control0.c | 2 +- src/core/proto/proto_ext_or.c | 2 +- src/core/proto/proto_http.c | 2 +- src/core/proto/proto_socks.c | 2 +- src/feature/control/control.c | 2 +- src/feature/stats/geoip_stats.c | 2 +- src/include.am | 1 + src/lib/buf/.may_include | 10 ++++++++++ src/lib/{container => buf}/buffers.c | 2 +- src/lib/{container => buf}/buffers.h | 0 src/lib/buf/include.am | 17 +++++++++++++++++ src/lib/compress/.may_include | 1 + src/lib/compress/compress_buf.c | 2 +- src/lib/container/.may_include | 3 --- src/lib/container/include.am | 2 -- src/lib/net/.may_include | 1 + src/lib/net/buffers_net.c | 2 +- src/lib/tls/.may_include | 1 + src/lib/tls/buffers_tls.c | 2 +- src/test/fuzz/fuzz_http.c | 2 +- src/test/fuzz/fuzz_http_connect.c | 2 +- src/test/fuzz/fuzz_socks.c | 2 +- src/test/test.c | 2 +- src/test/test_buffers.c | 2 +- src/test/test_channelpadding.c | 2 +- src/test/test_channeltls.c | 2 +- src/test/test_extorport.c | 2 +- src/test/test_helpers.c | 2 +- src/test/test_oom.c | 2 +- src/test/test_proto_http.c | 2 +- src/test/test_proto_misc.c | 2 +- src/test/test_routerlist.c | 2 +- src/test/test_socks.c | 2 +- src/test/test_util.c | 2 +- 47 files changed, 71 insertions(+), 41 deletions(-) create mode 100644 src/lib/buf/.may_include rename src/lib/{container => buf}/buffers.c (99%) rename src/lib/{container => buf}/buffers.h (100%) create mode 100644 src/lib/buf/include.am diff --git a/.gitignore b/.gitignore index ee2de376a6..e5d021f30d 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,8 @@ uptime-*.json # /src/lib /src/lib/libcurve25519_donna.a +/src/lib/libtor-buf.a +/src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a /src/lib/libtor-container.a diff --git a/Makefile.am b/Makefile.am index cb76edfa2f..99dec36e36 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,7 @@ endif TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ + src/lib/libtor-buf.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -72,6 +73,7 @@ if UNITTESTS_ENABLED TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ + src/lib/libtor-buf-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ diff --git a/src/app/main/main.c b/src/app/main/main.c index b8dcb852d2..653a393fe4 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -67,7 +67,7 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" #include "lib/geoip/geoip.h" diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 1198a01ad9..8058077cd5 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -57,7 +57,7 @@ #define CONNECTION_PRIVATE #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/err/backtrace.h" diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..8edee9d29d 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -95,7 +95,7 @@ #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/err/backtrace.h" #include "lib/tls/buffers_tls.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 35efc6541f..0aa21000a1 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -94,7 +94,7 @@ #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "ht.h" diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 58aefcf8f2..a69c907319 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -97,7 +97,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_util.h" #include "core/or/cell_st.h" diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 65f4e28c92..f231fda123 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -22,7 +22,7 @@ **/ #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). diff --git a/src/core/or/or.h b/src/core/or/or.h index acf092c8dc..0c4995cfd6 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -26,7 +26,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/container/map.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/container/smartlist.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rsa.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 510d96c648..2e92f2a55d 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -49,7 +49,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "feature/client/circpathbias.h" #include "core/or/circuitbuild.h" diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index 326e0d65d9..937e7e45db 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -9,7 +9,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/scheduler.h" #include "core/mainloop/mainloop.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #define TOR_CHANNEL_INTERNAL_ #include "core/or/channeltls.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index f112ea6357..3ed0f1a5e2 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -4,7 +4,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 49b07663d7..70278cd488 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_cell.h" #include "core/or/connection_or.h" diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index d26c69753a..a770a061a7 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_control0.h" /** Return 1 iff buf looks more like it has an (obsolete) v0 controller diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index 04bec4e6b9..fe36f6b396 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_ext_or.h" diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 37298d1284..4ce9ba02f5 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -6,7 +6,7 @@ #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" /** Return true if cmd looks like a HTTP (proxy) request. */ diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index e23da7730b..4071f34f0d 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -6,7 +6,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "app/config/config.h" diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b31b448e96..11e722c3a8 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -87,7 +87,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 1a4f8ddfb0..3106c6c82c 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "ht.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/dnsserv.h" diff --git a/src/include.am b/src/include.am index 8279499936..9070a69a03 100644 --- a/src/include.am +++ b/src/include.am @@ -1,5 +1,6 @@ include src/ext/include.am include src/lib/arch/include.am +include src/lib/buf/include.am include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c similarity index 99% rename from src/lib/container/buffers.c rename to src/lib/buf/buffers.c index 87d782d7ef..495c5ec453 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h similarity index 100% rename from src/lib/container/buffers.h rename to src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..3338c3dbdb --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,17 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 4870259ec9..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 63ee9e0102..ecf76ee078 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..76e5843728 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -11,8 +11,5 @@ lib/testsupport/testsupport.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - ht.h siphash.h diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..032e4033da 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ src/lib/container/order.h \ diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index f93f0e1552..d34aaed2ca 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -3,6 +3,7 @@ siphash.h ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..b0936e9928 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 79301bc318..069181b70e 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index 69ae4f7fc0..b4059292ea 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 06483282bc..4341bfabae 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/dircache.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index ca007a2c7f..e03d9e29d8 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 14c25304b1..2d93bea924 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/err/backtrace.h" #include "lib/log/log.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..a654030fac 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -37,7 +37,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "lib/compress/compress.h" diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 477066f699..85e7b8d90a 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/tls/tortls.h" #include "lib/compress/compress.h" diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 0fd60d0a92..bdd7c5f0a6 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -21,7 +21,7 @@ #include "test/log_test_helpers.h" #include "lib/tls/tortls.h" #include "lib/evloop/timers.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/cell_st.h" #include "feature/nodelist/networkstatus_st.h" diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 787a30a85d..44d623561b 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -8,7 +8,7 @@ #define TOR_CHANNEL_INTERNAL_ #include "core/or/or.h" #include "lib/net/address.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 71be9131c7..432a9ea5e3 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -5,7 +5,7 @@ #define EXT_ORPORT_PRIVATE #define MAINLOOP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 6ac73bff5c..b7bda16494 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -14,7 +14,7 @@ #include "orconfig.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index 313a6b3114..f84dc0764b 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -8,7 +8,7 @@ #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "lib/evloop/compat_libevent.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index 1cfa0a752c..b4e8278423 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index 1fcb763421..f7f6f69667 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_cell.h" diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 1071a095fe..167b7a35ce 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -46,7 +46,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "lib/encoding/confline.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "test/test.h" #include "test/test_dir_common.h" diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 7f6d8a48f1..d430f4329b 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index bcface64fd..2b4d64e42e 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -13,7 +13,7 @@ #define SUBPROCESS_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/transports.h" -- GitLab From c0a7527eb8590b39f11dd8b0ae18794dfc63a934 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 18:14:16 -0500 Subject: [PATCH 0138/1724] Remove dependency on lib/net from lib/sandbox. This was trivial, and the easiest way to remove the remaining .may_include circularities. --- src/app/main/main.c | 1 + src/lib/sandbox/.may_include | 1 - src/lib/sandbox/sandbox.c | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index 653a393fe4..ec86c46397 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1402,6 +1402,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) tor_free_all(0); return -1; } + tor_make_getaddrinfo_cache_active(); // registering libevent rng #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..e8ba5bb73c 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,7 +5,6 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 6f074bb4e1..4d832f106d 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,7 +38,6 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" @@ -1553,7 +1552,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); -- GitLab From e429e31ad1c3000d814f3172fcd3f7c433ee3219 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:27:43 -0500 Subject: [PATCH 0139/1724] Normalize .may_include to always have paths, and paths to include --- src/lib/container/.may_include | 6 +++--- src/lib/container/bloomfilt.c | 2 +- src/lib/container/map.c | 2 +- src/lib/container/map.h | 2 +- src/lib/crypt_ops/.may_include | 4 ++-- src/lib/crypt_ops/crypto_init.c | 2 +- src/lib/crypt_ops/digestset.c | 2 +- src/lib/evloop/.may_include | 4 ++-- src/lib/evloop/timers.c | 3 ++- src/lib/evloop/workqueue.c | 2 +- src/lib/fs/.may_include | 2 +- src/lib/malloc/.may_include | 2 +- src/lib/net/.may_include | 4 ++-- src/lib/net/address.c | 2 +- src/lib/net/resolve.c | 4 ++-- src/lib/process/.may_include | 2 +- src/lib/process/waitpid.c | 2 +- src/lib/sandbox/.may_include | 6 +++--- src/lib/sandbox/sandbox.c | 6 +++--- src/lib/smartlist_core/.may_include | 2 +- src/lib/string/.may_include | 4 ++-- src/lib/string/compat_string.c | 4 ++-- src/lib/term/.may_include | 3 +-- src/lib/term/getpass.c | 2 +- src/lib/tls/.may_include | 5 ++--- src/lib/tls/tortls_openssl.c | 2 +- 26 files changed, 40 insertions(+), 41 deletions(-) diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 76e5843728..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,9 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index ea2d2917c7..a64fcb9300 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/map.c b/src/lib/container/map.c index 85e074e10a..137e316920 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include #include -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called maptype. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index ff71622682..4f427fe767 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index 352fde858c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -13,7 +13,7 @@ lib/intmath/*.h lib/sandbox/*.h lib/string/*.h lib/subsys/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -22,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 4c4cc3e43b..f9943939fb 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -24,7 +24,7 @@ #include "lib/subsys/subsys.h" -#include "siphash.h" +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 89dd377a9c..84516e0172 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..273de7bb94 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -12,5 +12,5 @@ lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h \ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h \ No newline at end of file diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index 3603bf1a7d..6743b6af51 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 5471f87b04..b36a02da5e 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include #include diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index d34aaed2ca..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,6 +1,6 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h lib/buf/*.h diff --git a/src/lib/net/address.c b/src/lib/net/address.c index a351b9df28..240201d7b6 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -53,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 7c8df3e307..01f7882964 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -16,8 +16,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index a2d57a52f3..3a5d849fe7 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -15,4 +15,4 @@ lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h \ No newline at end of file +ext/ht.h \ No newline at end of file diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 32ba4530da..46d30bf50e 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index e8ba5bb73c..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -9,6 +9,6 @@ lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 4d832f106d..9477818553 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -41,9 +41,9 @@ #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index eae82fdae0..b3f1e0fd96 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index 27a27179b6..a473fb765b 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 069181b70e..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -13,7 +13,6 @@ lib/malloc/*.h lib/net/*.h lib/string/*.h lib/subsys/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 63f6259a6c..ddeaabc416 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -461,7 +461,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" -- GitLab From d32795bb6ebdfbf05180d7552d4cd18cb33ddcee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:34:47 -0500 Subject: [PATCH 0140/1724] Make "ext" participate in may_include. Also, resolve a circular dependency involving the use of lib/log by csiphash.c. --- src/ext/.may_include | 10 ++++++++++ src/ext/csiphash.c | 10 +++++----- src/ext/readpassphrase.c | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 src/ext/.may_include diff --git a/src/ext/.may_include b/src/ext/.may_include new file mode 100644 index 0000000000..1eafff2eeb --- /dev/null +++ b/src/ext/.may_include @@ -0,0 +1,10 @@ + +orconfig.h + +lib/err/*.h +lib/cc/*.h + +tinytest*.h +ext/siphash.h +ext/byteorder.h +ext/tor_readpassphrase.h \ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index a6a9846db4..af8559a476 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -30,12 +30,12 @@ */ #include "lib/cc/torint.h" -#include "lib/log/util_bug.h" +#include "lib/err/torerr.h" -#include "siphash.h" +#include "ext/siphash.h" #include #include -#include "byteorder.h" +#include "ext/byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) @@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0; static struct sipkey the_siphash_key; uint64_t siphash24g(const void *src, unsigned long src_sz) { - tor_assert(the_siphash_key_is_set); + raw_assert(the_siphash_key_is_set); return siphash24(src, src_sz, &the_siphash_key); } void siphash_set_global_key(const struct sipkey *key) { - tor_assert(! the_siphash_key_is_set); + raw_assert(! the_siphash_key_is_set); the_siphash_key.k0 = key->k0; the_siphash_key.k1 = key->k1; the_siphash_key_is_set = 1; diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c index e0df05d7b7..16611af1e2 100644 --- a/src/ext/readpassphrase.c +++ b/src/ext/readpassphrase.c @@ -30,7 +30,7 @@ #include #include #include -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #include #include #include -- GitLab From 4ecd5da3065d83af0743669dfcadd3cb01356cee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:38:09 -0500 Subject: [PATCH 0141/1724] Add .may_include to ext/timeouts. --- src/ext/timeouts/.may_include | 6 ++++++ src/ext/timeouts/timeout.c | 5 ++--- src/ext/timeouts/timeout.h | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 src/ext/timeouts/.may_include diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include new file mode 100644 index 0000000000..42f44befd4 --- /dev/null +++ b/src/ext/timeouts/.may_include @@ -0,0 +1,6 @@ +orconfig.h + +ext/tor_queue.h +timeout-bitops.c +timeout-debug.h +timeout.h diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index d4b514d2c5..07d06772c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -38,7 +38,7 @@ #include /* errno */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ #include "timeout.h" @@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) { timeout = MIN(_timeout, timeout); } - relmask <<= WHEEL_BIT; + relmask <<= WHEEL_BIT; relmask |= WHEEL_MASK; } @@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) { TIMEOUT_PUBLIC int timeout_v_api(void) { return TIMEOUT_V_API; } /* timeout_version() */ - diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h index b35874e153..1ed309fd08 100644 --- a/src/ext/timeouts/timeout.h +++ b/src/ext/timeouts/timeout.h @@ -31,7 +31,7 @@ #include /* PRIu64 PRIx64 PRIX64 uint64_t */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ /* @@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); /* true if on timing wheel, false otherwise */ - + TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); /* true if on expired queue, false otherwise */ -- GitLab From c3c8c926bfc8e44e28f3a404e3452f014f6bcbaa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:30:46 -0500 Subject: [PATCH 0142/1724] Update the check-includes script to enforce some naming and no-circularity rules --- scripts/maint/checkIncludes.py | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 46a3f39638..ecda3fc565 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -33,6 +33,9 @@ else: def open_file(fname): return open(fname, 'r', encoding='utf-8') +def warn(msg): + print(msg, file=sys.stderr) + def err(msg): """ Declare that an error has happened, and remember that there has been an error. """ @@ -48,14 +51,34 @@ def fname_is_c(fname): INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') RULES_FNAME = ".may_include" +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + class Rules(object): """ A 'Rules' object is the parsed version of a .may_include file. """ def __init__(self, dirpath): self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath self.patterns = [] self.usedPatterns = set() def addPattern(self, pattern): + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) self.patterns.append(pattern) def includeOk(self, path): @@ -86,6 +109,20 @@ class Rules(object): if p not in self.usedPatterns: print("Pattern {} in {} was never used.".format(p, self.dirpath)) + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + def load_include_rules(fname): """ Read a rules file from 'fname', and return it as a Rules object. """ result = Rules(os.path.split(fname)[0]) @@ -99,6 +136,8 @@ def load_include_rules(fname): list_unused = False +uses_dirs = { } + for dirpath, dirnames, fnames in os.walk("src"): if ".may_include" in fnames: rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) @@ -108,8 +147,33 @@ for dirpath, dirnames, fnames in os.walk("src"): if list_unused: rules.noteUnusedRules() + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + if trouble: err( """To change which includes are allowed in a C file, edit the {} files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) + +all_levels = [] + +n = 0 +while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level: + print(n, cur_level) + if n > 100: + break + +if uses_dirs: + print("Circular dependencies in here somewhere:", uses_dirs) + sys.exit(1) -- GitLab From fae29f7b424460353b3a5ab44e1394fdc28cc4c4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:59:23 -0500 Subject: [PATCH 0143/1724] Make the topological-sort output off by default --- scripts/maint/checkIncludes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index ecda3fc565..3afd9bbebe 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -135,6 +135,7 @@ def load_include_rules(fname): return result list_unused = False +log_sorted_levels = False uses_dirs = { } @@ -169,11 +170,12 @@ while uses_dirs: for k in cur_level: del uses_dirs[k] n += 1 - if cur_level: + if cur_level and log_sorted_levels: print(n, cur_level) if n > 100: break if uses_dirs: - print("Circular dependencies in here somewhere:", uses_dirs) + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) sys.exit(1) -- GitLab From 5090fecaca61c87d52559342d6488ee8bb8fd20b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 11:02:03 -0500 Subject: [PATCH 0144/1724] changes file for no-circular-dependencies stuff (28362) --- changes/ticket28362 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ticket28362 diff --git a/changes/ticket28362 b/changes/ticket28362 new file mode 100644 index 0000000000..4ac22d50f2 --- /dev/null +++ b/changes/ticket28362 @@ -0,0 +1,6 @@ + o Code simplification and refactoring: + - The .may_include files that we use to describe our + directory-by-directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. + Our checkIncludes.py tool now enforces this. + Closes ticket 28362. -- GitLab From 6ca29ea4094c7cb5696cd41d3994dd2fb230fe9a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 12:14:24 -0500 Subject: [PATCH 0145/1724] Add libtor-buf-testing to build.rs --- src/rust/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/build.rs b/src/rust/build.rs index bf566c56bf..5626b35f75 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -149,8 +149,9 @@ pub fn main() { cfg.component("tor-sandbox-testing"); cfg.component("tor-encoding-testing"); cfg.component("tor-fs-testing"); - cfg.component("tor-time-testing"); cfg.component("tor-net-testing"); + cfg.component("tor-buf-testing"); + cfg.component("tor-time-testing"); cfg.component("tor-thread-testing"); cfg.component("tor-memarea-testing"); cfg.component("tor-log-testing"); -- GitLab From e420154ce77d9169bc72c373bfc7e29cb7245723 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 14:39:20 -0500 Subject: [PATCH 0146/1724] Add an include to main.c --- src/app/main/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/main/main.c b/src/app/main/main.c index ec86c46397..59ab4ce408 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -71,6 +71,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" #include "lib/geoip/geoip.h" +#include "lib/net/resolve.h" #include "lib/process/waitpid.h" -- GitLab From eaff47352a1ef607f6d97e9b35bd6d5eefdfcb26 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 17:02:15 -0500 Subject: [PATCH 0147/1724] Make sure sandbox-related getaddrinfo() functions always exist. --- src/lib/net/resolve.c | 9 +++++++++ src/lib/net/resolve.h | 2 +- src/lib/sandbox/sandbox.c | 5 ----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 01f7882964..95c1b171b5 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } +#else +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} #endif diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index bf870c44c4..39157aaa67 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) @@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); #endif diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 9477818553..ea738b273e 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -1798,9 +1798,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ -- GitLab From 2070765c7c062c505358d0f1c83f2846181d1667 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 16:09:13 -0500 Subject: [PATCH 0148/1724] Use macros to make the periodic event table less verbose. --- src/core/mainloop/mainloop.c | 78 ++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..12820888f2 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1370,74 +1370,66 @@ CALLBACK(write_stats_file); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) +#define CALLBACK(name, r, f) \ + PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { /* Everyone needs to run those. */ - CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(add_entropy, ALL, 0), + CALLBACK(check_expired_networkstatus, ALL, 0), + CALLBACK(clean_caches, ALL, 0), + CALLBACK(fetch_networkstatus, ALL, 0), + CALLBACK(heartbeat, ALL, 0), + CALLBACK(launch_descriptor_fetches, ALL, FL(NEED_NET)), + CALLBACK(reset_padding_counts, ALL, 0), + CALLBACK(retry_listeners, ALL, FL(NEED_NET)), + CALLBACK(save_state, ALL, 0), + CALLBACK(rotate_x509_certificate, ALL, 0), + CALLBACK(write_stats_file, ALL, 0), /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), + CALLBACK(check_ed_keys, ROUTER, 0), + CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), + CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), + CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), + CALLBACK(retry_dns, ROUTER, 0), + CALLBACK(rotate_onion_key, ROUTER, 0), /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(downrate_stability, AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), + CALLBACK(save_stability, AUTHORITIES, 0), /* Directory authority only. */ - CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), - CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_authority_cert, DIRAUTH, 0), + CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), /* Relay only. */ - CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), + CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), /* Bridge only. */ - CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + CALLBACK(rend_cache_failure_clean, CLIENT, 0), /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), /* Directory server only. */ - CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + CALLBACK(clean_consdiffmgr, DIRSERVER, 0), END_OF_PERIODIC_EVENTS }; #undef CALLBACK +#undef FL /* These are pointers to members of periodic_events[] that are used to * implement particular callbacks. We keep them separate here so that we -- GitLab From 6d84972eb8e27d5e9f1adea36fcc9a9879d718ad Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 16:24:10 -0500 Subject: [PATCH 0149/1724] Add a function to schedule a periodic event once, then disable it --- src/core/mainloop/mainloop.c | 6 +++++- src/core/mainloop/periodic.c | 22 +++++++++++++++++----- src/core/mainloop/periodic.h | 7 ++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 12820888f2..42f6fb50c7 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1600,7 +1600,11 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - periodic_event_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE) { + periodic_event_flush_and_disable(item); + } else { + periodic_event_disable(item); + } } } } diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c1785eb38f..c290c3744e 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -45,10 +45,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) periodic_event_item_t *event = data; tor_assert(ev == event->ev); - if (BUG(!periodic_event_is_enabled(event))) { - return; - } - time_t now = time(NULL); update_current_time(now); const or_options_t *options = get_options(); @@ -57,7 +53,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) int next_interval = 0; if (!periodic_event_is_enabled(event)) { - /* The event got disabled from inside its callback; no need to + /* The event got disabled from inside its callback, or before: no need to * reschedule. */ return; } @@ -172,3 +168,19 @@ periodic_event_disable(periodic_event_item_t *event) mainloop_event_cancel(event->ev); event->enabled = 0; } + +/** + * Disable an event, then schedule it to run once. + * Do nothing if the event was already disabled. + */ +void +periodic_event_flush_and_disable(periodic_event_item_t *event) +{ + tor_assert(event); + if (!periodic_event_is_enabled(event)) + return; + + periodic_event_disable(event); + + mainloop_event_activate(event->ev); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 4c8c3c96cc..7c71be7bc4 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -39,6 +39,11 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) +/* Indicate that it the event is enabled, it event needs to be run once before + * it becomes disabled. + */ +#define PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE (1U << 1) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update last_action_time and be polled @@ -83,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); +void periodic_event_flush_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ - -- GitLab From b9a88bd53ae79a29c292275381bc7dbaa3804034 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 07:27:31 -0500 Subject: [PATCH 0150/1724] Add new "ALL" and "NET_PARTICIPANT" roles for periodic events The previous "ALL" role was the OR of a bunch of other roles, which is a mistake: it's better if "ALL" means "all". The "NET_PARTICIPANT" role refers to the anything that is actively building circuits, downloading directory information, and participating in the Tor network. For now, it is set to !net_is_disabled(), but we're going to use it to implement a new "extra dormant mode". Closes ticket 28336. --- src/core/mainloop/mainloop.c | 36 ++++++++++++++++++------- src/core/mainloop/periodic.h | 7 +++-- src/test/test_periodic_event.c | 49 +++++++++++++++++++++------------- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 42f6fb50c7..3c3f441a91 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1377,16 +1377,27 @@ CALLBACK(write_stats_file); STATIC periodic_event_item_t periodic_events[] = { /* Everyone needs to run those. */ CALLBACK(add_entropy, ALL, 0), - CALLBACK(check_expired_networkstatus, ALL, 0), - CALLBACK(clean_caches, ALL, 0), - CALLBACK(fetch_networkstatus, ALL, 0), CALLBACK(heartbeat, ALL, 0), - CALLBACK(launch_descriptor_fetches, ALL, FL(NEED_NET)), - CALLBACK(reset_padding_counts, ALL, 0), + + /* XXXX Do we have a reason to do this on a callback? */ CALLBACK(retry_listeners, ALL, FL(NEED_NET)), - CALLBACK(save_state, ALL, 0), - CALLBACK(rotate_x509_certificate, ALL, 0), - CALLBACK(write_stats_file, ALL, 0), + + /* We need to do these if we're participating in the Tor network. */ + CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), + CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + + /* We need to do these if we're participating in the Tor network, and + * immediately before we stop. */ + CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + + /* XXXX investigate this. */ + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + + /* XXXX investigate this. */ + CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), @@ -1418,7 +1429,8 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, CLIENT, 0), + /* XXXX this could be restricted to CLIENT even. */ + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), @@ -1477,7 +1489,7 @@ get_my_roles(const or_options_t *options) { tor_assert(options); - int roles = 0; + int roles = PERIODIC_EVENT_ROLE_ALL; int is_bridge = options->BridgeRelay; int is_relay = server_mode(options); int is_dirauth = authdir_mode_v3(options); @@ -1492,6 +1504,9 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; + /* We actually want a better definition here for our work on dormancy. */ + int is_net_participant = ! net_is_disabled(); + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; @@ -1499,6 +1514,7 @@ get_my_roles(const or_options_t *options) if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; return roles; } diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 7c71be7bc4..23459ff2b3 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -16,6 +16,9 @@ #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 7) +#define PERIODIC_EVENT_ROLE_ALL (1U << 8) + /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ @@ -25,10 +28,6 @@ /* Authorities that is both bridge and directory. */ #define PERIODIC_EVENT_ROLE_AUTHORITIES \ (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) -/* All roles. */ -#define PERIODIC_EVENT_ROLE_ALL \ - (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) /* * Event flags which can change the behavior of an event. diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 86dedd85d8..f63adf8e3a 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -19,6 +19,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/periodic.h" /** Helper function: This is replaced in some tests for the event callbacks so @@ -59,7 +60,9 @@ test_pe_initialize(void *arg) tt_u64_op(item->last_action_time, OP_EQ, 0); /* Every event must have role(s) assign to it. This is done statically. */ tt_u64_op(item->roles, OP_NE, 0); - tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } done: @@ -106,13 +109,12 @@ test_pe_launch(void *arg) /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); + int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); - } else { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); - } + int should_be_enabled = !!(item->roles & mask); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. tt_u64_op(item->last_action_time, OP_EQ, 0); } @@ -124,7 +126,8 @@ test_pe_launch(void *arg) unsigned roles = get_my_roles(options); tt_uint_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| + PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; @@ -144,17 +147,21 @@ test_pe_launch(void *arg) /* Disable everything and we'll enable them ALL. */ options->SocksPort_set = 0; options->ORPort_set = 0; + options->DisableNetwork = 1; periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } /* Enable everything. */ options->SocksPort_set = 1; options->ORPort_set = 1; options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + options->DisableNetwork = 0; register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -188,41 +195,46 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + const int ALL = PERIODIC_EVENT_ROLE_ALL; + /* Nothing configured, should be no roles. */ + tt_assert(net_is_disabled()); roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Indicate we have a SocksPort, roles should be come Client. */ options->SocksPort_set = 1; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL); /* Now, we'll add a ORPort so should now be a Relay + Client. */ options->ORPort_set = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_DIRSERVER | ALL)); /* Now add a Bridge. */ options->BridgeRelay = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); /* Unset client so we can solely test Router role. */ options->SocksPort_set = 0; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL); /* Reset options so we can test authorities. */ options->SocksPort_set = 0; options->ORPort_set = 0; options->BridgeRelay = 0; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Now upgrade to Dirauth. */ options->DirPort_set = 1; @@ -230,7 +242,7 @@ test_pe_get_roles(void *arg) options->V3AuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Now Bridge Authority. */ @@ -238,7 +250,7 @@ test_pe_get_roles(void *arg) options->BridgeAuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Move that bridge auth to become a relay. */ @@ -246,7 +258,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY - | PERIODIC_EVENT_ROLE_DIRSERVER)); + | PERIODIC_EVENT_ROLE_DIRSERVER|ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* And now an Hidden service. */ @@ -257,7 +269,8 @@ test_pe_get_roles(void *arg) remove_service(get_hs_service_map(), &service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); done: -- GitLab From db53bfe8f74dad1b45ba381a5ee3366148a30237 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 11:14:50 -0500 Subject: [PATCH 0151/1724] Annotate 1/s callback elements with NET_PARTICIPANT status. --- src/core/mainloop/mainloop.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 3c3f441a91..7e5e5d0efb 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1379,8 +1379,9 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), - /* XXXX Do we have a reason to do this on a callback? */ - CALLBACK(retry_listeners, ALL, FL(NEED_NET)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at + * all? For now, if we're dormant, we can let our listeners decay. */ + CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), /* We need to do these if we're participating in the Tor network. */ CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), @@ -1393,10 +1394,10 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. */ + /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. */ + /* XXXX investigate this. ???? */ CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ @@ -1423,13 +1424,13 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more /* Bridge only. */ CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - /* XXXX this could be restricted to CLIENT even. */ + /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Bridge Authority only. */ @@ -1730,18 +1731,24 @@ run_scheduled_events(time_t now) * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. */ + // TODO: Refactor or rewrite, or NET_PARTICIPANT. Needs separate wakeup + // handling. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ + // TODO: NET_PARTICIPANT circuit_upgrade_circuits_from_guard_wait(); if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ + // TODO: client+NET_PARTICIPANT. + // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { + // TODO: refactor or rewrite; NET_PARTICIPANT accounting_run_housekeeping(now); } @@ -1752,6 +1759,7 @@ run_scheduled_events(time_t now) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); @@ -1760,10 +1768,12 @@ run_scheduled_events(time_t now) * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_ap_expire_beginning(); /* 3c. And expire connections that we've held open for too long. */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_expire_held_open(); /* 4. Every second, we try a new circuit if there are no valid @@ -1773,19 +1783,24 @@ run_scheduled_events(time_t now) */ const int have_dir_info = router_have_minimum_dir_info(); if (have_dir_info && !net_is_disabled()) { + // TODO: NET_PARTICIPANT. circuit_build_needed_circs(now); } else { + // TODO: NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_old_circs_as_needed(now); } /* 5. We do housekeeping for each connection... */ + // TODO: NET_PARTICIPANT channel_update_bad_for_new_circs(NULL, 0); int i; for (i=0;i Date: Tue, 13 Nov 2018 08:22:58 -0500 Subject: [PATCH 0152/1724] Move control_per_second_events() into a callback with its own role Part of making extra-dormant mode work; closes ticket 28421. --- src/core/mainloop/mainloop.c | 29 +++++++++++++++++++++++------ src/core/mainloop/periodic.h | 5 +++-- src/feature/control/control.c | 2 +- src/test/test_periodic_event.c | 3 ++- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7e5e5d0efb..a9d5d8155a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1366,6 +1366,7 @@ CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); +CALLBACK(control_per_second_events); #undef CALLBACK @@ -1439,6 +1440,9 @@ STATIC periodic_event_item_t periodic_events[] = { /* Directory server only. */ CALLBACK(clean_consdiffmgr, DIRSERVER, 0), + /* Controller with per-second events only. */ + CALLBACK(control_per_second_events, CONTROLEV, 0), + END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1498,6 +1502,8 @@ get_my_roles(const or_options_t *options) int is_hidden_service = !!hs_service_get_num_services() || !!rend_num_services(); int is_dirserver = dir_server_mode(options); + int sending_control_events = control_any_per_second_event_enabled(); + /* We also consider tor to have the role of a client if the ControlPort is * set because a lot of things can be done over the control port which * requires tor to have basic functionnalities. */ @@ -1516,6 +1522,7 @@ get_my_roles(const or_options_t *options) if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; + if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV; return roles; } @@ -2524,6 +2531,21 @@ hs_service_callback(time_t now, const or_options_t *options) return 1; } +/* + * Periodic callback: Send once-per-second events to the controller(s). + * This is called every second. + */ +static int +control_per_second_events_callback(time_t now, const or_options_t *options) +{ + (void) options; + (void) now; + + control_per_second_events(); + + return 1; +} + /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; @@ -2546,8 +2568,7 @@ reschedule_per_second_timer(void) tor_assert(second_timer); } - const bool run_per_second_events = - control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + const bool run_per_second_events = ! net_is_completely_disabled(); if (run_per_second_events) { periodic_timer_launch(second_timer, &one_second); @@ -2640,10 +2661,6 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ update_current_time(now); - // TODO XXXX Turn this into a separate event. - /* Maybe some controller events are ready to fire */ - control_per_second_events(); - run_scheduled_events(now); } diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 23459ff2b3..52d5450ee8 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -15,9 +15,10 @@ #define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_CONTROLEV (1U << 7) -#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 7) -#define PERIODIC_EVENT_ROLE_ALL (1U << 8) +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8) +#define PERIODIC_EVENT_ROLE_ALL (1U << 9) /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b31b448e96..a5b6ab3bf5 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -368,7 +368,7 @@ control_update_global_event_mask(void) control_get_bytes_rw_last_sec(&r, &w); } if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - reschedule_per_second_timer(); + rescan_periodic_events(get_options()); } #undef NEWLY_ENABLED diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index f63adf8e3a..6a3e320b2e 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -172,7 +172,8 @@ test_pe_launch(void *arg) for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, + (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } done: -- GitLab From e535ec8542a1d42243c0b6ae28036aec8262269b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 08:36:38 -0500 Subject: [PATCH 0153/1724] Remove run_scheduled_events() as a separate function. (There was nothing else in second_elapsed_callbck() that couldn't go here.) --- src/core/mainloop/mainloop.c | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index a9d5d8155a..ebf9735d49 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1727,13 +1727,22 @@ safe_timer_diff(time_t now, time_t next) } /** Perform regular maintenance tasks. This function gets run once per - * second by second_elapsed_callback(). + * second. */ static void -run_scheduled_events(time_t now) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { + (void) timer; + (void) arg; + const time_t now = time(NULL); const or_options_t *options = get_options(); + /* We don't need to do this once-per-second any more: time-updating is + * only in this callback _because it is a callback_. It should be fine + * to disable this callback, and the time will still get updated. + */ + update_current_time(now); + /* 0. See if we've been asked to shut down and our timeout has * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. @@ -2642,28 +2651,6 @@ update_current_time(time_t now) current_second = now; } -/** Libevent callback: invoked once every second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) -{ - /* XXXX This could be sensibly refactored into multiple callbacks, and we - * could use Libevent's timers for this rather than checking the current - * time against a bunch of timeouts every second. */ - time_t now; - (void)timer; - (void)arg; - - now = time(NULL); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - run_scheduled_events(now); -} - #ifdef HAVE_SYSTEMD_209 static periodic_timer_t *systemd_watchdog_timer = NULL; -- GitLab From 303e5c70e08d72614ea95fd0c5aad7e05852b6b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 09:04:11 -0500 Subject: [PATCH 0154/1724] Move the responsibility for delayed shutdown into the mainloop This is part of 28422, so we don't have to call consider_hibernation() once per second when we're dormant. This commit does not remove delayed shutdown from hibernate.c: it uses it as a backup shutdown mechanism, in case the regular shutdown timer mechanism fails for some reason. --- src/core/mainloop/mainloop.c | 36 ++++++++++++++++++++++++++----- src/core/mainloop/mainloop.h | 2 ++ src/feature/hibernate/hibernate.c | 20 +++++++++++------ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ebf9735d49..d5f3cb13f9 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1703,6 +1703,30 @@ mainloop_schedule_postloop_cleanup(void) mainloop_event_activate(postloop_cleanup_ev); } +/** Event to run 'scheduled_shutdown_cb' */ +static mainloop_event_t *scheduled_shutdown_ev=NULL; + +/** Callback: run a scheduled shutdown */ +static void +scheduled_shutdown_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + tor_shutdown_event_loop_and_exit(0); +} + +/** Schedule the mainloop to exit after delay_sec seconds. */ +void +mainloop_schedule_shutdown(int delay_sec) +{ + const struct timeval delay_tv = { delay_sec, 0 }; + if (! scheduled_shutdown_ev) { + scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL); + } + mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between now and next, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1743,12 +1767,13 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ update_current_time(now); - /* 0. See if we've been asked to shut down and our timeout has - * expired; or if our bandwidth limits are exhausted and we - * should hibernate; or if it's time to wake up from hibernation. + /* 0. See if our bandwidth limits are exhausted and we should hibernate; or + * if it's time to wake up from hibernation; or if we have a scheduled + * shutdown and it's time to run it. + * + * Note: we have redundant mechanisms to handle the */ - // TODO: Refactor or rewrite, or NET_PARTICIPANT. Needs separate wakeup - // handling. + // TODO: NET_PARTICIPANT. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ @@ -2936,6 +2961,7 @@ tor_mainloop_free_all(void) mainloop_event_free(schedule_active_linked_connections_event); mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); + mainloop_event_free(scheduled_shutdown_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 632733d9a6..6f7b716858 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -86,6 +86,8 @@ void reschedule_per_second_timer(void); void do_signewnym(time_t); time_t get_last_signewnym_time(void); +void mainloop_schedule_shutdown(int delay_sec); + void tor_init_connection_lists(void); void initialize_mainloop_events(void); void tor_mainloop_free_all(void); diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 6f8795cecc..968c39dd6d 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -66,8 +66,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if - * we aren't shutting down. */ +/** If we are shutting down, when do we plan finally exit? Set to 0 if we + * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed + * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; /** A timed event that we'll use when it's time to wake up from @@ -867,7 +868,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now) log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); - shutdown_time = time(NULL) + options->ShutdownWaitLength; + /* We add an arbitrary delay here so that even if something goes wrong + * with the mainloop shutdown code, we can still shutdown from + * consider_hibernation() if we call it... but so that the + * mainloop_schedule_shutdown() mechanism will be the first one called. + */ + shutdown_time = time(NULL) + options->ShutdownWaitLength + 5; + mainloop_schedule_shutdown(options->ShutdownWaitLength); #ifdef HAVE_SYSTEMD /* tell systemd that we may need more than the default 90 seconds to shut * down so they don't kill us. add some extra time to actually finish @@ -1096,11 +1103,12 @@ consider_hibernation(time_t now) hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval - * elapses. */ + * elapses. The mainloop was supposed to catch this via + * mainloop_schedule_shutdown(), but apparently it didn't. */ if (hibernate_state == HIBERNATE_STATE_EXITING) { tor_assert(shutdown_time); if (shutdown_time <= now) { - log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting."); tor_shutdown_event_loop_and_exit(0); } return; /* if exiting soon, don't worry about bandwidth limits */ @@ -1112,7 +1120,7 @@ consider_hibernation(time_t now) if (hibernate_end_time > now && accounting_enabled) { /* If we're hibernating, don't wake up until it's time, regardless of * whether we're in a new interval. */ - return ; + return; } else { hibernate_end_time_elapsed(now); } -- GitLab From 4bf79fa4fa99fe66f6b1ad413cbf475325803e04 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 09:30:51 -0500 Subject: [PATCH 0155/1724] Turn second_elapsed_callback into a normal periodic event. --- src/app/config/config.c | 3 -- src/core/mainloop/mainloop.c | 81 +++++++------------------------ src/core/mainloop/mainloop.h | 1 - src/feature/hibernate/hibernate.c | 2 - 4 files changed, 17 insertions(+), 70 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 45a23d67d5..8aa0c1f4bd 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -1993,9 +1993,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* See whether we need to enable/disable our once-a-second timer. */ - reschedule_per_second_timer(); - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index d5f3cb13f9..176399b333 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -205,7 +205,6 @@ static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); static void conn_write_callback(evutil_socket_t fd, short event, void *_conn); -static void second_elapsed_callback(periodic_timer_t *timer, void *args); static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; @@ -1367,6 +1366,7 @@ CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); +CALLBACK(second_elapsed); #undef CALLBACK @@ -1380,6 +1380,11 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), + /* This is a legacy catch-all callback that runs once per second if + * we are online and active. */ + CALLBACK(second_elapsed, NET_PARTICIPANT, + FL(NEED_NET)|FL(FLUSH_ON_DISABLE)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), @@ -1753,43 +1758,30 @@ safe_timer_diff(time_t now, time_t next) /** Perform regular maintenance tasks. This function gets run once per * second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) +static int +second_elapsed_callback(time_t now, const or_options_t *options) { - (void) timer; - (void) arg; - const time_t now = time(NULL); - const or_options_t *options = get_options(); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - /* 0. See if our bandwidth limits are exhausted and we should hibernate; or - * if it's time to wake up from hibernation; or if we have a scheduled - * shutdown and it's time to run it. + /* 0. See if our bandwidth limits are exhausted and we should hibernate * - * Note: we have redundant mechanisms to handle the + * Note: we have redundant mechanisms to handle the case where it's + * time to wake up from hibernation; or where we have a scheduled + * shutdown and it's time to run it, but this will also handle those. */ - // TODO: NET_PARTICIPANT. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ - // TODO: NET_PARTICIPANT circuit_upgrade_circuits_from_guard_wait(); if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ - // TODO: client+NET_PARTICIPANT. + // TODO: client // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { - // TODO: refactor or rewrite; NET_PARTICIPANT + // TODO: refactor or rewrite? accounting_run_housekeeping(now); } @@ -1809,12 +1801,10 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_ap_expire_beginning(); /* 3c. And expire connections that we've held open for too long. */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_expire_held_open(); /* 4. Every second, we try a new circuit if there are no valid @@ -1824,26 +1814,24 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ const int have_dir_info = router_have_minimum_dir_info(); if (have_dir_info && !net_is_disabled()) { - // TODO: NET_PARTICIPANT. circuit_build_needed_circs(now); } else { - // TODO: NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_old_circs_as_needed(now); } /* 5. We do housekeeping for each connection... */ - // TODO: NET_PARTICIPANT channel_update_bad_for_new_circs(NULL, 0); int i; for (i=0;i Date: Tue, 13 Nov 2018 10:43:08 -0500 Subject: [PATCH 0156/1724] reset_padding_counts is only once per 24h; it can be all. --- src/core/mainloop/mainloop.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 176399b333..6df51062a7 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1376,9 +1376,11 @@ CALLBACK(second_elapsed); #define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { - /* Everyone needs to run those. */ + /* Everyone needs to run these. They need to have very long timeouts for + * that to be safe. */ CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), + CALLBACK(reset_padding_counts, ALL, 0), /* This is a legacy catch-all callback that runs once per second if * we are online and active. */ @@ -1403,9 +1405,6 @@ STATIC periodic_event_item_t periodic_events[] = { /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. ???? */ - CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), CALLBACK(check_ed_keys, ROUTER, 0), -- GitLab From ccbb36048f5167b9d5011b7c8b0d2c346ce567e8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 10:44:04 -0500 Subject: [PATCH 0157/1724] write_stats_file() is indeed NET_PARTICIPANT; remove comment. --- src/core/mainloop/mainloop.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 6df51062a7..e67ebdb4a8 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1401,8 +1401,6 @@ STATIC periodic_event_item_t periodic_events[] = { * immediately before we stop. */ CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - - /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ -- GitLab From 2c15b6538123047c258987b00475fa658ca14878 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:33:46 -0500 Subject: [PATCH 0158/1724] Make the NET_PARTICIPANT role dependent on user activity This patch implements all of 28337, except for the part where we turn off the role if we've been idle for a long time. --- src/app/main/main.c | 15 +++++++ src/core/mainloop/connection.c | 3 ++ src/core/mainloop/mainloop.c | 34 ++++++++++++++- src/core/mainloop/mainloop.h | 1 + src/core/mainloop/netstatus.c | 73 +++++++++++++++++++++++++++++++++ src/core/mainloop/netstatus.h | 7 ++++ src/core/or/or.h | 2 + src/feature/client/dnsserv.c | 4 ++ src/feature/control/control.c | 2 + src/test/test_compat_libevent.c | 1 - src/test/test_periodic_event.c | 11 ++++- 11 files changed, 149 insertions(+), 4 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index b8dcb852d2..03b3a95d03 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -303,6 +303,19 @@ process_signal(int sig) log_heartbeat(time(NULL)); control_event_signal(sig); break; + case SIGACTIVE: + /* "SIGACTIVE" counts as ersatz user activity. */ + note_user_activity(approx_time()); + control_event_signal(sig); + break; + case SIGDORMANT: + /* "SIGDORMANT" means to ignore past user activity */ + log_notice(LD_GENERAL, "Going dormant because of controller request."); + reset_user_activity(0); + set_network_participation(false); + schedule_rescan_periodic_events(); + control_event_signal(sig); + break; } } @@ -472,6 +485,8 @@ static struct { { SIGNEWNYM, 0, NULL }, { SIGCLEARDNSCACHE, 0, NULL }, { SIGHEARTBEAT, 0, NULL }, + { SIGACTIVE, 0, NULL }, + { SIGDORMANT, 0, NULL }, { -1, -1, NULL } }; diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 1198a01ad9..e0f1680c91 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1874,6 +1874,9 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; + /* Any incoming connection on an entry port counts as user activity. */ + note_user_activity(approx_time()); + switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index e67ebdb4a8..f18db2898a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1513,8 +1513,7 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; - /* We actually want a better definition here for our work on dormancy. */ - int is_net_participant = ! net_is_disabled(); + int is_net_participant = is_participating_on_network(); if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; @@ -1592,6 +1591,30 @@ teardown_periodic_events(void) periodic_events_initialized = 0; } +static mainloop_event_t *rescan_periodic_events_ev = NULL; + +/** Callback: rescan the periodic event list. */ +static void +rescan_periodic_events_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + rescan_periodic_events(get_options()); +} + +/** + * Schedule an event that will rescan which periodic events should run. + **/ +void +schedule_rescan_periodic_events(void) +{ + if (!rescan_periodic_events_ev) { + rescan_periodic_events_ev = + mainloop_event_new(rescan_periodic_events_cb, NULL); + } + mainloop_event_activate(rescan_periodic_events_ev); +} + /** Do a pass at all our periodic events, disable those we don't need anymore * and enable those we need now using the given options. */ void @@ -2714,6 +2737,12 @@ initialize_mainloop_events(void) int do_main_loop(void) { + /* For now, starting Tor always counts as user activity. Later, we might + * have an option to control this. + */ + reset_user_activity(approx_time()); + set_network_participation(true); + /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ @@ -2912,6 +2941,7 @@ tor_mainloop_free_all(void) mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); mainloop_event_free(scheduled_shutdown_ev); + mainloop_event_free(rescan_periodic_events_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index be642d81f9..7f27ef9a52 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,6 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); +void schedule_rescan_periodic_events(void); void update_current_time(time_t now); diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index f026474494..ed7c952dcd 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -6,6 +6,7 @@ #include "core/or/or.h" #include "core/mainloop/netstatus.h" +#include "core/mainloop/mainloop.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" @@ -26,3 +27,75 @@ net_is_completely_disabled(void) { return get_options()->DisableNetwork || we_are_fully_hibernating(); } + +/** + * The time at which we've last seen "user activity" -- that is, any activity + * that should keep us as a participant on the network. + */ +static time_t last_user_activity_seen = 0; + +/** + * True iff we are currently a "network participant" -- that is, we + * are building circuits, fetching directory information, and so on. + **/ +static bool participating_on_network = false; + +/** + * Record the fact that we have seen "user activity" at the time now. Move + * "last activity seen" time forwards, but never backwards. + * + * If we were previously not participating on the network, set our + * participation status to true, and launch periodic events as appropriate. + **/ +void +note_user_activity(time_t now) +{ + last_user_activity_seen = MAX(now, last_user_activity_seen); + + if (! participating_on_network) { + log_notice(LD_GENERAL, "Tor is no longer dormant."); + set_network_participation(true); + schedule_rescan_periodic_events(); + } +} + +/** + * Change the time at which "user activitiy" was last seen to now. + * + * Unlike note_user_actity, this function sets the time without checking + * whether it is in the past, and without causing any rescan of periodic events + * or change in participation status. + */ +void +reset_user_activity(time_t now) +{ + last_user_activity_seen = now; +} + +/** + * Return the most recent time at which we recorded "user activity". + **/ +time_t +get_last_user_activity_time(void) +{ + return last_user_activity_seen; +} + +/** + * Set the field that remembers whether we are currently participating on the + * network. Does not schedule or un-schedule periodic events. + **/ +void +set_network_participation(bool participation) +{ + participating_on_network = participation; +} + +/** + * Return true iff we are currently participating on the network. + **/ +bool +is_participating_on_network(void) +{ + return participating_on_network; +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index e9310c2929..58c994fd14 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -10,4 +10,11 @@ int net_is_disabled(void); int net_is_completely_disabled(void); +void note_user_activity(time_t now); +void reset_user_activity(time_t now); +time_t get_last_user_activity_time(void); + +void set_network_participation(bool participation); +bool is_participating_on_network(void); + #endif diff --git a/src/core/or/or.h b/src/core/or/or.h index acf092c8dc..e4b374b122 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -97,6 +97,8 @@ struct curve25519_public_key_t; #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 #define SIGHEARTBEAT 131 +#define SIGACTIVE 132 +#define SIGDORMANT 133 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index ea4951f915..e5abe5c6aa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -28,6 +28,7 @@ #include "core/or/connection_edge.h" #include "feature/control/control.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/policies.h" #include "feature/control/control_connection_st.h" @@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse, edge_connection_t *conn; char *q_name; + /* Launching a request for a user counts as user activity. */ + note_user_activity(approx_time()); + /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); entry_conn->entry_cfg.dns_request = 1; diff --git a/src/feature/control/control.c b/src/feature/control/control.c index a5b6ab3bf5..1344d66a3d 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1681,6 +1681,8 @@ static const struct signal_t signal_table[] = { { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, { SIGHEARTBEAT, "HEARTBEAT"}, + { SIGACTIVE, "ACTIVE" }, + { SIGDORMANT, "DORMANT" }, { 0, NULL }, }; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 3f505d013b..ade76bdb07 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -187,4 +187,3 @@ struct testcase_t compat_libevent_tests[] = { TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 6a3e320b2e..f3d518eb7b 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -51,6 +51,8 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + set_network_participation(false); + rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ for (int i = 0; periodic_events[i].name; ++i) { @@ -82,6 +84,8 @@ test_pe_launch(void *arg) * network gets enabled. */ consider_hibernation(time(NULL)); + set_network_participation(true); + /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ @@ -93,6 +97,7 @@ test_pe_launch(void *arg) options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); + #if 0 /* Lets make sure that before intialization, we can't scan the periodic * events list and launch them. Lets try by being a Client. */ @@ -148,6 +153,7 @@ test_pe_launch(void *arg) options->SocksPort_set = 0; options->ORPort_set = 0; options->DisableNetwork = 1; + set_network_participation(false); periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { @@ -162,6 +168,7 @@ test_pe_launch(void *arg) options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; options->DisableNetwork = 0; + set_network_participation(true); register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -195,8 +202,10 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + set_network_participation(true); - const int ALL = PERIODIC_EVENT_ROLE_ALL; + const int ALL = PERIODIC_EVENT_ROLE_ALL | + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; /* Nothing configured, should be no roles. */ tt_assert(net_is_disabled()); -- GitLab From ce6209cee4a113c6a224f0c98244852354ccdb40 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:51:53 -0500 Subject: [PATCH 0159/1724] Add a periodic event to become dormant. This event makes us become dormant if we have seen no activity in a long time. Note that being any kind of a server, or running an onion service, always counts as being active. Note that right now, just having an open stream that Tor did not open on its own (for a directory request) counts as "being active", so if you have an idle ssh connection, that will keep Tor from becoming dormant. Many of the features here should become configurable; I'd like feedback on which. --- src/core/mainloop/connection.c | 10 +++++++ src/core/mainloop/connection.h | 1 + src/core/mainloop/mainloop.c | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index e0f1680c91..25224fd999 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4429,6 +4429,16 @@ connection_get_by_type_state(int type, int state) CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } +/** + * Return a connection of type type that is not an internally linked + * connection, and is not marked for close. + **/ +connection_t * +connection_get_by_type_nonlinked(int type) +{ + CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); +} + /** Return a connection of type type that has rendquery equal * to rendquery, and that is not marked for close. If state * is non-zero, conn must be of that state too. diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index b569bb038e..9f1a23c6f2 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -240,6 +240,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_nonlinked(int type); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index f18db2898a..e6dee94fcc 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1367,6 +1367,7 @@ CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); +CALLBACK(check_network_participation); #undef CALLBACK @@ -1396,6 +1397,7 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + CALLBACK(check_network_participation, NET_PARTICIPANT, 0), /* We need to do these if we're participating in the Tor network, and * immediately before we stop. */ @@ -1998,6 +2000,54 @@ add_entropy_callback(time_t now, const or_options_t *options) return ENTROPY_INTERVAL; } +/** Periodic callback: if there has been no network usage in a while, + * enter a dormant state. */ +static int +check_network_participation_callback(time_t now, const or_options_t *options) +{ + /* If we're a server, we can't become dormant. */ + if (server_mode(options)) { + goto found_activity; + } + + /* If we're running an onion service, we can't become dormant. */ + /* XXXX this would be nice to change, so that we can be dormant with a + * service. */ + if (hs_service_get_num_services() || rend_num_services()) { + goto found_activity; + } + + /* XXXX Add an option to never become dormant. */ + + /* If we have any currently open entry streams other than "linked" + * connections used for directory requests, those count as user activity. + */ + /* XXXX make this configurable? */ + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } + + /* XXXX Make this configurable? */ +/** How often do we check whether we have had network activity? */ +#define CHECK_PARTICIPATION_INTERVAL (5*60) + + /** Become dormant if there has been no user activity in this long. */ + /* XXXX make this configurable! */ +#define BECOME_DORMANT_AFTER_INACTIVITY (24*60*60) + if (get_last_user_activity_time() + BECOME_DORMANT_AFTER_INACTIVITY >= now) { + log_notice(LD_GENERAL, "No user activity in a long time: becoming" + " dormant."); + set_network_participation(false); + rescan_periodic_events(options); + } + + return CHECK_PARTICIPATION_INTERVAL; + + found_activity: + note_user_activity(now); + return CHECK_PARTICIPATION_INTERVAL; +} + /** * Periodic callback: if we're an authority, make sure we test * the routers on the network for reachability. -- GitLab From d0e6abd0876f0d4adb0b82f5528220a81b34962e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:57:18 -0500 Subject: [PATCH 0160/1724] Reset dormancy status when the clock jumps. --- src/core/mainloop/mainloop.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index e6dee94fcc..2a68e8c098 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2674,6 +2674,11 @@ update_current_time(time_t now) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); + + /* Don't go dormant just because we jumped in time. */ + if (is_participating_on_network()) { + reset_user_activity(now); + } } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2695,6 +2700,11 @@ update_current_time(time_t now) if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); } + + /* Don't go dormant just because we jumped in time. */ + if (is_participating_on_network()) { + reset_user_activity(now); + } } else if (seconds_elapsed > 0) { stats_n_seconds_working += seconds_elapsed; } -- GitLab From 2f28cd1dc8e797b140271e5c33b9e4f823f7f2d8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Nov 2018 13:42:09 -0500 Subject: [PATCH 0161/1724] Rename and fix docs on FLUSH_ON_DISABLE Also rename "...flush_and_disable()" to "...schedule_and_disable()" --- src/core/mainloop/mainloop.c | 16 ++++++++-------- src/core/mainloop/periodic.c | 2 +- src/core/mainloop/periodic.h | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2a68e8c098..9f45f3c869 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1386,7 +1386,7 @@ STATIC periodic_event_item_t periodic_events[] = { /* This is a legacy catch-all callback that runs once per second if * we are online and active. */ CALLBACK(second_elapsed, NET_PARTICIPANT, - FL(NEED_NET)|FL(FLUSH_ON_DISABLE)), + FL(NEED_NET)|FL(RUN_ON_DISABLE)), /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ @@ -1401,9 +1401,9 @@ STATIC periodic_event_item_t periodic_events[] = { /* We need to do these if we're participating in the Tor network, and * immediately before we stop. */ - CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), @@ -1436,7 +1436,7 @@ STATIC periodic_event_item_t periodic_events[] = { /* Client only. */ /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ - CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), @@ -1651,8 +1651,8 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - if (item->flags & PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE) { - periodic_event_flush_and_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); } else { periodic_event_disable(item); } @@ -1814,7 +1814,7 @@ second_elapsed_callback(time_t now, const or_options_t *options) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE + // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c290c3744e..9f9b178e43 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -174,7 +174,7 @@ periodic_event_disable(periodic_event_item_t *event) * Do nothing if the event was already disabled. */ void -periodic_event_flush_and_disable(periodic_event_item_t *event) +periodic_event_schedule_and_disable(periodic_event_item_t *event) { tor_assert(event); if (!periodic_event_is_enabled(event)) diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 52d5450ee8..05ba4297f3 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -39,10 +39,10 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) -/* Indicate that it the event is enabled, it event needs to be run once before +/* Indicate that if the event is enabled, it needs to be run once before * it becomes disabled. */ -#define PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE (1U << 1) +#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE (1U << 1) /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return @@ -88,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); -void periodic_event_flush_and_disable(periodic_event_item_t *event); +void periodic_event_schedule_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ -- GitLab From 53ccdb6945f0d4a9b27a9939211a3c9125ca4427 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Nov 2018 15:05:05 -0500 Subject: [PATCH 0162/1724] Make sure that we are always a net participant when being a server Otherwise, if we're dormant, and we set ORPort, nothing makes us become non-dormant. --- src/core/mainloop/mainloop.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 9f45f3c869..2d12e26485 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1515,7 +1515,8 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; - int is_net_participant = is_participating_on_network(); + int is_net_participant = is_participating_on_network() || + is_relay || is_hidden_service; if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; -- GitLab From ce1b018ab742492e96b5bdae049a13c2faaefdbf Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 17:44:41 +0200 Subject: [PATCH 0163/1724] Fix shellcheck SC2006 warnings in 'coverage' script --- scripts/test/coverage | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index 59d468ee1e..e611a4be1b 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,19 +8,19 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=`basename $fn` - DN=`dirname $fn` - F=`echo $BN | sed -e 's/\.c$//;'` + BN=$(basename $fn) + DN=$(dirname $fn) + F=$(echo $BN | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=`echo ${DN}/src_*-${F}.o` + ONS=$(echo ${DN}/src_*-${F}.o) ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] then for on in $ONS; do # We should have a gcno file - GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'` + GCNO=$(echo $on | sed -e 's/\.o$/\.gcno/;') if [ -e $GCNO ] then # No need to test for gcda, since gcov assumes no execution -- GitLab From 7c04b00e65d67a38f119f4d3e50e8a47236e202f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 17:50:43 +0200 Subject: [PATCH 0164/1724] Fix most instances of SC2086 warning --- scripts/test/coverage | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index e611a4be1b..180a2d6dcb 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,9 +8,9 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=$(basename $fn) - DN=$(dirname $fn) - F=$(echo $BN | sed -e 's/\.c$//;') + BN=$(basename "$fn") + DN=$(dirname "$fn") + F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names ONS=$(echo ${DN}/src_*-${F}.o) @@ -20,18 +20,18 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do then for on in $ONS; do # We should have a gcno file - GCNO=$(echo $on | sed -e 's/\.o$/\.gcno/;') - if [ -e $GCNO ] + GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;') + if [ -e "$GCNO" ] then # No need to test for gcda, since gcov assumes no execution # if it's absent - rm -f $GC - gcov -o $on $fn - if [ -e $GC ] + rm -f "$GC" + gcov -o "$on" "$fn" + if [ -e "$GC" ] then if [ -d "$dst" ] then - mv $GC $dst/$GC + mv "$GC" "$dst"/"$GC" fi else echo "gcov -o $on $fn didn't make a .gcov file" -- GitLab From 4fc584f20e5aae8f7b89f169690c17d5bce84d41 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 18:04:07 +0200 Subject: [PATCH 0165/1724] Fix one last SC2086 --- scripts/test/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index 180a2d6dcb..7a417cdc89 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -13,7 +13,7 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=$(echo ${DN}/src_*-${F}.o) + ONS=$(echo "${DN}"/src_*-"${F}".o) ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] -- GitLab From 8cb817cc5e0cc36cc0218da53bc4c1f277ccd8cc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 18:56:20 +0200 Subject: [PATCH 0166/1724] Add changes file --- changes/ticket28008 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28008 diff --git a/changes/ticket28008 b/changes/ticket28008 new file mode 100644 index 0000000000..1f0de1a14d --- /dev/null +++ b/changes/ticket28008 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in scripts/test/coverage. Resolves issue + 28008. -- GitLab From d9f7cb3f6202291d38d611165f70e0c1c33b3e92 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 19:18:49 +0200 Subject: [PATCH 0167/1724] Fix shellcheck issues in cov-diff - SC2231, SC2006 and SC2086 --- changes/ticket28009 | 3 +++ scripts/test/cov-diff | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ticket28009 diff --git a/changes/ticket28009 b/changes/ticket28009 new file mode 100644 index 0000000000..1d986d4211 --- /dev/null +++ b/changes/ticket28009 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in cov-diff script. Resolves issue + 28009. diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 6179dff63e..f3ca856888 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -7,9 +7,9 @@ DIRA="$1" DIRB="$2" -for B in $DIRB/*; do - A=$DIRA/`basename $B` - if [ -f $A ]; then +for B in "$DIRB"/*; do + A=$DIRA/$(basename "$B") + if [ -f "$A" ]; then perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" else cat /dev/null > "$A.tmp" -- GitLab From bf82389e198a0cc6c7e078cada7f35a9cbf65fe1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 16 Nov 2018 11:32:52 -0500 Subject: [PATCH 0168/1724] Copy 0.3.5.5-alpha changelog into master --- ChangeLog | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/ChangeLog b/ChangeLog index cdf7249059..4c776f7163 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,70 @@ +Changes in version 0.3.5.5-alpha - 2018-11-16 + Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases, + including several that we hope to backport to older release series in + the future. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (documentation): + - Make Doxygen work again after the code movement in the 0.3.5 + source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion services): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on an NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + Changes in version 0.3.5.4-alpha - 2018-11-08 Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and improves our continuous integration support. It continues our attempts -- GitLab From 411780d563e4d08e12d8aef0bbe8915f48011cca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 17 Nov 2018 10:19:25 +0200 Subject: [PATCH 0169/1724] Make ROUTERLIST_PRUNING_INTERVAL 1 hr. --- src/core/mainloop/mainloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index be19136130..0b9d4e0889 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2231,7 +2231,7 @@ retry_dns_callback(time_t now, const or_options_t *options) static int prune_old_routers_callback(time_t now, const or_options_t *options) { -#define ROUTERLIST_PRUNING_INTERVAL (60) // 1 minute. +#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour. (void)now; (void)options; -- GitLab From c8c4c3dffa71b4bbc9e7cabfee2124fb5e19ad39 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 17 Nov 2018 10:27:10 +0200 Subject: [PATCH 0170/1724] fixup! Make ROUTERLIST_PRUNING_INTERVAL 1 hr. --- changes/bug27929 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/bug27929 b/changes/bug27929 index a97d18f202..dab57a2eca 100644 --- a/changes/bug27929 +++ b/changes/bug27929 @@ -1,5 +1,5 @@ o Minor bugfixes (periodic events): - Refrain from calling routerlist_remove_old_routers() from check_descriptor_callback(). Instead, create a new periodic - event that will run every minute even if Tor is not configured + event that will run once every hour even if Tor is not configured as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. -- GitLab From 83be4d2bbd7a4ed584f42d043558a4026c4a449d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 28 Aug 2018 11:34:31 -0400 Subject: [PATCH 0171/1724] Backend for compact node-family representation. This representation is meant to save memory in microdescriptors -- we can't use it in routerinfo_t yet, since those families need to be encoded losslessly for directory voting to work. This representation saves memory in three ways: 1. It uses only one allocation per family. (The old way used a smartlist (2 allocs) plus one strdup per entry.) 2. It stores identity digests in binary, not hex. 3. It keeps families in a canonical format, memoizes, and reference-counts them. Part of #27359. --- src/core/include.am | 3 + src/feature/nodelist/nodefamily.c | 373 +++++++++++++++++++++++++++ src/feature/nodelist/nodefamily.h | 47 ++++ src/feature/nodelist/nodefamily_st.h | 48 ++++ src/test/test_nodelist.c | 251 +++++++++++++++++- 5 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 src/feature/nodelist/nodefamily.c create mode 100644 src/feature/nodelist/nodefamily.h create mode 100644 src/feature/nodelist/nodefamily_st.h diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..003ef84216 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -106,6 +106,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nickname.c \ + src/feature/nodelist/nodefamily.c \ src/feature/nodelist/nodelist.c \ src/feature/nodelist/node_select.c \ src/feature/nodelist/routerinfo.c \ @@ -340,6 +341,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c new file mode 100644 index 0000000000..6b504c0ac4 --- /dev/null +++ b/src/feature/nodelist/nodefamily.c @@ -0,0 +1,373 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.c + * \brief Code to manipulate encoded, reference-counted node families. We + * use these tricks to save space, since these families would otherwise + * require a large number of tiny allocations. + **/ + +#include "core/or/or.h" +#include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" +#include "feature/nodelist/nodefamily_st.h" +#include "feature/nodelist/nodelist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" + +#include "ht.h" +#include "siphash.h" + +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/defs/digest_sizes.h" +#include "lib/log/util_bug.h" + +#include +#include + +/** + * Allocate and return a blank node family with space to hold n_members + * members. + */ +static nodefamily_t * +nodefamily_alloc(int n_members) +{ + size_t alloc_len = offsetof(nodefamily_t, family_members) + + NODEFAMILY_ARRAY_SIZE(n_members); + nodefamily_t *nf = tor_malloc_zero(alloc_len); + nf->n_members = n_members; + return nf; +} + +/** + * Hashtable hash implementation. + */ +static inline unsigned int +nodefamily_hash(const nodefamily_t *nf) +{ + return (unsigned) siphash24g(nf->family_members, + NODEFAMILY_ARRAY_SIZE(nf->n_members)); +} + +/** + * Hashtable equality implementation. + */ +static inline unsigned int +nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b) +{ + return (a->n_members == b->n_members) && + fast_memeq(a->family_members, b->family_members, + NODEFAMILY_ARRAY_SIZE(a->n_members)); +} + +static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families + = HT_INITIALIZER(); + +HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + nodefamily_eq) +HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + node_family_eq, 0.6, tor_reallocarray_, tor_free_) + +/** + * Parse the family declaration in s, returning the canonical + * nodefamily_t for its members. Return NULL on error. + * + * If rsa_id_self is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * If NF_WARN_MALFORMED is set in flags, warn about any + * elements that we can't parse. (By default, we log at info.) + * + * If NF_REJECT_MALFORMED is set in flags, treat any unparseable + * elements as an error. (By default, we simply omit them.) + **/ +nodefamily_t * +nodefamily_parse(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * qsort helper for encoded nodefamily elements. + **/ +static int +compare_members(const void *a, const void *b) +{ + return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN); +} + +/** + * Parse the member strings in members, returning their canonical + * nodefamily_t. Return NULL on error. + * + * If rsa_id_self is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * The flags element is interpreted as in nodefamily_parse(). + **/ +nodefamily_t * +nodefamily_from_members(const smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags) +{ + const int n_self = rsa_id_self ? 1 : 0; + int n_bad_elements = 0; + int n_members = smartlist_len(members) + n_self; + nodefamily_t *tmp = nodefamily_alloc(n_members); + uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0); + + SMARTLIST_FOREACH_BEGIN(members, const char *, cp) { + bool bad_element = true; + if (is_legal_nickname(cp)) { + ptr[0] = NODEFAMILY_BY_NICKNAME; + tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname + memcpy(ptr+1, cp, strlen(cp)); + bad_element = false; + } else if (is_legal_hexdigest(cp)) { + char digest_buf[DIGEST_LEN]; + char nn_buf[MAX_NICKNAME_LEN+1]; + char nn_char=0; + if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) { + bad_element = false; + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, digest_buf, DIGEST_LEN); + } + } + + if (bad_element) { + const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO; + log_fn(severity, LD_GENERAL, + "Bad element %s while parsing a node family.", + escaped(cp)); + ++n_bad_elements; + } else { + ptr += NODEFAMILY_MEMBER_LEN; + } + } SMARTLIST_FOREACH_END(cp); + + if (n_bad_elements && (flags & NF_REJECT_MALFORMED)) + goto err; + + if (rsa_id_self) { + /* Add self. */ + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, rsa_id_self, DIGEST_LEN); + } + + n_members -= n_bad_elements; + + /* Sort tmp into canonical order. */ + qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN, + compare_members); + + /* Remove duplicates. */ + int i; + for (i = 0; i < n_members-1; ++i) { + uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i); + uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1); + if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) { + memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN); + --n_members; + --i; + } + } + int n_members_alloc = tmp->n_members; + tmp->n_members = n_members; + + /* See if we already allocated this family. */ + nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp); + if (found) { + /* If we did, great: incref it and return it. */ + ++found->refcnt; + tor_free(tmp); + return found; + } else { + /* If not, insert it into the hashtable. */ + if (n_members_alloc != n_members) { + /* Compact the family if needed */ + nodefamily_t *tmp2 = nodefamily_alloc(n_members); + memcpy(tmp2->family_members, tmp->family_members, + n_members * NODEFAMILY_MEMBER_LEN); + tor_free(tmp); + tmp = tmp2; + } + + tmp->refcnt = 1; + HT_INSERT(nodefamily_map, &the_node_families, tmp); + return tmp; + } + + err: + tor_free(tmp); + return NULL; +} + +/** + * Drop our reference to family, freeing it if there are no more + * references. + */ +void +nodefamily_free_(nodefamily_t *family) +{ + if (family == NULL) + return; + + --family->refcnt; + + if (family->refcnt == 0) { + HT_REMOVE(nodefamily_map, &the_node_families, family); + tor_free(family); + } +} + +/** + * Return true iff family contains the SHA1 RSA1024 identity + * rsa_id. + */ +bool +nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + if (ptr[0] == NODEFAMILY_BY_RSA_ID && + fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) { + return true; + } + } + return false; +} + +/** + * Return true iff family contains the nickname name. + */ +bool +nodefamily_contains_nickname(const nodefamily_t *family, + const char *name) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + // note that the strcasecmp() is safe because there is always at least one + // NUL in the encoded nickname, because all legal nicknames are less than + // DIGEST_LEN bytes long. + if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) { + return true; + } + } + return false; +} + +/** + * Return true if family contains the nickname or the RSA ID for + * node + **/ +bool +nodefamily_contains_node(const nodefamily_t *family, + const node_t *node) +{ + return + nodefamily_contains_nickname(family, node_get_nickname(node)) + || + nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node)); +} + +/** + * Look up every entry in family, and add add the corresponding + * node_t to out. + **/ +void +nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + smartlist_t *out) +{ + if (!family) + return; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + const node_t *node = NULL; + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED); + break; + case NODEFAMILY_BY_RSA_ID: + node = node_get_by_id((char*)ptr+1); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + if (node) + smartlist_add(out, (void *)node); + } +} + +/** + * Encode family as a space-separated string. + */ +char * +nodefamily_format(const nodefamily_t *family) +{ + if (!family) + return tor_strdup(""); + + unsigned i; + smartlist_t *sl = smartlist_new(); + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + smartlist_add_strdup(sl, (char*)ptr+1); + break; + case NODEFAMILY_BY_RSA_ID: { + char buf[HEX_DIGEST_LEN+2]; + buf[0]='$'; + base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + smartlist_add_strdup(sl, buf); + break; + } + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + } + + char *result = smartlist_join_strings(sl, " ", 0, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Free all storage held in the nodefamily map. + **/ +void +nodefamily_free_all(void) +{ + HT_CLEAR(nodefamily_map, &the_node_families); +} diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h new file mode 100644 index 0000000000..342f161a07 --- /dev/null +++ b/src/feature/nodelist/nodefamily.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.h + * \brief Header file for nodefamily.c. + **/ + +#ifndef TOR_NODEFAMILY_H +#define TOR_NODEFAMILY_H + +#include "lib/malloc/malloc.h" +#include + +typedef struct nodefamily_t nodefamily_t; +struct node_t; +struct smartlist_t; + +#define NF_WARN_MALFORMED (1u<<0) +#define NF_REJECT_MALFORMED (1u<<1) + +nodefamily_t *nodefamily_parse(const char *s, + const uint8_t *rsa_id_self, + unsigned flags); +nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags); +void nodefamily_free_(nodefamily_t *family); +#define nodefamily_free(family) \ + FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) + +bool nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id); +bool nodefamily_contains_nickname(const nodefamily_t *family, + const char *name); +bool nodefamily_contains_node(const nodefamily_t *family, + const struct node_t *node); +void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + struct smartlist_t *out); +char *nodefamily_format(const nodefamily_t *family); + +void nodefamily_free_all(void); + +#endif diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h new file mode 100644 index 0000000000..f88ada494a --- /dev/null +++ b/src/feature/nodelist/nodefamily_st.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NODEFAMILY_ST_H +#define TOR_NODEFAMILY_ST_H + +#include "orconfig.h" +#include "ht.h" + +struct nodefamily_t { + /** Entry for this nodefamily_t within the hashtable. */ + HT_ENTRY(nodefamily_t) ht_ent; + /** Reference count. (The hashtable is not treated as a reference */ + uint32_t refcnt; + /** Number of items encoded in family_members. */ + uint32_t n_members; + /* A byte-array encoding the members of this family. We encode each member + * as one byte to indicate whether it's a nickname or a fingerprint, plus + * DIGEST_LEN bytes of data. The entries are lexically sorted. + */ + uint8_t family_members[FLEXIBLE_ARRAY_MEMBER]; +}; + +#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) + +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 0 +/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. + */ +#define NODEFAMILY_BY_RSA_ID 1 + +/** + * Number of bytes to allocate in the array for a nodefamily_t with N members. + **/ +#define NODEFAMILY_ARRAY_SIZE(n) \ + ((n) * NODEFAMILY_MEMBER_LEN) + +/** + * Pointer to the i'th member of nf, as encoded. + */ +#define NODEFAMILY_MEMBER_PTR(nf, i) \ + (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) + +#endif diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 1af6db68ec..2dbd949b7d 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -9,12 +9,14 @@ #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodefamily_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -231,6 +233,251 @@ test_nodelist_ed_id(void *arg) #undef N_NODES } +static void +test_nodelist_nodefamily(void *arg) +{ + (void)arg; + /* hex ID digests */ + const char h1[] = "5B435D6869206861206C65207363617270652070"; + const char h2[] = "75C3B220616E6461726520696E206769726F2061"; + const char h3[] = "2074726F766172206461206D616E67696172652C"; + const char h4[] = "206D656E747265206E6F6E2076616C65206C2769"; + const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669"; + + /* binary ID digests */ + uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN], + d5[DIGEST_LEN]; + base16_decode((char*)d1, sizeof(d1), h1, strlen(h1)); + base16_decode((char*)d2, sizeof(d2), h2, strlen(h2)); + base16_decode((char*)d3, sizeof(d3), h3, strlen(h3)); + base16_decode((char*)d4, sizeof(d4), h4, strlen(h4)); + base16_decode((char*)d5, sizeof(d5), h5, strlen(h5)); + + char *enc=NULL, *enc2=NULL; + + nodefamily_t *nf1 = NULL; + nodefamily_t *nf2 = NULL; + nodefamily_t *nf3 = NULL; + + /* Make sure that sorting and de-duplication work. */ + tor_asprintf(&enc, "$%s hello", h1); + nf1 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf1); + tor_free(enc); + + tor_asprintf(&enc, "hello hello $%s hello", h1); + nf2 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf2); + tt_ptr_op(nf1, OP_EQ, nf2); + tor_free(enc); + + tor_asprintf(&enc, "%s $%s hello", h1, h1); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(! nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_nickname(nf1, "hello")); + tt_assert(nodefamily_contains_nickname(nf1, "HELLO")); + tt_assert(! nodefamily_contains_nickname(nf1, "goodbye")); + + tt_int_op(nf1->refcnt, OP_EQ, 3); + nodefamily_free(nf3); + tt_int_op(nf1->refcnt, OP_EQ, 2); + + /* Try parsing with a provided self RSA digest. */ + nf3 = nodefamily_parse("hello ", d1, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + + /* Do we get the expected result when we re-encode? */ + tor_asprintf(&enc, "hello $%s", h1); + enc2 = nodefamily_format(nf1); + tt_str_op(enc2, OP_EQ, enc); + tor_free(enc2); + tor_free(enc); + + /* Make sure that we get a different result if we give a different digest. */ + nodefamily_free(nf3); + tor_asprintf(&enc, "hello $%s hello", h3); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_NE, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf3, d3)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d2)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d1)); + tt_assert(nodefamily_contains_nickname(nf3, "hello")); + tt_assert(! nodefamily_contains_nickname(nf3, "goodbye")); + + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + + /* Try one with several digests, all with nicknames appended, in different + formats. */ + tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4); + nf1 = nodefamily_parse(enc, d5, 0); + tt_assert(nf1); + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_rsa_id(nf1, d3)); + tt_assert(nodefamily_contains_rsa_id(nf1, d4)); + tt_assert(nodefamily_contains_rsa_id(nf1, d5)); + /* Nicknames aren't preserved when ids are present, since node naming is + * deprecated */ + tt_assert(! nodefamily_contains_nickname(nf3, "res")); + tor_free(enc); + tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2); + enc2 = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, enc2); + tor_free(enc); + tor_free(enc2); + + /* Try ones where we parse the empty string. */ + nf2 = nodefamily_parse("", NULL, 0); + nf3 = nodefamily_parse("", d4, 0); + tt_assert(nf2); + tt_assert(nf3); + tt_ptr_op(nf2, OP_NE, nf3); + + tt_assert(! nodefamily_contains_rsa_id(nf2, d4)); + tt_assert(nodefamily_contains_rsa_id(nf3, d4)); + tt_assert(! nodefamily_contains_rsa_id(nf2, d5)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d5)); + tt_assert(! nodefamily_contains_nickname(nf2, "fred")); + tt_assert(! nodefamily_contains_nickname(nf3, "bosco")); + + /* The NULL family should contain nothing. */ + tt_assert(! nodefamily_contains_rsa_id(NULL, d4)); + tt_assert(! nodefamily_contains_rsa_id(NULL, d5)); + + done: + tor_free(enc); + tor_free(enc2); + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); +} + +static void +test_nodelist_nodefamily_parse_err(void *arg) +{ + (void)arg; + nodefamily_t *nf1 = NULL; + char *enc = NULL; + const char *semibogus = + "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname + "$jkASDFLkjsadfjhkl " // not hex + "$7468696e67732d696e2d7468656d73656c766573 " // ok + "reticulatogranulate "// ok + "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex + "$616273656e746d696e6465646e6573736573" // too short for hex + ; + + setup_capture_of_logs(LOG_WARN); + + // We only get two items when we parse this. + for (int reject = 0; reject <= 1; ++reject) { + for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) { + unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0; + flags |= reject ? NF_REJECT_MALFORMED : 0; + nf1 = nodefamily_parse(semibogus, NULL, flags); + if (reject) { + tt_assert(nf1 == NULL); + } else { + tt_assert(nf1); + enc = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, + "reticulatogranulate " + "$7468696E67732D696E2D7468656D73656C766573"); + tor_free(enc); + } + + if (log_at_warn) { + expect_log_msg_containing("$616273656e746d696e6465646e6573736573"); + expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf"); + } else { + tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0); + } + mock_clean_saved_logs(); + } + } + + done: + tor_free(enc); + nodefamily_free(nf1); + teardown_capture_of_logs(); +} + +static const node_t * +mock_node_get_by_id(const char *id) +{ + if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN)) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + memcpy(fake_node->identity, id, DIGEST_LEN); + return fake_node; +} + +static const node_t * +mock_node_get_by_nickname(const char *nn, unsigned flags) +{ + (void)flags; + if (!strcmp(nn, "nonesuch")) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + strlcpy(fake_node->identity, nn, DIGEST_LEN); + return fake_node; +} + +static void +test_nodelist_nodefamily_lookup(void *arg) +{ + (void)arg; + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + smartlist_t *sl = smartlist_new(); + nodefamily_t *nf1 = NULL; + char *mem_op_hex_tmp = NULL; + + // 'null' is allowed. + nodefamily_add_nodes_to_smartlist(NULL, sl); + tt_int_op(smartlist_len(sl), OP_EQ, 0); + + // Try a real family + nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE " + "$2121212121212121212121212121212121212121 " + "$3333333333333333333333333333333333333333 " + "erewhon nonesuch", NULL, 0); + tt_assert(nf1); + nodefamily_add_nodes_to_smartlist(nf1, sl); + // There were 5 elements; 2 were dropped because the mocked lookup failed. + tt_int_op(smartlist_len(sl), OP_EQ, 3); + + const node_t *n = smartlist_get(sl, 0); + tt_str_op(n->identity, OP_EQ, "erewhon"); + n = smartlist_get(sl, 1); + test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); + n = smartlist_get(sl, 2); + test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node)); + smartlist_free(sl); + nodefamily_free(nf1); + tor_free(mem_op_hex_tmp); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -239,6 +486,8 @@ struct testcase_t nodelist_tests[] = { NODE(node_get_verbose_nickname_not_named, TT_FORK), NODE(node_is_dir, TT_FORK), NODE(ed_id, TT_FORK), + NODE(nodefamily, TT_FORK), + NODE(nodefamily_parse_err, TT_FORK), + NODE(nodefamily_lookup, TT_FORK), END_OF_TESTCASES }; - -- GitLab From 426c9561c5f5bc5f38a42f3e46437db59fcdc7c0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 23 Oct 2018 19:55:12 -0400 Subject: [PATCH 0172/1724] Use nodefamily_t in microdescriptors. Closes ticket 27359. --- changes/ticket27359 | 3 + src/feature/dirparse/microdesc_parse.c | 16 ++-- src/feature/nodelist/microdesc.c | 6 +- src/feature/nodelist/microdesc_st.h | 5 +- src/feature/nodelist/nodelist.c | 110 ++++++++++++++++--------- src/feature/nodelist/nodelist.h | 1 - src/test/test_microdesc.c | 8 +- 7 files changed, 89 insertions(+), 60 deletions(-) create mode 100644 changes/ticket27359 diff --git a/changes/ticket27359 b/changes/ticket27359 new file mode 100644 index 0000000000..bddc90634d --- /dev/null +++ b/changes/ticket27359 @@ -0,0 +1,3 @@ + o Minor features (memory usage): + - Store microdescriptor family lists with a more compact representation + to save memory. Closes ticket 27359. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index aebff5a35f..8ad9626377 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -18,6 +18,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/relay/router.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = { T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), - T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("family", K_FAMILY, CONCAT_ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), T01("p6", K_P6, CONCAT_ARGS, NO_OBJ ), A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), @@ -222,16 +223,9 @@ microdescs_parse_from_string(const char *s, const char *eos, } if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - int i; - md->family = smartlist_new(); - for (i=0;in_args;++i) { - if (!is_legal_nickname_or_hexdigest(tok->args[i])) { - log_warn(LD_DIR, "Illegal nickname %s in family line", - escaped(tok->args[i])); - goto next; - } - smartlist_add_strdup(md->family, tok->args[i]); - } + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); } if ((tok = find_opt_by_keyword(tokens, K_P))) { diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 146c772daf..f331d5e109 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -23,6 +23,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -882,10 +883,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); - if (md->family) { - SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); - smartlist_free(md->family); - } + nodefamily_free(md->family); short_policy_free(md->exit_policy); short_policy_free(md->ipv6_exit_policy); diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index d23da13137..30c896181d 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -9,6 +9,7 @@ struct curve25519_public_key_t; struct ed25519_public_key_t; +struct nodefamily_t; struct short_policy_t; /** A microdescriptor is the smallest amount of information needed to build a @@ -69,8 +70,8 @@ struct microdesc_t { tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; - /** As routerinfo_t.family */ - smartlist_t *family; + /** As routerinfo_t.family, with readable members parsed. */ + struct nodefamily_t *family; /** IPv4 exit policy summary */ struct short_policy_t *exit_policy; /** IPv6 exit policy summary */ diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index a98a5c8655..3994c8d072 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" @@ -1504,19 +1505,6 @@ node_is_me(const node_t *node) return router_digest_is_me(node->identity); } -/** Return node declared family (as a list of names), or NULL if - * the node didn't declare a family. */ -const smartlist_t * -node_get_declared_family(const node_t *node) -{ - if (node->ri && node->ri->declared_family) - return node->ri->declared_family; - else if (node->md && node->md->family) - return node->md->family; - else - return NULL; -} - /* Does this node have a valid IPv6 address? * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for * checking specific ports. */ @@ -1905,6 +1893,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) return 0; } +/** Return true iff n1's declared family contains n2. */ +static int +node_family_contains(const node_t *n1, const node_t *n2) +{ + if (n1->ri && n1->ri->declared_family) { + return node_in_nickname_smartlist(n1->ri->declared_family, n2); + } else if (n1->md) { + return nodefamily_contains_node(n1->md->family, n2); + } else { + return 0; + } +} + +/** + * Return true iff node has declared a nonempty family. + **/ +static bool +node_has_declared_family(const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + return true; + } + + if (node->md && node->md->family) { + return true; + } + + return false; +} + +/** + * Add to out every node_t that is listed by node as being in + * its family. (Note that these nodes are not in node's family unless they + * also agree that node is in their family.) + **/ +static void +node_lookup_declared_family(smartlist_t *out, const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) { + const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED); + if (n2) { + smartlist_add(out, (node_t *)n2); + } + } SMARTLIST_FOREACH_END(name); + return; + } + + if (node->md && node->md->family) { + nodefamily_add_nodes_to_smartlist(node->md->family, out); + } +} + /** Return true iff r1 and r2 are in the same family, but not the same * router. */ int @@ -1922,14 +1965,9 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) } /* Are they in the same family because the agree they are? */ - { - const smartlist_t *f1, *f2; - f1 = node_get_declared_family(node1); - f2 = node_get_declared_family(node2); - if (f1 && f2 && - node_in_nickname_smartlist(f1, node2) && - node_in_nickname_smartlist(f2, node1)) - return 1; + if (node_family_contains(node1, node2) && + node_family_contains(node2, node1)) { + return 1; } /* Are they in the same option because the user says they are? */ @@ -1957,13 +1995,10 @@ void nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) { const smartlist_t *all_nodes = nodelist_get_list(); - const smartlist_t *declared_family; const or_options_t *options = get_options(); tor_assert(node); - declared_family = node_get_declared_family(node); - /* Let's make sure that we have the node itself, if it's a real node. */ { const node_t *real_node = node_get_by_id(node->identity); @@ -1984,25 +2019,20 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) } SMARTLIST_FOREACH_END(node2); } - /* Now, add all nodes in the declared_family of this node, if they + /* Now, add all nodes in the declared family of this node, if they * also declare this node to be in their family. */ - if (declared_family) { + if (node_has_declared_family(node)) { + smartlist_t *declared_family = smartlist_new(); + node_lookup_declared_family(declared_family, node); + /* Add every r such that router declares familyness with node, and node * declares familyhood with router. */ - SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) { - const node_t *node2; - const smartlist_t *family2; - if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED))) - continue; - if (!(family2 = node_get_declared_family(node2))) - continue; - SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) { - if (node_nickname_matches(node, name2)) { - smartlist_add(sl, (void*)node2); - break; - } - } SMARTLIST_FOREACH_END(name2); - } SMARTLIST_FOREACH_END(name); + SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) { + if (node_family_contains(node2, node)) { + smartlist_add(sl, (void*)node2); + } + } SMARTLIST_FOREACH_END(node2); + smartlist_free(declared_family); } /* If the user declared any families locally, honor those too. */ diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index cf7658cf9d..87ea544db2 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -68,7 +68,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -const smartlist_t *node_get_declared_family(const node_t *node); const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, const struct ed25519_public_key_t *id); diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 8ede2690ec..3318408d53 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -11,6 +11,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -70,6 +71,7 @@ test_md_cache(void *data) const char *test_md3_noannotation = strchr(test_md3, '\n')+1; time_t time1, time2, time3; char *fn = NULL, *s = NULL; + char *encoded_family = NULL; (void)data; options = get_options_mutable(); @@ -172,8 +174,9 @@ test_md_cache(void *data) tt_ptr_op(md1->family, OP_EQ, NULL); tt_ptr_op(md3->family, OP_NE, NULL); - tt_int_op(smartlist_len(md3->family), OP_EQ, 3); - tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX"); + + encoded_family = nodefamily_format(md3->family); + tt_str_op(encoded_family, OP_EQ, "nodeX nodeY nodeZ"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); @@ -254,6 +257,7 @@ test_md_cache(void *data) smartlist_free(wanted); tor_free(s); tor_free(fn); + tor_free(encoded_family); } static const char truncated_md[] = -- GitLab From aa1d767e6b98efa4ffa7d39dba2272949aae2f65 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 12:46:59 -0500 Subject: [PATCH 0173/1724] Aim for 100% test coverage on nodefamily.c --- src/test/test_nodelist.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 2dbd949b7d..7e0df56df5 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -259,6 +259,10 @@ test_nodelist_nodefamily(void *arg) nodefamily_t *nf2 = NULL; nodefamily_t *nf3 = NULL; + enc = nodefamily_format(NULL); + tt_str_op(enc, OP_EQ, ""); + tor_free(enc); + /* Make sure that sorting and de-duplication work. */ tor_asprintf(&enc, "$%s hello", h1); nf1 = nodefamily_parse(enc, NULL, 0); @@ -361,6 +365,7 @@ test_nodelist_nodefamily(void *arg) nodefamily_free(nf1); nodefamily_free(nf2); nodefamily_free(nf3); + nodefamily_free_all(); } static void -- GitLab From 4f9548f89339fe1fab1863ac2b2c72d09f7224ef Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 12:55:23 -0500 Subject: [PATCH 0174/1724] Expose more nodelist.c functions to tests --- src/feature/nodelist/nodelist.c | 10 +++++----- src/feature/nodelist/nodelist.h | 10 ++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 3994c8d072..d070f31c12 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1870,7 +1870,7 @@ addrs_in_same_network_family(const tor_addr_t *a1, * (case-insensitive), or if node's identity key digest * matches a hexadecimal value stored in nickname. Return * false otherwise. */ -static int +STATIC int node_nickname_matches(const node_t *node, const char *nickname) { const char *n = node_get_nickname(node); @@ -1882,7 +1882,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff node is named by some nickname in lst. */ -static inline int +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1894,7 +1894,7 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) } /** Return true iff n1's declared family contains n2. */ -static int +STATIC int node_family_contains(const node_t *n1, const node_t *n2) { if (n1->ri && n1->ri->declared_family) { @@ -1909,7 +1909,7 @@ node_family_contains(const node_t *n1, const node_t *n2) /** * Return true iff node has declared a nonempty family. **/ -static bool +STATIC bool node_has_declared_family(const node_t *node) { if (node->ri && node->ri->declared_family && @@ -1929,7 +1929,7 @@ node_has_declared_family(const node_t *node) * its family. (Note that these nodes are not in node's family unless they * also agree that node is in their family.) **/ -static void +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node) { if (node->ri && node->ri->declared_family && diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index 87ea544db2..32300eb00c 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -154,10 +154,16 @@ int count_loading_descriptors_progress(void); #ifdef NODELIST_PRIVATE +STATIC int node_nickname_matches(const node_t *node, const char *nickname); +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, + const node_t *node); +STATIC int node_family_contains(const node_t *n1, const node_t *n2); +STATIC bool node_has_declared_family(const node_t *node); +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node); + #ifdef TOR_UNIT_TESTS -STATIC void -node_set_hsdir_index(node_t *node, const networkstatus_t *ns); +STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns); #endif /* defined(TOR_UNIT_TESTS) */ -- GitLab From 0e762c0cf5260040c38e93b4d4204be3f6746301 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 13:35:52 -0500 Subject: [PATCH 0175/1724] Test new functions in nodelist.c --- src/test/test_nodelist.c | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 7e0df56df5..0287be3305 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -6,6 +6,8 @@ * \brief Unit tests for nodelist related functions. **/ +#define NODELIST_PRIVATE + #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" @@ -483,6 +485,131 @@ test_nodelist_nodefamily_lookup(void *arg) tor_free(mem_op_hex_tmp); } +static void +test_nodelist_nickname_matches(void *arg) +{ + (void)arg; + node_t mock_node; + routerstatus_t mock_rs; + memset(&mock_node, 0, sizeof(mock_node)); + memset(&mock_rs, 0, sizeof(mock_rs)); + + strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN); + +#define match(x) tt_assert(node_nickname_matches(&mock_node, (x))) +#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x))) + + match("evilgeniuses"); + match("EvilGeniuses"); + match("EvilGeniuses"); + match("2e666f7261626574746572746f6d6f72726f772e"); + match("2E666F7261626574746572746F6D6F72726F772E"); + match("$2e666f7261626574746572746f6d6f72726f772e"); + match("$2E666F7261626574746572746F6D6F72726F772E"); + match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses"); + match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES"); + + no_match("evilgenius"); + no_match("evilgeniuseses"); + no_match("evil.genius"); + no_match("$2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("$2E666F7261626574746572746F6D6F72726F772E~fred"); + no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES"); + done: + ; +} + +static void +test_nodelist_node_nodefamily(void *arg) +{ + (void)arg; + node_t mock_node1; + routerstatus_t mock_rs; + microdesc_t mock_md; + + node_t mock_node2; + routerinfo_t mock_ri; + + smartlist_t *nodes=smartlist_new(); + + memset(&mock_node1, 0, sizeof(mock_node1)); + memset(&mock_node2, 0, sizeof(mock_node2)); + memset(&mock_rs, 0, sizeof(mock_rs)); + memset(&mock_md, 0, sizeof(mock_md)); + memset(&mock_ri, 0, sizeof(mock_ri)); + + mock_node1.rs = &mock_rs; + mock_node1.md = &mock_md; + + mock_node2.ri = &mock_ri; + + strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname)); + mock_ri.nickname = tor_strdup("nodetwo"); + + memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN); + memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN); + + // empty families. + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain nodes, but not these nodes + mock_ri.declared_family = smartlist_new(); + smartlist_add(mock_ri.declared_family, (char*)"NodeThree"); + mock_md.family = nodefamily_parse("NodeFour", NULL, 0); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain one another. + smartlist_add(mock_ri.declared_family, (char*) + "4e6f64654f6e654e6f6465314e6f64654f6e6531"); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + nodefamily_free(mock_md.family); + mock_md.family = nodefamily_parse( + "NodeFour " + "5365636f6e644e6f64655765277265546573746e", NULL, 0); + tt_assert(node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + // Try looking up families now. + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + + node_lookup_declared_family(nodes, &mock_node1); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + const node_t *n = smartlist_get(nodes, 0); + tt_str_op(n->identity, OP_EQ, "NodeFour"); + n = smartlist_get(nodes, 1); + tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + + // free, try the other one. + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_clear(nodes); + + node_lookup_declared_family(nodes, &mock_node2); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + n = smartlist_get(nodes, 0); + tt_str_op(n->identity, OP_EQ, "NodeThree"); + n = smartlist_get(nodes, 1); + // This gets a truncated hex hex ID since it was looked up by name + tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + smartlist_free(mock_ri.declared_family); + nodefamily_free(mock_md.family); + tor_free(mock_ri.nickname); + // use tor_free, these aren't real nodes + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_free(nodes); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -494,5 +621,7 @@ struct testcase_t nodelist_tests[] = { NODE(nodefamily, TT_FORK), NODE(nodefamily_parse_err, TT_FORK), NODE(nodefamily_lookup, TT_FORK), + NODE(nickname_matches, 0), + NODE(node_nodefamily, TT_FORK), END_OF_TESTCASES }; -- GitLab From 71651ea4aa507ee50865bd9584873e0c9a422c14 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 7 Apr 2015 16:36:05 +0200 Subject: [PATCH 0176/1724] Complain if net.inet.ip.random_id is not set on FreeBSD-based servers Apparently a couple of operators haven't gotten the memos [0] yet and it looks like FreeBSD's default value will not change any time soon [1]. [0]: https://lists.torproject.org/pipermail/tor-relays/2014-March/004199.html https://lists.torproject.org/pipermail/tor-relays/2014-November/005687.html https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=195828 [1]: https://lists.freebsd.org/pipermail/freebsd-net/2015-April/041942.html --- src/app/config/config.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/app/config/config.c b/src/app/config/config.c index 45a23d67d5..56fca15499 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -157,6 +157,10 @@ #include "core/or/connection_st.h" #include "core/or/port_cfg_st.h" +#ifdef __FreeBSD__ +#include +#endif + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -3383,6 +3387,23 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); +#ifdef __FreeBSD__ + if (server_mode(options)) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#endif + /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { if (quiet_level == 0) -- GitLab From d020124138cc0d16e685bfd35dd388a4db7f68af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 19 Nov 2018 16:33:06 +0100 Subject: [PATCH 0177/1724] Add changes file for #28518. See: https://bugs.torproject.org/28518 --- changes/bug28518 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug28518 diff --git a/changes/bug28518 b/changes/bug28518 new file mode 100644 index 0000000000..d7ebab29bb --- /dev/null +++ b/changes/bug28518 @@ -0,0 +1,4 @@ + o Minor features (FreeBSD): + - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID + randomization) is disabled on their relay if it is running on FreeBSD + based operating systems. Closes ticket 28518. -- GitLab From ffee0a6384e751486bb4ca2752b6a00527b923ca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 15:40:52 +0200 Subject: [PATCH 0178/1724] Add pre-push git hook to prevent fixup and squash commits from ending up in master --- changes/ticket27993 | 3 +++ scripts/maint/pre-push | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 changes/ticket27993 create mode 100755 scripts/maint/pre-push diff --git a/changes/ticket27993 b/changes/ticket27993 new file mode 100644 index 0000000000..78ee7c2054 --- /dev/null +++ b/changes/ticket27993 @@ -0,0 +1,3 @@ + o Minor features (developer tooling): + - Provide git hook script to prevent "fixup!" and "squash!" commits from + ending up in master. Closes ticket 27993. diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push new file mode 100755 index 0000000000..2cf1837b8d --- /dev/null +++ b/scripts/maint/pre-push @@ -0,0 +1,56 @@ +#!/bin/sh + +# git pre-push hook script to prevent "fixup!" and "squash!" commit +# from ending up in master, or in any branch if CUR_BRANCH check is removed. +# It is meant to be placed in .git/hooks directory. +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +z40=0000000000000000000000000000000000000000 + +CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$CUR_BRANCH" != "master" ] +then + exit 0 +fi + +echo "Running pre-push hook" + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found squash! commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 + -- GitLab From 7dd515b0e0a12cfd9a324a878f7ca6b6b84b24cb Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:22:47 +0200 Subject: [PATCH 0179/1724] Silence SC1117 warnings --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index d2680856b8..1e927dd0b1 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -27,19 +27,19 @@ VENDORED="$TOPLEVEL/src/ext/rust/crates" CARGO=$(which cargo) if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" + printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" fi if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED" + printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" fi if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\n" + printf "Error: cargo must be installed and in your \$PATH\\n" fi if test -z $(cargo --list | grep vendor) ; then - printf "Error: cargo-vendor not installed\n" + printf "Error: cargo-vendor not installed\\n" fi $CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED -- GitLab From b2053cfc44f4876a52f8d71f2308077c6e39498d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:37:30 +0200 Subject: [PATCH 0180/1724] Also disallow fixup/squash commits in maint-* and release-* --- scripts/maint/pre-push | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push index 2cf1837b8d..26c48c4e21 100755 --- a/scripts/maint/pre-push +++ b/scripts/maint/pre-push @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # git pre-push hook script to prevent "fixup!" and "squash!" commit # from ending up in master, or in any branch if CUR_BRANCH check is removed. @@ -10,7 +10,8 @@ z40=0000000000000000000000000000000000000000 CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) -if [ "$CUR_BRANCH" != "master" ] +if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] then exit 0 fi -- GitLab From 27e982b470f4d401a9be687609897ab3320591e1 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:55:35 +0200 Subject: [PATCH 0181/1724] Fix SC2046 and SC2086 warnings --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index 1e927dd0b1..0b303d5382 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,8 +20,8 @@ set -e -HERE=$(dirname $(realpath $0)) -TOPLEVEL=$(dirname $(dirname $HERE)) +HERE=$(dirname "$(realpath "$0")") +TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" CARGO=$(which cargo) @@ -38,8 +38,8 @@ if test -z "$CARGO" ; then printf "Error: cargo must be installed and in your \$PATH\\n" fi -if test -z $(cargo --list | grep vendor) ; then +if test -z "$(cargo --list | grep vendor)" ; then printf "Error: cargo-vendor not installed\\n" fi -$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED +$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" -- GitLab From ba2a9988ccd40b9c2ef1f16cb7960191b4131d78 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:57:57 +0200 Subject: [PATCH 0182/1724] Fix SC2230 warning --- scripts/maint/updateRustDependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index 0b303d5382..6d0587351f 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -24,7 +24,7 @@ HERE=$(dirname "$(realpath "$0")") TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=$(which cargo) +CARGO=$(command -v cargo) if ! test -f "$TOML" ; then printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" -- GitLab From befcd6ab7f53e8d5d825949547e2530fdc91b177 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 17:00:32 +0200 Subject: [PATCH 0183/1724] Add changes file for #28012 --- changes/ticket28012 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28012 diff --git a/changes/ticket28012 b/changes/ticket28012 new file mode 100644 index 0000000000..b2fe83e02a --- /dev/null +++ b/changes/ticket28012 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. -- GitLab From 469f47ef8dc8b18104108f0437c860ec88fca6ad Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 21 Nov 2018 07:38:38 -0500 Subject: [PATCH 0184/1724] Fix a fun heisenbug in memoize_protover_flags() After we clear the protover map for getting full, we need to re-create it, since we are about to use it. This is a bugfix for bug 28558. It is a bugfix for the code from ticket 27225, which is not in any released Tor. Found by Google OSS-Fuzz, as issue 11475. --- src/core/or/versions.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 6f8eea7a67..5d4effcaf8 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -399,6 +399,7 @@ memoize_protover_summary(protover_summary_flags_t *out, if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { protover_summary_cache_free_all(); + protover_summary_map = strmap_new(); } const protover_summary_flags_t *cached = -- GitLab From 6d2c2be291e682bff985a357f3703adf1cd03774 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 21 Nov 2018 20:36:08 -0600 Subject: [PATCH 0185/1724] Add CrackerboxPalace fallback --- scripts/maint/fallback.whitelist | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 94db9272fd..23c7be58a5 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1070,3 +1070,6 @@ # Email sent directly to Phoul 192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 + +# https://lists.torproject.org/pipermail/tor-relays/2018-November/016610.html +24.117.194.80:80 orport=443 id=B6C4C9A43658F686F8892CA5666717532F72979C -- GitLab From 3741f9e524a2d3bd7239ca865d6169fd1e3dddb5 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 21 Nov 2018 20:15:21 +1000 Subject: [PATCH 0186/1724] Fix a comment typo in test_hs_common.c --- src/test/test_hs_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 95f7ed14ba..6198573f22 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -630,7 +630,7 @@ test_disaster_srv(void *arg) get_disaster_srv(1, srv_one); get_disaster_srv(2, srv_two); - /* Check that the cached ones where updated */ + /* Check that the cached ones were updated */ tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN); tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN); -- GitLab From 997a8b0ca7a4f4eafa2cd09582c4bcc0507e2fa6 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 22 Nov 2018 00:53:57 +0000 Subject: [PATCH 0187/1724] Create a temporary directory for tor's DataDirectory in test_rebind. Fixes #28562. While here, put the argument count test and usage message _before_ we attempt to read from sys.argv. --- src/test/test_rebind.py | 10 +++++++--- src/test/test_rebind.sh | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index c63341a681..2f366b2716 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -67,12 +67,19 @@ socks_port = pick_random_port() assert control_port != 0 assert socks_port != 0 +if len(sys.argv) < 3: + fail('Usage: %s ' % sys.argv[0]) + if not os.path.exists(sys.argv[1]): fail('ERROR: cannot find tor at %s' % sys.argv[1]) +if not os.path.exists(sys.argv[2]): + fail('ERROR: cannot find datadir at %s' % sys.argv[2]) tor_path = sys.argv[1] +data_dir = sys.argv[2] tor_process = subprocess.Popen([tor_path, + '-DataDirectory', data_dir, '-ControlPort', '127.0.0.1:{}'.format(control_port), '-SOCKSPort', '127.0.0.1:{}'.format(socks_port), '-FetchServerDescriptors', '0'], @@ -82,9 +89,6 @@ tor_process = subprocess.Popen([tor_path, if tor_process == None: fail('ERROR: running tor failed') -if len(sys.argv) < 2: - fail('Usage: %s ' % sys.argv[0]) - wait_for_log('Opened Control listener on') try_connecting_to_socksport() diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index 76eb9f2e4d..498072de35 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -14,6 +14,19 @@ fi exitcode=0 -"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" || exitcode=1 +tmpdir= +clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; } +trap clean EXIT HUP INT TERM + +tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`" +if [ -z "$tmpdir" ]; then + echo >&2 mktemp failed + exit 2 +elif [ ! -d "$tmpdir" ]; then + echo >&2 mktemp failed to make a directory + exit 3 +fi + +"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" "$tmpdir" || exitcode=1 exit ${exitcode} -- GitLab From 0e9a963b6b87282011fe204e81b5c2530153a935 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 10:30:15 -0500 Subject: [PATCH 0188/1724] Revise nodefamily.c to match proposal 298 Prop298 says that family entries should be formatted with $hexids in uppercase, nicknames in lower case, $hexid~names truncated, and everything sorted lexically. These changes implement that ordering for nodefamily.c. We don't _strictly speaking_ need to nodefamily.c formatting use this for prop298 microdesc generation, but it seems silly to have two separate canonicalization algorithms. --- src/feature/nodelist/nodefamily.c | 2 ++ src/feature/nodelist/nodefamily_st.h | 8 ++++---- src/test/test_microdesc.c | 2 +- src/test/test_nodelist.c | 18 +++++++++--------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c index 6b504c0ac4..29659ed93d 100644 --- a/src/feature/nodelist/nodefamily.c +++ b/src/feature/nodelist/nodefamily.c @@ -135,6 +135,7 @@ nodefamily_from_members(const smartlist_t *members, ptr[0] = NODEFAMILY_BY_NICKNAME; tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname memcpy(ptr+1, cp, strlen(cp)); + tor_strlower((char*) ptr+1); bad_element = false; } else if (is_legal_hexdigest(cp)) { char digest_buf[DIGEST_LEN]; @@ -346,6 +347,7 @@ nodefamily_format(const nodefamily_t *family) char buf[HEX_DIGEST_LEN+2]; buf[0]='$'; base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + tor_strupper(buf); smartlist_add_strdup(sl, buf); break; } diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h index f88ada494a..a498b4b3b9 100644 --- a/src/feature/nodelist/nodefamily_st.h +++ b/src/feature/nodelist/nodefamily_st.h @@ -26,12 +26,12 @@ struct nodefamily_t { #define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) -/** Tag byte, indicates that the following bytes are a NUL-padded nickname. - */ -#define NODEFAMILY_BY_NICKNAME 0 /** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. */ -#define NODEFAMILY_BY_RSA_ID 1 +#define NODEFAMILY_BY_RSA_ID 0 +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 1 /** * Number of bytes to allocate in the array for a nodefamily_t with N members. diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 3318408d53..debb11155a 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -176,7 +176,7 @@ test_md_cache(void *data) tt_ptr_op(md3->family, OP_NE, NULL); encoded_family = nodefamily_format(md3->family); - tt_str_op(encoded_family, OP_EQ, "nodeX nodeY nodeZ"); + tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 0287be3305..afbcc60ac0 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -299,7 +299,7 @@ test_nodelist_nodefamily(void *arg) tt_ptr_op(nf1, OP_EQ, nf3); /* Do we get the expected result when we re-encode? */ - tor_asprintf(&enc, "hello $%s", h1); + tor_asprintf(&enc, "$%s hello", h1); enc2 = nodefamily_format(nf1); tt_str_op(enc2, OP_EQ, enc); tor_free(enc2); @@ -399,8 +399,8 @@ test_nodelist_nodefamily_parse_err(void *arg) tt_assert(nf1); enc = nodefamily_format(nf1); tt_str_op(enc, OP_EQ, - "reticulatogranulate " - "$7468696E67732D696E2D7468656D73656C766573"); + "$7468696E67732D696E2D7468656D73656C766573 " + "reticulatogranulate"); tor_free(enc); } @@ -470,11 +470,11 @@ test_nodelist_nodefamily_lookup(void *arg) tt_int_op(smartlist_len(sl), OP_EQ, 3); const node_t *n = smartlist_get(sl, 0); - tt_str_op(n->identity, OP_EQ, "erewhon"); - n = smartlist_get(sl, 1); test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); - n = smartlist_get(sl, 2); + n = smartlist_get(sl, 1); test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + n = smartlist_get(sl, 2); + tt_str_op(n->identity, OP_EQ, "erewhon"); done: UNMOCK(node_get_by_nickname); @@ -583,9 +583,9 @@ test_nodelist_node_nodefamily(void *arg) node_lookup_declared_family(nodes, &mock_node1); tt_int_op(smartlist_len(nodes), OP_EQ, 2); const node_t *n = smartlist_get(nodes, 0); - tt_str_op(n->identity, OP_EQ, "NodeFour"); - n = smartlist_get(nodes, 1); tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "nodefour"); // free, try the other one. SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); @@ -594,9 +594,9 @@ test_nodelist_node_nodefamily(void *arg) node_lookup_declared_family(nodes, &mock_node2); tt_int_op(smartlist_len(nodes), OP_EQ, 2); n = smartlist_get(nodes, 0); + // This gets a truncated hex hex ID since it was looked up by name tt_str_op(n->identity, OP_EQ, "NodeThree"); n = smartlist_get(nodes, 1); - // This gets a truncated hex hex ID since it was looked up by name tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); done: -- GitLab From d29e3a02d57aef402a1aaf9747ef44393b043d98 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 10:53:38 -0500 Subject: [PATCH 0189/1724] Add a function to canonicalize nodefamilies per prop298 This is the same as the regular canonical nodefamily format, except that unrecognized elements are preserved. --- src/feature/nodelist/nodefamily.c | 45 +++++++++++++++++++++++++++++-- src/feature/nodelist/nodefamily.h | 5 +++- src/test/test_nodelist.c | 31 +++++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c index 29659ed93d..944ad54755 100644 --- a/src/feature/nodelist/nodefamily.c +++ b/src/feature/nodelist/nodefamily.c @@ -93,12 +93,46 @@ nodefamily_parse(const char *s, const uint8_t *rsa_id_self, { smartlist_t *sl = smartlist_new(); smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL); SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); return result; } +/** + * Canonicalize the family list s, returning a newly allocated string. + * + * The canonicalization rules are fully specified in dir-spec.txt, but, + * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are + * truncated, nicknames are put into lowercase, unrecognized entries are left + * alone, and everything is sorted. + **/ +char * +nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_t *result_members = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags, + result_members); + + char *formatted = nodefamily_format(nf); + smartlist_split_string(result_members, formatted, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + smartlist_sort_strings(result_members); + char *combined = smartlist_join_strings(result_members, " ", 0, NULL); + + nodefamily_free(nf); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp)); + smartlist_free(result_members); + tor_free(formatted); + + return combined; +} + /** * qsort helper for encoded nodefamily elements. **/ @@ -117,11 +151,15 @@ compare_members(const void *a, const void *b) * family declaration if it is not there already. * * The flags element is interpreted as in nodefamily_parse(). + * + * If unrecognized is provided, fill it copies of any unrecognized + * members. (Note that malformed $hexids are not considered unrecognized.) **/ nodefamily_t * nodefamily_from_members(const smartlist_t *members, const uint8_t *rsa_id_self, - unsigned flags) + unsigned flags, + smartlist_t *unrecognized_out) { const int n_self = rsa_id_self ? 1 : 0; int n_bad_elements = 0; @@ -146,6 +184,9 @@ nodefamily_from_members(const smartlist_t *members, ptr[0] = NODEFAMILY_BY_RSA_ID; memcpy(ptr+1, digest_buf, DIGEST_LEN); } + } else { + if (unrecognized_out) + smartlist_add_strdup(unrecognized_out, cp); } if (bad_element) { diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h index 342f161a07..ea1076876d 100644 --- a/src/feature/nodelist/nodefamily.h +++ b/src/feature/nodelist/nodefamily.h @@ -27,7 +27,8 @@ nodefamily_t *nodefamily_parse(const char *s, unsigned flags); nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, const uint8_t *rsa_id_self, - unsigned flags); + unsigned flags, + smartlist_t *unrecognized_out); void nodefamily_free_(nodefamily_t *family); #define nodefamily_free(family) \ FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) @@ -41,6 +42,8 @@ bool nodefamily_contains_node(const nodefamily_t *family, void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, struct smartlist_t *out); char *nodefamily_format(const nodefamily_t *family); +char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags); void nodefamily_free_all(void); diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index afbcc60ac0..ed919f4edf 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -610,6 +610,36 @@ test_nodelist_node_nodefamily(void *arg) smartlist_free(nodes); } +static void +test_nodelist_nodefamily_canonicalize(void *arg) +{ + (void)arg; + char *c = NULL; + + c = nodefamily_canonicalize("", NULL, 0); + tt_str_op(c, OP_EQ, ""); + tor_free(c); + + uint8_t own_id[20]; + memset(own_id, 0, sizeof(own_id)); + c = nodefamily_canonicalize( + "alice BOB caroL %potrzebie !!!@#@# " + "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred " + "ffffffffffffffffffffffffffffffffffffffff " + "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0); + tt_str_op(c, OP_EQ, + "!!!@#@# " + "$0000000000000000000000000000000000000000 " + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB " + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC " + "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF " + "%potrzebie " + "alice bob carol"); + + done: + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -623,5 +653,6 @@ struct testcase_t nodelist_tests[] = { NODE(nodefamily_lookup, TT_FORK), NODE(nickname_matches, 0), NODE(node_nodefamily, TT_FORK), + NODE(nodefamily_canonicalize, 0), END_OF_TESTCASES }; -- GitLab From 0a0c612b79cb5230cc70922634a37f73a1c87c10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 11:51:41 -0500 Subject: [PATCH 0190/1724] Add a consensus method in which md families get canonicalized. Implements prop298. Closes ticket 28266. --- changes/ticket28266 | 5 +++++ src/feature/dirauth/dirvote.c | 18 +++++++++++++++--- src/feature/dirauth/dirvote.h | 8 +++++++- src/test/test_microdesc.c | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 changes/ticket28266 diff --git a/changes/ticket28266 b/changes/ticket28266 new file mode 100644 index 0000000000..d90dc74f76 --- /dev/null +++ b/changes/ticket28266 @@ -0,0 +1,5 @@ + o Minor features (directory authority): + - Directory authorities support a new consensus algorithm, + under which microdescriptor entries are encoded in a canonical + form. This improves their compressibility in transit and on the client. + Closes ticket 28266; implements proposal 298. diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 066a9e6e8a..6c4e12cd51 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -28,6 +28,7 @@ #include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -3799,8 +3800,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); - if (family) - smartlist_add_asprintf(chunks, "family %s\n", family); + if (family) { + if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) { + smartlist_add_asprintf(chunks, "family %s\n", family); + } else { + const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest; + char *canonical_family = nodefamily_canonicalize(family, id, 0); + smartlist_add_asprintf(chunks, "family %s\n", canonical_family); + tor_free(canonical_family); + } + } if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); @@ -3898,7 +3907,10 @@ static const struct consensus_method_range_t { int high; } microdesc_consensus_methods[] = { {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, - {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, + MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1}, + {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS, + MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a21e9f3455..6afb6047ff 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -57,7 +57,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 28 +#define MAX_SUPPORTED_CONSENSUS_METHOD 29 /** Lowest consensus method where authorities vote on required/recommended * protocols. */ @@ -79,6 +79,12 @@ * addresses. See #23828 and #20916. */ #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28 +/** + * Lowest consensus method where microdescriptor lines are put in canonical + * form for improved compressibility and ease of storage. See proposal 298. + **/ +#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index debb11155a..fd79aee6be 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -421,6 +421,28 @@ static const char test_md2_21[] = "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; +static const char test_md2_withfamily_28[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family OtherNode !Strange\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + +static const char test_md2_withfamily_29[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + static void test_md_generate(void *arg) { @@ -451,6 +473,17 @@ test_md_generate(void *arg) tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, &ri->cache_info.signing_key_cert->signing_key)); + // Try family encoding. + microdesc_free(md); + ri->declared_family = smartlist_new(); + smartlist_add_strdup(ri->declared_family, "OtherNode !Strange"); + md = dirvote_create_microdescriptor(ri, 28); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_28); + + microdesc_free(md); + md = dirvote_create_microdescriptor(ri, 29); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_29); + done: microdesc_free(md); routerinfo_free(ri); -- GitLab From f82eb6269ff738d136aff66cff59b51b9c56f731 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 12:07:54 -0500 Subject: [PATCH 0191/1724] Extract get_declared_family() into its own function. This will help as we refactor it. --- src/feature/relay/router.c | 114 +++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9d61ced11c..3f153e3a2d 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1807,6 +1807,71 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** + * Return a new smartlist containing the family members configured in + * options. Warn about invalid or missing entries. Return NULL + * if this relay should not declare a family. + **/ +static smartlist_t * +get_declared_family(const or_options_t *options) +{ + if (!options->MyFamily) + return NULL; + + if (options->BridgeRelay) + return NULL; + + if (!warned_nonexistent_family) + warned_nonexistent_family = smartlist_new(); + + smartlist_t *declared_family = smartlist_new(); + config_line_t *family; + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (!strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself, that's redundant */ + else + member = node_get_by_nickname(name, 0); + if (!member) { + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_nonexistent_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "I have no descriptor for the router named \"%s\" in my " + "declared family; I'll use the nickname as is, but " + "this may confuse clients.", name); + else + log_warn(LD_CONFIG, "There is a router named \"%s\" in my " + "declared family, but that isn't a legal nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_nonexistent_family, name); + } + if (is_legal) { + smartlist_add_strdup(declared_family, name); + } + } else if (router_digest_is_me(member->identity)) { + /* Don't list ourself in our own family; that's redundant */ + /* XXX shouldn't be possible */ + } else { + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(declared_family, fp); + if (smartlist_contains_string(warned_nonexistent_family, name)) + smartlist_string_remove(warned_nonexistent_family, name); + } + } + + /* remove duplicates from the list */ + smartlist_sort_strings(declared_family); + smartlist_uniq_strings(declared_family); + + return declared_family; +} + /** Build a fresh routerinfo, signed server descriptor, and extra-info document * for this OR. Set r to the generated routerinfo, e to the generated * extra-info document. Return 0 on success, -1 on temporary error. Failure to @@ -1921,54 +1986,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - if (options->MyFamily && ! options->BridgeRelay) { - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); - ri->declared_family = smartlist_new(); - config_line_t *family; - for (family = options->MyFamily; family; family = family->next) { - char *name = family->value; - const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ - else - member = node_get_by_nickname(name, 0); - if (!member) { - int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && - !is_legal_hexdigest(name)) { - if (is_legal) - log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); - else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " - "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); - } - if (is_legal) { - smartlist_add_strdup(ri->declared_family, name); - } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ - } else { - char *fp = tor_malloc(HEX_DIGEST_LEN+2); - fp[0] = '$'; - base16_encode(fp+1,HEX_DIGEST_LEN+1, - member->identity, DIGEST_LEN); - smartlist_add(ri->declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); - } - } - - /* remove duplicates from the list */ - smartlist_sort_strings(ri->declared_family); - smartlist_uniq_strings(ri->declared_family); - } + ri->declared_family = get_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); -- GitLab From 05dee063c8d74437c792bdee2432293f97b6307c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 16:35:58 -0500 Subject: [PATCH 0192/1724] Emit router families in canonical form This patch has routers use the same canonicalization logic as authorities when encoding their family lists. Additionally, they now warn if any router in their list is given by nickname, since that's error-prone. This patch also adds some long-overdue tests for family formatting. --- changes/ticket28266 | 5 ++ src/feature/relay/router.c | 135 ++++++++++++++++++++++++-------- src/feature/relay/router.h | 4 + src/test/test_router.c | 155 +++++++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 33 deletions(-) diff --git a/changes/ticket28266 b/changes/ticket28266 index d90dc74f76..e0bc171080 100644 --- a/changes/ticket28266 +++ b/changes/ticket28266 @@ -3,3 +3,8 @@ under which microdescriptor entries are encoded in a canonical form. This improves their compressibility in transit and on the client. Closes ticket 28266; implements proposal 298. + + o Minor features (relay): + - When listing relay families, list them in canonical form including the + relay's own identity, and try to give a more useful set of warnings. + Part of ticket 28266 and proposal 298. diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 3f153e3a2d..d57fcc3a44 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -340,6 +341,16 @@ set_server_identity_key(crypto_pk_t *k) } } +#ifdef TOR_UNIT_TESTS +/** Testing only -- set the server's RSA identity digest to + * be digest */ +void +set_server_identity_key_digest_testing(const uint8_t *digest) +{ + memcpy(server_identitykey_digest, digest, DIGEST_LEN); +} +#endif + /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ static void @@ -1691,10 +1702,6 @@ router_get_descriptor_gen_reason(void) return desc_gen_reason; } -/** A list of nicknames that we've warned about including in our family - * declaration verbatim rather than as digests. */ -static smartlist_t *warned_nonexistent_family = NULL; - static int router_guess_address_from_dir_headers(uint32_t *guess); /** Make a current best guess at our address, either because @@ -1807,13 +1814,17 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** A list of nicknames that we've warned about including in our family, + * for one reason or another. */ +static smartlist_t *warned_family = NULL; + /** * Return a new smartlist containing the family members configured in * options. Warn about invalid or missing entries. Return NULL * if this relay should not declare a family. **/ -static smartlist_t * -get_declared_family(const or_options_t *options) +STATIC smartlist_t * +get_my_declared_family(const or_options_t *options) { if (!options->MyFamily) return NULL; @@ -1821,55 +1832,112 @@ get_declared_family(const or_options_t *options) if (options->BridgeRelay) return NULL; - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); + if (!warned_family) + warned_family = smartlist_new(); smartlist_t *declared_family = smartlist_new(); config_line_t *family; + + /* First we try to get the whole family in the form of hexdigests. */ for (family = options->MyFamily; family; family = family->next) { char *name = family->value; const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ + if (options->Nickname && !strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself by nickname, that's redundant */ else member = node_get_by_nickname(name, 0); + if (!member) { + /* This node doesn't seem to exist, so warn about it if it is not + * a hexdigest. */ int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && + if (!smartlist_contains_string(warned_family, name) && !is_legal_hexdigest(name)) { if (is_legal) log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); + "There is a router named %s in my declared family, but " + "I have no descriptor for it. I'll use the nickname " + "as is, but this may confuse clients. Please list it " + "by identity digest instead.", escaped(name)); else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but that isn't a legal digest or nickname. " "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); + smartlist_add_strdup(warned_family, name); } if (is_legal) { smartlist_add_strdup(declared_family, name); } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ } else { + /* List the node by digest. */ char *fp = tor_malloc(HEX_DIGEST_LEN+2); fp[0] = '$'; base16_encode(fp+1,HEX_DIGEST_LEN+1, member->identity, DIGEST_LEN); smartlist_add(declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); + + if (! is_legal_hexdigest(name) && + !smartlist_contains_string(warned_family, name)) { + /* Warn if this node was not specified by hexdigest. */ + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but it wasn't listed by digest. Please consider " + "saying %s instead, if that's what you meant.", + escaped(name), fp); + smartlist_add_strdup(warned_family, name); + } } } - /* remove duplicates from the list */ - smartlist_sort_strings(declared_family); - smartlist_uniq_strings(declared_family); + /* Now declared_family should have the closest we can come to the + * identities that the user wanted. + * + * Unlike older versions of Tor, we _do_ include our own identity: this + * helps microdescriptor compression, and helps in-memory compression + * on clients. */ + nodefamily_t *nf = nodefamily_from_members(declared_family, + router_get_my_id_digest(), + NF_WARN_MALFORMED, + NULL); + SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s)); + smartlist_free(declared_family); + if (!nf) { + return NULL; + } + + char *s = nodefamily_format(nf); + nodefamily_free(nf); + + smartlist_t *result = smartlist_new(); + smartlist_split_string(result, s, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(s); + + if (smartlist_len(result) == 1) { + /* This is a one-element list containing only ourself; instead return + * nothing */ + const char *singleton = smartlist_get(result, 0); + bool is_me = false; + if (singleton[0] == '$') { + char d[DIGEST_LEN]; + int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1)); + if (n == DIGEST_LEN && + fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) { + is_me = true; + } + } + if (!is_me) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Found a singleton family list with an element " + "that wasn't us! Element was %s", escaped(singleton)); + // LCOV_EXCL_STOP + } else { + SMARTLIST_FOREACH(result, char *, cp, tor_free(cp)); + smartlist_free(result); + return NULL; + } + } - return declared_family; + return result; } /** Build a fresh routerinfo, signed server descriptor, and extra-info document @@ -1986,7 +2054,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - ri->declared_family = get_declared_family(options); + ri->declared_family = get_my_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -3077,9 +3145,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, void router_reset_warnings(void) { - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_clear(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_family); } } @@ -3103,11 +3171,12 @@ router_free_all(void) memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_free(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_free(warned_family); } } + /* From the given RSA key object, convert it to ASN-1 encoded format and set * the newly allocated object in onion_pkey_out. The length of the key is set * in onion_pkey_len_out. */ diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 4575172afb..872eed1f46 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,10 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +STATIC smartlist_t *get_my_declared_family(const or_options_t *options); +#ifdef TOR_UNIT_TESTS +void set_server_identity_key_digest_testing(const uint8_t *digest); +#endif #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/test/test_router.c b/src/test/test_router.c index 921ec42904..b92e96ff3e 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,16 +7,21 @@ * \brief Unittests for code in router.c **/ +#define CONFIG_PRIVATE +#define ROUTER_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/encoding/confline.h" /* Test suite stuff */ #include "test/test.h" @@ -231,11 +236,161 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static node_t fake_node; +static const node_t * +mock_node_get_by_nickname(const char *name, unsigned flags) +{ + (void)flags; + if (!strcasecmp(name, "crumpet")) + return &fake_node; + else + return NULL; +} + +static void +test_router_get_my_family(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + smartlist_t *sl = NULL; + char *join = NULL; + // Overwrite the result of router_get_my_identity_digest(). This + // happens to be okay, but only for testing. + set_server_identity_key_digest_testing( + (const uint8_t*)"holeinthebottomofthe"); + + setup_capture_of_logs(LOG_WARN); + + // No family listed -- so there's no list. + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + +#define CLEAR() do { \ + if (sl) { \ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); \ + smartlist_free(sl); \ + } \ + tor_free(join); \ + mock_clean_saved_logs(); \ + } while (0) + + // Add a single nice friendly hex member. This should be enough + // to have our own ID added. + tt_ptr_op(options->MyFamily, OP_EQ, NULL); + config_line_append(&options->MyFamily, "MyFamily", + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 2); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Add a hex member with a ~. The ~ part should get removed. + config_line_append(&options->MyFamily, "MyFamily", + "$0123456789abcdef0123456789abcdef01234567~Muffin"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 3); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Nickname lookup will fail, so a nickname will appear verbatim. + config_line_append(&options->MyFamily, "MyFamily", + "BAGEL"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + expect_single_log_msg_containing( + "There is a router named \"BAGEL\" in my declared family, but " + "I have no descriptor for it."); + CLEAR(); + + // A bogus digest should fail entirely. + config_line_append(&options->MyFamily, "MyFamily", + "$painauchocolat"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. + expect_single_log_msg_containing( + "There is a router named \"$painauchocolat\" in my declared " + "family, but that isn't a legal digest or nickname. Skipping it."); + CLEAR(); + + // Let's introduce a node we can look up by nickname + memset(&fake_node, 0, sizeof(fake_node)); + memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN); + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + + config_line_append(&options->MyFamily, "MyFamily", + "CRUmpeT"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 5); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. Some with "$painauchocolat". + expect_single_log_msg_containing( + "There is a router named \"CRUmpeT\" in my declared " + "family, but it wasn't listed by digest. Please consider saying " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's " + "what you meant."); + CLEAR(); + UNMOCK(node_get_by_nickname); + + // Try a singleton list containing only us: It should give us NULL. + config_free_lines(options->MyFamily); + config_line_append(&options->MyFamily, "MyFamily", + "$686F6C65696E746865626F74746F6D6F66746865"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + + done: + or_options_free(options); + teardown_capture_of_logs(); + CLEAR(); + UNMOCK(node_get_by_nickname); + +#undef CLEAR +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(get_my_family, TT_FORK), END_OF_TESTCASES }; -- GitLab From b16d6453adde2b13d2aa95a74ec3c09a259f185f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 20:03:48 -0500 Subject: [PATCH 0193/1724] Rewrite updateVersions script in Python, add datestamp functionality. This updateVersions.pl script was one of the only essential perl scripts left in out maint system, and was the only one that used autoconf to fill in the script. This script adds a feature to define an APPROX_RELEASE_DATE macro that is updated when the version changes. We'll use this to implement prop297, so that we have an accurate view of when a release date happens. --- Makefile.am | 2 +- configure.ac | 10 ++- doc/HACKING/ReleasingTor.md | 6 +- scripts/maint/updateVersions.pl.in | 59 ------------- scripts/maint/update_versions.py | 133 +++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 66 deletions(-) delete mode 100755 scripts/maint/updateVersions.pl.in create mode 100755 scripts/maint/update_versions.py diff --git a/Makefile.am b/Makefile.am index 803e9d00df..cbe94ad937 100644 --- a/Makefile.am +++ b/Makefile.am @@ -416,7 +416,7 @@ endif .PHONY: update-versions update-versions: - $(PERL) $(top_builddir)/scripts/maint/updateVersions.pl + abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py .PHONY: callgraph callgraph: diff --git a/configure.ac b/configure.ac index 31e41c3bbc..7f0d375440 100644 --- a/configure.ac +++ b/configure.ac @@ -8,6 +8,15 @@ AC_INIT([tor],[0.4.0.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) +# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING. +# +# The update_versions.py script updates this definition when the +# version number changes. Tor uses it to make sure that it +# only shuts down for missing "required protocols" when those protocols +# are listed as required by a consensus after this date. +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-15"], # for 0.4.0.0-alpha-dev + [Approximate date when this software was released. (Updated when the version changes.)]) + # "foreign" means we don't follow GNU package layout standards # "1.11" means we require automake version 1.11 or newer # "subdir-objects" means put .o files in the same directory as the .c files @@ -2417,7 +2426,6 @@ AC_CONFIG_FILES([ src/config/torrc.minimal src/rust/.cargo/config scripts/maint/checkOptionDocs.pl - scripts/maint/updateVersions.pl ]) if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index b5444afa96..b260cdbb19 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -131,13 +131,9 @@ new Tor release: === III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `perl scripts/maint/updateVersions.pl` to update version numbers in other + `make update-versions` to update version numbers in other places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. - (NOTE: To bump the version number, edit `configure.ac`, and then run - either `make`, or `perl scripts/maint/updateVersions.pl`, depending on - your version.) - When you merge the maint branch forward to the next maint branch, or into master, merge it with "-s ours" to avoid a needless version bump. diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in deleted file mode 100755 index 65c51a1f2d..0000000000 --- a/scripts/maint/updateVersions.pl.in +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/perl -w - -$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac'; -$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h'; -$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in'; - -$quiet = 1; - -sub demand { - my $fn = shift; - die "Missing file $fn" unless (-f $fn); -} - -demand($CONFIGURE_IN); -demand($ORCONFIG_H); -demand($TOR_NSI); - -# extract version from configure.ac - -open(F, $CONFIGURE_IN) or die "$!"; -$version = undef; -while () { - if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) { - $version = $1; - last; - } -} -die "No version found" unless $version; -print "Tor version is $version\n" unless $quiet; -close F; - -sub correctversion { - my ($fn, $defchar) = @_; - undef $/; - open(F, $fn) or die "$!"; - my $s = ; - close F; - if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) { - $oldver = $1; - if ($oldver ne $version) { - print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n"; - $line = $defchar . "define VERSION \"$version\""; - open(F, ">$fn.bak"); - print F $s; - close F; - $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m; - open(F, ">$fn"); - print F $s; - close F; - } else { - print "$fn has the correct version. Good.\n" unless $quiet; - } - } else { - print "Didn't find a version line in $fn -- uh oh.\n"; - } -} - -correctversion($TOR_NSI, "!"); -correctversion($ORCONFIG_H, "#"); diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py new file mode 100755 index 0000000000..8067f2c6c8 --- /dev/null +++ b/scripts/maint/update_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import io +import os +import re +import sys +import time + +def P(path): + """ + Give 'path' as a path relative to the abs_top_srcdir environment + variable. + """ + return os.path.join( + os.environ.get('abs_top_srcdir', "."), + path) + +def warn(msg): + """ + Print an warning message. + """ + print("WARNING: {}".format(msg), file=sys.stderr) + +def find_version(infile): + """ + Given an open file (or some other iterator of lines) holding a + configure.ac file, find the current version line. + """ + for line in infile: + m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line) + if m: + return m.group(1) + + return None + +def update_version_in(infile, outfile, regex, versionline): + """ + Copy every line from infile to outfile. If any line matches 'regex', + replace it with 'versionline'. Return True if any line was changed; + false otherwise. + + 'versionline' is either a string -- in which case it is used literally, + or a function that receives the output of 'regex.match'. + """ + found = False + have_changed = False + for line in infile: + m = regex.match(line) + if m: + found = True + oldline = line + if type(versionline) == type(u""): + line = versionline + else: + line = versionline(m) + if not line.endswith("\n"): + line += "\n" + if oldline != line: + have_changed = True + outfile.write(line) + + if not found: + warn("didn't find any version line to replace in {}".format(infile.name)) + + return have_changed + +def replace_on_change(fname, change): + """ + If "change" is true, replace fname with fname.tmp. Otherwise, + delete fname.tmp. Log what we're doing to stderr. + """ + if not change: + print("No change in {}".format(fname)) + os.unlink(fname+".tmp") + else: + print("Updating {}".format(fname)) + os.rename(fname+".tmp", fname) + + +def update_file(fname, + regex, + versionline, + encoding="utf-8"): + """ + Replace any line matching 'regex' in 'fname' with 'versionline'. + Do not modify 'fname' if there are no changes made. Use the + provided encoding to read and write. + """ + with io.open(fname, "r", encoding=encoding) as f, \ + io.open(fname+".tmp", "w", encoding=encoding) as outf: + have_changed = update_version_in(f, outf, regex, versionline) + + replace_on_change(fname, have_changed) + +# Find out our version +with open("configure.ac") as f: + version = find_version(f) + +# If we have no version, we can't proceed. +if version == None: + print("No version found in configure.ac", file=sys.stderr()) + sys.exit(1) + +print("The version is {}".format(version)) + +today = time.strftime("%Y-%m-%d", time.gmtime()) + +# In configure.ac, we replace the definition of APPROX_RELEASE_DATE +# with "{today} for {version}", but only if the version does not match +# what is already there. +def replace_fn(m): + if m.group(1) != version: + # The version changed -- we change the date. + return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version) + else: + # No changes. + return m.group(0) +update_file(P("configure.ac"), + re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'), + replace_fn) + +# In tor-mingw.nsi.in, we replace the definition of VERSION. +update_file(P("contrib/win32build/tor-mingw.nsi.in"), + re.compile(r'!define VERSION .*'), + u'!define VERSION "{}"'.format(version), + encoding="iso-8859-1") + +# In src/win32/orconfig.h, we replace the definition of VERSION. +update_file(P("src/win32/orconfig.h"), + re.compile(r'#define VERSION .*'), + u'#define VERSION "{}"'.format(version)) -- GitLab From 7da06e43da96f4253a756af546f27f03141b3784 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 20:40:48 -0500 Subject: [PATCH 0194/1724] No longer exit for missing required protocolversions on an old consensus Specifically, if the consensus is older than the (estimted or measured) release date for this version of tor, we assume that the required versions may have changed in between that consensus and this release. Implements ticket 27735 and proposal 297. --- changes/prop297 | 7 +++++++ src/core/or/versions.c | 19 +++++++++++++++++++ src/core/or/versions.h | 2 ++ src/feature/nodelist/networkstatus.c | 5 ++++- 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changes/prop297 diff --git a/changes/prop297 b/changes/prop297 new file mode 100644 index 0000000000..4f93b232d2 --- /dev/null +++ b/changes/prop297 @@ -0,0 +1,7 @@ + o Minor features (required protocols): + - Tor no longer exits if it is missing a required protocol, if the + consensus that requires the protocol predates the release date of the + version of Tor. This change prevents Tor releases from exiting because + of an old cached consensus, on the theory that a newer cached + consensus might not require the protocol. Implements proposal 297; + closes ticket 27735. diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 5d4effcaf8..d978935f4f 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -16,6 +16,25 @@ #include "core/or/tor_version_st.h" +/** + * Return the approximate date when this release came out, or was + * scheduled to come out, according to the APPROX_RELEASE_DATE set in + * configure.ac + **/ +time_t +tor_get_approx_release_date(void) +{ + char tbuf[ISO_TIME_LEN+1]; + tor_snprintf(tbuf, sizeof(tbuf), + "%s 00:00:00", APPROX_RELEASE_DATE); + time_t result = 0; + int r = parse_iso_time(tbuf, &result); + if (BUG(r < 0)) { + result = 0; + } + return result; +} + /** Return VS_RECOMMENDED if myversion is contained in * versionlist. Else, return VS_EMPTY if versionlist has no * entries. Else, return VS_OLD if every member of diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 4fc50a0018..acd8998918 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -26,6 +26,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; +time_t tor_get_approx_release_date(void); + version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_parse_platform(const char *platform, diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index f1def9afb1..a25a539cd8 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2684,6 +2684,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, const char *required, *recommended; char *missing = NULL; + const bool consensus_postdates_this_release = + ns->valid_after >= tor_get_approx_release_date(); + tor_assert(warning_out); if (client_mode) { @@ -2701,7 +2704,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, "%s on the Tor network. The missing protocols are: %s", func, missing); tor_free(missing); - return 1; + return consensus_postdates_this_release ? 1 : 0; } if (! protover_all_supported(recommended, &missing)) { -- GitLab From 36f808c9364bb2e6781e555697eac977d754e905 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:05:13 -0500 Subject: [PATCH 0195/1724] Vote on the StaleDesc flag from prop293 The StaleDesc flag tells relays that they need to upload a new descriptor soon, or they will drop out of the consensus. --- src/feature/dirauth/dirvote.c | 3 ++- src/feature/dirauth/voteflags.c | 4 ++++ src/feature/dirauth/voteflags.h | 3 +++ src/feature/dirparse/ns_parse.c | 2 ++ src/feature/nodelist/fmt_routerstatus.c | 3 ++- src/feature/nodelist/routerstatus_st.h | 2 ++ 6 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 066a9e6e8a..aa4242f678 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -4620,7 +4620,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + "Authority Exit Fast Guard Stable V2Dir Valid HSDir " + "StaleDesc", 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 4a24dcb50d..5adf21ad11 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -593,6 +593,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + rs->is_staledesc = + (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; + if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 2f0e061ea4..743a666cc8 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -25,6 +25,9 @@ void set_routerstatus_from_routerinfo(routerstatus_t *rs, void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef VOTEFLAGS_PRIVATE +/** Any descriptor older than this age causes the authorities to set the + * StaleDesc flag. */ +#define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); #endif diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 3fccec1540..e0cdb2d46d 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -434,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_hs_dir = 1; } else if (!strcmp(tok->args[i], "V2Dir")) { rs->is_v2_dir = 1; + } else if (!strcmp(tok->args[i], "StaleDesc")) { + rs->is_staledesc = 1; } } /* These are implied true by having been included in a consensus made diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index e70aeb2950..b1d4a48038 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -135,7 +135,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -145,6 +145,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", + rs->is_staledesc?" StaleDesc":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 714aa27435..ea06587799 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -47,6 +47,8 @@ struct routerstatus_t { unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort * or it claims to accept tunnelled dir requests. */ + unsigned int is_staledesc:1; /** True iff the authorities think this router + * should upload a new descriptor soon. */ unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ -- GitLab From 881b85cf328b5fbcd0108ecb0320374f638b9e47 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:12:20 -0500 Subject: [PATCH 0196/1724] Treat the StaleDesc flag as making our descriptor dirty. Relay side of prop293. --- src/feature/relay/router.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9d61ced11c..2d4ab9b0a0 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -59,6 +59,7 @@ #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/authority_cert_st.h" #include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -2134,7 +2135,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now) /* Now we see whether we want to be retrying frequently or no. The * rule here is that we'll retry frequently if we aren't listed in the * live consensus we have, or if the publication time of the - * descriptor listed for us in the consensus is very old. */ + * descriptor listed for us in the consensus is very old, or if the + * consensus lists us as "stale" and we haven't regenerated since the + * consensus was published. */ ns = networkstatus_get_live_consensus(now); if (ns) { rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); @@ -2142,6 +2145,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now) retry_fast_reason = "not listed in consensus"; else if (rs->published_on < slow_cutoff) retry_fast_reason = "version listed in consensus is quite old"; + else if (rs->is_staledesc && ns->valid_after > desc_clean_since) + retry_fast_reason = "listed as stale in consensus"; } if (retry_fast_reason && desc_clean_since < fast_cutoff) -- GitLab From 439ffcefd57031153e49e605389a1c218e737180 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:19:08 -0500 Subject: [PATCH 0197/1724] changes file for prop293 / ticket 26770 --- changes/ticket26770 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changes/ticket26770 diff --git a/changes/ticket26770 b/changes/ticket26770 new file mode 100644 index 0000000000..7f3e92e9dd --- /dev/null +++ b/changes/ticket26770 @@ -0,0 +1,8 @@ + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a relay's + descriptor is so old that the relay should upload again soon. Relays + understand this flag, and treat it as a signal to upload a new + descriptor. This flag will eventually let us remove the 'published' + date from routerstatus entries, and save a great deal of space in our + consensus diffs. Closes ticket 26770; implements proposal 293. + -- GitLab From e92f90019145c09fd8b1292b0198db1c2ba3a9b5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 26 Nov 2018 17:32:29 +0200 Subject: [PATCH 0198/1724] Try silencing Coverity false positive CID 1441482 Bugfix on 469f47ef8dc8b18104108f0437c860ec88fca6ad; bug not in any released Tor version. --- src/core/or/versions.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 5d4effcaf8..a1336c7a78 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -399,6 +399,7 @@ memoize_protover_summary(protover_summary_flags_t *out, if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { protover_summary_cache_free_all(); + tor_assert(protover_summary_map == NULL); protover_summary_map = strmap_new(); } -- GitLab From 30f8b49d3b7917a15237724a0d0b913d27e876e2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 10:09:11 +0300 Subject: [PATCH 0199/1724] Silence SC2034 shellcheck checker for EXTRA_CHECKERS and NOISY_CHECKERS variables --- scripts/test/scan-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 8d126cbcee..e111566bf7 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -33,6 +33,7 @@ CHECKERS="\ -enable-checker security.insecureAPI.strcpy \ " +# shellcheck disable=SC2034 # These have high false-positive rates. EXTRA_CHECKERS="\ -enable-checker alpha.security.ArrayBoundV2 \ @@ -40,6 +41,7 @@ EXTRA_CHECKERS="\ -enable-checker alpha.core.CastSize \ " +# shellcheck disable=SC2034 # These don't seem to generate anything useful NOISY_CHECKERS="\ -enable-checker alpha.clone.CloneChecker \ -- GitLab From d6eafd06a9797d2747b0bd5988893c292076ef1f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 10:37:27 +0300 Subject: [PATCH 0200/1724] Add changes file --- changes/ticket28007 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28007 diff --git a/changes/ticket28007 b/changes/ticket28007 new file mode 100644 index 0000000000..1ac87862eb --- /dev/null +++ b/changes/ticket28007 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Cleanup scan-build.sh to silence shellcheck warnings. + Closes ticket 28007. -- GitLab From 59001a69c96a2f14821d312b10482c0854c6d7dd Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 26 Nov 2018 18:03:36 +0200 Subject: [PATCH 0201/1724] Address SC2086 in scan-build.sh We can safely silence SC2086 warning on $CHECKERS, as contents of that is hardcoded into script, and we don't want to require Bash to use Bash array here. Double-quote $OUTPUTARG, as it's value depends on environment variable. --- scripts/test/scan-build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index e111566bf7..9a63383804 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -54,6 +54,7 @@ else OUTPUTARG="" fi +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ ./configure @@ -63,13 +64,15 @@ scan-build \ # Make this not get scanned for dead assignments, since it has lots of # dead assignments we don't care about. +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ -disable-checker deadcode.DeadStores \ make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a +# shellcheck disable=SC2086 scan-build \ - $CHECKERS $OUTPUTARG \ + $CHECKERS "$OUTPUTARG" \ make -j5 -k CHECKERS="\ -- GitLab From 3743f7969587079a2f2bb03d0b7e5038557fd64a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 15 Nov 2018 13:16:58 -0500 Subject: [PATCH 0202/1724] Add options to control dormant-client feature. The DormantClientTimeout option controls how long Tor will wait before going dormant. It also provides a way to disable the feature by setting DormantClientTimeout to e.g. "50 years". The DormantTimeoutDisabledByIdleStreams option controls whether open but inactive streams count as "client activity". To implement it, I had to make it so that reading or writing on a client stream *always* counts as activity. Closes ticket 28429. --- doc/tor.1.txt | 13 +++++++++++++ src/app/config/config.c | 6 ++++++ src/app/config/or_options_st.h | 10 ++++++++++ src/core/mainloop/mainloop.c | 19 ++++++++++--------- src/core/or/connection_edge.c | 11 +++++++++++ src/test/test_options.c | 1 + 6 files changed, 51 insertions(+), 9 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index b147ad68aa..47bddea097 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1789,6 +1789,19 @@ The following options are useful only for clients (that is, if Try this many simultaneous connections to download a consensus before waiting for one to complete, timeout, or error out. (Default: 3) +[[DormantClientTimeout]] **DormantClientTimeout** __N__ **minutes**|**hours**|**days**|**weeks**:: + If Tor spends this much time without any client activity, + enter a dormant state where automatic circuits are not built, and + directory information is not fetched. + Does not affect servers or onion services. Must be at least 10 minutes. + (Default: 24 hours) + +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams **0**|**1**:: + If true, then any open client stream (even one not reading or writing) + counts as client activity for the purpose of DormantClientTimeout. + If false, then only network activity counts. (Default: 1) + + SERVER OPTIONS -------------- diff --git a/src/app/config/config.c b/src/app/config/config.c index 8aa0c1f4bd..90eae50fdd 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -389,6 +389,8 @@ static config_var_t option_vars_[] = { OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), OBSOLETE("DNSListenAddress"), + V(DormantClientTimeout, INTERVAL, "24 hours"), + V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), @@ -3836,6 +3838,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "default."); } + if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) { + REJECT("DormantClientTimeout is too low. It must be at least 10 minutes."); + } + if (options->PathBiasNoticeRate > 1.0) { tor_asprintf(msg, "PathBiasNoticeRate is too high. " diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 3524b99b53..6cbc86ec18 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1072,6 +1072,16 @@ struct or_options_t { /** Autobool: Do we refuse single hop client rendezvous? */ int DoSRefuseSingleHopClientRendezvous; + + /** Interval: how long without activity does it take for a client + * to become dormant? + **/ + int DormantClientTimeout; + + /** Boolean: true if having an idle stream is sufficient to prevent a client + * from becoming dormant. + **/ + int DormantTimeoutDisabledByIdleStreams; }; #endif diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2d12e26485..1bd186d856 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2018,24 +2018,25 @@ check_network_participation_callback(time_t now, const or_options_t *options) goto found_activity; } - /* XXXX Add an option to never become dormant. */ - /* If we have any currently open entry streams other than "linked" * connections used for directory requests, those count as user activity. */ - /* XXXX make this configurable? */ - if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { - goto found_activity; + if (options->DormantTimeoutDisabledByIdleStreams) { + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } } /* XXXX Make this configurable? */ /** How often do we check whether we have had network activity? */ #define CHECK_PARTICIPATION_INTERVAL (5*60) - /** Become dormant if there has been no user activity in this long. */ - /* XXXX make this configurable! */ -#define BECOME_DORMANT_AFTER_INACTIVITY (24*60*60) - if (get_last_user_activity_time() + BECOME_DORMANT_AFTER_INACTIVITY >= now) { + /* Become dormant if there has been no user activity in a long time. + * (The funny checks below are in order to prevent overflow.) */ + time_t time_since_last_activity = 0; + if (get_last_user_activity_time() < now) + time_since_last_activity = now - get_last_user_activity_time(); + if (time_since_last_activity >= options->DormantClientTimeout) { log_notice(LD_GENERAL, "No user activity in a long time: becoming" " dormant."); set_network_participation(false); diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 58aefcf8f2..7b51313e8a 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -62,6 +62,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" @@ -297,6 +298,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } return 0; case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -751,6 +757,11 @@ connection_edge_flushed_some(edge_connection_t *conn) { switch (conn->base_.state) { case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: connection_edge_consider_sending_sendme(conn); break; diff --git a/src/test/test_options.c b/src/test/test_options.c index f14e620eeb..376d77626f 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -425,6 +425,7 @@ get_options_test_data(const char *conf) // with options_init(), but about a dozen tests break when I do that. // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" + result->opt->DormantClientTimeout = 1800; // must be over 600. rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); -- GitLab From 55512ef022de39770e0787e9dfc2e29e762652ae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Nov 2018 15:35:55 -0500 Subject: [PATCH 0203/1724] Test netstatus.c tracking of user participation status --- src/core/mainloop/mainloop.c | 4 +-- src/core/mainloop/mainloop.h | 2 +- src/test/test_mainloop.c | 57 +++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 1bd186d856..331f7021a4 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1608,8 +1608,8 @@ rescan_periodic_events_cb(mainloop_event_t *event, void *arg) /** * Schedule an event that will rescan which periodic events should run. **/ -void -schedule_rescan_periodic_events(void) +MOCK_IMPL(void, +schedule_rescan_periodic_events,(void)) { if (!rescan_periodic_events_ev) { rescan_periodic_events_ev = diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 7f27ef9a52..e5e730fc8e 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,7 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); -void schedule_rescan_periodic_events(void); +MOCK_DECL(void, schedule_rescan_periodic_events,(void)); void update_current_time(time_t now); diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 92ce2e9918..94b684fb3a 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -11,6 +11,7 @@ #include "core/or/or.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" static const uint64_t BILLION = 1000000000; @@ -131,12 +132,66 @@ test_mainloop_update_time_jumps(void *arg) monotime_disable_test_mocking(); } +static int schedule_rescan_called = 0; +static void +mock_schedule_rescan_periodic_events(void) +{ + ++schedule_rescan_called; +} + +static void +test_mainloop_user_activity(void *arg) +{ + (void)arg; + const time_t start = 1542658829; + update_approx_time(start); + + MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events); + + reset_user_activity(start); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + set_network_participation(false); + + // reset can move backwards and forwards, but does not change network + // participation. + reset_user_activity(start-10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10); + reset_user_activity(start+10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + tt_int_op(schedule_rescan_called, OP_EQ, 0); + tt_int_op(false, OP_EQ, is_participating_on_network()); + + // "note" can only move forward. Calling it from a non-participating + // state makes us rescan the periodic callbacks and set participation. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // Calling it again will move us forward, but not call rescan again. + note_user_activity(start+25); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We won't move backwards. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + done: + UNMOCK(schedule_rescan_periodic_events); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), + MAINLOOP_TEST(user_activity), END_OF_TESTCASES }; - -- GitLab From 02843c4a4e2fab9c5d9cdb95c425c37ff3d1a4ae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Nov 2018 16:31:50 -0500 Subject: [PATCH 0204/1724] Test for check_network_participation_callback() --- src/core/mainloop/connection.c | 4 +- src/core/mainloop/connection.h | 2 +- src/core/mainloop/mainloop.c | 3 +- src/core/mainloop/mainloop.h | 3 ++ src/feature/hs/hs_service.c | 4 +- src/feature/hs/hs_service.h | 2 +- src/test/test_mainloop.c | 88 ++++++++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 8 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 25224fd999..c1c7c3678b 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4433,8 +4433,8 @@ connection_get_by_type_state(int type, int state) * Return a connection of type type that is not an internally linked * connection, and is not marked for close. **/ -connection_t * -connection_get_by_type_nonlinked(int type) +MOCK_IMPL(connection_t *, +connection_get_by_type_nonlinked,(int type)) { CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); } diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 9f1a23c6f2..07b8df4138 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -240,7 +240,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_nonlinked(int type); +MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type)); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 331f7021a4..42df1038a8 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1367,7 +1367,6 @@ CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); -CALLBACK(check_network_participation); #undef CALLBACK @@ -2003,7 +2002,7 @@ add_entropy_callback(time_t now, const or_options_t *options) /** Periodic callback: if there has been no network usage in a while, * enter a dormant state. */ -static int +STATIC int check_network_participation_callback(time_t now, const or_options_t *options) { /* If we're a server, we can't become dormant. */ diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index e5e730fc8e..14e80ebb21 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -104,6 +104,9 @@ STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); +STATIC int check_network_participation_callback(time_t now, + const or_options_t *options); + #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index c288e28e80..ee0b64a969 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -3667,8 +3667,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk) } /* Return the number of service we have configured and usable. */ -unsigned int -hs_service_get_num_services(void) +MOCK_IMPL(unsigned int, +hs_service_get_num_services,(void)) { if (hs_service_map == NULL) { return 0; diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index a8a9faaea9..be1155bcd1 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -310,7 +310,7 @@ hs_service_t *hs_service_new(const or_options_t *options); void hs_service_free_(hs_service_t *service); #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s)) -unsigned int hs_service_get_num_services(void); +MOCK_DECL(unsigned int, hs_service_get_num_services,(void)); void hs_service_stage_services(const smartlist_t *service_list); int hs_service_load_all_keys(void); int hs_service_get_version_from_key(const hs_service_t *service); diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 94b684fb3a..8dfd5f619a 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -6,13 +6,21 @@ * \brief Tests for functions closely related to the Tor main loop */ +#define CONFIG_PRIVATE +#define MAINLOOP_PRIVATE + #include "test/test.h" #include "test/log_test_helpers.h" #include "core/or/or.h" +#include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" +#include "feature/hs/hs_service.h" + +#include "app/config/config.h" + static const uint64_t BILLION = 1000000000; static void @@ -186,6 +194,85 @@ test_mainloop_user_activity(void *arg) UNMOCK(schedule_rescan_periodic_events); } +static unsigned int +mock_get_num_services(void) +{ + return 1; +} + +static connection_t * +mock_connection_gbtu(int type) +{ + (void) type; + return (void *)"hello fellow connections"; +} + +static void +test_mainloop_check_participation(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + const time_t start = 1542658829; + const time_t ONE_DAY = 24*60*60; + + // Suppose we've been idle for a day or two + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + check_network_participation_callback(start, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + // suppose we've been idle for 2 days... but we are a server. + reset_user_activity(start - 2*ONE_DAY); + options->ORPort_set = 1; + set_network_participation(true); + check_network_participation_callback(start+2, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2); + options->ORPort_set = 0; + + // idle for 2 days, but we have a hidden service. + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + MOCK(hs_service_get_num_services, mock_get_num_services); + check_network_participation_callback(start+3, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3); + UNMOCK(hs_service_get_num_services); + + // idle for 2 days but we have at least one user connection + MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu); + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 1; + check_network_participation_callback(start+10, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + // as above, but DormantTimeoutDisabledByIdleStreams is not set + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 0; + check_network_participation_callback(start+13, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + UNMOCK(connection_get_by_type_nonlinked); + options->DormantTimeoutDisabledByIdleStreams = 1; + + // idle for 2 days but DormantClientTimeout is 3 days + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantClientTimeout = ONE_DAY * 3; + check_network_participation_callback(start+30, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + done: + or_options_free(options); + UNMOCK(hs_service_get_num_services); + UNMOCK(connection_get_by_type_nonlinked); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } @@ -193,5 +280,6 @@ struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), MAINLOOP_TEST(user_activity), + MAINLOOP_TEST(check_participation), END_OF_TESTCASES }; -- GitLab From e12fdeb18199925070f4e99e60f2c5bda7f6af82 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Nov 2018 16:39:44 -0500 Subject: [PATCH 0205/1724] Changes file for "Dormant Mode" (28335, 2149). --- changes/ticket28335 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changes/ticket28335 diff --git a/changes/ticket28335 b/changes/ticket28335 new file mode 100644 index 0000000000..eecf7c7fd9 --- /dev/null +++ b/changes/ticket28335 @@ -0,0 +1,7 @@ + o Major features (client): + - When Tor is running as a client, and it is unused for a long time, it + can now enter a "dormant" state. When Tor is dormant, it avoids + network activity and CPU wakeups until it is reawoken either by a user + request or by a controller command. For more information, see + the configuration options starting with "Dormant". Implements tickets + 2149 and 28335. -- GitLab From 2a3eef4404415f1bb90a86c9cc396b6dee89039c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:05:01 +0200 Subject: [PATCH 0206/1724] Remove unused `int pid` member of `managed_proxy_t`. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index d304dcd485..b80875c95c 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -97,8 +97,6 @@ typedef struct { /* A pointer to the process handle of this managed proxy. */ struct process_handle_t *process_handle; - int pid; /* The Process ID this managed proxy is using. */ - /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport * plugins that use it. */ -- GitLab From 5f26ae833eea79e56cb8cb8133b16a9cb0944ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:23:59 +0200 Subject: [PATCH 0207/1724] Refactor read_to_chunk() such that it supports both pipes and sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..bd420510ab 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -21,6 +21,7 @@ #endif #include +#include #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable @@ -30,27 +31,33 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to at_most bytes from the socket fd into +/** Read up to at_most bytes from the file descriptor fd into * chunk (which must be on buf). If we get an EOF, set * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, * and the number of bytes read otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -108,7 +115,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); check(); if (r < 0) return r; /* Error */ -- GitLab From 340260281a514ee92b4c40ee5ddf8728ac580a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:28:01 +0200 Subject: [PATCH 0208/1724] Refactor flush_chunk() to work on pipes as well as sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index bd420510ab..fc133a01fe 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -129,22 +129,26 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write sz bytes from chunk - * chunk of buffer buf onto socket s. On success, deduct - * the bytes written from *buf_flushlen. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * chunk of buffer buf onto file descriptor fd. On + * success, deduct the bytes written from *buf_flushlen. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -195,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); check(); if (r < 0) return r; -- GitLab From c71f9df07bb5f9f01a47a66f245c7794cb2bd839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:29:27 +0200 Subject: [PATCH 0209/1724] Refactor buf_flush_to_socket() into buf_flush_to_fd(). This patch refactors buf_flush_to_socket() into buf_flush_to_fd() and creates a specialization function for buf_flush_to_socket() that makes use of buf_flush_to_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index fc133a01fe..2ffca8b7da 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -166,15 +166,15 @@ flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from buf to the socket s. Write at most +/** Write data from buf to the file descriptor fd. Write at most * sz bytes, decrement *buf_flushlen by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -182,7 +182,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -199,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -211,3 +211,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from buf to the socket s. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} -- GitLab From 771930b84cb4aeeb75e175368108697bafd94a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:35:38 +0200 Subject: [PATCH 0210/1724] Refactor buf_read_from_socket() into buf_read_from_fd(). This patch refactors buf_read_from_socket() into buf_read_from_fd(), and creates a specialized function for buf_read_from_socket(), which uses buf_read_from_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 2ffca8b7da..7c81096e38 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -75,16 +75,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket s, writing onto end of buf. Read at most - * at_most bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor fd, writing onto end of buf. Read + * at most at_most bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *reached_eof to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -94,7 +95,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -115,7 +116,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -224,3 +226,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, { return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); } + +/** Read from socket s, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} -- GitLab From 31b3a6577c89492e94836f6e3b4bfc7051a3dc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:42:29 +0200 Subject: [PATCH 0211/1724] Add buf_flush_to_pipe() and buf_read_from_pipe(). This patch adds two new functions: buf_flush_to_pipe() and buf_read_from_pipe(), which makes use of our new buf_flush_to_fd() and buf_read_from_fd() functions. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 26 ++++++++++++++++++++++++++ src/lib/net/buffers_net.h | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 7c81096e38..1b65819dbb 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -239,3 +239,29 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, { return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); } + +/** Write data from buf to the pipe fd. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe fd, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 417f6f9413..8911b082a2 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ -- GitLab From 2b41b857bdabffeafbbf748189b22cd6ee818394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 15:15:45 +0200 Subject: [PATCH 0212/1724] Add LD_PROCESS as log domain. See: https://bugs.torproject.org/28179 --- doc/tor.1.txt | 3 ++- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index e94382b66b..42fe8f1bcc 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -669,7 +669,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, and + process. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 46107fe848..861de6e963 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 493409756f..1cd6087eb8 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,10 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 27 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ -- GitLab From bd29b3531a48b41f6eee5b4a29769a13ef199298 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Nov 2018 01:06:19 +1000 Subject: [PATCH 0213/1724] Dir: Refactor ns expiry check to remove duplicate code Instead of checking NS_EXPIRY_SLOP, use networkstatus_consensus_reasonably_live(). Preparation for 28591. --- src/core/mainloop/mainloop.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..af2bc8262f 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2028,11 +2028,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) (void)options; /* Check whether our networkstatus has expired. */ networkstatus_t *ns = networkstatus_get_latest_consensus(); - /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in - * networkstatus_get_reasonably_live_consensus(), but that value is way - * way too high. Arma: is the bridge issue there resolved yet? -NM */ -#define NS_EXPIRY_SLOP (24*60*60) - if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) && + /* Use reasonably live consensuses until they are no longer reasonably live. + */ + if (ns && !networkstatus_consensus_reasonably_live(ns, now) && router_have_minimum_dir_info()) { router_dir_info_changed(); } -- GitLab From 7a45bc74a430ff64f0696c77f075f9c830822413 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Nov 2018 00:46:39 +1000 Subject: [PATCH 0214/1724] Dir: when Tor's clock is behind, use a future consensus to bootstrap When Tor's clock is behind the clocks on the authorities, allow Tor to bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha. --- changes/bug28591 | 4 ++++ src/feature/nodelist/microdesc.c | 10 ++++---- src/feature/nodelist/networkstatus.c | 35 +++++++++++++++++++--------- src/feature/nodelist/networkstatus.h | 2 ++ src/test/test_entrynodes.c | 32 +++++++++++++++++-------- src/test/test_routerlist.c | 1 - 6 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 changes/bug28591 diff --git a/changes/bug28591 b/changes/bug28591 new file mode 100644 index 0000000000..3a1c96ac16 --- /dev/null +++ b/changes/bug28591 @@ -0,0 +1,4 @@ + o Minor bugfixes (client, bootstrap): + - When Tor's clock is behind the clocks on the authorities, allow Tor to + bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha. + diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 3f5085412f..8c5a9ee611 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -110,8 +110,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest) /* If we have a reasonably live consensus, then most of our dirservers should * still be caching all the microdescriptors in it. Reasonably live - * consensuses are up to a day old. But microdescriptors expire 7 days after - * the last consensus that referenced them. */ + * consensuses are up to a day old (or a day in the future). But + * microdescriptors expire 7 days after the last consensus that referenced + * them. */ if (!networkstatus_get_reasonably_live_consensus(approx_time(), FLAV_MICRODESC)) { return; @@ -544,8 +545,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) size_t bytes_dropped = 0; time_t now = time(NULL); - /* If we don't know a live consensus, don't believe last_listed values: we - * might be starting up after being down for a while. */ + /* If we don't know a reasonably live consensus, don't believe last_listed + * values: we might be starting up after being down for a while. */ if (! force && ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC)) return; @@ -973,6 +974,7 @@ update_microdesc_downloads(time_t now) if (directory_too_idle_to_fetch_descriptors(options, now)) return; + /* Give up if we don't have a reasonably live consensus. */ consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); if (!consensus) return; diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 51e720a984..a40c3dfb22 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1377,7 +1377,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor)) } /** Return the most recent consensus that we have downloaded, or NULL if we - * don't have one. */ + * don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus,(void)) { @@ -1388,7 +1388,7 @@ networkstatus_get_latest_consensus,(void)) } /** Return the latest consensus we have whose flavor matches f, or NULL - * if we don't have one. */ + * if we don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { @@ -1422,10 +1422,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now) return (ns->valid_after <= now && now <= ns->valid_until); } -/** Determine if consensus is valid or expired recently enough that - * we can still use it. +/** Determine if consensus is valid, or expired recently enough, or not + * too far in the future, so that we can still use it. * - * Return 1 if the consensus is reasonably live, or 0 if it is too old. + * Return 1 if the consensus is reasonably live, or 0 if it is too old or + * too new. */ int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, @@ -1434,29 +1435,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, if (BUG(!consensus)) return 0; - return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + return networkstatus_valid_after_is_reasonably_live(consensus->valid_after, + now) && + networkstatus_valid_until_is_reasonably_live(consensus->valid_until, now); } +#define REASONABLY_LIVE_TIME (24*60*60) + +/** As networkstatus_consensus_reasonably_live, but takes a valid_after + * time, and checks to see if it is in the past, or not too far in the future. + */ +int +networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now) +{ + return (now >= valid_after - REASONABLY_LIVE_TIME); +} + /** As networkstatus_consensus_reasonably_live, but takes a valid_until - * time rather than an entire consensus. */ + * time, and checks to see if it is in the future, or not too far in the past. + */ int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) return (now <= valid_until + REASONABLY_LIVE_TIME); } /** As networkstatus_get_live_consensus(), but is way more tolerant of expired - * consensuses. */ + * and future consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_reasonably_live_consensus,(time_t now, int flavor)) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(flavor); if (consensus && - consensus->valid_after <= now && networkstatus_consensus_reasonably_live(consensus, now)) return consensus; else @@ -2082,7 +2096,6 @@ networkstatus_set_current_consensus(const char *consensus, nodelist_set_consensus(c); - /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); /* Change the cell EWMA settings */ diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 7b1a0ff72f..6f1d15d532 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -87,6 +87,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); int networkstatus_is_live(const networkstatus_t *ns, time_t now); int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now); int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now); MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus, diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 52b87a84a8..348c642a23 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -127,6 +127,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) return 1; /* NOP */ } +#define REASONABLY_FUTURE " reasonably-future" +#define REASONABLY_PAST " reasonably-past" + /* Unittest setup function: Setup a fake network. */ static void * big_fake_network_setup(const struct testcase_t *testcase) @@ -138,9 +141,10 @@ big_fake_network_setup(const struct testcase_t *testcase) const int N_NODES = 271; const char *argument = testcase->setup_data; - int reasonably_live_consensus = 0; + int reasonably_future_consensus = 0, reasonably_past_consensus = 0; if (argument) { - reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL; + reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL; + reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL; } big_fake_net_nodes = smartlist_new(); @@ -198,11 +202,15 @@ big_fake_network_setup(const struct testcase_t *testcase) dummy_state = tor_malloc_zero(sizeof(or_state_t)); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); - if (reasonably_live_consensus) { - /* Make the dummy consensus valid from 4 hours ago, but expired an hour + if (reasonably_future_consensus) { + /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */ + dummy_consensus->valid_after = approx_time() + 6*3600; + dummy_consensus->valid_until = approx_time() + 7*3600; + } else if (reasonably_past_consensus) { + /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours * ago. */ - dummy_consensus->valid_after = approx_time() - 4*3600; - dummy_consensus->valid_until = approx_time() - 3600; + dummy_consensus->valid_after = approx_time() - 16*3600; + dummy_consensus->valid_until = approx_time() - 12*3600; } else { /* Make the dummy consensus valid for an hour either side of now. */ dummy_consensus->valid_after = approx_time() - 3600; @@ -3035,13 +3043,17 @@ static const struct testcase_setup_t upgrade_circuits = { #define BFN_TEST(name) \ EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &big_fake_network, (void*)("reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_PAST) } #define UPGRADE_TEST(name, arg) \ EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &upgrade_circuits, (void*)(arg " reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_PAST) } struct testcase_t entrynodes_tests[] = { NO_PREFIX_TEST(node_preferred_orport), diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 67af2fd484..6ba4877b91 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -301,7 +301,6 @@ test_router_pick_directory_server_impl(void *arg) tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until + 24*60*60)); /* These times are outside the test validity period */ - tt_assert(networkstatus_consensus_is_bootstrapping(now)); tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60)); tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60)); -- GitLab From e2b418bab5c3249fba7b430b942da67ddf8a43dc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 10:56:56 +0200 Subject: [PATCH 0215/1724] Rename to pre-push.git-hook --- scripts/maint/{pre-push => pre-push.git-hook} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/maint/{pre-push => pre-push.git-hook} (100%) diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push.git-hook similarity index 100% rename from scripts/maint/pre-push rename to scripts/maint/pre-push.git-hook -- GitLab From 7c6dc2888144e587bc8d62c7b2bddae4fb6606b9 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 11:10:30 +0200 Subject: [PATCH 0216/1724] Improve comment at the top --- scripts/maint/pre-push.git-hook | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 26c48c4e21..11f062b539 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -1,8 +1,10 @@ #!/bin/bash -# git pre-push hook script to prevent "fixup!" and "squash!" commit -# from ending up in master, or in any branch if CUR_BRANCH check is removed. -# It is meant to be placed in .git/hooks directory. +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# +# This is git pre-push hook script to prevent "fixup!" and "squash!" commits +# from ending up in upstream branches (master, release-* or maint-*). # # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample -- GitLab From 9c90bddc42467396909812746d5b4256adcb5d2d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 11:10:48 +0200 Subject: [PATCH 0217/1724] Mention --no-verify in error message --- scripts/maint/pre-push.git-hook | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 11f062b539..26296023fb 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -42,6 +42,7 @@ do if [ -n "$commit" ] then echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." exit 1 fi @@ -50,6 +51,7 @@ do if [ -n "$commit" ] then echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." exit 1 fi fi -- GitLab From a51dad4272de8da3d1379e72962a4a8bd500b688 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 29 Nov 2018 10:02:50 -0500 Subject: [PATCH 0218/1724] test: Fix a warning underflow in rend_cache/clean Because the test is adding entries to the "rend_cache" directly, the rend_cache_increment_allocation() was never called which made the rend_cache_clean() call trigger that underflow warning: rend_cache/clean: [forking] Nov 29 09:55:04.024 [warn] rend_cache_decrement_allocation(): Bug: Underflow in rend_cache_decrement_allocation (on Tor 0.4.0.0-alpha-dev 2240fe63feb9a8cf) The test is still good and valid. Fixes #28660 Signed-off-by: David Goulet --- src/test/test_rendcache.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index 394e28d785..2ace45d085 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -788,7 +788,9 @@ test_rend_cache_clean(void *data) desc_two->pk = pk_generate(1); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo2", two); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); tt_int_op(strmap_size(rend_cache), OP_EQ, 0); @@ -806,7 +808,9 @@ test_rend_cache_clean(void *data) desc_one->pk = pk_generate(0); desc_two->pk = pk_generate(1); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); strmap_set_lc(rend_cache, "foo2", two); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); -- GitLab From 1fe6507d29497d7bbd8f42f66f0ddd596078993e Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 19 Nov 2018 16:24:01 -0600 Subject: [PATCH 0219/1724] Split bootstrap event reporting out of control.c Part of ticket 27402. --- src/core/include.am | 1 + src/feature/control/control.c | 373 +--------------------- src/feature/control/control.h | 2 + src/feature/control/control_bootstrap.c | 403 ++++++++++++++++++++++++ 4 files changed, 408 insertions(+), 371 deletions(-) create mode 100644 src/feature/control/control_bootstrap.c diff --git a/src/core/include.am b/src/core/include.am index 48d75618ad..3cd6e83ed5 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -62,6 +62,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ src/feature/control/control.c \ + src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 94679dfd22..060320341c 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -179,13 +179,6 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -/** A sufficiently large size to record the last bootstrap phase string. */ -#define BOOTSTRAP_MSG_LEN 1024 - -/** What was the last bootstrap phase message we sent? We keep track - * of this so we can respond to getinfo status/bootstrap-phase queries. */ -static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -3044,7 +3037,7 @@ getinfo_helper_events(control_connection_t *control_conn, check_whether_orport_reachable(options) ? 1 : 0, check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = tor_strdup(last_sent_bootstrap_message); + *answer = control_event_boot_last_msg(); } else if (!strcmpstart(question, "status/version/")) { int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); @@ -7015,361 +7008,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Convert the name of a bootstrapping phase s into strings - * tag and summary suitable for display by the controller. */ -static int -bootstrap_status_to_string(bootstrap_status_t s, const char **tag, - const char **summary) -{ - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; - } - return 0; -} - -/** What percentage through the bootstrap process are we? We remember - * this so we can avoid sending redundant bootstrap status events, and - * so we can guess context for the bootstrap messages which are - * ambiguous. It starts at 'undef', but gets set to 'starting' while - * Tor initializes. */ -static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_percent, but only takes on the enumerated values in - * bootstrap_status_t. - */ -static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - -/** As bootstrap_percent, but holds the bootstrapping level at which we last - * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, - * to avoid flooding the log with a new message every time we get a few more - * microdescriptors */ -static int notice_bootstrap_percent = 0; - -/** How many problems have we had getting to the next bootstrapping phase? - * These include failure to establish a connection to a Tor relay, - * failures to finish the TLS handshake, failures to validate the - * consensus document, etc. */ -static int bootstrap_problems = 0; - -/** We only tell the controller once we've hit a threshold of problems - * for the current phase. */ -#define BOOTSTRAP_PROBLEM_THRESHOLD 10 - -/** When our bootstrapping progress level changes, but our bootstrapping - * status has not advanced, we only log at NOTICE when we have made at least - * this much progress. - */ -#define BOOTSTRAP_PCT_INCREMENT 5 - -/** Do the actual logging and notifications for - * control_event_bootstrap(). Doesn't change any state beyond that. - */ -static void -control_event_bootstrap_core(int loglevel, bootstrap_status_t status, - int progress) -{ - char buf[BOOTSTRAP_MSG_LEN]; - const char *tag, *summary; - - bootstrap_status_to_string(status, &tag, &summary); - /* Locally reset status if there's incremental progress */ - if (progress) - status = progress; - - tor_log(loglevel, LD_CONTROL, - "Bootstrapped %d%%: %s", status, summary); - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", - status, tag, summary); - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "NOTICE %s", buf); - control_event_client_status(LOG_NOTICE, "%s", buf); -} - -/** Called when Tor has made progress at bootstrapping its directory - * information and initial circuits. - * - * status is the new status, that is, what task we will be doing - * next. progress is zero if we just started this task, else it - * represents progress on the task. - */ -void -control_event_bootstrap(bootstrap_status_t status, int progress) -{ - int loglevel = LOG_NOTICE; - - if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ - - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - - if (status <= bootstrap_percent) { - /* If there's no new progress, return early. */ - if (!progress || progress <= bootstrap_percent) - return; - /* Log at INFO if not enough progress happened. */ - if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) - loglevel = LOG_INFO; - } - - control_event_bootstrap_core(loglevel, status, progress); - - if (status > bootstrap_percent) { - bootstrap_phase = status; /* new milestone reached */ - bootstrap_percent = status; - } - if (progress > bootstrap_percent) { - /* incremental progress within a milestone */ - bootstrap_percent = progress; - bootstrap_problems = 0; /* Progress! Reset our problem counter. */ - } - if (loglevel == LOG_NOTICE && - bootstrap_percent > notice_bootstrap_percent) { - /* Remember that we gave a notice at this level. */ - notice_bootstrap_percent = bootstrap_percent; - } -} - -/** Flag whether we've opened an OR_CONN yet */ -static int bootstrap_first_orconn = 0; - -/** Like bootstrap_phase, but for (possibly deferred) directory progress */ -static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_problems, but for (possibly deferred) directory progress */ -static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - -/** Defer directory info bootstrap events until we have successfully - * completed our first connection to a router. */ -void -control_event_boot_dir(bootstrap_status_t status, int progress) -{ - if (status > bootstrap_dir_progress) { - bootstrap_dir_progress = status; - bootstrap_dir_phase = status; - } - if (progress && progress >= bootstrap_dir_progress) { - bootstrap_dir_progress = progress; - } - - /* Don't report unless we have successfully opened at least one OR_CONN */ - if (!bootstrap_first_orconn) - return; - - control_event_bootstrap(status, progress); -} - -/** Set a flag to allow reporting of directory bootstrap progress. - * (Code that reports completion of an OR_CONN calls this.) Also, - * report directory progress so far. */ -void -control_event_boot_first_orconn(void) -{ - bootstrap_first_orconn = 1; - control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. warn gives a human-readable hint - * as to why, and reason provides a controller-facing short - * tag. conn is the connection that caused this problem and - * can be NULL if a connection cannot be easily identified. - */ -void -control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn) -{ - int status = bootstrap_percent; - const char *tag = "", *summary = ""; - char buf[BOOTSTRAP_MSG_LEN]; - const char *recommendation = "ignore"; - int severity; - char *or_id = NULL, *hostaddr = NULL; - or_connection_t *or_conn = NULL; - - /* bootstrap_percent must not be in "undefined" state here. */ - tor_assert(status >= 0); - - if (bootstrap_percent == 100) - return; /* already bootstrapped; nothing to be done here. */ - - bootstrap_problems++; - - if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) - dowarn = 1; - - /* Don't warn about our bootstrapping status if we are hibernating or - * shutting down. */ - if (we_are_hibernating()) - dowarn = 0; - - tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); - - severity = dowarn ? LOG_WARN : LOG_INFO; - - if (dowarn) - recommendation = "warn"; - - if (conn && conn->type == CONN_TYPE_OR) { - /* XXX TO_OR_CONN can't deal with const */ - or_conn = TO_OR_CONN((connection_t *)conn); - or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); - } else { - or_id = tor_strdup("?"); - } - - if (conn) - tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); - else - hostaddr = tor_strdup("?"); - - log_fn(severity, - LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s; host %s at %s)", - status, summary, warn, reason, - bootstrap_problems, recommendation, - or_id, hostaddr); - - connection_or_report_broken_states(severity, LD_HANDSHAKE); - - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", - bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, - recommendation, - or_id, hostaddr); - - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "WARN %s", buf); - control_event_client_status(LOG_WARN, "%s", buf); - - tor_free(hostaddr); - tor_free(or_id); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. warn gives a hint as to why, and - * reason provides an "or_conn_end_reason" tag. or_conn - * is the connection that caused this problem. - */ -MOCK_IMPL(void, -control_event_bootstrap_prob_or, (const char *warn, int reason, - or_connection_t *or_conn)) -{ - int dowarn = 0; - - if (or_conn->have_noted_bootstrap_problem) - return; - - or_conn->have_noted_bootstrap_problem = 1; - - if (reason == END_OR_CONN_REASON_NO_ROUTE) - dowarn = 1; - - /* If we are using bridges and all our OR connections are now - closed, it means that we totally failed to connect to our - bridges. Throw a warning. */ - if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) - dowarn = 1; - - control_event_bootstrap_problem(warn, - orconn_end_reason_to_control_string(reason), - TO_CONN(or_conn), dowarn); -} - /** We just generated a new summary of which countries we've seen clients * from recently. Send a copy to the controller in case it wants to * display it for the user. */ @@ -7886,17 +7524,10 @@ control_free_all(void) mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } - bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - notice_bootstrap_percent = 0; - bootstrap_problems = 0; - bootstrap_first_orconn = 0; - bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + control_event_bootstrap_reset(); authentication_cookie_is_set = 0; global_event_mask = 0; disable_log_messages = 0; - memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); } #ifdef TOR_UNIT_TESTS diff --git a/src/feature/control/control.h b/src/feature/control/control.h index cd5402d455..4ad245e8e5 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -200,6 +200,8 @@ void control_event_boot_dir(bootstrap_status_t status, int progress); void control_event_boot_first_orconn(void); void control_event_bootstrap_problem(const char *warn, const char *reason, const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c new file mode 100644 index 0000000000..58b342c219 --- /dev/null +++ b/src/feature/control/control_bootstrap.c @@ -0,0 +1,403 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_bootstrap.c + * \brief Provide bootstrap progress events for the control port. + */ +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "core/or/connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/hibernate/hibernate.h" +#include "feature/nodelist/nodelist.h" +#include "lib/malloc/malloc.h" + +/** A sufficiently large size to record the last bootstrap phase string. */ +#define BOOTSTRAP_MSG_LEN 1024 + +/** What was the last bootstrap phase message we sent? We keep track + * of this so we can respond to getinfo status/bootstrap-phase queries. */ +static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; + +/** Convert the name of a bootstrapping phase s into strings + * tag and summary suitable for display by the controller. */ +static int +bootstrap_status_to_string(bootstrap_status_t s, const char **tag, + const char **summary) +{ + switch (s) { + case BOOTSTRAP_STATUS_UNDEF: + *tag = "undef"; + *summary = "Undefined"; + break; + case BOOTSTRAP_STATUS_STARTING: + *tag = "starting"; + *summary = "Starting"; + break; + case BOOTSTRAP_STATUS_CONN_DIR: + *tag = "conn_dir"; + *summary = "Connecting to directory server"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE: + *tag = "status_handshake"; + *summary = "Finishing handshake"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE_DIR: + *tag = "handshake_dir"; + *summary = "Finishing handshake with directory server"; + break; + case BOOTSTRAP_STATUS_ONEHOP_CREATE: + *tag = "onehop_create"; + *summary = "Establishing an encrypted directory connection"; + break; + case BOOTSTRAP_STATUS_REQUESTING_STATUS: + *tag = "requesting_status"; + *summary = "Asking for networkstatus consensus"; + break; + case BOOTSTRAP_STATUS_LOADING_STATUS: + *tag = "loading_status"; + *summary = "Loading networkstatus consensus"; + break; + case BOOTSTRAP_STATUS_LOADING_KEYS: + *tag = "loading_keys"; + *summary = "Loading authority key certs"; + break; + case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: + *tag = "requesting_descriptors"; + /* XXXX this appears to incorrectly report internal on most loads */ + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Asking for relay descriptors for internal paths" : + "Asking for relay descriptors"; + break; + /* If we're sure there are no exits in the consensus, + * inform the controller by adding "internal" + * to the status summaries. + * (We only check this while loading descriptors, + * so we may not know in the earlier stages.) + * But if there are exits, we can't be sure whether + * we're creating internal or exit paths/circuits. + * XXXX Or should be use different tags or statuses + * for internal and exit/all? */ + case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: + *tag = "loading_descriptors"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Loading relay descriptors for internal paths" : + "Loading relay descriptors"; + break; + case BOOTSTRAP_STATUS_CONN_OR: + *tag = "conn_or"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Connecting to the Tor network internally" : + "Connecting to the Tor network"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE_OR: + *tag = "handshake_or"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Finishing handshake with first hop of internal circuit" : + "Finishing handshake with first hop"; + break; + case BOOTSTRAP_STATUS_CIRCUIT_CREATE: + *tag = "circuit_create"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Establishing an internal Tor circuit" : + "Establishing a Tor circuit"; + break; + case BOOTSTRAP_STATUS_DONE: + *tag = "done"; + *summary = "Done"; + break; + default: +// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); + *tag = *summary = "unknown"; + return -1; + } + return 0; +} + +/** What percentage through the bootstrap process are we? We remember + * this so we can avoid sending redundant bootstrap status events, and + * so we can guess context for the bootstrap messages which are + * ambiguous. It starts at 'undef', but gets set to 'starting' while + * Tor initializes. */ +static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_percent, but only takes on the enumerated values in + * bootstrap_status_t. + */ +static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + +/** How many problems have we had getting to the next bootstrapping phase? + * These include failure to establish a connection to a Tor relay, + * failures to finish the TLS handshake, failures to validate the + * consensus document, etc. */ +static int bootstrap_problems = 0; + +/** We only tell the controller once we've hit a threshold of problems + * for the current phase. */ +#define BOOTSTRAP_PROBLEM_THRESHOLD 10 + +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + +/** Do the actual logging and notifications for + * control_event_bootstrap(). Doesn't change any state beyond that. + */ +static void +control_event_bootstrap_core(int loglevel, bootstrap_status_t status, + int progress) +{ + char buf[BOOTSTRAP_MSG_LEN]; + const char *tag, *summary; + + bootstrap_status_to_string(status, &tag, &summary); + /* Locally reset status if there's incremental progress */ + if (progress) + status = progress; + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%%: %s", status, summary); + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", + status, tag, summary); + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "NOTICE %s", buf); + control_event_client_status(LOG_NOTICE, "%s", buf); +} + +/** Called when Tor has made progress at bootstrapping its directory + * information and initial circuits. + * + * status is the new status, that is, what task we will be doing + * next. progress is zero if we just started this task, else it + * represents progress on the task. + */ +void +control_event_bootstrap(bootstrap_status_t status, int progress) +{ + int loglevel = LOG_NOTICE; + + if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) + return; /* already bootstrapped; nothing to be done here. */ + + /* special case for handshaking status, since our TLS handshaking code + * can't distinguish what the connection is going to be for. */ + if (status == BOOTSTRAP_STATUS_HANDSHAKE) { + if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { + status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; + } else { + status = BOOTSTRAP_STATUS_HANDSHAKE_OR; + } + } + + if (status <= bootstrap_percent) { + /* If there's no new progress, return early. */ + if (!progress || progress <= bootstrap_percent) + return; + /* Log at INFO if not enough progress happened. */ + if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) + loglevel = LOG_INFO; + } + + control_event_bootstrap_core(loglevel, status, progress); + + if (status > bootstrap_percent) { + bootstrap_phase = status; /* new milestone reached */ + bootstrap_percent = status; + } + if (progress > bootstrap_percent) { + /* incremental progress within a milestone */ + bootstrap_percent = progress; + bootstrap_problems = 0; /* Progress! Reset our problem counter. */ + } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } +} + +/** Flag whether we've opened an OR_CONN yet */ +static int bootstrap_first_orconn = 0; + +/** Like bootstrap_phase, but for (possibly deferred) directory progress */ +static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_problems, but for (possibly deferred) directory progress */ +static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + +/** Defer directory info bootstrap events until we have successfully + * completed our first connection to a router. */ +void +control_event_boot_dir(bootstrap_status_t status, int progress) +{ + if (status > bootstrap_dir_progress) { + bootstrap_dir_progress = status; + bootstrap_dir_phase = status; + } + if (progress && progress >= bootstrap_dir_progress) { + bootstrap_dir_progress = progress; + } + + /* Don't report unless we have successfully opened at least one OR_CONN */ + if (!bootstrap_first_orconn) + return; + + control_event_bootstrap(status, progress); +} + +/** Set a flag to allow reporting of directory bootstrap progress. + * (Code that reports completion of an OR_CONN calls this.) Also, + * report directory progress so far. */ +void +control_event_boot_first_orconn(void) +{ + bootstrap_first_orconn = 1; + control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. warn gives a human-readable hint + * as to why, and reason provides a controller-facing short + * tag. conn is the connection that caused this problem and + * can be NULL if a connection cannot be easily identified. + */ +void +control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn) +{ + int status = bootstrap_percent; + const char *tag = "", *summary = ""; + char buf[BOOTSTRAP_MSG_LEN]; + const char *recommendation = "ignore"; + int severity; + char *or_id = NULL, *hostaddr = NULL; + or_connection_t *or_conn = NULL; + + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + + if (bootstrap_percent == 100) + return; /* already bootstrapped; nothing to be done here. */ + + bootstrap_problems++; + + if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) + dowarn = 1; + + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ + if (we_are_hibernating()) + dowarn = 0; + + tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); + + severity = dowarn ? LOG_WARN : LOG_INFO; + + if (dowarn) + recommendation = "warn"; + + if (conn && conn->type == CONN_TYPE_OR) { + /* XXX TO_OR_CONN can't deal with const */ + or_conn = TO_OR_CONN((connection_t *)conn); + or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); + } else { + or_id = tor_strdup("?"); + } + + if (conn) + tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); + else + hostaddr = tor_strdup("?"); + + log_fn(severity, + LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " + "count %d; recommendation %s; host %s at %s)", + status, summary, warn, reason, + bootstrap_problems, recommendation, + or_id, hostaddr); + + connection_or_report_broken_states(severity, LD_HANDSHAKE); + + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", + bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, + recommendation, + or_id, hostaddr); + + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "WARN %s", buf); + control_event_client_status(LOG_WARN, "%s", buf); + + tor_free(hostaddr); + tor_free(or_id); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. warn gives a hint as to why, and + * reason provides an "or_conn_end_reason" tag. or_conn + * is the connection that caused this problem. + */ +MOCK_IMPL(void, +control_event_bootstrap_prob_or, (const char *warn, int reason, + or_connection_t *or_conn)) +{ + int dowarn = 0; + + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + + if (reason == END_OR_CONN_REASON_NO_ROUTE) + dowarn = 1; + + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) + dowarn = 1; + + control_event_bootstrap_problem(warn, + orconn_end_reason_to_control_string(reason), + TO_CONN(or_conn), dowarn); +} + +/** Return a copy of the last sent bootstrap message. */ +char * +control_event_boot_last_msg(void) +{ + return tor_strdup(last_sent_bootstrap_message); +} + +/** Reset bootstrap tracking state. */ +void +control_event_bootstrap_reset(void) +{ + bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + notice_bootstrap_percent = 0; + bootstrap_problems = 0; + bootstrap_first_orconn = 0; + bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); +} -- GitLab From 7685f8ad35f8662e8af149ae4e530677646f88a0 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 19 Nov 2018 19:43:23 -0600 Subject: [PATCH 0220/1724] Use table lookup for bootstrap_status_to_string It also no longer distinguishes the case of internal-only paths, which was often wrong anyway. Closes ticket 27402. --- changes/ticket27402 | 10 ++ src/feature/control/control_bootstrap.c | 129 ++++++++---------------- 2 files changed, 52 insertions(+), 87 deletions(-) create mode 100644 changes/ticket27402 diff --git a/changes/ticket27402 b/changes/ticket27402 new file mode 100644 index 0000000000..b79fb56760 --- /dev/null +++ b/changes/ticket27402 @@ -0,0 +1,10 @@ + o Minor feature (bootstrap): + - When reporting bootstrap progress, stop distinguishing between + situations where it seems that only internal paths are available + and situations where it seems that external paths are available. + Previously, tor would often erroneously report that it had only + internal paths. Closes ticket 27402. + + o Code simplification and refactoring: + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 58b342c219..7299757f91 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -16,7 +16,6 @@ #include "core/or/reasons.h" #include "feature/control/control.h" #include "feature/hibernate/hibernate.h" -#include "feature/nodelist/nodelist.h" #include "lib/malloc/malloc.h" /** A sufficiently large size to record the last bootstrap phase string. */ @@ -26,99 +25,55 @@ * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; +/** Table to convert bootstrap statuses to strings. */ +static const struct { + bootstrap_status_t status; + const char *tag; + const char *summary; +} boot_to_str_tab[] = { + { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, + { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, + { BOOTSTRAP_STATUS_CONN_DIR, "conn_dir", "Connecting to directory server" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "status_handshake", "Finishing handshake" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DIR, "handshake_dir", + "Finishing handshake with directory server" }, + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", + "Establishing an encrypted directory connection" }, + { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", + "Asking for networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status", + "Loading networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys", + "Loading authority key certs" }, + { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors", + "Asking for relay descriptors" }, + { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", + "Loading relay descriptors" }, + { BOOTSTRAP_STATUS_CONN_OR, "conn_or", "Connecting to the Tor network" }, + { BOOTSTRAP_STATUS_HANDSHAKE_OR, "handshake_or", + "Finishing handshake with first hop" }, + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", + "Establishing a Tor circuit" }, + { BOOTSTRAP_STATUS_DONE, "done", "Done" }, +}; +#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0])) + /** Convert the name of a bootstrapping phase s into strings * tag and summary suitable for display by the controller. */ static int bootstrap_status_to_string(bootstrap_status_t s, const char **tag, const char **summary) { - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; + for (size_t i = 0; i < N_BOOT_TO_STR; i++) { + if (s == boot_to_str_tab[i].status) { + *tag = boot_to_str_tab[i].tag; + *summary = boot_to_str_tab[i].summary; + return 0; + } } - return 0; + + *tag = *summary = "unknown"; + return -1; } /** What percentage through the bootstrap process are we? We remember -- GitLab From 701eaef980de4f7dbb5c31c4fee9b7e1e266d7a1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 1 Dec 2018 11:36:03 -0500 Subject: [PATCH 0221/1724] Move net.inet.ip.random_id code to lib/net/ --- src/app/config/config.c | 21 +-------------------- src/lib/net/socket.c | 29 +++++++++++++++++++++++++++++ src/lib/net/socket.h | 1 + 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index d907b07136..50f3793d6c 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -157,10 +157,6 @@ #include "core/or/connection_st.h" #include "core/or/port_cfg_st.h" -#ifdef __FreeBSD__ -#include -#endif - #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -3386,22 +3382,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); -#ifdef __FreeBSD__ - if (server_mode(options)) { - int random_id_state; - size_t state_size = sizeof(random_id_state); - - if (sysctlbyname("net.inet.ip.random_id", &random_id_state, - &state_size, NULL, 0)) { - log_warn(LD_CONFIG, - "Failed to figure out if IP ids are randomized."); - } else if (random_id_state == 0) { - log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " - "Please consider setting the net.inet.ip.random_id sysctl, " - "so your relay makes it harder to figure out how busy it is."); - } - } -#endif + check_network_configuration(server_mode(options)); /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index cd7c9685cd..8940e00591 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include #include +#ifdef __FreeBSD__ +#include +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else + (void) server_mode; +#endif +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 2b87441fc6..822b9975e6 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); -- GitLab From a2bb172225e1eecc9e93da655a0d316c4b214ff6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 19:38:30 +0300 Subject: [PATCH 0222/1724] tor-resolve: generate SOCKS4a request with trunnel --- src/tools/tor-resolve.c | 75 ++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 95411d173d..4ba372b66c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,6 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" +#include "socks5.h" #include #include @@ -49,11 +50,68 @@ static void usage(void) ATTR_NORETURN; +/** + * Set out to a pointer to newly allocated buffer containing + * SOCKS4a RESOLVE request with username and hostname. + * Return number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks4a_resolve_request(uint8_t **out, + const char *username, + const char *hostname) +{ + tor_assert(out); + tor_assert(username); + tor_assert(hostname); + + const char *errmsg = NULL; + uint8_t *output = NULL; + socks4_client_request_t *rq = socks4_client_request_new(); + + socks4_client_request_set_version(rq, 4); + socks4_client_request_set_command(rq, CMD_RESOLVE); + socks4_client_request_set_port(rq, 0); + socks4_client_request_set_addr(rq, 0x00000001u); + socks4_client_request_set_username(rq, username); + socks4_client_request_set_socks4a_addr_hostname(rq, hostname); + + errmsg = socks4_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks4_client_request_encoded_len(rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encoded_len failed"; + goto cleanup; + } + + output = tor_malloc(encoded_len); + memset(output, 0, encoded_len); + + encoded_len = socks4_client_request_encode(output, encoded_len, rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encode failed"; + goto cleanup; + } + + *out = output; + + cleanup: + socks4_client_request_free(rq); + if (errmsg) { + log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg); + *out = NULL; + tor_free(output); + } + return errmsg ? -1 : encoded_len; +} + /** Set *out to a newly allocated SOCKS4a resolve request with * username and hostname as provided. Return the number * of bytes in the request. */ static ssize_t -build_socks_resolve_request(char **out, +build_socks_resolve_request(uint8_t **out, const char *username, const char *hostname, int reverse, @@ -65,14 +123,7 @@ build_socks_resolve_request(char **out, tor_assert(hostname); if (version == 4) { - len = 8 + strlen(username) + 1 + strlen(hostname) + 1; - *out = tor_malloc(len); - (*out)[0] = 4; /* SOCKS version 4 */ - (*out)[1] = '\xF0'; /* Command: resolve. */ - set_uint16((*out)+2, htons(0)); /* port: 0. */ - set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */ - memcpy((*out)+8, username, strlen(username)+1); - memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1); + return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { int is_ip_address; tor_addr_t addr; @@ -205,7 +256,7 @@ do_resolve(const char *hostname, int s = -1; struct sockaddr_storage ss; socklen_t socklen; - char *req = NULL; + uint8_t *req = NULL; ssize_t len = 0; tor_assert(hostname); @@ -231,7 +282,7 @@ do_resolve(const char *hostname, if (version == 5) { char method_buf[2]; - if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) { + if (write_all_to_socket(s, (const char *)"\x05\x01\x00", 3) != 3) { log_err(LD_NET, "Error sending SOCKS5 method list."); goto err; } @@ -257,7 +308,7 @@ do_resolve(const char *hostname, tor_assert(!req); goto err; } - if (write_all_to_socket(s, req, len) != len) { + if (write_all_to_socket(s, (const char *)req, len) != len) { log_sock_error("sending SOCKS request", s); tor_free(req); goto err; -- GitLab From 30582b940e22897e68c50d4d0c9c36f82472e9b2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 19:39:00 +0300 Subject: [PATCH 0223/1724] tor-resolve: link tor-resolve binary with trunnel lib --- src/tools/include.am | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tools/include.am b/src/tools/include.am index f7aa7e0d1e..352d4b5328 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c -src_tools_tor_resolve_LDFLAGS = +src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ + $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ -- GitLab From 83af6d6149bb589be01a14b1d800614c1beb9f18 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 21:42:38 +0300 Subject: [PATCH 0224/1724] tor-resolve: Use trunnel code for SOCKS5 request generation --- src/tools/tor-resolve.c | 158 +++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 36 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 4ba372b66c..c255a71123 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -107,6 +107,123 @@ build_socks4a_resolve_request(uint8_t **out, return errmsg ? -1 : encoded_len; } +#define SOCKS5_ATYPE_HOSTNAME 0x03 +#define SOCKS5_ATYPE_IPV4 0x01 +#define SOCKS5_ATYPE_IPV6 0x04 + +/** + * Set out to pointer to newly allocated buffer containing + * SOCKS5 RESOLVE/RESOLVE_PTR request with given hostname. + * Generate a reverse request if reverse is true. + * Return the number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks5_resolve_request(uint8_t **out, + const char *hostname, + int reverse) +{ + const char *errmsg = NULL; + uint8_t *outbuf = NULL; + int is_ip_address; + tor_addr_t addr; + size_t addrlen; + int ipv6; + is_ip_address = tor_addr_parse(&addr, hostname) != -1; + if (!is_ip_address && reverse) { + log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); + return -1; + } + ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; + addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); + if (addrlen > UINT8_MAX) { + log_err(LD_GENERAL, "Hostname is too long!"); + return -1; + } + + socks5_client_request_t *rq = socks5_client_request_new(); + + socks5_client_request_set_version(rq, 5); + /* RESOLVE_PTR or RESOLVE */ + socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR : + CMD_RESOLVE); + socks5_client_request_set_reserved(rq, 0); + + uint8_t atype = SOCKS5_ATYPE_HOSTNAME; + if (reverse) + atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4; + + socks5_client_request_set_atype(rq, atype); + + switch (atype) { + case SOCKS5_ATYPE_IPV4: { + socks5_client_request_set_dest_addr_ipv4(rq, + tor_addr_to_ipv4h(&addr)); + } break; + case SOCKS5_ATYPE_IPV6: { + uint8_t *ipv6_array = + socks5_client_request_getarray_dest_addr_ipv6(rq); + + tor_assert(ipv6_array); + + memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16); + } break; + + case SOCKS5_ATYPE_HOSTNAME: { + domainname_t *dn = domainname_new(); + domainname_set_len(dn, addrlen - 1); + domainname_setlen_name(dn, addrlen - 1); + char *dn_buf = domainname_getarray_name(dn); + memcpy(dn_buf, hostname, addrlen - 1); + + errmsg = domainname_check(dn); + + if (errmsg) { + domainname_free(dn); + goto cleanup; + } else { + socks5_client_request_set_dest_addr_domainname(rq, dn); + } + } break; + default: + tor_assert_unreached(); + break; + } + + socks5_client_request_set_dest_port(rq, 0); + + errmsg = socks5_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks5_client_request_encoded_len(rq); + if (encoded_len < 0) { + errmsg = "Cannot predict encoded length"; + goto cleanup; + } + + outbuf = tor_malloc(encoded_len); + memset(outbuf, 0, encoded_len); + + encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq); + if (encoded_len < 0) { + errmsg = "encoding failed"; + goto cleanup; + } + + *out = outbuf; + + cleanup: + socks5_client_request_free(rq); + if (errmsg) { + tor_free(outbuf); + log_err(LD_NET, "build_socks5_resolve_request failed with error: %s", + errmsg); + } + + return errmsg ? -1 : encoded_len; +} + /** Set *out to a newly allocated SOCKS4a resolve request with * username and hostname as provided. Return the number * of bytes in the request. */ @@ -117,51 +234,20 @@ build_socks_resolve_request(uint8_t **out, int reverse, int version) { - size_t len = 0; tor_assert(out); tor_assert(username); tor_assert(hostname); + tor_assert(version == 4 || version == 5); + if (version == 4) { return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { - int is_ip_address; - tor_addr_t addr; - size_t addrlen; - int ipv6; - is_ip_address = tor_addr_parse(&addr, hostname) != -1; - if (!is_ip_address && reverse) { - log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); - return -1; - } - ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; - addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); - if (addrlen > UINT8_MAX) { - log_err(LD_GENERAL, "Hostname is too long!"); - return -1; - } - len = 6 + addrlen; - *out = tor_malloc(len); - (*out)[0] = 5; /* SOCKS version 5 */ - (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */ - (*out)[2] = 0; /* reserved. */ - if (reverse) { - (*out)[3] = ipv6 ? 4 : 1; - if (ipv6) - memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16); - else - set_uint32((*out)+4, tor_addr_to_ipv4n(&addr)); - } else { - (*out)[3] = 3; - (*out)[4] = (char)(uint8_t)(addrlen - 1); - memcpy((*out)+5, hostname, addrlen - 1); - } - set_uint16((*out)+4+addrlen, 0); /* port */ - } else { - tor_assert(0); + return build_socks5_resolve_request(out, hostname, reverse); } - return len; + tor_assert_unreached(); + return -1; } static void -- GitLab From d49baa77b5586ecb473a822deb60dbe70c6769c8 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 11:18:43 +0300 Subject: [PATCH 0225/1724] Allow socks4_server_reply version to be 0 (for tor-resolve) --- src/trunnel/socks5.c | 13 ++++++------- src/trunnel/socks5.trunnel | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 9e5f6fcfed..057a52b042 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1694,7 +1694,6 @@ socks4_server_reply_new(void) socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t)); if (NULL == val) return NULL; - val->version = 4; return val; } @@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp) int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val) { - if (! ((val == 4))) { + if (! ((val == 0 || val == 4))) { TRUNNEL_SET_ERROR_CODE(inp); return -1; } @@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) return "Integer out of bounds"; return NULL; } @@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj) return -1; - /* Length of u8 version IN [4] */ + /* Length of u8 version IN [0, 4] */ result += 1; /* Length of u8 status */ @@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 version IN [4] */ + /* Encode u8 version IN [0, 4] */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, ssize_t result = 0; (void)result; - /* Parse u8 version IN [4] */ + /* Parse u8 version IN [0, 4] */ CHECK_REMAINING(1, truncated); obj->version = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) goto fail; /* Parse u8 status */ diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel index b86ec03b9d..17fa2ed996 100644 --- a/src/trunnel/socks5.trunnel +++ b/src/trunnel/socks5.trunnel @@ -86,7 +86,7 @@ struct socks4_client_request { } struct socks4_server_reply { - u8 version IN [4]; + u8 version IN [0,4]; u8 status; u16 port; u32 addr; -- GitLab From 1051969a1d7f642c9062afefd760577d0eda35f5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 11:30:01 +0300 Subject: [PATCH 0226/1724] tor-resolve: parse SOCKS4a reply --- src/tools/tor-resolve.c | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index c255a71123..659d511793 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -271,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname, const char *response, size_t len, tor_addr_t *addr_out) { + int result = 0; uint8_t status; tor_assert(response); tor_assert(addr_out); - if (len < RESPONSE_LEN_4) { + socks4_server_reply_t *reply; + + ssize_t parsed = socks4_server_reply_parse(&reply, + (const uint8_t *)response, + len); + + if (parsed == -1) { + log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response"); + result = -1; goto cleanup; + } + + if (parsed == -2) { log_warn(LD_PROTOCOL,"Truncated socks response."); - return -1; + result = -1; goto cleanup; } - if (((uint8_t)response[0])!=0) { /* version: 0 */ + + if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */ log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format."); - return -1; + result = -1; goto cleanup; } - status = (uint8_t)response[1]; - if (get_uint16(response+2)!=0) { /* port: 0 */ + if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */ log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format."); - return -1; + result = -1; goto cleanup; } + status = socks4_server_reply_get_status(reply); if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { onion_warning(hostname); - return -1; + result = -1; goto cleanup; } - return -1; + result = -1; goto cleanup; } - tor_addr_from_ipv4n(addr_out, get_uint32(response+4)); - return 0; + tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply)); + + cleanup: + socks4_server_reply_free(reply); + return result; } /* It would be nice to let someone know what SOCKS5 issue a user may have */ @@ -418,6 +434,7 @@ do_resolve(const char *hostname, log_err(LD_NET, "Error reading SOCKS5 response."); goto err; } + if (reply_buf[0] != 5) { log_err(LD_NET, "Bad SOCKS5 reply version."); goto err; -- GitLab From 8b9d6581f6d00ff1c5739174e6f2ea47b7a393a7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 13:12:51 +0300 Subject: [PATCH 0227/1724] tor-resolve: Rework SOCKS5 method negotiation client part with trunnel --- src/tools/tor-resolve.c | 50 ++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 659d511793..7aa3871b6d 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -383,23 +383,57 @@ do_resolve(const char *hostname, } if (version == 5) { - char method_buf[2]; - if (write_all_to_socket(s, (const char *)"\x05\x01\x00", 3) != 3) { + socks5_client_version_t *v = socks5_client_version_new(); + + socks5_client_version_set_version(v, 5); + socks5_client_version_set_n_methods(v, 1); + socks5_client_version_setlen_methods(v, 1); + socks5_client_version_set_methods(v, 0, 0x00); + + tor_assert(!socks5_client_version_check(v)); + ssize_t encoded_len = socks5_client_version_encoded_len(v); + tor_assert(encoded_len > 0); + + uint8_t *buf = tor_malloc(encoded_len); + encoded_len = socks5_client_version_encode(buf, encoded_len, v); + tor_assert(encoded_len > 0); + + socks5_client_version_free(v); + + if (write_all_to_socket(s, (const char *)buf, + encoded_len) != encoded_len) { log_err(LD_NET, "Error sending SOCKS5 method list."); + tor_free(buf); + goto err; } - if (read_all_from_socket(s, method_buf, 2) != 2) { + + tor_free(buf); + + uint8_t method_buf[2]; + + if (read_all_from_socket(s, (char *)method_buf, 2) != 2) { log_err(LD_NET, "Error reading SOCKS5 methods."); goto err; } - if (method_buf[0] != '\x05') { - log_err(LD_NET, "Unrecognized socks version: %u", - (unsigned)method_buf[0]); + + socks5_server_method_t *m; + ssize_t parsed = socks5_server_method_parse(&m, method_buf, + sizeof(method_buf)); + + if (parsed < 2) { + log_err(LD_NET, "Failed to parse SOCKS5 method selection " + "message"); goto err; } - if (method_buf[1] != '\x00') { + + uint8_t method = socks5_server_method_get_method(m); + + socks5_server_method_free(m); + + if (method != 0x00) { log_err(LD_NET, "Unrecognized socks authentication method: %u", - (unsigned)method_buf[1]); + (unsigned)method); goto err; } } -- GitLab From 39e158db365fe3c033c3f8f99a2de1bfed3b707a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 15:03:08 +0300 Subject: [PATCH 0228/1724] tor-resolve: Rework SOCKS5 response parsing with trunnel --- src/tools/tor-resolve.c | 78 +++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 7aa3871b6d..ba83f3e3b4 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,7 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" -#include "socks5.h" +#include "trunnel/socks5.h" #include #include @@ -463,56 +463,66 @@ do_resolve(const char *hostname, goto err; } } else { - char reply_buf[16]; - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading SOCKS5 response."); + uint8_t reply_buf[512]; + + len = read_all_from_socket(s, (char *)reply_buf, + sizeof(reply_buf)); + + socks5_server_reply_t *reply; + + ssize_t parsed = socks5_server_reply_parse(&reply, + reply_buf, + len); + if (parsed == -1) { + log_err(LD_NET, "Failed parsing SOCKS5 response"); goto err; } - if (reply_buf[0] != 5) { - log_err(LD_NET, "Bad SOCKS5 reply version."); + if (parsed == -2) { + log_err(LD_NET, "Truncated SOCKS5 response"); goto err; } + /* Give a user some useful feedback about SOCKS5 errors */ - if (reply_buf[1] != 0) { + uint8_t reply_field = socks5_server_reply_get_reply(reply); + if (reply_field != 0) { log_warn(LD_NET,"Got SOCKS5 status response '%u': %s", - (unsigned)reply_buf[1], - socks5_reason_to_string(reply_buf[1])); - if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) { + (unsigned)reply_field, + socks5_reason_to_string(reply_field)); + if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { onion_warning(hostname); } + + socks5_server_reply_free(reply); goto err; } - if (reply_buf[3] == 1) { + + uint8_t atype = socks5_server_reply_get_atype(reply); + + if (atype == SOCKS5_ATYPE_IPV4) { /* IPv4 address */ - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf)); - } else if (reply_buf[3] == 4) { + tor_addr_from_ipv4h(result_addr, + socks5_server_reply_get_bind_addr_ipv4(reply)); + } else if (atype == SOCKS5_ATYPE_IPV6) { /* IPv6 address */ - if (read_all_from_socket(s, reply_buf, 16) != 16) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv6_bytes(result_addr, reply_buf); - } else if (reply_buf[3] == 3) { + tor_addr_from_ipv6_bytes(result_addr, + (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply)); + } else if (atype == SOCKS5_ATYPE_HOSTNAME) { /* Domain name */ - size_t result_len; - if (read_all_from_socket(s, reply_buf, 1) != 1) { - log_err(LD_NET, "Error reading address_length in socks5 response."); - goto err; - } - result_len = *(uint8_t*)(reply_buf); + domainname_t *dn = + socks5_server_reply_get_bind_addr_domainname(reply); + + size_t result_len = (size_t)domainname_get_len(dn); + *result_hostname = tor_malloc(result_len+1); - if (read_all_from_socket(s, *result_hostname, result_len) - != (int) result_len) { - log_err(LD_NET, "Error reading hostname in socks5 response."); - goto err; - } + + strncpy(*result_hostname, (char *)domainname_getstr_name(dn), + result_len); + (*result_hostname)[result_len] = '\0'; } + + socks5_server_reply_free(reply); } tor_close_socket(s); -- GitLab From 353d2a091d245146576d33a703fe3674c29e0654 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 13 Sep 2018 09:57:43 +0300 Subject: [PATCH 0229/1724] Fix coverage build --- src/tools/include.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/include.am b/src/tools/include.am index 352d4b5328..72dfe6017c 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -17,8 +17,11 @@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_cov_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_TESTING_LIBS) \ + $(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif -- GitLab From 009205dabe4ba7aba43dffead4087ca94ef9a8e3 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 13 Sep 2018 10:18:06 +0300 Subject: [PATCH 0230/1724] Add changes file --- changes/ticket27620 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket27620 diff --git a/changes/ticket27620 b/changes/ticket27620 new file mode 100644 index 0000000000..6c491696d0 --- /dev/null +++ b/changes/ticket27620 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. -- GitLab From 822cb93cab59e9735e2efda70bc88c47cc92c498 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 26 Sep 2018 19:14:33 -0400 Subject: [PATCH 0231/1724] Add new option ClientAutoIPv6ORPort to switch between IPv4 and IPv6 OR ports --- changes/ticket27490 | 6 ++++++ doc/tor.1.txt | 6 ++++++ src/app/config/config.c | 1 + src/app/config/or_options_st.h | 3 +++ src/core/mainloop/connection.c | 5 +++++ src/core/or/policies.c | 15 ++++++++++++++- src/feature/client/bridges.c | 3 ++- 7 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 changes/ticket27490 diff --git a/changes/ticket27490 b/changes/ticket27490 new file mode 100644 index 0000000000..523477dfea --- /dev/null +++ b/changes/ticket27490 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - We add an option ClientAutoIPv6ORPort which makes clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + Closes ticket 27490. Patch by Neel Chauhan. + diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 406372433f..bd4dbbcbdb 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1748,6 +1748,12 @@ The following options are useful only for clients (that is, if other clients prefer IPv4. Other things may influence the choice. This option breaks a tie to the favor of IPv6. (Default: auto) +[[ClientAutoIPv6ORPort]] **ClientAutoIPv6ORPort** **0**|**1**:: + If this option is set to 1, Tor clients randomly prefer a node's IPv4 or + IPv6 ORPort. The random preference is set every time a node is loaded + from a new consensus or bridge config. When this option is set to 1, + **ClientPreferIPv6ORPort** is ignored. (Default: 0) + [[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__:: Tor clients don't build circuits for user traffic until they know about enough of the network so that they could potentially construct diff --git a/src/app/config/config.c b/src/app/config/config.c index 01b48e3c5f..6a510c56da 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -332,6 +332,7 @@ static config_var_t option_vars_[] = { V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), + V(ClientAutoIPv6ORPort, BOOL, "0"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 3524b99b53..ff3d30d7ec 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -666,6 +666,9 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, prefer an IPv4 or IPv6 OR port at random. */ + int ClientAutoIPv6ORPort; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 4231bec014..9f8169082c 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -2069,6 +2069,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn) return; } + if (fascist_firewall_use_ipv6(options)) { + log_info(LD_NET, "Our outgoing connection is using IPv%d.", + tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4); + } + /* Check if we couldn't satisfy an address family preference */ if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 3443a17107..e51a49cf60 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -28,6 +28,7 @@ #include "feature/nodelist/routerparse.h" #include "feature/stats/geoip.h" #include "ht.h" +#include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" #include "core/or/addr_policy_st.h" @@ -487,6 +488,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) return -1; } +/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address + * family. Return 0 for IPv4, and 1 for IPv6. */ +static int +fascist_firewall_rand_prefer_ipv6_addr(void) +{ + /* TODO: Check for failures, and infer our preference based on this. */ + return crypto_rand_int(2); +} + /** Do we prefer to connect to IPv6 ORPorts? * Use node_ipv6_or_preferred() whenever possible: it supports bridge client * per-node IPv6 preferences. @@ -501,7 +511,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options) } /* We can use both IPv4 and IPv6 - which do we prefer? */ - if (options->ClientPreferIPv6ORPort == 1) { + if (options->ClientAutoIPv6ORPort == 1) { + /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */ + return fascist_firewall_rand_prefer_ipv6_addr(); + } else if (options->ClientPreferIPv6ORPort == 1) { return 1; } diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index e8afb5a924..e3ed288415 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - if (options->ClientPreferIPv6ORPort == -1) { + if (options->ClientPreferIPv6ORPort == -1 || + options->ClientAutoIPv6ORPort == 0) { /* Mark which address to use based on which bridge_t we got. */ node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && !tor_addr_is_null(&node->ri->ipv6_addr)); -- GitLab From 81f2828d67f0853c028617511f3846c02f0d6628 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 26 Sep 2018 19:19:11 -0400 Subject: [PATCH 0232/1724] In fascist_firewall_use_ipv6(), say we can use IPv6 if ClientAutoIPv6ORPort is 1 --- src/core/or/policies.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index e51a49cf60..6da369bf36 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -461,7 +461,8 @@ fascist_firewall_use_ipv6(const or_options_t *options) * ClientPreferIPv6DirPort is deprecated, but check it anyway. */ return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 || options->ClientPreferIPv6ORPort == 1 || - options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1); + options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 || + options->ClientAutoIPv6ORPort == 1); } /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and -- GitLab From ad031b64ce355ac3af15adb716a4322858dce4a8 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Oct 2018 20:14:27 -0400 Subject: [PATCH 0233/1724] Add regression test for ClientAutoIPv6ORPort --- src/core/or/policies.c | 4 ++-- src/core/or/policies.h | 1 + src/test/test_policy.c | 50 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 6da369bf36..c630c2fbdc 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -491,8 +491,8 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) /* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address * family. Return 0 for IPv4, and 1 for IPv6. */ -static int -fascist_firewall_rand_prefer_ipv6_addr(void) +MOCK_IMPL(int, +fascist_firewall_rand_prefer_ipv6_addr, (void)) { /* TODO: Check for failures, and infer our preference based on this. */ return crypto_rand_int(2); diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 7da3ba031f..0c64ecf378 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -70,6 +70,7 @@ typedef struct short_policy_t { int firewall_is_fascist_or(void); int firewall_is_fascist_dir(void); int fascist_firewall_use_ipv6(const or_options_t *options); +MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void)); int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 6a07e5b1f8..cc9023d6a6 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -2023,6 +2023,20 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/** Mock the preferred address function to return zero (prefer IPv4). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) +{ + return 0; +} + +/** Mock the preferred address function to return one (prefer IPv6). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void) +{ + return 1; +} + /** Run unit tests for fascist_firewall_choose_address */ static void test_policies_fascist_firewall_choose_address(void *arg) @@ -2421,6 +2435,42 @@ test_policies_fascist_firewall_choose_address(void *arg) CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1, ipv4_dir_ap); + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv4_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv4_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv6_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv6_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + done: UNMOCK(get_options); } -- GitLab From 0015d0084262ddc60098b15d6cdf30ea66d59a53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 1 Dec 2018 20:46:06 -0500 Subject: [PATCH 0234/1724] Use tor_strdup() in place of malloc+strncpy+terminate. --- src/tools/tor-resolve.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index ba83f3e3b4..803ed26b3b 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -512,14 +512,7 @@ do_resolve(const char *hostname, domainname_t *dn = socks5_server_reply_get_bind_addr_domainname(reply); - size_t result_len = (size_t)domainname_get_len(dn); - - *result_hostname = tor_malloc(result_len+1); - - strncpy(*result_hostname, (char *)domainname_getstr_name(dn), - result_len); - - (*result_hostname)[result_len] = '\0'; + *result_hostname = tor_strdup(domainname_getstr_name(dn)); } socks5_server_reply_free(reply); -- GitLab From 612b21b8eadea9209b62877e5796d8b461916536 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Dec 2018 10:19:34 +1000 Subject: [PATCH 0235/1724] comment: replace cached-routers with cached-descriptors cached-routers has been gone for a long time --- src/feature/nodelist/routerlist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index a8c9003e68..b4d56459df 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0; * * From time to time, we replace "cached-descriptors" with a new file * containing only the live, non-superseded descriptors, and clear - * cached-routers.new. + * cached-descriptors.new. * * On startup, we read both files. */ -- GitLab From 90ce8d0a5896348bf0acf284e6e7fdef5b64e813 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Dec 2018 10:22:49 +1000 Subject: [PATCH 0236/1724] doc: remove cached-routers from the man page cached-routers was removed from the code a long time ago --- doc/tor.1.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index b147ad68aa..6c4746aa52 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -3251,10 +3251,6 @@ __CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**:: router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-microdescs file. -__CacheDirectory__**/cached-routers** and **cached-routers.new**:: - Obsolete versions of cached-descriptors and cached-descriptors.new. When - Tor can't find the newer files, it looks here instead. - __DataDirectory__**/state**:: A set of persistent key-value mappings. These are documented in the file. These include: -- GitLab From 9369152aae9527cc3764cac8688f258b11bd503d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:28:32 +0200 Subject: [PATCH 0237/1724] Check that new listener connection is actually listening --- src/core/mainloop/connection.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index eaad45c81a..66a8b5147f 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1460,6 +1460,19 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } + +#ifndef __APPLE__ + int value; + socklen_t len = sizeof(value); + + if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) { + if (value == 0) { + log_err(LD_NET, "Could not listen on %s - " + "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address); + goto err; + } + } +#endif /* __APPLE__ */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", -- GitLab From 25f3b8244582c49db71c618c3859d7e7a89cd762 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:40:37 +0200 Subject: [PATCH 0238/1724] More logging for #28229 --- src/core/mainloop/connection.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 66a8b5147f..9d5bad7d8d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -2907,6 +2907,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) retval = -1; #ifdef ENABLE_LISTENER_REBIND + if (smartlist_len(replacements)) + log_debug(LD_NET, "%d replacements - starting rebinding loop.", + smartlist_len(replacements)); + SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) { int addr_in_use = 0; int skip = 0; @@ -2918,8 +2922,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) connection_listener_new_for_port(r->new_port, &skip, &addr_in_use); connection_t *old_conn = r->old_conn; - if (skip) + if (skip) { + log_debug(LD_NET, "Skipping creating new listener for %s:%d", + old_conn->address, old_conn->port); continue; + } connection_close_immediate(old_conn); connection_mark_for_close(old_conn); -- GitLab From db9ab3754a0a45fb421211ea9b815b4449999c66 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:49:33 +0200 Subject: [PATCH 0239/1724] Print error message we get from socket.connect_ex when it fails --- src/test/test_rebind.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index 6f759b717e..7d0ccaf628 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -19,9 +19,10 @@ def fail(msg): def try_connecting_to_socksport(): socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if socks_socket.connect_ex(('127.0.0.1', socks_port)): + e = socks_socket.connect_ex(('127.0.0.1', socks_port)) + if e: tor_process.terminate() - fail('Cannot connect to SOCKSPort') + fail('Cannot connect to SOCKSPort: error ' + os.strerror(e)) socks_socket.close() def wait_for_log(s): -- GitLab From 894d207f844cf5fa5816432abfe8d1c02763cd59 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 15:19:33 +0200 Subject: [PATCH 0240/1724] manpage: Mention that adding new HS is unsupported if Sandbox is enabled --- changes/doc28560 | 3 +++ doc/tor.1.txt | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changes/doc28560 diff --git a/changes/doc28560 b/changes/doc28560 new file mode 100644 index 0000000000..c3356bda0a --- /dev/null +++ b/changes/doc28560 @@ -0,0 +1,3 @@ + o Documentation (hidden service manpage, sandbox): + - Mention that you cannot add new Onion Service if Tor is already + running with Sandbox enabled. Closes ticket 28560. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8e6ec7f1a2..dbfb6358c4 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -610,7 +610,8 @@ GENERAL OPTIONS Logs ServerDNSResolvConfFile Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). + ORPort are not allowed). Launching new Onion Services through Control + Port is not supported with current syscall sandboxing implementation. ClientOnionAuthDir and any files in it won't reload on HUP signal. (Default: 0) @@ -2821,6 +2822,8 @@ The following options are used to configure a hidden service. Store data files for a hidden service in DIRECTORY. Every hidden service must have a separate directory. You may use this option multiple times to specify multiple services. If DIRECTORY does not exist, Tor will create it. + Please note that you cannot add new Onion Service to already running Tor + instance if **Sandbox** is enabled. (Note: in current versions of Tor, if DIRECTORY is a relative path, it will be relative to the current working directory of Tor instance, not to its DataDirectory. Do not -- GitLab From 32213fa9ad8a306e1f3bade1c95b1ad95a136bd9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 12:18:45 -0500 Subject: [PATCH 0241/1724] Keep list of dirauth flags in sync between dirvote.c and fuzz_vrs.c Suggested by Teor on PR --- src/feature/dirauth/dirvote.c | 20 ++++++++++++++++++-- src/feature/dirauth/dirvote.h | 3 +++ src/test/fuzz/fuzz_vrs.c | 8 ++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index aa4242f678..c6a2193087 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -4372,6 +4372,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * forget to add it to this clause. */ } +/** Space-separated list of all the flags that we will always vote on. */ +const char DIRVOTE_UNIVERSAL_FLAGS[] = + "Authority " + "Exit " + "Fast " + "Guard " + "HSDir " + "Stable " + "StaleDesc " + "V2Dir " + "Valid"; +/** Space-separated list of all flags that we may or may not vote on, + * depending on our configuration. */ +const char DIRVOTE_OPTIONAL_FLAGS[] = + "BadExit " + "Running"; + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -4620,8 +4637,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir " - "StaleDesc", + DIRVOTE_UNIVERSAL_FLAGS, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a21e9f3455..f99cc2d2bf 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -92,6 +92,9 @@ /** Maximum size of a line in a vote. */ #define MAX_BW_FILE_HEADERS_LINE_LEN 1024 +extern const char DIRVOTE_UNIVERSAL_FLAGS[]; +extern const char DIRVOTE_OPTIONAL_FLAGS[]; + /* * Public API. Used outside of the dirauth subsystem. * diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 3c6d205a3f..f0d90d7cc6 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -3,6 +3,7 @@ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE #include "core/or/or.h" +#include "feature/dirauth/dirvote.h" #include "feature/dirparse/ns_parse.h" #include "feature/dirparse/unparseable.h" #include "lib/memarea/memarea.h" @@ -35,9 +36,12 @@ fuzz_init(void) dummy_vote = tor_malloc_zero(sizeof(*dummy_vote)); dummy_vote->known_flags = smartlist_new(); smartlist_split_string(dummy_vote->known_flags, - "Authority BadExit Exit Fast Guard HSDir " - "NoEdConsensus Running Stable V2Dir Valid", + DIRVOTE_UNIVERSAL_FLAGS, " ", 0, 0); + smartlist_split_string(dummy_vote->known_flags, + DIRVOTE_OPTIONAL_FLAGS, + " ", 0, 0); + smartlist_sort_strings(dummy_vote->known_flags); return 0; } -- GitLab From 417a324a8577b0c61185c9d06eb72fbc483d984b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 12:34:29 -0500 Subject: [PATCH 0242/1724] Make input argument const in set_routerstatus_from_routerinfo. --- src/feature/dirauth/voteflags.c | 4 ++-- src/feature/dirauth/voteflags.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 5adf21ad11..aab322d96f 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -95,7 +95,7 @@ real_uptime(const routerinfo_t *router, time_t now) */ static int dirserv_thinks_router_is_unreliable(time_t now, - routerinfo_t *router, + const routerinfo_t *router, int need_uptime, int need_capacity) { if (need_uptime) { @@ -541,7 +541,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, + const routerinfo_t *ri, time_t now, int listbadexits) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 743a666cc8..8dce9fbb04 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -19,7 +19,8 @@ int running_long_enough_to_decide_unreachable(void); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, time_t now, + const routerinfo_t *ri, + time_t now, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); -- GitLab From 92af8e511394685f7a39224ec1f73ce79fcf2c42 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 13:22:23 -0500 Subject: [PATCH 0243/1724] Add a framework for testing set_routerstatus_from_routerinfo(). Additionally, use it to test that is_staledesc is set correctly. Eventually we'll want to test all the other flags, but I'm aiming for only adding coverage on the changed code here. --- src/test/include.am | 1 + src/test/test.c | 3 +- src/test/test.h | 1 + src/test/test_voting_flags.c | 192 +++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 src/test/test_voting_flags.c diff --git a/src/test/include.am b/src/test/include.am index e5eae56e25..d0f71fa666 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -177,6 +177,7 @@ src_test_test_SOURCES += \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_flags.c \ src/test/test_voting_schedule.c \ src/test/test_x509.c \ src/test/test_helpers.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..bbb00da788 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -866,7 +866,8 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "dir/", dir_tests }, { "dir/md/", microdesc_tests }, - { "dir/voting-schedule/", voting_schedule_tests }, + { "dir/voting/flags/", voting_flags_tests }, + { "dir/voting/schedule/", voting_schedule_tests }, { "dir_handle_get/", dir_handle_get_tests }, { "dns/", dns_tests }, { "dos/", dos_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..1d8f79717f 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -265,6 +265,7 @@ extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; extern struct testcase_t util_tests[]; +extern struct testcase_t voting_flags_tests[]; extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t x509_tests[]; diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c new file mode 100644 index 0000000000..0c4cedb516 --- /dev/null +++ b/src/test/test_voting_flags.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define VOTEFLAGS_PRIVATE + +#include "core/or/or.h" + +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "app/config/config.h" + +#include "test/test.h" + +typedef struct { + time_t now; + routerinfo_t ri; + node_t node; + + routerstatus_t expected; +} flag_vote_test_cfg_t; + +static void +setup_cfg(flag_vote_test_cfg_t *c) +{ + memset(c, 0, sizeof(*c)); + + c->now = approx_time(); + + c->ri.nickname = (char *) "testing100"; + strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); + + memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST256_LEN); + + c->ri.cache_info.published_on = c->now - 100; + c->expected.published_on = c->now - 100; + + c->ri.addr = 0x7f010105; + c->expected.addr = 0x7f010105; + c->ri.or_port = 9090; + c->expected.or_port = 9090; + + tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6); + tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6); + + // By default we have no loaded information about stability or speed, + // so we'll default to voting "yeah sure." on these two. + c->expected.is_fast = 1; + c->expected.is_stable = 1; +} + +static bool +check_result(flag_vote_test_cfg_t *c) +{ + bool result = false; + routerstatus_t rs; + memset(&rs, 0, sizeof(rs)); + set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + + tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); + tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); + + // identity_digest and descriptor_digest are not set here. + + tt_uint_op(rs.addr, OP_EQ, c->expected.addr); + tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port); + tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port); + + tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr)); + tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport); + +#define FLAG(flagname) \ + tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname) + + FLAG(is_authority); + FLAG(is_exit); + FLAG(is_stable); + FLAG(is_fast); + FLAG(is_flagged_running); + FLAG(is_named); + FLAG(is_unnamed); + FLAG(is_valid); + FLAG(is_possible_guard); + FLAG(is_bad_exit); + FLAG(is_hs_dir); + FLAG(is_v2_dir); + FLAG(is_staledesc); + FLAG(has_bandwidth); + FLAG(has_exitsummary); + FLAG(bw_is_unmeasured); + + result = true; + + done: + return result; +} + +static void +test_voting_flags_minimal(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + check_result(cfg); +} + +static void +test_voting_flags_ipv6(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + + tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6); + cfg->ri.ipv6_orport = 9091; + // no change in expected results, since we aren't set up with ipv6 + // connectivity. + if (!check_result(cfg)) + goto done; + + get_options_mutable()->AuthDirHasIPv6Connectivity = 1; + // no change in expected results, since last_reachable6 won't be set. + if (!check_result(cfg)) + goto done; + + cfg->node.last_reachable6 = cfg->now - 10; + // now that lastreachable6 is set, we expect to see the result. + tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6); + cfg->expected.ipv6_orport = 9091; + if (!check_result(cfg)) + goto done; + done: + ; +} + +static void +test_voting_flags_staledesc(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + time_t now = cfg->now; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10; + // no change in expectations for is_staledesc + if (!check_result(cfg)) + goto done; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.is_staledesc = 1; + if (!check_result(cfg)) + goto done; + + done: + ; +} + +static void * +setup_voting_flags_test(const struct testcase_t *testcase) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + setup_cfg(cfg); + return cfg; +} + +static int +teardown_voting_flags_test(const struct testcase_t *testcase, void *arg) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = arg; + tor_free(cfg); + return 1; +} + +static const struct testcase_setup_t voting_flags_setup = { + .setup_fn = setup_voting_flags_test, + .cleanup_fn = teardown_voting_flags_test, +}; + +#define T(name,flags) \ + { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL } + +struct testcase_t voting_flags_tests[] = { + T(minimal, 0), + T(ipv6, TT_FORK), + // TODO: Add more of these tests. + T(staledesc, TT_FORK), + END_OF_TESTCASES +}; + -- GitLab From 31a6d9f49997cfb1266a55b742a15706ae16db5e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 13:40:08 -0500 Subject: [PATCH 0244/1724] Add tests for parsing each routerstatus flag. --- src/test/test_dir.c | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 26ba269abd..5cdbd877ce 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -6052,6 +6052,80 @@ test_dir_find_dl_min_delay(void* data) mock_options = NULL; } +static void +test_dir_matching_flags(void *arg) +{ + (void) arg; + routerstatus_t *rs_noflags = NULL; + routerstatus_t *rs = NULL; + char *s = NULL; + + smartlist_t *tokens = smartlist_new(); + memarea_t *area = memarea_new(); + + int expected_val_when_unused = 0; + + const char *ex_noflags = + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " + "192.168.0.1 9001 0\n" + "m thisoneislongerbecauseitisa256bitmddigest33\n" + "s\n"; + const char *cp = ex_noflags; + rs_noflags = routerstatus_parse_entry_from_string( + area, &cp, + cp + strlen(cp), + tokens, NULL, NULL, + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); + tt_assert(rs_noflags); + +#define FLAG(string, field) STMT_BEGIN { \ + tor_asprintf(&s,\ + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \ + "192.168.0.1 9001 0\n" \ + "m thisoneislongerbecauseitisa256bitmddigest33\n" \ + "s %s\n", string); \ + cp = s; \ + rs = routerstatus_parse_entry_from_string( \ + area, &cp, \ + cp + strlen(cp), \ + tokens, NULL, NULL, \ + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); \ + /* the field should usually be 0 when no flags are listed */ \ + tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused); \ + /* the field should be 1 when this flags islisted */ \ + tt_int_op(rs->field, OP_EQ, 1); \ + tor_free(s); \ + routerstatus_free(rs); \ +} STMT_END + + FLAG("Authority", is_authority); + FLAG("BadExit", is_bad_exit); + FLAG("Exit", is_exit); + FLAG("Fast", is_fast); + FLAG("Guard", is_possible_guard); + FLAG("HSDir", is_hs_dir); + FLAG("Stable", is_stable); + FLAG("StaleDesc", is_staledesc); + FLAG("V2Dir", is_v2_dir); + + // These flags are assumed to be set whether they're declared or not. + expected_val_when_unused = 1; + FLAG("Running", is_flagged_running); + FLAG("Valid", is_valid); + expected_val_when_unused = 0; + + // These flags are no longer used, but still parsed. + FLAG("Named", is_named); + FLAG("Unnamed", is_unnamed); + + done: + tor_free(s); + routerstatus_free(rs); + routerstatus_free(rs_noflags); + memarea_drop_all(area); + smartlist_free(tokens); +} + static void test_dir_assumed_flags(void *arg) { @@ -6377,6 +6451,7 @@ struct testcase_t dir_tests[] = { DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), + DIR(matching_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), DIR(networkstatus_consensus_has_ipv6, TT_FORK), -- GitLab From b25b8150c282c28d3fa23330891fda7adbbfe584 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 11:56:05 -0500 Subject: [PATCH 0245/1724] Remember in our state file how long we've spent since user activity Rather than initializing the "Dormant" status to "off" and the "last activity" count to "now", initialize them based on our state file: stay dormant if we were dormant, or remember the amount of time we've spent inactive. --- src/app/config/or_state_st.h | 7 ++++++ src/app/config/statefile.c | 8 +++++++ src/core/mainloop/mainloop.c | 7 +----- src/core/mainloop/netstatus.c | 41 +++++++++++++++++++++++++++++++++++ src/core/mainloop/netstatus.h | 3 +++ 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index d95df6236b..00968d3731 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -87,6 +87,13 @@ struct or_state_t { /** When did we last rotate our onion key? "0" for 'no idea'. */ time_t LastRotatedOnionKey; + + /** Number of minutes since the last user-initiated request (as defined by + * the dormant net-status system.) Set to zero if we are dormant. */ + int MinutesSinceUserActivity; + /** True if we were dormant when we last wrote the file; false if we + * weren't. "auto" on initial startup. */ + int Dormant; }; #endif diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 4ba7be1519..97b96f1149 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -34,6 +34,7 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "feature/client/entrynodes.h" @@ -132,6 +133,9 @@ static config_var_t state_vars_[] = { VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + V(MinutesSinceUserActivity, UINT, NULL), + V(Dormant, AUTOBOOL, "auto"), + END_OF_CONFIG_VARS }; @@ -309,6 +313,8 @@ or_state_set(or_state_t *new_state) get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } + netstatus_load_from_state(global_state, time(NULL)); + return ret; } @@ -500,6 +506,8 @@ or_state_save(time_t now) entry_guards_update_state(global_state); rep_hist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); + netstatus_flush_to_state(global_state, now); + if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2e2ae876d4..cd955e0ca0 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1627,6 +1627,7 @@ schedule_rescan_periodic_events,(void)) void rescan_periodic_events(const or_options_t *options) { + puts("RESCAN"); tor_assert(options); /* Avoid scanning the event list if we haven't initialized it yet. This is @@ -2818,12 +2819,6 @@ initialize_mainloop_events(void) int do_main_loop(void) { - /* For now, starting Tor always counts as user activity. Later, we might - * have an option to control this. - */ - reset_user_activity(approx_time()); - set_network_participation(true); - /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index ed7c952dcd..59fd8f8037 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -10,6 +10,8 @@ #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "app/config/or_state_st.h" + /** Return true iff our network is in some sense disabled or shutting down: * either we're hibernating, entering hibernation, or the network is turned * off with DisableNetwork. */ @@ -31,6 +33,10 @@ net_is_completely_disabled(void) /** * The time at which we've last seen "user activity" -- that is, any activity * that should keep us as a participant on the network. + * + * This is not actually the true time. We will adjust this forward if + * our clock jumps, or if Tor is shut down for a while, so that the time + * since our last activity remains as it was before the jump or shutdown. */ static time_t last_user_activity_seen = 0; @@ -99,3 +105,38 @@ is_participating_on_network(void) { return participating_on_network; } + +/** + * Update 'state' with the last time at which we were active on the network. + **/ +void +netstatus_flush_to_state(or_state_t *state, time_t now) +{ + state->Dormant = ! participating_on_network; + if (participating_on_network) { + time_t sec_since_activity = MAX(0, now - last_user_activity_seen); + state->MinutesSinceUserActivity = (int)(sec_since_activity / 60); + } else { + state->MinutesSinceUserActivity = 0; + } +} + +/** + * Update our current view of network participation from an or_state_t object. + **/ +void +netstatus_load_from_state(const or_state_t *state, time_t now) +{ + time_t last_activity; + if (state->Dormant == -1) { // Initial setup. + last_activity = now; + participating_on_network = true; + } else if (state->Dormant) { + last_activity = 0; + participating_on_network = false; + } else { + last_activity = now - 60 * state->MinutesSinceUserActivity; + participating_on_network = true; + } + reset_user_activity(last_activity); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index 58c994fd14..4b008e4cfa 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -17,4 +17,7 @@ time_t get_last_user_activity_time(void); void set_network_participation(bool participation); bool is_participating_on_network(void); +void netstatus_flush_to_state(or_state_t *state, time_t now); +void netstatus_load_from_state(const or_state_t *state, time_t now); + #endif -- GitLab From 4f558843151f0c49c077d3d9a1cd8a187904a024 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:07:23 -0500 Subject: [PATCH 0246/1724] Add an option to start tor in dormant mode for the first time. --- doc/tor.1.txt | 11 ++++++++++- src/app/config/config.c | 1 + src/app/config/or_options_st.h | 4 ++++ src/core/mainloop/netstatus.c | 10 ++++++++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8e6ec7f1a2..82c9e34de4 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1803,11 +1803,20 @@ The following options are useful only for clients (that is, if Does not affect servers or onion services. Must be at least 10 minutes. (Default: 24 hours) -[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams **0**|**1**:: +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams** **0**|**1**:: If true, then any open client stream (even one not reading or writing) counts as client activity for the purpose of DormantClientTimeout. If false, then only network activity counts. (Default: 1) +[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**:: + If true, then the first time Tor starts up with a fresh DataDirectory, + it starts in dormant mode, and takes no actions until the user has made + a request. (This mode is recommended if installing a Tor client for a + user who might not actually use it.) If false, Tor bootstraps the first + time it is started, whether it sees a user request or not. + + + After the first time Tor starts, it begins in dormant mode if it was + dormant before, and not otherwise. (Default: 0) SERVER OPTIONS -------------- diff --git a/src/app/config/config.c b/src/app/config/config.c index d40e362b32..dcefa3d6a4 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -392,6 +392,7 @@ static config_var_t option_vars_[] = { OBSOLETE("DNSListenAddress"), V(DormantClientTimeout, INTERVAL, "24 hours"), V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), + V(DormantOnFirstStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 9065248a9c..c2bc1079a5 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1085,6 +1085,10 @@ struct or_options_t { * from becoming dormant. **/ int DormantTimeoutDisabledByIdleStreams; + + /** Boolean: true if Tor should be dormant the first time it starts with + * a datadirectory; false otherwise. */ + int DormantOnFirstStartup; }; #endif diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 59fd8f8037..2426baae34 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -129,8 +129,14 @@ netstatus_load_from_state(const or_state_t *state, time_t now) { time_t last_activity; if (state->Dormant == -1) { // Initial setup. - last_activity = now; - participating_on_network = true; + if (get_options()->DormantOnFirstStartup) { + last_activity = 0; + participating_on_network = false; + } else { + // Start up as active, treat activity as happening now. + last_activity = now; + participating_on_network = true; + } } else if (state->Dormant) { last_activity = 0; participating_on_network = false; -- GitLab From 1f6d7bc4afb5dd00c84c21c9323c2e2109f2378e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 16 Nov 2018 11:51:58 -0500 Subject: [PATCH 0247/1724] Update ReleasingTor.md to reflect current practice See #28479. --- doc/HACKING/ReleasingTor.md | 109 ++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index b5444afa96..3073cfb108 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -20,30 +20,29 @@ new Tor release: === I. Make sure it works -1. Use it for a while, as a client, as a relay, as a hidden service, - and as a directory authority. See if it has any obvious bugs, and - resolve those. +1. Make sure that CI passes: have a look at Travis, Appveyor, and + Jenkins. Make sure you're looking at the right branches. - As applicable, merge the `maint-X` branch into the `release-X` branch. - But you've been doing that all along, right? + If there are any unexplained failures, try to fix them or figure them + out. -2. Are all of the jenkins builders happy? See jenkins.torproject.org. +2. Verify that there are no big outstanding issues. You might find such + issues -- - What about the bsd buildbots? - See http://buildbot.pixelminers.net/builders/ + * On Trac - What about Coverity Scan? + * On coverity scan - What about clang scan-build? + * On OSS-Fuzz - Does `make distcheck` complain? +3. Run checks that aren't covered above, including: - How about `make test-stem` and `make test-network` and - `make test-network-all`? + * clang scan-build. (See the script in ./scripts/test/scan_build.sh) - - Are all those tests still happy with --enable-expensive-hardening ? + * make test-network and make test-network-all (with + --enable-expensive-hardening) - Any memory leaks? + * Running Tor yourself and making sure that it actually works for you. === II. Write a changelog @@ -55,11 +54,14 @@ new Tor release: of them and reordering to focus on what users and funders would find interesting and understandable. - To do this, first run `./scripts/maint/lintChanges.py changes/*` and - fix as many warnings as you can. Then run `./scripts/maint/sortChanges.py - changes/* > changelog.in` to combine headings and sort the entries. - After that, it's time to hand-edit and fix the issues that lintChanges - can't find: + To do this, run + `./scripts/maint/sortChanges.py changes/* > changelog.in` + to combine headings and sort the entries. Copy the changelog.in file + into the ChangeLog. Run 'format_changelog.py' (see below) to clean + up the line breaks. + + After that, it's time to hand-edit and fix the issues that + lintChanges can't find: 1. Within each section, sort by "version it's a bugfix on", else by numerical ticket order. @@ -68,8 +70,6 @@ new Tor release: Make stuff very terse - Make sure each section name ends with a colon - Describe the user-visible problem right away Mention relevant config options by name. If they're rare or unusual, @@ -81,6 +81,8 @@ new Tor release: "Relays", not "servers" or "nodes" or "Tor relays". + "Onion services", not "hidden services". + "Stop FOOing", not "Fix a bug where we would FOO". Try not to let any given section be longer than about a page. Break up @@ -106,6 +108,8 @@ new Tor release: changelog: instead, look up the corrected versions that were merged into ChangeLog in the master branch, and use those. + Add "backport from X.Y.Z" in the section header for these entries. + 2. Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding @@ -142,33 +146,50 @@ new Tor release: master, merge it with "-s ours" to avoid a needless version bump. 2. Make distcheck, put the tarball up in somewhere (how about your - homedir on your homedir on people.torproject.org?) , and tell `#tor` - about it. Wait a while to see if anybody has problems building it. - (Though jenkins is usually pretty good about catching these things.) + homedir on your homedir on people.torproject.org?) , and tell `#tor-dev` + about it. + + If you want, wait until at least one person has built it + successfully. (We used to say "wait for others to test it", but our + CI has successfully caught these kinds of errors for the last several + years.) + + +3. Make sure that the new version is recommended in the latest consensus. + (Otherwise, users will get confused when it complains to them + about its status.) + + If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see + item 0.1 at the start of this document. === IV. Commit, upload, announce 1. Sign the tarball, then sign and push the git tag: gpg -ba - git tag -u tor-0.3.x.y-status - git push origin tag tor-0.3.x.y-status + git tag -s tor-0.4.x.y- + git push origin tag tor-0.4.x.y- + + (You must do this before you update the website: the website scripts + rely on finding the version by tag.) - (You must do this before you update the website: it relies on finding - the version by tag.) + (If your default PGP key is not the one you want to sign with, then say + "-u " instead of "-s".) 2. scp the tarball and its sig to the dist website, i.e. - `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want - it to go live, you run "static-update-component dist.torproject.org" - on dist-master. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. In the webwml.git repository, `include/versions.wmi` and `Makefile` - to note the new version. + to note the new version. Push these changes to master. (NOTE: Due to #17805, there can only be one stable version listed at once. Nonetheless, do not call your version "alpha" if it is stable, or people will get confused.) + (NOTE: It will take a while for the website update scripts to update + the website.) + 3. Email the packagers (cc'ing tor-team) that a new tarball is up. The current list of packagers is: @@ -186,29 +207,35 @@ new Tor release: Also, email tor-packagers@lists.torproject.org. + Mention where to download the tarball (https://dist.torproject.org). + + Include a link to the changelog. + + 4. Add the version number to Trac. To do this, go to Trac, log in, select "Admin" near the top of the screen, then select "Versions" from the menu on the left. At the right, there will be an "Add version" box. By convention, we enter the version in the form "Tor: - 0.2.2.23-alpha" (or whatever the version is), and we select the date as + 0.4.0.1-alpha" (or whatever the version is), and we select the date as the date in the ChangeLog. -5. Double-check: did the version get recommended in the consensus yet? Is - the website updated? If not, don't announce until they have the - up-to-date versions, or people will get confused. +5. Wait for the download page to be updated. (If you don't do this before you + announce, people will be confused.) 6. Mail the release blurb and ChangeLog to tor-talk (development release) or tor-announce (stable). Post the changelog on the blog as well. You can generate a - blog-formatted version of the changelog with the -B option to - format-changelog. + blog-formatted version of the changelog with + `./scripts/maint/format_changelog.py --B` When you post, include an estimate of when the next TorBrowser releases will come out that include this Tor release. This will usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it can vary. + For templates to use when announcing, see: + https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates === V. Aftermath and cleanup @@ -216,7 +243,7 @@ new Tor release: `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate). +2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + master branch. 3. Keep an eye on the blog post, to moderate comments and answer questions. - -- GitLab From c01507a5fe73d0e49902a8abaa1f73112ae1fefa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Dec 2018 08:14:21 -0500 Subject: [PATCH 0248/1724] remember why we are doing getsockopt() --- src/core/mainloop/connection.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 9d5bad7d8d..55a1dd361e 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1462,6 +1462,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, } #ifndef __APPLE__ + /* This code was introduced to help debug #28229. */ int value; socklen_t len = sizeof(value); -- GitLab From a2f81b644b2a4679c634744d3830cdb6397f0814 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 14:06:53 -0500 Subject: [PATCH 0249/1724] Write tests for mark_my_descriptor_dirty_if_too_old() --- src/feature/nodelist/networkstatus.c | 4 +- src/feature/nodelist/networkstatus.h | 5 +- src/feature/relay/router.c | 4 +- src/feature/relay/router.h | 4 ++ src/test/test_router.c | 98 ++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index f1def9afb1..a730f7d834 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -724,8 +724,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) /** Return the entry in ns for the identity digest digest, or * NULL if none was found. */ -const routerstatus_t * -networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +MOCK_IMPL(const routerstatus_t *, +networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest)) { return networkstatus_vote_find_mutable_entry(ns, digest); } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 8802de2d65..3fd151cf37 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); +MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,( + networkstatus_t *ns, + const char *digest)); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 2d4ab9b0a0..ef433db8bc 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1471,9 +1471,9 @@ static extrainfo_t *desc_extrainfo = NULL; static const char *desc_gen_reason = "uninitialized reason"; /** Since when has our descriptor been "clean"? 0 if we need to regenerate it * now. */ -static time_t desc_clean_since = 0; +STATIC time_t desc_clean_since = 0; /** Why did we mark the descriptor dirty? */ -static const char *desc_dirty_reason = "Tor just started"; +STATIC const char *desc_dirty_reason = "Tor just started"; /** Boolean: do we need to regenerate the above? */ static int desc_needs_upload = 0; diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 4575172afb..217511df9f 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,10 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +#ifdef TOR_UNIT_TESTS +extern time_t desc_clean_since; +extern const char *desc_dirty_reason; +#endif #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/test/test_router.c b/src/test/test_router.c index 921ec42904..18740dcb84 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,12 +7,17 @@ * \brief Unittests for code in router.c **/ +#define ROUTER_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" @@ -231,11 +236,104 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static networkstatus_t *mock_ns = NULL; +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void)now; + return mock_ns; +} + +static routerstatus_t *mock_rs = NULL; +static const routerstatus_t * +mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +{ + (void)ns; + (void)digest; + return mock_rs; +} + +static void +test_router_mark_if_too_old(void *arg) +{ + (void)arg; + time_t now = approx_time(); + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry); + + routerstatus_t rs; + networkstatus_t ns; + memset(&rs, 0, sizeof(rs)); + memset(&ns, 0, sizeof(ns)); + mock_ns = &ns; + mock_ns->valid_after = now-3600; + mock_rs = &rs; + mock_rs->published_on = now - 10; + + // no reason to mark this time. + desc_clean_since = now-10; + desc_dirty_reason = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + + // Doesn't appear in consensus? Still don't mark it. + mock_ns = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + mock_ns = &ns; + + // No new descriptor in a long time? Mark it. + desc_clean_since = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor"); + + // Version in consensus published a long time ago? We won't mark it + // if it's been clean for only a short time. + desc_clean_since = now - 10; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now - 10); + + // ... but if it's been clean a while, we mark. + desc_clean_since = now - 2 * 3600; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "version listed in consensus is quite old"); + + // same deal if we're marked stale. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 10; + mock_rs->is_staledesc = 1; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "listed as stale in consensus"); + + // same deal if we're absent from the consensus. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "not listed in consensus"); + + done: + UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_vote_find_entry); +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(mark_if_too_old, TT_FORK), END_OF_TESTCASES }; -- GitLab From c31346ffb444065820407670ecf64b925e3775fc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 6 Dec 2018 15:26:34 +0200 Subject: [PATCH 0250/1724] Print Python version during each Travis CI job --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b5713d6933..2ea529e252 100644 --- a/.travis.yml +++ b/.travis.yml @@ -193,6 +193,9 @@ install: - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi + ## Get python version + - python --version + ## run stem tests if they are enabled. - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi script: -- GitLab From ecaecaddd89ce0d89e5f90effbf066065ccccd10 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 6 Dec 2018 15:31:33 +0200 Subject: [PATCH 0251/1724] Add changes file --- changes/ticket28551 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28551 diff --git a/changes/ticket28551 b/changes/ticket28551 new file mode 100644 index 0000000000..46ba9d713b --- /dev/null +++ b/changes/ticket28551 @@ -0,0 +1,3 @@ + o Minor features (Continuous Integration): + - Log Python version during each Travis CI job. Resolves issue + 28551. -- GitLab From d9f36d3e929e5acebbf483e908d0c90aeabf6558 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:21:17 +1000 Subject: [PATCH 0252/1724] Fallbacks: minor script comment changes --- scripts/maint/updateFallbackDirs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 0ea3992d8f..cf923bfa32 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -18,8 +18,8 @@ # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis. # -# Then read the logs to make sure the fallbacks aren't dominated by a single -# netblock or port. +# After running this script, read the logs to make sure the fallbacks aren't +# dominated by a single netblock or port. # Script by weasel, April 2015 # Portions by gsathya & karsten, 2013 @@ -39,8 +39,6 @@ import urllib import urllib2 import hashlib import dateutil.parser -# bson_lazy provides bson -#from bson import json_util import copy import re @@ -1400,7 +1398,7 @@ class CandidateList(dict): each line's key/value pairs are placed in a dictonary, (of string -> string key/value pairs), and these dictionaries are placed in an array. - comments start with # and are ignored """ + comments start with # and are ignored. """ file_data = file_obj['data'] file_name = file_obj['name'] relaylist = [] -- GitLab From 766fd6cf7666cecc25a94e6cfe46b8b27188ff5d Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:29:39 +1000 Subject: [PATCH 0253/1724] Fallbacks: use a 24 hour consensus expiry tolerance Tor clients will use a consensus that expired up to 24 hours ago. Clients on 0.3.5.5-alpha? and earlier won't select guards from an expired consensus, but they can still bootstrap if they have existing guards. Update the fallback expiry tolerance to match tor's checks. Part of 28768, follow-up on 24661. --- changes/ticket28768 | 4 ++++ scripts/maint/updateFallbackDirs.py | 29 +++++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 changes/ticket28768 diff --git a/changes/ticket28768 b/changes/ticket28768 new file mode 100644 index 0000000000..ce991c8a42 --- /dev/null +++ b/changes/ticket28768 @@ -0,0 +1,4 @@ + o Minor features (fallback directory mirrors): + - Accept fallbacks that deliver reasonably live consensuses. + (Consensuses that expired less than 24 hours ago.) + Closes ticket 28768. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index cf923bfa32..41c4da675e 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -98,19 +98,24 @@ MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS # Clients have been using microdesc consensuses by default for a while now DOWNLOAD_MICRODESC_CONSENSUS = True -# If a relay delivers an expired consensus, if it expired less than this many -# seconds ago, we still allow the relay. This should never be less than -90, -# as all directory mirrors should have downloaded a consensus 90 minutes -# before it expires. It should never be more than 24 hours, because clients -# reject consensuses that are older than REASONABLY_LIVE_TIME. -# For the consensus expiry check to be accurate, the machine running this -# script needs an accurate clock. +# If a relay delivers an invalid consensus, if it expired less than this many +# seconds ago, accept the relay as a fallback. For the consensus expiry check +# to be accurate, the machine running this script needs an accurate clock. # -# Relays on 0.3.0 and later return a 404 when they are about to serve an -# expired consensus. This makes them fail the download check. -# We use a tolerance of 0, so that 0.2.x series relays also fail the download -# check if they serve an expired consensus. -CONSENSUS_EXPIRY_TOLERANCE = 0 +# Relays on 0.3.0 and later return a 404 when they are about to serve a +# consensus that expired more than 24 hours ago. 0.2.9 and earlier relays +# will serve consensuses that are very old. +# +# A 404 makes relays fail the download check. We use a tolerance of 24 hours, +# so that 0.2.9 relays also fail the download check if they serve a consensus +# that is not reasonably live. +# +# CONSENSUS_EXPIRY_TOLERANCE should never be more than 24 hours, because +# clients reject consensuses that are older than REASONABLY_LIVE_TIME. Clients +# on 0.3.5.5-alpha? and earlier also won't select guards from consensuses that +# have expired, but can bootstrap if they already have guards in their state +# file. +CONSENSUS_EXPIRY_TOLERANCE = 24*60*60 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False -- GitLab From c3fe405e217d1551b3a58f2469f05650dd9d7579 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:41:28 +1000 Subject: [PATCH 0254/1724] Fallbacks: use a 24 hour future consensus tolerance Tor clients on 0.3.5.6-rc? and later will use a consensus that will become valid up to 24 hours in the future. Clients on 0.3.5.5-alpha? and earlier won't accept future consensuses. Update the fallback expiry tolerance to match tor's checks. Part of 28768, follow-up on 28591. --- changes/ticket28768 | 4 ++-- scripts/maint/updateFallbackDirs.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/changes/ticket28768 b/changes/ticket28768 index ce991c8a42..27d90febc8 100644 --- a/changes/ticket28768 +++ b/changes/ticket28768 @@ -1,4 +1,4 @@ o Minor features (fallback directory mirrors): - Accept fallbacks that deliver reasonably live consensuses. - (Consensuses that expired less than 24 hours ago.) - Closes ticket 28768. + (Consensuses that will become valid less than 24 hours in the future, + or that expired less than 24 hours ago.) Closes ticket 28768. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 41c4da675e..142d468273 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -98,14 +98,19 @@ MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS # Clients have been using microdesc consensuses by default for a while now DOWNLOAD_MICRODESC_CONSENSUS = True -# If a relay delivers an invalid consensus, if it expired less than this many -# seconds ago, accept the relay as a fallback. For the consensus expiry check -# to be accurate, the machine running this script needs an accurate clock. +# If a relay delivers an invalid consensus, if it will become valid less than +# this many seconds in the future, or expired less than this many seconds ago, +# accept the relay as a fallback. For the consensus expiry check to be +# accurate, the machine running this script needs an accurate clock. # # Relays on 0.3.0 and later return a 404 when they are about to serve a # consensus that expired more than 24 hours ago. 0.2.9 and earlier relays # will serve consensuses that are very old. # +# Relays on 0.3.5.6-rc? and later return a 404 when they are about to serve a +# consensus that will become valid more than 24 hours in the future. Older +# relays don't serve future consensuses. +# # A 404 makes relays fail the download check. We use a tolerance of 24 hours, # so that 0.2.9 relays also fail the download check if they serve a consensus # that is not reasonably live. @@ -1127,6 +1132,7 @@ class Candidate(object): ).run()[0] end = datetime.datetime.utcnow() time_since_expiry = (end - consensus.valid_until).total_seconds() + time_until_valid = (consensus.valid_after - end).total_seconds() except Exception, stem_error: end = datetime.datetime.utcnow() log_excluded('Unable to retrieve a consensus from %s: %s', nickname, @@ -1151,6 +1157,15 @@ class Candidate(object): status += ', invalid' level = logging.WARNING download_failed = True + elif (time_until_valid > 0): + status = 'future consensus, valid in %ds'%(int(time_until_valid)) + if time_until_valid <= CONSENSUS_EXPIRY_TOLERANCE: + status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + level = logging.INFO + else: + status += ', invalid' + level = logging.WARNING + download_failed = True else: status = 'ok' level = logging.DEBUG -- GitLab From 7f3a7d9a2713ecddaf0cd8e08e054de4c2870792 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 15:53:13 +1000 Subject: [PATCH 0255/1724] Fallbacks: Rename CONSENSUS_EXPIRY_TOLERANCE to REASONABLY_LIVE_TIME Cleanup after 28768. --- scripts/maint/updateFallbackDirs.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 142d468273..914d121345 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -115,12 +115,12 @@ DOWNLOAD_MICRODESC_CONSENSUS = True # so that 0.2.9 relays also fail the download check if they serve a consensus # that is not reasonably live. # -# CONSENSUS_EXPIRY_TOLERANCE should never be more than 24 hours, because -# clients reject consensuses that are older than REASONABLY_LIVE_TIME. Clients -# on 0.3.5.5-alpha? and earlier also won't select guards from consensuses that -# have expired, but can bootstrap if they already have guards in their state -# file. -CONSENSUS_EXPIRY_TOLERANCE = 24*60*60 +# REASONABLY_LIVE_TIME should never be more than Tor's REASONABLY_LIVE_TIME, +# (24 hours), because clients reject consensuses that are older than that. +# Clients on 0.3.5.5-alpha? and earlier also won't select guards from +# consensuses that have expired, but can bootstrap if they already have guards +# in their state file. +REASONABLY_LIVE_TIME = 24*60*60 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False @@ -1150,8 +1150,8 @@ class Candidate(object): download_failed = True elif (time_since_expiry > 0): status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry)) - if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + if time_since_expiry <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) level = logging.INFO else: status += ', invalid' @@ -1159,8 +1159,8 @@ class Candidate(object): download_failed = True elif (time_until_valid > 0): status = 'future consensus, valid in %ds'%(int(time_until_valid)) - if time_until_valid <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + if time_until_valid <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) level = logging.INFO else: status += ', invalid' -- GitLab From 75b5cc047d3e257701de6dfab9f80d358025fb95 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 11:16:15 +1000 Subject: [PATCH 0256/1724] Fallbacks: refactor is_in_whitelist() to take an exact match boolean No behaviour change. Preparation for 24838. --- scripts/maint/fallback.whitelist | 23 +-- scripts/maint/updateFallbackDirs.py | 220 ++++++++++++++++++++-------- 2 files changed, 167 insertions(+), 76 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 79551948c6..97291d73be 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1,34 +1,21 @@ # updateFallbackDirs.py directory mirror whitelist -# -# Format: -# IPv4:DirPort orport= id= [ ipv6=: ] -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... -# + # All attributes must match for the directory mirror to be included. # If the fallback has an ipv6 key, the whitelist line must also have # it, and vice versa, otherwise they don't match. -# (The blacklist overrides the whitelist.) # To replace this list with the hard-coded fallback list (for testing), use -# a command similar to: +# "updateFallbackDirs.py check_existing", or a command similar to: # cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ # tr -d '\n' | \ # sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ # > scripts/maint/fallback.whitelist -# -# When testing before a release, exclusions due to changed details will result -# in a warning, unless the IPv4 address or port change happened recently. -# Then it is only logged at info level, as part of the eligibility check. -# Exclusions due to stability also are only shown at info level. -# -# Add the number of selected, slow, and excluded relays, and compare that to -# the number of hard-coded relays. If it's less, use info-level logs to find -# out why each of the missing relays was excluded. # If a relay operator wants their relay to be a FallbackDir, # enter the following information here: -# : orport= id= [ ipv6=: ] +# : orport= id= ( ipv6=[]: )? +# or use: +# scripts/maint/generateFallbackDirLine.py fingerprint ... # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html # https://trac.torproject.org/projects/tor/ticket/22321#comment:22 diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 914d121345..b334352152 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -920,61 +920,155 @@ class Candidate(object): return False return True - def is_in_whitelist(self, relaylist): - """ A fallback matches if each key in the whitelist line matches: + def id_matches(self, id, exact=False): + """ Does this fallback's id match id? + exact is ignored. """ + return self._fpr == id + + def ipv4_addr_matches(self, ipv4_addr, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + exact is ignored. """ + return self.dirip == ipv4_addr + + def ipv4_dirport_matches(self, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 dirport match ipv4_dirport? + If exact is False, always return True. """ + if exact: + return self.dirport == int(ipv4_dirport) + else: + return True + + def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_dirport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport, + exact=exact) + else: + return ipv4_match + + def ipv4_orport_matches(self, ipv4_orport, exact=False): + """ Does this fallback's IPv4 orport match ipv4_orport? + If exact is False, always return True. """ + if exact: + return self.orport == int(ipv4_orport) + else: + return True + + def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_orport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_orport_matches(ipv4_orport, + exact=exact) + else: + return ipv4_match + + def ipv6_addr_matches(self, ipv6_addr, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + Both addresses must be present to match. + exact is ignored. """ + if self.has_ipv6() and ipv6_addr is not None: + # Check that we have a bracketed IPv6 address without a port + assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']')) + return self.ipv6addr == ipv6_addr + else: + return False + + def ipv6_orport_matches(self, ipv6_orport, exact=False): + """ Does this fallback's IPv6 orport match ipv6_orport? + Both ports must be present to match. + If exact is False, always return True. """ + if exact: + return (self.has_ipv6() and ipv6_orport is not None and + self.ipv6orport == int(ipv6_orport)) + else: + return True + + def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + If exact is True, also check ipv6_orport. """ + ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact) + if exact: + return ipv6_match and self.ipv6_orport_matches(ipv6_orport, + exact=exact) + else: + return ipv6_match + + def entry_matches_exact(self, entry): + """ Is entry an exact match for this fallback? + A fallback is an exact match for entry if each key in entry matches: ipv4 dirport orport id - ipv6 address and port (if present) + ipv6 address and port (if present in the fallback or the whitelist) If the fallback has an ipv6 key, the whitelist line must also have - it, and vice versa, otherwise they don't match. """ - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) + it, otherwise they don't match. + + Logs a warning-level message if the fallback would be an exact match, + but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport + have changed. """ + if not self.id_matches(entry['id'], exact=True): + # can't log here unless we match an IP and port, because every relay's + # fingerprint is compared to every entry's fingerprint + if self.ipv4_and_orport_matches(entry['ipv4'], + entry['orport'], + exact=True): + logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + + '%s?', entry['id'], self.dirip, self.orport, + self._fpr) + if self.ipv6_and_orport_matches(entry.get('ipv6_addr'), + entry.get('ipv6_orport'), + exact=True): + logging.warning('%s excluded: has OR %s changed fingerprint to ' + + '%s?', entry['id'], entry['ipv6'], self._fpr) + return False + if not self.ipv4_addr_matches(entry['ipv4'], exact=True): + logging.warning('%s excluded: has it changed IPv4 from %s to %s?', + self._fpr, entry['ipv4'], self.dirip) + return False + if not self.ipv4_dirport_matches(entry['dirport'], exact=True): + logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), + self.dirip, self.dirport) + return False + if not self.ipv4_orport_matches(entry['orport'], exact=True): + logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['orport']), + self.dirip, self.orport) + return False + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if not self.ipv6_and_orport_matches(entry['ipv6_addr'], + entry['ipv6_orport'], + exact=True): + logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + + 'to %s:%d?', self._fpr, entry['ipv6'], + self.ipv6addr, self.ipv6orport) + return False + # if the fallback has an IPv6 address but the whitelist entry + # doesn't, or vice versa, the whitelist entry doesn't match + elif entry.has_key('ipv6') and not self.has_ipv6(): + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) + return False + elif not entry.has_key('ipv6') and self.has_ipv6(): + logging.warning('%s excluded: has it gained an IPv6 address %s:%d?', + self._fpr, self.ipv6addr, self.ipv6orport) + return False + return True + + def is_in_whitelist(self, relaylist, exact=False): + """ If exact is True (existing fallback list), check if this fallback is + an exact match for any whitelist entry, using entry_matches_exact(). + """ for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], ipv6, self._fpr) - continue - if entry['ipv4'] != self.dirip: - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - continue - if int(entry['dirport']) != self.dirport: - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - continue - if int(entry['orport']) != self.orport: - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - continue - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != ipv6: - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s?', self._fpr, entry['ipv6'], ipv6) - continue - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - continue - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s?', - self._fpr, ipv6) - continue - return True + if exact: + if self.entry_matches_exact(entry): + return True return False def cw_to_bw_factor(self): @@ -1458,18 +1552,28 @@ class CandidateList(dict): relay_entry['dirport'] = ipv4_maybe_dirport_split[1] elif kvl == 2: relay_entry[key_value_split[0]] = key_value_split[1] + # split ipv6 addresses and orports + if key_value_split[0] == 'ipv6': + ipv6_orport_split = key_value_split[1].rsplit(':', 1) + ipv6l = len(ipv6_orport_split) + if ipv6l != 2: + print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( + file_name, item) + relay_entry['ipv6_addr'] = ipv6_orport_split[0] + relay_entry['ipv6_orport'] = ipv6_orport_split[1] relaylist.append(relay_entry) return relaylist - # apply the fallback whitelist - def apply_filter_lists(self, whitelist_obj): + def apply_filter_lists(self, whitelist_obj, exact=False): + """ Apply the fallback whitelist_obj to this fallback list, + passing exact to is_in_whitelist(). """ excluded_count = 0 logging.debug('Applying whitelist') # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) filtered_fallbacks = [] for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist) + in_whitelist = f.is_in_whitelist(whitelist, exact=exact) if in_whitelist: # include filtered_fallbacks.append(f) @@ -2082,14 +2186,14 @@ def process_existing(): logging.getLogger('stem').setLevel(logging.INFO) whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), 'name': FALLBACK_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=True) def process_default(): logging.basicConfig(level=logging.WARNING) logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=True) ## Main Function def main(): @@ -2110,10 +2214,10 @@ def log_excluded(msg, *args): else: logging.info(msg, *args) -def list_fallbacks(whitelist): +def list_fallbacks(whitelist, exact=False): """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays """ - + fallback directory criteria for each of the relays, + passing exact to apply_filter_lists(). """ print "/* type=fallback */" print ("/* version={} */" .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) @@ -2153,7 +2257,7 @@ def list_fallbacks(whitelist): # warning that the details have changed from those in the whitelist. # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist) + excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) -- GitLab From 6bc5c06dc25630c5a5f97f1da2af29f88683d243 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 13:23:57 +1000 Subject: [PATCH 0257/1724] Fallbacks: accept relays that are a fuzzy match to the whitelist If a relay matches at least one fingerprint, IPv4 address, or IPv6 address in the fallback whitelist, it can become a fallback. This reduces the work required to keep the list up to date. Closes ticket 28768. --- changes/ticket24838 | 6 ++++++ scripts/maint/fallback.whitelist | 26 +++++++++++++------------ scripts/maint/updateFallbackDirs.py | 30 +++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 changes/ticket24838 diff --git a/changes/ticket24838 b/changes/ticket24838 new file mode 100644 index 0000000000..d068e31b91 --- /dev/null +++ b/changes/ticket24838 @@ -0,0 +1,6 @@ + o Minor features (fallback directory mirrors): + - Accept relays that are a fuzzy match to a fallback whitelist entry. + If a relay matches at least one fingerprint, IPv4 address, or IPv6 + address in the fallback whitelist, it can become a fallback. This + reduces the work required to keep the list up to date. + Closes ticket 24838. diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 97291d73be..eb7fd92d4b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1,16 +1,18 @@ # updateFallbackDirs.py directory mirror whitelist - -# All attributes must match for the directory mirror to be included. -# If the fallback has an ipv6 key, the whitelist line must also have -# it, and vice versa, otherwise they don't match. - -# To replace this list with the hard-coded fallback list (for testing), use -# "updateFallbackDirs.py check_existing", or a command similar to: -# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ -# tr -d '\n' | \ -# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ -# > scripts/maint/fallback.whitelist - +# +# At least one of these keys must match for a directory mirror to be included +# in the fallback list: +# id +# ipv4 +# ipv6 +# The ports and nickname are ignored. Missing or extra ipv6 addresses +# are ignored. +# +# The latest relay details from Onionoo are included in the generated list. +# +# To check the hard-coded fallback list (for testing), use: +# $ updateFallbackDirs.py check_existing +# # If a relay operator wants their relay to be a FallbackDir, # enter the following information here: # : orport= id= ( ipv6=[]: )? diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index b334352152..14372d0e83 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1061,14 +1061,40 @@ class Candidate(object): return False return True + def entry_matches_fuzzy(self, entry): + """ Is entry a fuzzy match for this fallback? + A fallback is a fuzzy match for entry if at least one of these keys + in entry matches: + id + ipv4 + ipv6 (if present in both the fallback and whitelist) + The ports and nickname are ignored. Missing or extra ipv6 addresses + are ignored. + + Doesn't log any warning messages. """ + if self.id_matches(entry['id'], exact=False): + return True + if self.ipv4_addr_matches(entry['ipv4'], exact=False): + return True + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False): + return True + return False + def is_in_whitelist(self, relaylist, exact=False): """ If exact is True (existing fallback list), check if this fallback is an exact match for any whitelist entry, using entry_matches_exact(). - """ + + If exact is False (new fallback whitelist), check if this fallback is + a fuzzy match for any whitelist entry, using entry_matches_fuzzy(). """ for entry in relaylist: if exact: if self.entry_matches_exact(entry): return True + else: + if self.entry_matches_fuzzy(entry): + return True return False def cw_to_bw_factor(self): @@ -2193,7 +2219,7 @@ def process_default(): logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist, exact=True) + list_fallbacks(whitelist, exact=False) ## Main Function def main(): -- GitLab From d9b9c1fa76dacacd502141aeca08c5a9722d0898 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 17:03:46 +1000 Subject: [PATCH 0258/1724] Changes file for 24805 --- changes/ticket24805 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket24805 diff --git a/changes/ticket24805 b/changes/ticket24805 new file mode 100644 index 0000000000..4ba6f6ecd4 --- /dev/null +++ b/changes/ticket24805 @@ -0,0 +1,3 @@ + o Minor features (fallback directory list): + - Update the fallback whitelist based on operator opt-ins and opt-outs. + Closes ticket 24805, patch by Phoul. -- GitLab From da264f7c766b332f596a92766f7625c4a17abf70 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 8 Dec 2018 15:15:20 +0200 Subject: [PATCH 0259/1724] Let's not double-quote OUTPUTARG after all --- scripts/test/scan-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 9a63383804..26e05ff101 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -72,7 +72,7 @@ scan-build \ # shellcheck disable=SC2086 scan-build \ - $CHECKERS "$OUTPUTARG" \ + $CHECKERS $OUTPUTARG \ make -j5 -k CHECKERS="\ -- GitLab From f0a8664677b8e4a3503172d6e7564da33496be8f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 6 Dec 2018 14:13:29 -0500 Subject: [PATCH 0260/1724] Add code to parse K=V lines into config_line_t format. Closes ticket 28755 --- src/lib/encoding/.may_include | 1 + src/lib/encoding/include.am | 2 + src/lib/encoding/kvline.c | 239 ++++++++++++++++++++++++++++++++++ src/lib/encoding/kvline.h | 24 ++++ src/test/test_config.c | 78 +++++++++++ 5 files changed, 344 insertions(+) create mode 100644 src/lib/encoding/kvline.c create mode 100644 src/lib/encoding/kvline.h diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..83e9211b6f 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -9,6 +9,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ src/lib/encoding/time_fmt.c @@ -22,5 +23,6 @@ noinst_HEADERS += \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..11ff4f0f96 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include +#include +#include + +/** Return true iff we need to quote and escape the string s to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in line is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the all the lines in line can be encoded + * using flags. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in line as a series of 'Key=Value' + * pairs, using the provided flags to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in flags, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in flags, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a line containing a series of space-separated 'Key=Value' + * pairs, using the provided flags to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in flags, then (double-)quoted values are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in flags, then values without keys are + * allowed. Otherwise, such values are not allowed. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + const char *cp = line, *cplast = NULL; + bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + bool quoted = (flags & KV_QUOTED) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything, this code is broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + + /* Find the key. */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + cp = unescape_string(cp, &val, &len); + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val; + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..3272cc1754 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/test/test_config.c b/src/test/test_config.c index dae4d83766..5140c3c1a8 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -54,6 +54,7 @@ #include "lib/meminfo/meminfo.h" #include "lib/net/gethostname.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #ifdef HAVE_UNISTD_H #include @@ -5813,6 +5814,82 @@ test_config_extended_fmt(void *arg) config_free_lines(lines); } +static void +test_config_kvline_parse(void *arg) +{ + (void)arg; + + config_line_t *lines = NULL; + char *enc = NULL; + + lines = kvline_parse("A=B CD=EF", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "A"); + tt_str_op(lines->value, OP_EQ, "B"); + tt_str_op(lines->next->key, OP_EQ, "CD"); + tt_str_op(lines->next->value, OP_EQ, "EF"); + enc = kvline_encode(lines, 0); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB CDE=F", 0); + tt_assert(! lines); + + lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, ""); + tt_str_op(lines->value, OP_EQ, "AB"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F"); + tt_assert(lines); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB CDE=F"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=C CDE=\"F G\"", 0); + tt_assert(!lines); + + lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "C"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F G"); + tt_str_op(lines->next->next->key, OP_EQ, ""); + tt_str_op(lines->next->next->value, OP_EQ, "GHI"); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(! lines); + + lines = kvline_parse("AB=", KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + + done: + config_free_lines(lines); + tor_free(enc); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -5864,5 +5941,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(include_opened_file_list, 0), CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), + CONFIG_TEST(kvline_parse, 0), END_OF_TESTCASES }; -- GitLab From 845e8dbe5904b4b2d3eb2db9e2681ea2f4d98008 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Dec 2018 10:00:26 -0500 Subject: [PATCH 0261/1724] Fuzzing module for various string operations, currently focusing on encoding and decoding. There are bunches of places where we don't want to invest in a full fuzzer, but we would like to make sure that some string operation can handle all its possible inputs. This fuzzer uses the first byte of its input to decide what to do with the rest of the input. Right now, all the possibilities are decoding a string, and seeing whether it is decodeable. If it is, we try to re-encode it and do the whole thing again, to make sure we get the same result. This turned up a lot of bugs in the key-value parser, and I think it will help in other cases too. Closes ticket 28808. --- scripts/codegen/fuzzing_include_am.py | 1 + src/test/fuzz/fuzz_strops.c | 247 ++++++++++++++++++++++++++ src/test/fuzz/include.am | 29 +++ 3 files changed, 277 insertions(+) create mode 100644 src/test/fuzz/fuzz_strops.c diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 3c948d87cf..5b51158843 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -13,6 +13,7 @@ FUZZERS = """ iptsv2 microdesc socks + strops vrs """ diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c new file mode 100644 index 0000000000..5da590acfa --- /dev/null +++ b/src/test/fuzz/fuzz_strops.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file fuzz_strops.c + * \brief Fuzzers for various string encoding/decoding operations + **/ + +#include "orconfig.h" + +#include "lib/cc/torint.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/confline.h" +#include "lib/malloc/malloc.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" +#include "lib/intmath/muldiv.h" + +#include "test/fuzz/fuzzing.h" + +#include +#include + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +typedef struct chunk_t { + uint8_t *buf; + size_t len; +} chunk_t; + +#define chunk_free(ch) \ + FREE_AND_NULL(chunk_t, chunk_free_, (ch)) + +static chunk_t * +chunk_new(size_t len) +{ + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = tor_malloc(len); + ch->len = len; + return ch; +} +static void +chunk_free_(chunk_t *ch) +{ + if (!ch) + return; + tor_free(ch->buf); + tor_free(ch); +} +static bool +chunk_eq(const chunk_t *a, const chunk_t *b) +{ + return a->len == b->len && fast_memeq(a->buf, b->buf, a->len); +} + +static chunk_t * +b16_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2)); + int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b16_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len * 2 + 1); + base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + return ch; +} + +#if 0 +static chunk_t * +b32_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX + int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; // XXXX we need some way to get the actual length of + // XXXX the output here. + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b32_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(base32_encoded_size(inp->len)); + base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + ch->len = strlen((char *) ch->buf); + return ch; +} +#endif + +static chunk_t * +b64_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter. + int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b64_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len)); + base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0); + ch->len = strlen((char *) ch->buf); + return ch; +} + +static chunk_t * +c_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + char *r = NULL; + (void) unescape_string(s, &r, &ch->len); + tor_free(s); + ch->buf = (uint8_t*) r; + if (!ch->buf) { + tor_free(ch); + } + return ch; +} +static chunk_t * +c_enc(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = (uint8_t*)esc_for_log(s); + tor_free(s); + ch->len = strlen((char*)ch->buf); + return ch; +} + +static int kv_flags = 0; +static config_line_t * +kv_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + config_line_t *res = kvline_parse(s, kv_flags); + tor_free(s); + return res; +} +static chunk_t * +kv_enc(const config_line_t *inp) +{ + char *s = kvline_encode(inp, kv_flags); + if (!s) + return NULL; + chunk_t *res = tor_malloc(sizeof(chunk_t)); + res->buf = (uint8_t*)s; + res->len = strlen(s); + return res; +} + +/* Given an encoder function, a decoder function, and a function to free + * the decoded object, check whether any string that successfully decoded + * will then survive an encode-decode-encode round-trip unchanged. + */ +#define ENCODE_ROUNDTRIP(E,D,FREE) \ + STMT_BEGIN { \ + bool err = false; \ + a = D(&inp); \ + if (!a) \ + return 0; \ + b = E(a); \ + tor_assert(b); \ + c = D(b); \ + tor_assert(c); \ + d = E(c); \ + tor_assert(d); \ + if (!chunk_eq(b,d)) { \ + printf("Unequal chunks: %s\n", \ + hex_str((char*)b->buf, b->len)); \ + printf(" vs %s\n", \ + hex_str((char*)d->buf, d->len)); \ + err = true; \ + } \ + FREE(a); \ + chunk_free(b); \ + FREE(c); \ + chunk_free(d); \ + tor_assert(!err); \ + } STMT_END + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + if (!data_size) + return 0; + + chunk_t inp = { (uint8_t*)stdin_buf, data_size }; + chunk_t *b=NULL,*d=NULL; + void *a=NULL,*c=NULL; + + switch (stdin_buf[0]) { + case 0: + ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); + break; + case 1: + /* + XXXX see notes above about our base-32 functions. + ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); + */ + break; + case 2: + ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); + break; + case 3: + ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_); + break; + case 5: + kv_flags = KV_QUOTED|KV_OMIT_KEYS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 6: + kv_flags = 0; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + } + + return 0; +} diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 27eeced8c5..d0711f05d6 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -152,6 +152,16 @@ src_test_fuzz_fuzz_socks_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS) endif +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_strops_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_strops.c +src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS) +endif + if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ @@ -176,6 +186,7 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-socks \ + src/test/fuzz/fuzz-strops \ src/test/fuzz/fuzz-vrs endif @@ -290,6 +301,15 @@ src_test_fuzz_lf_fuzz_socks_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS) endif +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_strops_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS) +endif + if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) @@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-socks \ + src/test/fuzz/lf-fuzz-strops \ src/test/fuzz/lf-fuzz-vrs else @@ -405,6 +426,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) @@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-socks.a \ + src/test/fuzz/liboss-fuzz-strops.a \ src/test/fuzz/liboss-fuzz-vrs.a else -- GitLab From 5200df85570dae7dede1473a169398e83ff64a37 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Dec 2018 17:44:53 -0500 Subject: [PATCH 0262/1724] Copy the nss-related changes into fuzzing_include_am.py. --- scripts/codegen/fuzzing_include_am.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 5b51158843..a944584453 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -24,12 +24,12 @@ FUZZING_CPPFLAGS = \ FUZZING_CFLAGS = \ $(AM_CFLAGS) $(TEST_CFLAGS) FUZZING_LDFLAG = \ - @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ + @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ $(TOR_INTERNAL_TESTING_LIBS) \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ -- GitLab From 53855d72b71cca7c61124bbdfac0a93b1386cb43 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 12 Dec 2018 13:23:46 -0500 Subject: [PATCH 0263/1724] man: Add that "GETINFO address" won't work with "Sandbox 1" Patch by "wagon". Closes #28538 Signed-off-by: David Goulet --- doc/tor.1.txt | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index c886e37613..5b73e25e16 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -598,21 +598,26 @@ GENERAL OPTIONS Otherwise the sandbox will be disabled. The option is currently an experimental feature. It only works on Linux-based operating systems, and only when Tor has been built with the libseccomp library. This option - can not be changed while tor is running. + can not be changed while tor is running. + + - When the Sandbox is 1, the following options can not be changed when tor + When the **Sandbox** is 1, the following options can not be changed when tor is running: - Address - ConnLimit - CookieAuthFile - DirPortFrontPage - ExtORPortCookieAuthFile - Logs - ServerDNSResolvConfFile - Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). Launching new Onion Services through Control - Port is not supported with current syscall sandboxing implementation. - ClientOnionAuthDir and any files in it won't reload on HUP signal. + **Address**, + **ConnLimit**, + **CookieAuthFile**, + **DirPortFrontPage**, + **ExtORPortCookieAuthFile**, + **Logs**, + **ServerDNSResolvConfFile**, + **ClientOnionAuthDir** (and any files in it won't reload on HUP signal). + + + Launching new Onion Services through the control port is not supported + with current syscall sandboxing implementation. + + + Tor must remain in client or server mode (some changes to **ClientOnly** + and **ORPort** are not allowed). Currently, if **Sandbox** is 1, + **ControlPort** command "GETINFO address" will not work. + + (Default: 0) [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]:: -- GitLab From c037bf58173db56766381a7c1cd5973789f0fd0f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:11:42 -0500 Subject: [PATCH 0264/1724] changes file for ticket26864 --- changes/ticket28624 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket28624 diff --git a/changes/ticket28624 b/changes/ticket28624 new file mode 100644 index 0000000000..353f962be9 --- /dev/null +++ b/changes/ticket28624 @@ -0,0 +1,5 @@ + o Minor features (battery management, client, dormant mode): + - The client's memory of whether it is "dormant", and how long it has + spend idle, persists across invocations. Implements ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use if + they expect that in many cases, Tor will be installed but not used. -- GitLab From b5c04173c88369d0c4cdaf8a34c2474dc25c79fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:21:14 -0500 Subject: [PATCH 0265/1724] Change interaction between dormant mode and clock jumps. When the clock jumps, and we have a record of last user activity, adjust that record. This way if I'm inactive for 10 minutes and then the laptop is sleeping for an hour, I'll still count as having been inactive for 10 minutes. Previously, we treat every jump as if it were activity, which is ridiculous, and would prevent a Tor instance with a jumpy clock from ever going dormant. --- src/core/mainloop/mainloop.c | 21 +++++++++++---------- src/core/mainloop/netstatus.c | 12 ++++++++++++ src/core/mainloop/netstatus.h | 1 + src/lib/intmath/cmp.h | 3 +++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index cd955e0ca0..6b9c03798e 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1627,7 +1627,6 @@ schedule_rescan_periodic_events,(void)) void rescan_periodic_events(const or_options_t *options) { - puts("RESCAN"); tor_assert(options); /* Avoid scanning the event list if we haven't initialized it yet. This is @@ -2687,6 +2686,17 @@ update_current_time(time_t now) memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); monotime_coarse_get(¤t_second_last_changed); + /** How much clock jumping means that we should adjust our idea of when + * to go dormant? */ +#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20 + + /* Don't go dormant early or late just because we jumped in time. */ + if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) { + if (is_participating_on_network()) { + netstatus_note_clock_jumped(seconds_elapsed); + } + } + /** How much clock jumping do we tolerate? */ #define NUM_JUMPED_SECONDS_BEFORE_WARN 100 @@ -2697,10 +2707,6 @@ update_current_time(time_t now) // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); - /* Don't go dormant just because we jumped in time. */ - if (is_participating_on_network()) { - reset_user_activity(now); - } } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2722,11 +2728,6 @@ update_current_time(time_t now) if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); } - - /* Don't go dormant just because we jumped in time. */ - if (is_participating_on_network()) { - reset_user_activity(now); - } } else if (seconds_elapsed > 0) { stats_n_seconds_working += seconds_elapsed; } diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 2426baae34..d1989cb839 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -146,3 +146,15 @@ netstatus_load_from_state(const or_state_t *state, time_t now) } reset_user_activity(last_activity); } + +/** + * Adjust the time at which the user was last active by seconds_diff + * in response to a clock jump. + */ +void +netstatus_note_clock_jumped(time_t seconds_diff) +{ + time_t last_active = get_last_user_activity_time(); + if (last_active) + reset_user_activity(last_active + seconds_diff); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index 4b008e4cfa..9a0fa410fd 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -19,5 +19,6 @@ bool is_participating_on_network(void); void netstatus_flush_to_state(or_state_t *state, time_t now); void netstatus_load_from_state(const or_state_t *state, time_t now); +void netstatus_note_clock_jumped(time_t seconds_diff); #endif diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index 16952bee3e..11b6fdf98e 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of x, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ -- GitLab From e3b7fd2a8129f0d2a7879976e57496b6fd4a7d00 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 17:52:06 -0500 Subject: [PATCH 0266/1724] Unit tests for back-end functions for persistent dormant state --- src/test/test_mainloop.c | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 8dfd5f619a..d797417912 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -8,6 +8,7 @@ #define CONFIG_PRIVATE #define MAINLOOP_PRIVATE +#define STATEFILE_PRIVATE #include "test/test.h" #include "test/log_test_helpers.h" @@ -20,6 +21,8 @@ #include "feature/hs/hs_service.h" #include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/config/or_state_st.h" static const uint64_t BILLION = 1000000000; @@ -190,6 +193,13 @@ test_mainloop_user_activity(void *arg) tt_int_op(true, OP_EQ, is_participating_on_network()); tt_int_op(schedule_rescan_called, OP_EQ, 1); + // We _will_ adjust if the clock jumps though. + netstatus_note_clock_jumped(500); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525); + + netstatus_note_clock_jumped(-400); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125); + done: UNMOCK(schedule_rescan_periodic_events); } @@ -273,6 +283,70 @@ test_mainloop_check_participation(void *arg) UNMOCK(connection_get_by_type_nonlinked); } +static void +test_mainloop_dormant_load_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + reset_user_activity(0); + set_network_participation(false); + + // When we construct a new state, it starts out in "auto" mode. + tt_int_op(state->Dormant, OP_EQ, -1); + + // Initializing from "auto" makes us start out (by default) non-Dormant, + // with activity right now. + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + // Initializing from dormant clears the last user activity time, and + // makes us dormant. + state->Dormant = 1; + netstatus_load_from_state(state, start); + tt_assert(! is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, 0); + + // Initializing from non-dormant sets the last user activity time, and + // makes us non-dormant. + state->Dormant = 0; + state->MinutesSinceUserActivity = 123; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + + done: + or_state_free(state); +} + +static void +test_mainloop_dormant_save_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + // Can we save a non-dormant state correctly? + reset_user_activity(start - 1000); + set_network_participation(true); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 0); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60); + + // Can we save a dormant state correctly? + set_network_participation(false); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 1); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0); + + done: + or_state_free(state); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } @@ -281,5 +355,7 @@ struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_jumps), MAINLOOP_TEST(user_activity), MAINLOOP_TEST(check_participation), + MAINLOOP_TEST(dormant_load_state), + MAINLOOP_TEST(dormant_save_state), END_OF_TESTCASES }; -- GitLab From 4bc3983f64ab2a60a79127f47573836373ddf587 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 19:35:02 -0500 Subject: [PATCH 0267/1724] Add a DROPOWNERSHIP controller command to undo TAKEOWNERSHIP. Closes ticket 28843. --- changes/ticket28843 | 3 +++ src/feature/control/control.c | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 changes/ticket28843 diff --git a/changes/ticket28843 b/changes/ticket28843 new file mode 100644 index 0000000000..00905293c4 --- /dev/null +++ b/changes/ticket28843 @@ -0,0 +1,3 @@ + o Minor features (controller): + - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. + Implements ticket 28843. diff --git a/src/feature/control/control.c b/src/feature/control/control.c index c2da7fe48b..50f43c0f44 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1741,6 +1741,26 @@ handle_control_takeownership(control_connection_t *conn, uint32_t len, return 0; } +/** Called when we get a DROPOWNERSHIP command. Mark this connection + * as a non-owning connection, so that we will not exit if the connection + * closes. */ +static int +handle_control_dropownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 0; + + log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + /** Return true iff addr is unusable as a mapaddress target because of * containing funny characters. */ static int @@ -5548,6 +5568,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { if (handle_control_takeownership(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { + if (handle_control_dropownership(conn, cmd_data_len, args)) + return -1; } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { if (handle_control_mapaddress(conn, cmd_data_len, args)) return -1; -- GitLab From 325348b360fc8a51d1344d918c1bb20017f1e863 Mon Sep 17 00:00:00 2001 From: Rob Jansen Date: Wed, 12 Dec 2018 15:27:45 -0500 Subject: [PATCH 0268/1724] allow any value for HearbeatPeriod in testing Tor networks --- src/app/config/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index dcefa3d6a4..071c33d530 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3897,7 +3897,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HeartbeatPeriod && - options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD && + !options->TestingTorNetwork) { log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; -- GitLab From 06046c726fb09ec745d2a565abfef28097856a19 Mon Sep 17 00:00:00 2001 From: Matt Traudt Date: Fri, 14 Dec 2018 09:24:19 -0500 Subject: [PATCH 0269/1724] Add changes file for <30min HeartbeatPeriod patch --- changes/ticket28840 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28840 diff --git a/changes/ticket28840 b/changes/ticket28840 new file mode 100644 index 0000000000..05d3fbb94c --- /dev/null +++ b/changes/ticket28840 @@ -0,0 +1,3 @@ + o Minor features (testing): + - Allow HeartbeatPeriod of less than 30 minutes in testing Tor networks. + Closes ticket 28840, patch by robgjansen -- GitLab From cf7342ab6f653c2dc49134024d668b06bac2c96c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:45:22 -0500 Subject: [PATCH 0270/1724] Add a benchmark for parsing a microdescriptor --- src/test/bench.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/bench.c b/src/test/bench.c index ff8707d41c..f8680c3ab6 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -39,6 +39,9 @@ #include "lib/crypt_ops/digestset.h" #include "lib/crypt_ops/crypto_init.h" +#include "feature/dirparse/microdesc_parse.h" +#include "feature/nodelist/microdesc.h" + #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; static inline uint64_t @@ -639,6 +642,41 @@ bench_ecdh_p224(void) } #endif +static void +bench_md_parse(void) +{ + uint64_t start, end; + const int N = 100000; + // selected arbitrarily + const char md_text[] = + "@last-listed 2018-12-14 18:14:14\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n" + "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n" + "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n" + "family $00E9A86E7733240E60D8435A7BBD634A23894098 " + "$329BD7545DEEEBBDC8C4285F243916F248972102 " + "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D " + "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n" + "p accept 53,80,443,5222-5223,25565\n" + "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n"; + + reset_perftime(); + start = perftime(); + for (int i = 0; i < N; ++i) { + smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1, + SAVED_IN_CACHE, NULL); + SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md)); + smartlist_free(s); + } + + end = perftime(); + printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N)); +} + typedef void (*bench_fn)(void); typedef struct benchmark_t { @@ -666,6 +704,8 @@ static struct benchmark_t benchmarks[] = { ENT(ecdh_p256), ENT(ecdh_p224), #endif + + ENT(md_parse), {NULL,NULL,0} }; -- GitLab From 3c35c0d441cc25f750524056113970a376d8432c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:07:55 -0500 Subject: [PATCH 0271/1724] Add a function to provide an upper bound on base64 decoded length --- src/lib/encoding/binascii.c | 12 ++++++++++++ src/lib/encoding/binascii.h | 1 + src/test/test_util_format.c | 4 +++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index e9140d9432..067db075ad 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string srclen. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 23cbaa7070..c71ba65dfb 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 85d8a8e62e..fd57125b86 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg) base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); + base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, BASE64_ENCODE_MULTILINE); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, BASE64_ENCODE_MULTILINE)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); } done: @@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = { { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL }, END_OF_TESTCASES }; - -- GitLab From 6dc90d290daa29b4ff2c7692be3a2ed64f25dfc1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:11:58 -0500 Subject: [PATCH 0272/1724] Use 25% less RAM for base64-encoded directory objects We were allocating N bytes to decode an N-byte base64 encoding, when 3N/4 would have been enough. --- src/feature/dirparse/parsecommon.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..91b775533b 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -393,8 +393,9 @@ get_next_token(memarea_t *area, RET_ERR("Couldn't parse private key."); } else { /* If it's something else, try to base64-decode it */ int r; - tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */ - r = base64_decode(tok->object_body, next-*s, *s, next-*s); + size_t maxsize = base64_decode_maxsize(next-*s); + tok->object_body = ALLOC(maxsize); + r = base64_decode(tok->object_body, maxsize, *s, next-*s); if (r<0) RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; -- GitLab From 9dc53bc68f5e038c9531e3b12a58026d4007f652 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 14:48:12 -0500 Subject: [PATCH 0273/1724] Remove a needless memset() in get_token_arguments() I believe we originally added this for "just in case" safety, but it isn't actually needed -- we never copy uninitialized stack here. What's more, this one memset is showing up on our startup profiles, so we ought to remove it. Closes ticket 28852. --- changes/ticket28852 | 4 ++++ src/feature/dirparse/parsecommon.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket28852 diff --git a/changes/ticket28852 b/changes/ticket28852 new file mode 100644 index 0000000000..a58cc3ba0e --- /dev/null +++ b/changes/ticket28852 @@ -0,0 +1,4 @@ + o Minor features (performance): + - Remove a needless memset() call from get_token_arguments, + thereby speeding up the tokenization of directory objects by about + 20%. Closes ticket 28852. diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..5280f2ed2f 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -169,7 +169,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok, char *cp = mem; int j = 0; char *args[MAX_ARGS]; - memset(args, 0, sizeof(args)); while (*cp) { if (j == MAX_ARGS) return -1; -- GitLab From 3dd1f064a7d5708585f88beffaf3897ba6555208 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 16:07:10 -0500 Subject: [PATCH 0274/1724] Rewrite the core of parse_short_policy() to be faster. The old implementation did some funky out-of-order lexing, and tended to parse every port twice if the %d-%d pattern didn't match. Closes ticket 28853. --- changes/ticket28853 | 3 ++ src/core/or/policies.c | 80 ++++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 38 deletions(-) create mode 100644 changes/ticket28853 diff --git a/changes/ticket28853 b/changes/ticket28853 new file mode 100644 index 0000000000..e76f6bd8c9 --- /dev/null +++ b/changes/ticket28853 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Replace parse_short_policy() with a faster implementation, to improve + microdescriptor parsing time. Closes ticket 28853. diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 123fc8566e..bffdb1fddd 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -2720,7 +2720,7 @@ parse_short_policy(const char *summary) int is_accept; int n_entries; short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */ - const char *next; + char *next; if (!strcmpstart(summary, "accept ")) { is_accept = 1; @@ -2735,57 +2735,56 @@ parse_short_policy(const char *summary) n_entries = 0; for ( ; *summary; summary = next) { - const char *comma = strchr(summary, ','); - unsigned low, high; - char dummy; - char ent_buf[32]; - size_t len; - - next = comma ? comma+1 : strchr(summary, '\0'); - len = comma ? (size_t)(comma - summary) : strlen(summary); - if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s", escaped(orig_summary)); return NULL; } - if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) { - /* unrecognized entry format. skip it. */ - continue; - } - if (len < 1) { - /* empty; skip it. */ - /* XXX This happens to be unreachable, since if len==0, then *summary is - * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */ - continue; + unsigned low, high; + int ok; + low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next); + if (!ok) { + if (! TOR_ISDIGIT(*summary) || *summary == ',') { + /* Unrecognized format: skip it. */ + goto skip_ent; + } else { + goto bad_ent; + } } - memcpy(ent_buf, summary, len); - ent_buf[len] = '\0'; + switch (*next) { + case ',': + ++next; + /* fall through */ + case '\0': + high = low; + break; + case '-': + high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next); + if (!ok) + goto bad_ent; + + if (*next == ',') + ++next; + else if (*next != '\0') + goto bad_ent; - if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) { - if (low<1 || low>65535 || high<1 || high>65535 || low>high) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) { - if (low<1 || low>65535) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - high = low; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", - escaped(orig_summary)); - return NULL; + break; + default: + goto bad_ent; } entries[n_entries].min_port = low; entries[n_entries].max_port = high; n_entries++; + + continue; + skip_ent: + next = strchr(next, ','); + if (!next) + break; + ++next; } if (n_entries == 0) { @@ -2806,6 +2805,11 @@ parse_short_policy(const char *summary) result->n_entries = n_entries; memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries); return result; + + bad_ent: + log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", + escaped(orig_summary)); + return NULL; } /** Write policy back out into a string. */ -- GitLab From 3bec371d04b291ec0351f01eeafdc842ea2a51d6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 16 Dec 2018 10:05:35 +0200 Subject: [PATCH 0275/1724] Refrain from hardcoding address length and type in netinfo.trunnel --- src/trunnel/netinfo.c | 30 ++++++------------------------ src/trunnel/netinfo.trunnel | 6 +++--- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c index 170e9bf034..de389eb13f 100644 --- a/src/trunnel/netinfo.c +++ b/src/trunnel/netinfo.c @@ -34,8 +34,6 @@ netinfo_addr_new(void) netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); if (NULL == val) return NULL; - val->addr_type = NETINFO_ADDR_TYPE_IPV4; - val->len = 4; return val; } @@ -65,10 +63,6 @@ netinfo_addr_get_addr_type(const netinfo_addr_t *inp) int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) { - if (! ((val == NETINFO_ADDR_TYPE_IPV4 || val == NETINFO_ADDR_TYPE_IPV6))) { - TRUNNEL_SET_ERROR_CODE(inp); - return -1; - } inp->addr_type = val; return 0; } @@ -80,10 +74,6 @@ netinfo_addr_get_len(const netinfo_addr_t *inp) int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) { - if (! ((val == 4 || val == 16))) { - TRUNNEL_SET_ERROR_CODE(inp); - return -1; - } inp->len = val; return 0; } @@ -141,10 +131,6 @@ netinfo_addr_check(const netinfo_addr_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) - return "Integer out of bounds"; - if (! (obj->len == 4 || obj->len == 16)) - return "Integer out of bounds"; switch (obj->addr_type) { case NETINFO_ADDR_TYPE_IPV4: @@ -169,10 +155,10 @@ netinfo_addr_encoded_len(const netinfo_addr_t *obj) return -1; - /* Length of u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Length of u8 addr_type */ result += 1; - /* Length of u8 len IN [4, 16] */ + /* Length of u8 len */ result += 1; switch (obj->addr_type) { @@ -219,14 +205,14 @@ netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *o trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Encode u8 addr_type */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; trunnel_set_uint8(ptr, (obj->addr_type)); written += 1; ptr += 1; - /* Encode u8 len IN [4, 16] */ + /* Encode u8 len */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -296,19 +282,15 @@ netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t ssize_t result = 0; (void)result; - /* Parse u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Parse u8 addr_type */ CHECK_REMAINING(1, truncated); obj->addr_type = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) - goto fail; - /* Parse u8 len IN [4, 16] */ + /* Parse u8 len */ CHECK_REMAINING(1, truncated); obj->len = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->len == 4 || obj->len == 16)) - goto fail; /* Parse union addr[addr_type] */ switch (obj->addr_type) { diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel index 21afcd136e..83c3a9e40f 100644 --- a/src/trunnel/netinfo.trunnel +++ b/src/trunnel/netinfo.trunnel @@ -5,14 +5,14 @@ const NETINFO_ADDR_TYPE_IPV4 = 4; const NETINFO_ADDR_TYPE_IPV6 = 6; struct netinfo_addr { - u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6]; - u8 len IN [4, 16]; + u8 addr_type; + u8 len; union addr[addr_type] { NETINFO_ADDR_TYPE_IPV4: u32 ipv4; NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; default: fail; }; - + } struct netinfo_cell { -- GitLab From 5b2acbec0e50a1858c43d0cafe9c4696152cde27 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 16 Dec 2018 10:19:37 +0200 Subject: [PATCH 0276/1724] Refrain from closing connection if found one unrecognized address in NETINFO cell --- src/core/or/channeltls.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 392e78506e..8f407d5e15 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1809,10 +1809,8 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Bad address in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - netinfo_cell_free(netinfo_cell); - return; + "Bad address in netinfo cell; Skipping."); + continue; } /* A relay can connect from anywhere and be canonical, so * long as it tells you from where it came. This may sound a bit -- GitLab From a0fad3981e9d072188c6ba19f25bdd78962330d8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:03:04 -0500 Subject: [PATCH 0277/1724] Replace use of strcmp_len() with new mem_eq_token(). The strcmp_len() function was somewhat misconceived, since we're only using it to test whether a length+extent string is equal to a NUL-terminated string or not. By simplifying it and making it inlined, we should be able to make it a little faster. (It *does* show up in profiles.) Closes ticket 28856. --- src/feature/dirparse/parsecommon.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..a69957e74a 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -15,6 +15,7 @@ #include "lib/string/printf.h" #include "lib/memarea/memarea.h" #include "lib/crypt_ops/crypto_rsa.h" +#include "lib/ctime/di_ops.h" #include @@ -251,6 +252,16 @@ token_check_object(memarea_t *area, const char *kwd, return tok; } +/** Return true iff the memlen-byte chunk of memory at + * memlen is the same length as token, and their + * contents are equal. */ +static bool +mem_eq_token(const void *mem, size_t memlen, const char *token) +{ + size_t len = strlen(token); + return memlen == len && fast_memeq(mem, token, len); +} + /** Helper function: read the next token from *s, advance *s to the end of the * token, and return the parsed token. Parse *s according to the list * of tokens in table. @@ -290,7 +301,7 @@ get_next_token(memarea_t *area, next = find_whitespace_eos(*s, eol); - if (!strcmp_len(*s, "opt", next-*s)) { + if (mem_eq_token(*s, next-*s, "opt")) { /* Skip past an "opt" at the start of the line. */ *s = eat_whitespace_eos_no_nl(next, eol); next = find_whitespace_eos(*s, eol); @@ -301,7 +312,7 @@ get_next_token(memarea_t *area, /* Search the table for the appropriate entry. (I tried a binary search * instead, but it wasn't any faster.) */ for (i = 0; table[i].t ; ++i) { - if (!strcmp_len(*s, table[i].t, next-*s)) { + if (mem_eq_token(*s, next-*s, table[i].t)) { /* We've found the keyword. */ kwd = table[i].t; tok->tp = table[i].v; @@ -354,7 +365,7 @@ get_next_token(memarea_t *area, obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ - strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ + !mem_eq_token(eol-5, 5, "-----") || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ RET_ERR("Malformed object: bad begin line"); } @@ -373,8 +384,8 @@ get_next_token(memarea_t *area, eol = eos; /* Validate the ending tag, which should be 9 + NAME + 5 + eol */ if ((size_t)(eol-next) != 9+obname_len+5 || - strcmp_len(next+9, tok->object_type, obname_len) || - strcmp_len(eol-5, "-----", 5)) { + !mem_eq_token(next+9, obname_len, tok->object_type) || + !mem_eq_token(eol-5, 5, "-----")) { tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s", tok->object_type); ebuf[sizeof(ebuf)-1] = '\0'; -- GitLab From 29254812a3428da4ffeaede5d211364d942f94ab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:04:25 -0500 Subject: [PATCH 0278/1724] Remove strcmp_len(): it is now unused (See 28856.) --- src/lib/string/util_string.c | 15 --------------- src/lib/string/util_string.h | 1 - src/test/test_util.c | 9 --------- 3 files changed, 25 deletions(-) diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index e76e73046f..36e19d029c 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -212,21 +212,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string s1 with s2, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 99467a27c3..6541afa4cb 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -33,7 +33,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); diff --git a/src/test/test_util.c b/src/test/test_util.c index 2b4d64e42e..7ac1a1ec42 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2153,15 +2153,6 @@ test_util_strmisc(void *arg) tt_int_op(strcmp_opt(NULL, "foo"), OP_LT, 0); tt_int_op(strcmp_opt("foo", NULL), OP_GT, 0); - /* Test strcmp_len */ - tt_int_op(strcmp_len("foo", "bar", 3), OP_GT, 0); - tt_int_op(strcmp_len("foo", "bar", 2), OP_LT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */ - tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0); /* Really stop at len */ - tt_int_op(strcmp_len("blah", "", 4), OP_GT, 0); - tt_int_op(strcmp_len("blah", "", 0), OP_EQ, 0); - done: tor_free(cp_tmp); } -- GitLab From 82fb40c8dc2f21753298559e79d898add80bf6b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:26:57 -0500 Subject: [PATCH 0279/1724] Fix dead-assignment warnings in test_config.c Found by scan-build. --- src/test/test_config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/test_config.c b/src/test/test_config.c index 5140c3c1a8..67a43d669e 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -5783,6 +5783,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, "is back here"); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL); lp = lp->next; + tt_assert(!lp); config_free_lines(lines); /* Try with the "extended" flag enabled. */ @@ -5809,6 +5810,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, ""); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR); lp = lp->next; + tt_assert(!lp); done: config_free_lines(lines); -- GitLab From f50558ce8ce33339b86ac642d92a27430e066d62 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:27:40 -0500 Subject: [PATCH 0280/1724] Fix dead-assignment warning in test_shared_random.c --- src/test/test_shared_random.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 433661f128..16e28afafe 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -309,6 +309,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC", &mock_consensus.valid_after); + tt_int_op(retval, OP_EQ, 0); retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC", ¤t_time); -- GitLab From d58a597a55dba2ac468cbdef5172ecda1dc2344f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:28:08 -0500 Subject: [PATCH 0281/1724] Fix dead assignment warning in test_hs_service.c --- src/test/test_hs_service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index ee2d71aa75..79cbd2e5d4 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1396,7 +1396,6 @@ static void test_build_update_descriptors(void *arg) { int ret; - time_t now = time(NULL); node_t *node; hs_service_t *service; hs_service_intro_point_t *ip_cur, *ip_next; @@ -1422,7 +1421,8 @@ test_build_update_descriptors(void *arg) voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); update_approx_time(mock_ns.valid_after+1); - now = mock_ns.valid_after+1; + + time_t now = mock_ns.valid_after+1; /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); -- GitLab From ce3d501040fb994bb3c7dd82b21e02648621d3f1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:28:24 -0500 Subject: [PATCH 0282/1724] Fix null-pointer-deref warning from scan-build in test_hs_service.c --- src/test/test_hs_service.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 79cbd2e5d4..b2aafc1cd6 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1654,6 +1654,9 @@ test_build_descriptors(void *arg) service->desc_current = NULL; build_all_descriptors(now); + tt_assert(service->desc_current); + tt_assert(service->desc_current->desc); + hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); -- GitLab From 35509978dd4985901431abe895d1443e75afc00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:25:11 +0100 Subject: [PATCH 0283/1724] Add new Process subsystem. This patch adds a new Process subsystem for running external programs in the background of Tor. The design is focused around a new type named `process_t` which have an API that allows the developer to easily write code that interacts with the given child process. These interactions includes: - Easy API for writing output to the child process's standard input handle. - Receive callbacks whenever the child has output on either its standard output or standard error handles. - Receive callback when the child process terminates. We also support two different "protocols" for handling output from the child process. The default protocol is the "line" protocol where the process output callbacks will be invoked only when there is complete lines (either "\r\n" or "\n" terminated). We also support the "raw" protocol where the read callbacks will get whatever the operating system delivered to us in a single read operation. This patch does not include any operating system backends, but the Unix and Windows backends will be included in separate commits. See: https://bugs.torproject.org/28179 --- src/app/main/main.c | 6 + src/lib/process/.may_include | 5 +- src/lib/process/include.am | 2 + src/lib/process/process.c | 644 +++++++++++++++++++++++++++++++++++ src/lib/process/process.h | 127 +++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_process.c | 558 ++++++++++++++++++++++++++++++ 9 files changed, 1343 insertions(+), 2 deletions(-) create mode 100644 src/lib/process/process.c create mode 100644 src/lib/process/process.h create mode 100644 src/test/test_process.c diff --git a/src/app/main/main.c b/src/app/main/main.c index 03b3a95d03..15b6b48ff2 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -73,6 +73,7 @@ #include "lib/geoip/geoip.h" #include "lib/process/waitpid.h" +#include "lib/process/process.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -557,6 +558,10 @@ tor_init(int argc, char *argv[]) rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + + /* Initialize Process subsystem. */ + process_init(); + /* Initialize the HS subsystem. */ hs_init(); @@ -784,6 +789,7 @@ tor_free_all(int postfork) circuitmux_ewma_free_all(); accounting_free_all(); protover_summary_cache_free_all(); + process_free_all(); if (!postfork) { config_free_all(); diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index a2d57a52f3..b1c50c24a6 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -4,8 +4,9 @@ lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h @@ -15,4 +16,4 @@ lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h \ No newline at end of file +ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 2aa30cc3c1..1d213d1c00 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,6 +9,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -24,6 +25,7 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..d3967a52da --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,644 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include +#endif + +/** A list of all process_t instances currently allocated. */ +static smartlist_t *processes; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; +}; + +/** Convert a given process status in status to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in protocol to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, process_free_all() should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * process_t containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various process_set_*() + * methods to configure it and run the process using process_exec(). Use + * command to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in process. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * callback function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use process_get_data() in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with process_set_data() + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use tor_free() + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** Set the given key/value pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated process_environment_t containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +/** Write size bytes of data to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in exit_code. We mark the + * process as no longer running and calls the exit_callback with + * information about the process termination. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + if (process->exit_callback) { + process->exit_callback(process, exit_code); + } +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); +} + +/** This function calls the protocol handlers based on the value of + * process_get_protocol(process). Currently we call + * process_read_buffer() for PROCESS_PROTOCOL_RAW and + * process_read_lines() for PROCESS_PROTOCOL_LINE. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given buffer and passes it to + * the given callback function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given buffer + * and calls the given callback function whenever it has a complete + * line. Before calling callback we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..cf20a5d80b --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; + +typedef void (*process_read_callback_t)(process_t *, + char *, + size_t); +typedef void (*process_exit_callback_t)(process_t *, + process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/test/include.am b/src/test/include.am index e5eae56e25..482897c7a5 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -153,6 +153,7 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..b13c3ffbe2 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -898,6 +898,7 @@ struct testgroup_t testgroups[] = { { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..7c17389776 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -241,6 +241,7 @@ extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..2adbde7ad2 --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,558 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static void +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + return; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); + process_free_all(); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_int_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_int_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); + process_free_all(); +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; -- GitLab From 2e957027e28449d4c3254cc404d154f4bce41bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:43:27 +0100 Subject: [PATCH 0284/1724] Add Unix backend for the Process subsystem. This patch adds the Unix backend for the Process subsystem. The Unix backend attaches file descriptors from the child process's standard in, out and error to Tor's libevent based main loop using traditional Unix pipes. We use the already available `waitpid` module to get events whenever the child process terminates. See: https://bugs.torproject.org/28179 --- src/lib/process/include.am | 2 + src/lib/process/process.c | 43 +++ src/lib/process/process.h | 5 + src/lib/process/process_unix.c | 611 +++++++++++++++++++++++++++++++++ src/lib/process/process_unix.h | 64 ++++ src/test/test_process.c | 21 ++ 6 files changed, 746 insertions(+) create mode 100644 src/lib/process/process_unix.c create mode 100644 src/lib/process/process_unix.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 1d213d1c00..f068032681 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -10,6 +10,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/env.c \ src/lib/process/pidfile.c \ src/lib/process/process.c \ + src/lib/process/process_unix.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -26,6 +27,7 @@ noinst_HEADERS += \ src/lib/process/env.h \ src/lib/process/pidfile.h \ src/lib/process/process.h \ + src/lib/process/process_unix.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index d3967a52da..8d6a9d3fa0 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -15,6 +15,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/process/process.h" +#include "lib/process/process_unix.h" #include "lib/process/env.h" #ifdef HAVE_STDDEF_H @@ -64,6 +65,11 @@ struct process_t { /** Do we need to store some custom data with the process? */ void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#endif }; /** Convert a given process status in status to its string @@ -161,6 +167,11 @@ process_new(const char *command) process->stderr_buffer = buf_new(); process->stdin_buffer = buf_new(); +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#endif + smartlist_add(processes, process); return process; @@ -188,6 +199,11 @@ process_free_(process_t *process) buf_free(process->stderr_buffer); buf_free(process->stdin_buffer); +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#endif + smartlist_remove(processes, process); tor_free(process); @@ -204,6 +220,10 @@ process_exec(process_t *process) log_info(LD_PROCESS, "Starting new process: %s", process->command); +#ifndef _WIN32 + status = process_unix_exec(process); +#endif + /* Update our state. */ process_set_status(process, status); @@ -391,6 +411,17 @@ process_get_environment(const process_t *process) return process_environment_make(process->environment); } +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#endif + /** Write size bytes of data to the given process's standard * input. */ void @@ -510,7 +541,11 @@ MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else return 0; +#endif } /** This function is called whenever the Process backend have notified us that @@ -521,7 +556,11 @@ MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else return 0; +#endif } /** This function calls the backend function for the given process whenever @@ -531,6 +570,10 @@ MOCK_IMPL(STATIC void, process_write_stdin, { tor_assert(process); tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#endif } /** This function calls the protocol handlers based on the value of diff --git a/src/lib/process/process.h b/src/lib/process/process.h index cf20a5d80b..b17b8dac7c 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -95,6 +95,11 @@ void *process_get_data(const process_t *process); void process_set_status(process_t *process, process_status_t status); process_status_t process_get_status(const process_t *process); +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#endif + void process_write(process_t *process, const uint8_t *data, size_t size); void process_vprintf(process_t *process, diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..c3691f1851 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,611 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include + +#ifdef HAVE_STRING_H +#include +#endif + +#ifdef HAVE_ERRNO_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include +#endif + +#if HAVE_SIGNAL_H +#include +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated process_unix_t. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given unix_process. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Cleanup our events. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + process_unix_stop_writing(&unix_process->stdin_handle); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + retval = execve(argv[0], argv, env->unixoid_