Merge branch 'maint-0.4.0' into maint-0.4.1

changes/bug31408

+  o Major bugfixes (torrc):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any config
+      options. (But does contain comments or whitespace.)
+      Fixes bug 31408; bugfix on 0.3.1.1-alpha.

changes/bug31837

+  o Minor bugfixes (testing):
+    - When testing port rebinding, don't busy-wait for tor to log. Instead,
+      actually sleep for a short time before polling again. Also improve the
+      formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.

src/lib/fs/conffile.c

@@ -153,16 +153,18 @@ config_process_include(const char *path, int recursion_level, int extended,
   int rv = -1;
   SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
     config_line_t *included_config = NULL;
+    config_line_t *included_config_last = NULL;
     if (config_get_included_config(config_file, recursion_level, extended,
-                                   &included_config, list_last,
+                                   &included_config, &included_config_last,
                                    opened_lst) < 0) {
       goto done;
     }
 
     *next = included_config;
-    if (*list_last)
-      next = &(*list_last)->next;
-
+    if (included_config_last) {
+      next = &included_config_last->next;
+      *list_last = included_config_last;
+    }
   } SMARTLIST_FOREACH_END(config_file);
   *list = ret_list;
   rv = 0;

src/test/test_config.c

@@ -5287,6 +5287,73 @@ test_config_include_folder_order(void *data)
   tor_free(dir);
 }
 
+static void
+test_config_include_blank_file_last(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrcd = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_blank_file_last"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "aa_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "bb_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "cc_comment");
+  tt_int_op(write_str_to_file(path, "# comment only\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "Test 3\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 3);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(path);
+  tor_free(dir);
+}
+
 static void
 test_config_include_path_syntax(void *data)
 {
@@ -5982,6 +6049,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_recursion_before_after, 0),
   CONFIG_TEST(include_recursion_after_only, 0),
   CONFIG_TEST(include_folder_order, 0),
+  CONFIG_TEST(include_blank_file_last, 0),
   CONFIG_TEST(include_path_syntax, 0),
   CONFIG_TEST(include_not_processed, 0),
   CONFIG_TEST(include_has_include, 0),

src/test/test_rebind.py

@@ -32,15 +32,17 @@ def wait_for_log(s):
     cutoff = time.time() + LOG_TIMEOUT
     while time.time() < cutoff:
         l = tor_process.stdout.readline()
-        l = l.decode('utf8')
+        l = l.decode('utf8', 'backslashreplace')
         if s in l:
             logging.info('Tor logged: "{}"'.format(l.strip()))
             return
-        logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
         # readline() returns a blank string when there is no output
         # avoid busy-waiting
-        if len(s) == 0:
+        if len(l) == 0:
+            logging.debug('Tor has not logged anything, waiting for "{}"'.format(s))
             time.sleep(LOG_WAIT)
+        else:
+            logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
     fail('Could not find "{}" in logs after {} seconds'.format(s, LOG_TIMEOUT))
 
 def pick_random_port():
@@ -120,18 +122,18 @@ if control_socket.connect_ex(('127.0.0.1', control_port)):
     tor_process.terminate()
     fail('Cannot connect to ControlPort')
 
-control_socket.sendall('AUTHENTICATE \r\n'.encode('utf8'))
-control_socket.sendall('SETCONF SOCKSPort=0.0.0.0:{}\r\n'.format(socks_port).encode('utf8'))
+control_socket.sendall('AUTHENTICATE \r\n'.encode('ascii'))
+control_socket.sendall('SETCONF SOCKSPort=0.0.0.0:{}\r\n'.format(socks_port).encode('ascii'))
 wait_for_log('Opened Socks listener')
 
 try_connecting_to_socksport()
 
-control_socket.sendall('SETCONF SOCKSPort=127.0.0.1:{}\r\n'.format(socks_port).encode('utf8'))
+control_socket.sendall('SETCONF SOCKSPort=127.0.0.1:{}\r\n'.format(socks_port).encode('ascii'))
 wait_for_log('Opened Socks listener')
 
 try_connecting_to_socksport()
 
-control_socket.sendall('SIGNAL HALT\r\n'.encode('utf8'))
+control_socket.sendall('SIGNAL HALT\r\n'.encode('ascii'))
 
 wait_for_log('exiting cleanly')
 logging.info('OK')