Commit 34fa2c4d authored by Jigsaw52's avatar Jigsaw52
Browse files

Add support for patterns on %include #25140

Also adds generic tor_glob function to expand globs.
parent a7226ca0
......@@ -866,6 +866,7 @@ dnl Where do you live, libevent? And how do we call you?
if test "$bwin32" = "true"; then
TOR_LIB_WS32=-lws2_32
TOR_LIB_IPHLPAPI=-liphlpapi
TOR_LIB_SHLWAPI=-lshlwapi
# Some of the cargo-cults recommend -lwsock32 as well, but I don't
# think it's actually necessary.
TOR_LIB_GDI=-lgdi32
......@@ -878,6 +879,7 @@ fi
AC_SUBST(TOR_LIB_WS32)
AC_SUBST(TOR_LIB_GDI)
AC_SUBST(TOR_LIB_IPHLPAPI)
AC_SUBST(TOR_LIB_SHLWAPI)
AC_SUBST(TOR_LIB_USERENV)
tor_libevent_pkg_redhat="libevent"
......@@ -1646,7 +1648,8 @@ AC_CHECK_HEADERS([errno.h \
sys/utime.h \
sys/wait.h \
syslog.h \
utime.h])
utime.h \
glob.h])
AC_CHECK_HEADERS(sys/param.h)
......
......@@ -205,14 +205,22 @@ backslash character (\) before the end of the line. Comments can be used in
such multiline entries, but they must start at the beginning of a line.
Configuration options can be imported from files or folders using the %include
option with the value being a path. If the path is a file, the options from the
option with the value being a path. This path can have wildcards. Wildcards are
expanded first, using lexical order. Then, for each matching file or folder, the
following rules are followed: if the path is a file, the options from the
file will be parsed as if they were written where the %include option is. If
the path is a folder, all files on that folder will be parsed following lexical
order. Files starting with a dot are ignored. Files on subfolders are ignored.
order. Files starting with a dot are ignored. Files in subfolders are ignored.
The %include option can be used recursively.
New configuration files or directories cannot be added to already running Tor
instance if **Sandbox** is enabled.
The supported wildcards are * meaning any number of characters including none
and ? meaning exactly one character. These characters can be escaped by preceding
them with a backslash, except on Windows. Files starting with a dot are not matched
when expanding wildcards unless the starting dot is explicitly in the pattern, except
on Windows.
By default, an option on the command line overrides an option found in the
configuration file, and an option in a configuration file overrides one in
the defaults file.
......
......@@ -35,7 +35,7 @@ FUZZING_LIBS = \
$(rust_ldadd) \
@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
@TOR_SYSTEMD_LIBS@ \
@TOR_LZMA_LIBS@ \
@TOR_ZSTD_LIBS@
......
......@@ -18,7 +18,7 @@ src_app_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_li
src_app_tor_LDADD = $(TOR_INTERNAL_LIBS) \
$(rust_ldadd) \
@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
......@@ -29,7 +29,7 @@ src_app_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
src_app_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
src_app_tor_cov_LDADD = $(TOR_INTERNAL_TESTING_LIBS) \
@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ \
@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \
@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
endif
......@@ -242,11 +242,12 @@
#PublishServerDescriptor 0
## Configuration options can be imported from files or folders using the %include
## option with the value being a path. If the path is a file, the options from the
## file will be parsed as if they were written where the %include option is. If
## the path is a folder, all files on that folder will be parsed following lexical
## order. Files starting with a dot are ignored. Files on subfolders are ignored.
## option with the value being a path. This path can have wildcards. Wildcards are
## expanded first, using lexical order. Then, for each matching file or folder, the following
## rules are followed: if the path is a file, the options from the file will be parsed as if
## they were written where the %include option is. If the path is a folder, all files on that
## folder will be parsed following lexical order. Files starting with a dot are ignored. Files
## on subfolders are ignored.
## The %include option can be used recursively.
#%include /etc/torrc.d/
#%include /etc/torrc.custom
#%include /etc/torrc.d/*.conf
......@@ -21,6 +21,8 @@
#include "lib/malloc/malloc.h"
#include "lib/string/printf.h"
#include <stdbool.h>
static smartlist_t *config_get_file_list(const char *path,
smartlist_t *opened_files);
static int config_get_included_config(const char *path, int recursion_level,
......@@ -50,62 +52,109 @@ config_get_lines_include(const char *string, config_line_t **result,
opened_lst, 1, NULL, config_process_include);
}
/** Adds a list of configuration files present on <b>path</b> to
* <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
* only that file will be added to <b>file_list</b>. If it is a directory,
* all paths for files on that directory root (no recursion) except for files
* whose name starts with a dot will be added to <b>file_list</b>.
* <b>opened_files</b> will have a list of files opened by this function
* if provided. Return 0 on success, -1 on failure. Ignores empty files.
*/
/** Returns a list of paths obtained when expading globs in <b>pattern</b>. If
* <b>pattern</b> has no globs, returns a list with <b>pattern</b> if it is an
* existing path or NULL otherwise. If <b>opened_files</b> is provided, adds
* paths opened by glob to it. Returns NULL on failure. */
static smartlist_t *
config_get_file_list(const char *path, smartlist_t *opened_files)
expand_glob(const char *pattern, smartlist_t *opened_files)
{
smartlist_t *file_list = smartlist_new();
smartlist_t *matches = tor_glob(pattern);
if (!matches) {
return NULL;
}
if (opened_files) {
smartlist_add_strdup(opened_files, path);
// if it is not a glob, return error when the path is missing
if (!has_glob(pattern) && smartlist_len(matches) == 0) {
smartlist_free(matches);
return NULL;
}
file_status_t file_type = file_status(path);
if (file_type == FN_FILE) {
smartlist_add_strdup(file_list, path);
return file_list;
} else if (file_type == FN_DIR) {
smartlist_t *all_files = tor_listdir(path);
if (!all_files) {
smartlist_free(file_list);
if (opened_files) {
smartlist_t *glob_opened = get_glob_opened_files(pattern);
if (!glob_opened) {
SMARTLIST_FOREACH(matches, char *, f, tor_free(f));
smartlist_free(matches);
return NULL;
}
smartlist_sort_strings(all_files);
SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
if (f[0] == '.') {
tor_free(f);
continue;
}
smartlist_add_all(opened_files, glob_opened);
smartlist_free(glob_opened);
}
smartlist_sort_strings(matches);
return matches;
}
/** Returns a list of configuration files present on paths that match
* <b>pattern</b>. The pattern is expanded and then all the paths are
* processed. A path can be a file or a directory. If it is a file, that file
* will be added to the list to be returned. If it is a directory,
* all paths for files on that directory root (no recursion) except for files
* whose name starts with a dot will be added to the list to be returned.
* <b>opened_files</b> will have a list of files opened by this function
* if provided. Return NULL on failure. Ignores empty files.
*/
static smartlist_t *
config_get_file_list(const char *pattern, smartlist_t *opened_files)
{
smartlist_t *glob_matches = expand_glob(pattern, opened_files);
if (!glob_matches) {
return NULL;
}
char *fullname;
tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
tor_free(f);
bool error_found = false;
smartlist_t *file_list = smartlist_new();
SMARTLIST_FOREACH_BEGIN(glob_matches, char *, path) {
if (opened_files) {
smartlist_add_strdup(opened_files, path);
}
if (opened_files) {
smartlist_add_strdup(opened_files, fullname);
file_status_t file_type = file_status(path);
if (file_type == FN_FILE) {
smartlist_add_strdup(file_list, path);
} else if (file_type == FN_DIR) {
smartlist_t *all_files = tor_listdir(path);
if (!all_files) {
error_found = true;
break;
}
if (file_status(fullname) != FN_FILE) {
tor_free(fullname);
smartlist_sort_strings(all_files);
SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
if (f[0] == '.') {
continue;
}
char *fullname;
tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
if (opened_files) {
smartlist_add_strdup(opened_files, fullname);
}
if (file_status(fullname) != FN_FILE) {
tor_free(fullname);
continue;
}
smartlist_add(file_list, fullname);
} SMARTLIST_FOREACH_END(f);
SMARTLIST_FOREACH(all_files, char *, f, tor_free(f));
smartlist_free(all_files);
} else if (file_type == FN_EMPTY) {
continue;
}
smartlist_add(file_list, fullname);
} SMARTLIST_FOREACH_END(f);
smartlist_free(all_files);
return file_list;
} else if (file_type == FN_EMPTY) {
return file_list;
} else {
} else {
error_found = true;
break;
}
} SMARTLIST_FOREACH_END(path);
SMARTLIST_FOREACH(glob_matches, char *, f, tor_free(f));
smartlist_free(glob_matches);
if (error_found) {
SMARTLIST_FOREACH(file_list, char *, f, tor_free(f));
smartlist_free(file_list);
return NULL;
file_list = NULL;
}
return file_list;
}
/** Creates a list of config lines present on included <b>path</b>.
......@@ -133,19 +182,19 @@ config_get_included_config(const char *path, int recursion_level, int extended,
return 0;
}
/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
/** Process an %include <b>pattern</b> in a config file. Set <b>list</b> to the
* list of configuration settings obtained and <b>list_last</b> to the last
* element of the same list. <b>opened_lst</b> will have a list of opened
* files if provided. Return 0 on success, -1 on failure. */
static int
config_process_include(const char *path, int recursion_level, int extended,
config_process_include(const char *pattern, int recursion_level, int extended,
config_line_t **list, config_line_t **list_last,
smartlist_t *opened_lst)
{
config_line_t *ret_list = NULL;
config_line_t **next = &ret_list;
smartlist_t *config_files = config_get_file_list(path, opened_lst);
smartlist_t *config_files = config_get_file_list(pattern, opened_lst);
if (!config_files) {
return -1;
}
......
......@@ -247,6 +247,22 @@ file_status(const char *fname)
}
}
/** Returns true if <b>file_type</b> represents an existing file (even if
* empty). Returns false otherwise. */
bool
is_file(file_status_t file_type)
{
return file_type != FN_ERROR && file_type != FN_NOENT && file_type != FN_DIR;
}
/** Returns true if <b>file_type</b> represents an existing directory. Returns
* false otherwise. */
bool
is_dir(file_status_t file_type)
{
return file_type == FN_DIR;
}
/** Create a file named <b>fname</b> with the contents <b>str</b>. Overwrite
* the previous <b>fname</b> if possible. Return 0 on success, -1 on failure.
*
......
......@@ -55,6 +55,8 @@ MOCK_DECL(int,tor_unlink,(const char *pathname));
typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
file_status_t file_status(const char *filename);
bool is_file(file_status_t file_type);
bool is_dir(file_status_t file_type);
int64_t tor_get_avail_disk_space(const char *path);
......
......@@ -13,18 +13,44 @@
#include "lib/malloc/malloc.h"
#include "lib/log/log.h"
#include "lib/log/util_bug.h"
#include "lib/container/smartlist.h"
#include "lib/sandbox/sandbox.h"
#include "lib/string/printf.h"
#include "lib/string/util_string.h"
#include "lib/string/compat_ctype.h"
#include "lib/string/compat_string.h"
#include "lib/fs/files.h"
#include "lib/fs/dir.h"
#include "lib/fs/userdb.h"
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef _WIN32
#include <windows.h>
#include <shlwapi.h>
#else /* !(defined(_WIN32)) */
#include <dirent.h>
#include <glob.h>
#endif /* defined(_WIN32) */
#include <errno.h>
#include <string.h>
#ifdef _WIN32
#define IS_GLOB_CHAR(s,i) (((s)[(i)]) == '*' || ((s)[(i)]) == '?')
#else
#define IS_GLOB_CHAR(s,i) ((((s)[(i)]) == '*' || ((s)[(i)]) == '?') &&\
((i) == 0 || (s)[(i)-1] != '\\')) /* check escape */
#endif
/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
* enclosing quotes. Backslashes are not unescaped. Return the unquoted
* <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
......@@ -294,3 +320,273 @@ make_path_absolute(const char *fname)
return absfname;
#endif /* defined(_WIN32) */
}
/** Expands globs in <b>pattern</b> for the path fragment between
* <b>prev_sep</b> and <b>next_sep</b>. Returns NULL on failure. */
typedef struct smartlist_t * unglob_fn(const char *pattern, int prev_sep,
int next_sep);
/** Adds <b>path</b> to <b>result</b> if it exists and is a file type we can
* handle. Returns false if <b>path</b> is a file type we cannot handle,
* returns true otherwise. Used on tor_glob for WIN32. */
static bool
add_non_glob_path(const char *path, struct smartlist_t *result)
{
file_status_t file_type = file_status(path);
if (file_type == FN_ERROR) {
return false;
} else if (file_type != FN_NOENT) {
char *to_add = tor_strdup(path);
clean_fname_for_stat(to_add);
smartlist_add(result, to_add);
}
/* If WIN32 tor_glob is called with a non-existing path, we want it to
* return an empty list instead of error to match the regular version */
return true;
}
/** Auxiliary function used by get_glob_opened_files and WIN32 tor_glob.
* Returns a list of paths obtained from <b>pattern</b> using <b>unglob</b> to
* expand each path fragment. If <b>final</b> is true, the paths are the result
* of the glob expansion of <b>pattern</b> (implements tor_glob). Otherwise,
* the paths are the paths opened by glob while expanding <b>pattern</b>
* (implements get_glb_opened_files). Returns NULL on failure. */
static struct smartlist_t *
get_glob_paths(const char *pattern, unglob_fn unglob, bool final)
{
smartlist_t *result = smartlist_new();
int i, prev_sep = -1, next_sep = -1;
bool is_glob = false, error_found = false, is_sep = false, is_last = false;
// find first path fragment with globs
for (i = 0; pattern[i]; i++) {
is_glob = is_glob || IS_GLOB_CHAR(pattern, i);
is_last = !pattern[i+1];
is_sep = pattern[i] == *PATH_SEPARATOR || pattern[i] == '/';
if (is_sep || is_last) {
prev_sep = next_sep;
next_sep = i; // next_sep+1 is start of next fragment or end of string
if (is_glob) {
break;
}
}
}
if (!is_glob) { // pattern fully expanded or no glob in pattern
if (final && !add_non_glob_path(pattern, result)) {
error_found = true;
goto end;
}
return result;
}
if (!final) {
// add path before the glob to result
int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
char *path_until_glob = tor_strndup(pattern, len);
smartlist_add(result, path_until_glob);
}
smartlist_t *unglobbed_paths = unglob(pattern, prev_sep, next_sep);
if (!unglobbed_paths) {
error_found = true;
} else {
// for each path for current fragment, add the rest of the pattern
// and call recursively to get all expanded paths
SMARTLIST_FOREACH_BEGIN(unglobbed_paths, char *, current_path) {
char *next_path;
tor_asprintf(&next_path, "%s"PATH_SEPARATOR"%s", current_path,
&pattern[next_sep+1]);
smartlist_t *opened_next = get_glob_paths(next_path, unglob, final);
tor_free(next_path);
if (!opened_next) {
error_found = true;
break;
}
smartlist_add_all(result, opened_next);
smartlist_free(opened_next);
} SMARTLIST_FOREACH_END(current_path);
SMARTLIST_FOREACH(unglobbed_paths, char *, p, tor_free(p));
smartlist_free(unglobbed_paths);
}
end:
if (error_found) {
SMARTLIST_FOREACH(result, char *, p, tor_free(p));
smartlist_free(result);
result = NULL;
}
return result;
}
#ifdef _WIN32
/** Expands globs in <b>pattern</b> for the path fragment between
* <b>prev_sep</b> and <b>next_sep</b> using the WIN32 API. Returns NULL on
* failure. Used by the WIN32 implementation of tor_glob. */
static struct smartlist_t *
unglob_win32(const char *pattern, int prev_sep, int next_sep)
{
smartlist_t *result = smartlist_new();
int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
char *path_until_glob = tor_strndup(pattern, len);
if (!is_file(file_status(path_until_glob))) {
smartlist_t *filenames = tor_listdir(path_until_glob);
if (!filenames) {
smartlist_free(result);
result = NULL;
} else {
SMARTLIST_FOREACH_BEGIN(filenames, char *, filename) {
TCHAR tpattern[MAX_PATH] = {0};
TCHAR tfile[MAX_PATH] = {0};
char *full_path;
tor_asprintf(&full_path, "%s"PATH_SEPARATOR"%s",
path_until_glob, filename);
char *path_curr_glob = tor_strndup(pattern, next_sep + 1);
// *\ must return only dirs, remove \ from the pattern so it matches
if (is_dir(file_status(full_path))) {
clean_fname_for_stat(path_curr_glob);
}
#ifdef UNICODE
mbstowcs(tpattern, path_curr_glob, MAX_PATH);
mbstowcs(tfile, full_path, MAX_PATH);
#else
strlcpy(tpattern, path_curr_glob, MAX_PATH);
strlcpy(tfile, full_path, MAX_PATH);
#endif
if (PathMatchSpec(tfile, tpattern)) {
smartlist_add(result, full_path);
} else {
tor_free(full_path);
}
tor_free(path_curr_glob);
} SMARTLIST_FOREACH_END(filename);
SMARTLIST_FOREACH(filenames, char *, p, tor_free(p));
smartlist_free(filenames);
}
}
tor_free(path_until_glob);
return result;
}
#else /* !defined(_WIN32) */
/** Same as opendir but calls sandbox_intern_string before */
static DIR *
prot_opendir(const char *name)
{
return opendir(sandbox_intern_string(name));
}
/** Same as stat but calls sandbox_intern_string before */
static int
prot_stat(const char *pathname, struct stat *buf)
{
return stat(sandbox_intern_string(pathname), buf);
}
/** Same as lstat but calls sandbox_intern_string before */
static int
prot_lstat(const char *pathname, struct stat *buf)
{
return lstat(sandbox_intern_string(pathname), buf);
}
#endif /* defined(_WIN32) */
/** Return a new list containing the paths that match the pattern
* <b>pattern</b>. Return NULL on error.
*/
struct smartlist_t *
tor_glob(const char *pattern)
{
smartlist_t *result;
#ifdef _WIN32
// PathMatchSpec does not support forward slashes, change them to backslashes
char *pattern_normalized = tor_strdup(pattern);
tor_strreplacechar(pattern_normalized, '/', *PATH_SEPARATOR);
result = get_glob_paths(pattern_normalized, unglob_win32, true);
tor_free(pattern_normalized);
#else /* !(defined(_WIN32)) */
glob_t matches;
int flags = GLOB_ERR | GLOB_NOSORT;
#ifdef GLOB_ALTDIRFUNC
/* use functions that call sandbox_intern_string */
flags |= GLOB_ALTDIRFUNC;
typedef void *(*gl_opendir)(const char * name);
typedef struct dirent *(*gl_readdir)(void *);
typedef void (*gl_closedir)(void *);
matches.gl_opendir = (gl_opendir) &prot_opendir;
matches.gl_readdir = (gl_readdir) &readdir;
matches.gl_closedir = (gl_closedir) &closedir;
matches.gl_stat = &prot_stat;
matches.gl_lstat = &prot_lstat;
#endif /* defined(GLOB_ALTDIRFUNC) */
int ret = glob(pattern, flags, NULL, &matches);
if (ret == GLOB_NOMATCH) {
return smartlist_new();
} else if (ret != 0) {
return NULL;
}
result = smartlist_new();
size_t i;
for (i = 0; i < matches.gl_pathc; i++) {
char *match = tor_strdup(matches.gl_pathv[i]);
size_t len = strlen(match);
if (len > 0 && match[len-1] == *PATH_SEPARATOR) {
match[len-1] = '\0';
}
smartlist_add(result, match);
}
globfree(&matches);
#endif /* defined(_WIN32) */
return result;
}
/** Returns true if <b>s</b> contains characters that can be globbed.
* Returns false otherwise. */
bool
has_glob(const char *s)
{
int i;
for (i = 0; s[i]; i++) {
if (IS_GLOB_CHAR(s, i)) {
return true;
}
}
return false;
}
/** Expands globs in <b>pattern</b> for the path fragment between
* <b>prev_sep</b> and <b>next_sep</b> using tor_glob. Returns NULL on
* failure. Used by get_glob_opened_files. */
static struct smartlist_t *
unglob_opened_files(const char *pattern, int prev_sep, int next_sep)
{
(void)prev_sep;
smartlist_t *result = smartlist_new();
// if the following fragments have no globs, we're done
if (has_glob(&pattern[next_sep+1])) {
// if there is a glob after next_sep, we know it is a separator and not the
// last char and glob_path will have the path without the separator
char *glob_path = tor_strndup(pattern, next_sep);
smartlist_t *child_paths = tor_glob(glob_path);
tor_free(glob_path);
if (!child_paths) {
smartlist_free(result);
result = NULL;
} else {
smartlist_add_all(result, child_paths);
smartlist_free(child_paths);
}
}
return result;