New script to check includes for modularity violations

Includes configuration files to enforce these rules on lib and
common.  Of course, "common" *is* a modularity violation right now,
so these rules aren't as strict as I would like them to be.

scripts/maint/checkIncludes.py

+#!/usr/bin/python3
+
+import fnmatch
+import os
+import re
+import sys
+
+trouble = False
+
+def err(msg):
+    global trouble
+    trouble = True
+    print(msg, file=sys.stderr)
+
+def fname_is_c(fname):
+    return fname.endswith(".h") or fname.endswith(".c")
+
+INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
+RULES_FNAME = ".may_include"
+
+class Rules(object):
+    def __init__(self):
+        self.patterns = []
+
+    def addPattern(self, pattern):
+        self.patterns.append(pattern)
+
+    def includeOk(self, path):
+        for pattern in self.patterns:
+            if fnmatch.fnmatchcase(path, pattern):
+                return True
+        return False
+
+    def applyToLines(self, lines, context=""):
+        lineno = 0
+        for line in lines:
+            lineno += 1
+            m = INCLUDE_PATTERN.match(line)
+            if m:
+                include = m.group(1)
+                if not self.includeOk(include):
+                    err("Forbidden include of {} on line {}{}".format(
+                        include, lineno, context))
+
+    def applyToFile(self, fname):
+        with open(fname, 'r') as f:
+            #print(fname)
+            self.applyToLines(iter(f), " of {}".format(fname))
+
+def load_include_rules(fname):
+    result = Rules()
+    with open(fname, 'r') as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            result.addPattern(line)
+    return result
+
+for dirpath, dirnames, fnames in os.walk("src"):
+    if ".may_include" in fnames:
+        rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
+        for fname in fnames:
+            if fname_is_c(fname):
+                rules.applyToFile(os.path.join(dirpath,fname))
+
+if trouble:
+    err(
+"""To change which includes are allowed in a C file, edit the {} files in its
+enclosing directory.""".format(RULES_FNAME))

src/common/.may_include

+orconfig.h
+common/*.h
+lib/*/*.h
+
+# XXXX These all belong somewhere else
+ht.h
+linux_syscalls.inc
+micro-revision.i
+siphash.h
+src/ext/timeouts/timeout.c
+strlcat.c
+strlcpy.c
+tor_queue.h
+tor_readpassphrase.h

src/lib/cc/.may_include

+orconfig.h

src/lib/compress/.may_include

+orconfig.h
+lib/cc/*.h
+lib/compress/*.h
+
+# XXX I'd like to remove this.
+common/*.h

src/lib/crypt_ops/.may_include

+orconfig.h
+lib/cc/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/err/*.h
+lib/testsupport/testsupport.h
+
+trunnel/pwbox.h
+
+keccak-tiny/*.h
+ed25519/*.h
+
+# XXX I'd like to remove this.
+common/*.h

src/lib/ctime/.may_include

+orconfig.h
+lib/cc/*.h
+lib/ctime/*.h
+
+# XXXX I'd like to remove this
+common/util.h

src/lib/err/.may_include

+orconfig.h
+lib/cc/*.h
+lib/err/*.h

src/lib/tls/.may_include

+orconfig.h
+lib/cc/*.h
+lib/crypt_ops/*.h
+lib/err/*.h
+lib/testsupport/testsupport.h
+lib/tls/*.h
+
+ciphers.inc
+
+# XXX I'd like to remove this.
+common/*.h

src/lib/tls/buffers_tls.c

@@ -10,7 +10,6 @@
 #include "common/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "common/compat.h"
-#include "lib/compress/compress.h"
 #include "common/util.h"
 #include "lib/cc/torint.h"
 #include "common/torlog.h"
@@ -176,4 +175,3 @@ buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
   tor_assert(flushed < INT_MAX);
   return (int)flushed;
 }
-

src/lib/trace/.may_include

+orconfig.h
+lib/trace/*.h
+
+# XXXX
+common/torlog.h