Sanitize TCP ports in bridge descriptors. (ecb05389) · Commits · Legacy / gitolite / user / sebastian / metrics-db

CHANGELOG.md

+2 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,8 @@
		- Add support for Bifroest's bridge descriptor tarballs.
		- Use a shutdown hook that gives currently running modules up to 10
		minutes to finish properly, rather than killing them immediately.
		- Replace TCP ports with hashes in @type bridge-network-status 1.1
		and @type bridge-server-descriptor 1.2.

		* Minor changes
		- Remove quotes around base URL in index.json.

src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java

+65 −16

Original line number	Diff line number	Diff line
		@@ -138,7 +138,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		while ((line = br.readLine()) != null) {
		String[] parts = line.split(",");
		if ((line.length() != ("yyyy-MM,".length() + 31 * 2)
		&& line.length() != ("yyyy-MM,".length() + 50 * 2))
		&& line.length() != ("yyyy-MM,".length() + 50 * 2)
		&& line.length() != ("yyyy-MM,".length() + 83 * 2))
		\|\| parts.length != 2) {
		logger.warn("Invalid line in bridge-ip-secrets file "
		+ "starting with '" + line.substring(0, 7) + "'! "
		@@ -218,8 +219,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		scrubbedAddressPart = this.scrubIpv4Address(addressPart,
		fingerprintBytes, published);
		}
		String scrubbedPort = this.scrubTcpPort(portPart, fingerprintBytes,
		published);
		return (scrubbedAddressPart == null ? null :
		scrubbedAddressPart + ":" + portPart);
		scrubbedAddressPart + ":" + scrubbedPort);
		}

		private String scrubIpv4Address(String address, byte[] fingerprintBytes,
		@@ -334,14 +337,42 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		return sb.toString();
		}

		private String scrubTcpPort(String portString, byte[] fingerprintBytes,
		String published) throws IOException {
		if (portString.equals("0")) {
		return "0";
		} else if (this.replaceIpAddressesWithHashes) {
		if (this.persistenceProblemWithSecrets) {
		/* There's a persistence problem, so we shouldn't scrub more TCP
		* ports in this execution. */
		return null;
		}
		byte[] hashInput = new byte[2 + 20 + 33];
		int portNumber = Integer.parseInt(portString);
		hashInput[0] = (byte) (portNumber >> 8);
		hashInput[1] = (byte) portNumber;
		System.arraycopy(fingerprintBytes, 0, hashInput, 2, 20);
		String month = published.substring(0, "yyyy-MM".length());
		byte[] secret = this.getSecretForMonth(month);
		System.arraycopy(secret, 50, hashInput, 22, 33);
		byte[] hashOutput = DigestUtils.sha256(hashInput);
		int hashedPort = ((((hashOutput[0] & 0xFF) << 8)
		\| (hashOutput[1] & 0xFF)) >> 2) \| 0xC000;
		return String.valueOf(hashedPort);
		} else {
		return "1";
		}
		}

		private byte[] getSecretForMonth(String month) throws IOException {
		if (!this.secretsForHashingIpAddresses.containsKey(month)
		\|\| this.secretsForHashingIpAddresses.get(month).length == 31) {
		byte[] secret = new byte[50];
		\|\| this.secretsForHashingIpAddresses.get(month).length < 83) {
		byte[] secret = new byte[83];
		this.secureRandom.nextBytes(secret);
		if (this.secretsForHashingIpAddresses.containsKey(month)) {
		System.arraycopy(this.secretsForHashingIpAddresses.get(month), 0,
		secret, 0, 31);
		secret, 0,
		this.secretsForHashingIpAddresses.get(month).length);
		}
		if (month.compareTo(
		this.bridgeSanitizingCutOffTimestamp) < 0) {
		@@ -362,8 +393,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		bw.close();
		} catch (IOException e) {
		logger.warn("Could not store new secret "
		+ "to disk! Not calculating any IP address hashes in "
		+ "this execution!", e);
		+ "to disk! Not calculating any IP address or TCP port "
		+ "hashes in this execution!", e);
		this.persistenceProblemWithSecrets = true;
		throw new IOException(e);
		}
		@@ -471,11 +502,15 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		fingerprintBytes,
		descPublicationTime);
		String nickname = parts[1];
		String scrubbedOrPort = this.scrubTcpPort(orPort,
		fingerprintBytes, descPublicationTime);
		String scrubbedDirPort = this.scrubTcpPort(dirPort,
		fingerprintBytes, descPublicationTime);
		scrubbed.append("r " + nickname + " "
		+ hashedBridgeIdentityBase64 + " "
		+ hashedDescriptorIdentifier + " " + descPublicationTime
		+ " " + scrubbedAddress + " " + orPort + " " + dirPort
		+ "\n");
		+ " " + scrubbedAddress + " " + scrubbedOrPort + " "
		+ scrubbedDirPort + "\n");

		/* Sanitize any addresses in a lines using the fingerprint and
		* descriptor publication time from the previous r line. */
		@@ -555,7 +590,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		outputFile.getParentFile().mkdirs();
		BufferedWriter bw = new BufferedWriter(new FileWriter(
		outputFile));
		bw.write("@type bridge-network-status 1.0\n");
		bw.write("@type bridge-network-status 1.1\n");
		bw.write("published " + publicationTime + "\n");
		bw.write(header.toString());
		for (String scrubbed : scrubbedLines.values()) {
		@@ -595,6 +630,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		String hashedBridgeIdentity = null;
		String address = null;
		String routerLine = null;
		String scrubbedRouterLine = null;
		String scrubbedAddress = null;
		String masterKeyEd25519 = null;
		List<String> orAddresses = null;
		@@ -611,7 +647,12 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		* the bridge identity fingerprint for replacing the IP address in
		* the scrubbed version. */
		} else if (line.startsWith("router ")) {
		address = line.split(" ")[2];
		String[] parts = line.split(" ");
		if (parts.length != 6) {
		logger.warn("Invalid router line: '" + line + "'. Skipping.");
		return;
		}
		address = parts[2];
		routerLine = line;

		/* Store or-address parts in a list and sanitize them when we have
		@@ -671,6 +712,17 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		}
		}
		}
		String[] routerLineParts = routerLine.split(" ");
		String nickname = routerLineParts[1];
		String scrubbedOrPort = this.scrubTcpPort(routerLineParts[3],
		fingerprintBytes, published);
		String scrubbedDirPort = this.scrubTcpPort(routerLineParts[4],
		fingerprintBytes, published);
		String scrubbedSocksPort = this.scrubTcpPort(
		routerLineParts[5], fingerprintBytes, published);
		scrubbedRouterLine = String.format("router %s %s %s %s %s%n",
		nickname, scrubbedAddress, scrubbedOrPort,
		scrubbedDirPort, scrubbedSocksPort);
		} catch (IOException e) {
		/* There's a persistence problem, so we shouldn't scrub more
		* IP addresses in this execution. */
		@@ -692,10 +744,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		/* When we reach the signature, we're done. Write the sanitized
		* descriptor to disk below. */
		} else if (line.startsWith("router-signature")) {
		String[] routerLineParts = routerLine.split(" ");
		scrubbedDesc = "router " + routerLineParts[1] + " "
		+ scrubbedAddress + " " + routerLineParts[3] + " "
		+ routerLineParts[4] + " " + routerLineParts[5] + "\n";
		scrubbedDesc = scrubbedRouterLine;
		if (scrubbedOrAddresses != null) {
		for (String scrubbedOrAddress : scrubbedOrAddresses) {
		scrubbedDesc = scrubbedDesc += "or-address "
		@@ -915,7 +964,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		outputFile.getParentFile().mkdirs();
		BufferedWriter bw = new BufferedWriter(new FileWriter(
		outputFile, appendToFile));
		bw.write("@type bridge-server-descriptor 1.1\n");
		bw.write("@type bridge-server-descriptor 1.2\n");
		bw.write(scrubbedDesc);
		if (descriptorDigestSha256Base64 != null) {
		bw.write("router-digest-sha256 " + descriptorDigestSha256Base64

src/main/webapp/index.html

+40 −6

Original line number	Diff line number	Diff line
		@@ -166,14 +166,14 @@
		</tr>
		<tr>
		<td>Bridge Network Statuses</td>
		<td><tt>@type bridge-network-status 1.0</tt></td>
		<td><tt>@type bridge-network-status 1.1</tt></td>
		<td><a href="recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">recent</a></td>
		<td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td>
		<td><a href="#type-bridge-network-status" class="btn btn-default btn-xs">format</a></td>
		</tr>
		<tr>
		<td>Bridge Server Descriptors</td>
		<td><tt>@type bridge-server-descriptor 1.1</tt></td>
		<td><tt>@type bridge-server-descriptor 1.2</tt></td>
		<td><a href="recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">recent</a></td>
		<td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td>
		<td><a href="#type-bridge-server-descriptor" class="btn btn-default btn-xs">format</a></td>
		@@ -447,6 +447,28 @@ with <tt>xx:xxxx</tt> being the hex-formatted 3 byte output of a similar
		hash function as described for IPv4 addresses.
		The only differences are that the input <tt>IP address</tt> is 16 bytes
		long and the <tt>secret</tt> is only 19 bytes long.</li></ul>
		<li><b>Replace TCP port with TCP port hash:</b> It may be less obvious
		that TCP ports need to be sanitized, but an unusual TCP port used by a
		high-value bridge might still stand out and provide yet another way to
		locate and block the bridge.
		Therefore, each non-zero TCP port is replaced with a number in the range
		from 49152 to 65535 that is the result of
		<tt>H(port \| bridge identity \| secret)[:2] / 2^2 + 2^15 + 2^14</tt>
		written as decimal number.
		The input <tt>port</tt> is the 2-byte long binary representation of the
		TCP port.
		The <tt>bridge identity</tt> is the 20-byte long binary representation of
		the bridge's long-term identity fingerprint.
		The <tt>secret</tt> is a 33-byte long secure random string that changes
		once per month for all descriptors and statuses published in that month.
		<tt>H()</tt> is SHA-256.
		The <tt>[:2]</tt> operator means that we pick the 2 most significant bytes
		of the result.
		The subsequent integer division and additions make sure that sanitized
		ports are in the range from 49152 to 65535 which is reserved for private
		services.
		All operations assume inputs to be in network byte order.
		TCP ports that are 0 in the original descriptor are left unchanged.</li>
		<li><b>Replace contact information:</b> If there is contact information in
		a descriptor, the contact line is changed to
		<tt>somebody</tt>.</li>
		@@ -468,7 +490,7 @@ descriptor including all signatures.</li>
		</ol>

		<h3 id="type-bridge-network-status" class="hover">Bridge Network Statuses
		<small><tt>@type bridge-network-status 1.0</tt></small>
		<small><tt>@type bridge-network-status 1.1</tt></small>
		<a href="recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">recent</a>
		<a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a>
		<a href="#type-bridge-network-status" class="hover">#</a>
		@@ -480,10 +502,19 @@ statuses, but with only a <tt>published</tt> line in the header and
		without any lines in the footer.
		The bridge descriptor archive tarballs contain all bridge
		descriptors of a given month, not just network statuses.
		The format has changed over time to accomodate changes to the sanitizing
		process, with earlier versions being:
		</p>

		<ul>
		<li><font color="#666"><tt>@type bridge-network-status 1.0</tt> was the
		first version.</font></li>
		<li><tt>@type bridge-network-status 1.1</tt> introduced sanitized TCP
		ports.</li>
		</ul>

		<h3 id="type-bridge-server-descriptor" class="hover">Bridge Server descriptors
		<small><tt>@type bridge-server-descriptor 1.1</tt></small>
		<small><tt>@type bridge-server-descriptor 1.2</tt></small>
		<a href="recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">recent</a>
		<a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a>
		<a href="#type-bridge-server-descriptor" class="hover">#</a>
		@@ -511,9 +542,12 @@ number of sanitized bridge extra-info descriptors was raised.
		As a result, there may be sanitized bridge server descriptors with version
		<tt>@type bridge-server-descriptor 1.0</tt> with and without those
		lines.</font></li>
		<li><tt>@type bridge-server-descriptor 1.1</tt> added
		<li><font color="#666"><tt>@type bridge-server-descriptor 1.1</tt> added
		<tt>master-key-ed25519</tt> lines and <tt>router-digest-sha256</tt> to
		server descriptors published by bridges using an Ed25519 master key.</li>
		server descriptors published by bridges using an Ed25519 master
		key.</font></li>
		<li><tt>@type bridge-server-descriptor 1.2</tt> introduced sanitized TCP
		ports.</li>
		</ul>

		<h3 id="type-bridge-extra-info" class="hover">Bridge Extra-info Descriptors