Commit dc946fe1 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add new SnowflakeStats descriptor type.

Implements #29461.
parent 4d8a8ea7
......@@ -11,6 +11,7 @@
versions resolved by Ivy are the same as in Debian stretch with
few exceptions.
- Remove Cobertura from the build process.
- Add new SnowflakeStats descriptor type.
# Changes in version 2.6.2 - 2019-05-29
......
/* Copyright 2019 The Tor Project
* See LICENSE for licensing information */
package org.torproject.descriptor;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.SortedMap;
/**
* Contain aggregated information about snowflake proxies and snowflake clients.
*
* @since 2.7.0
*/
public interface SnowflakeStats extends Descriptor {
/**
* Return the end of the included measurement interval.
*
* @return End of the included measurement interval.
* @since 2.7.0
*/
LocalDateTime snowflakeStatsEnd();
/**
* Return the length of the included measurement interval.
*
* @return Length of the included measurement interval.
* @since 2.7.0
*/
Duration snowflakeStatsIntervalLength();
/**
* Return a list of mappings from two-letter country codes to the number of
* unique IP addresses of snowflake proxies that have polled.
*
* @return List of mappings from two-letter country codes to the number of
* unique IP addresses of snowflake proxies that have polled.
* @since 2.7.0
*/
Optional<SortedMap<String, Long>> snowflakeIps();
/**
* Return a count of the total number of unique IP addresses of snowflake
* proxies that have polled.
*
* @return Count of the total number of unique IP addresses of snowflake
* proxies that have polled.
* @since 2.7.0
*/
Optional<Long> snowflakeIpsTotal();
/**
* Return a count of the number of times a proxy has polled but received no
* client offer, rounded up to the nearest multiple of 8.
*
* @return Count of the number of times a proxy has polled but received no
* client offer, rounded up to the nearest multiple of 8.
* @since 2.7.0
*/
Optional<Long> snowflakeIdleCount();
/**
* Return a count of the number of times a client has requested a proxy from
* the broker but no proxies were available, rounded up to the nearest
* multiple of 8.
*
* @return Count of the number of times a client has requested a proxy from
* the broker but no proxies were available, rounded up to the nearest
* multiple of 8.
* @since 2.7.0
*/
Optional<Long> clientDeniedCount();
/**
* Return a count of the number of times a client successfully received a
* proxy from the broker, rounded up to the nearest multiple of 8.
*
* @return Count of the number of times a client successfully received a proxy
* from the broker, rounded up to the nearest multiple of 8.
* @since 2.7.0
*/
Optional<Long> clientSnowflakeMatchCount();
}
......@@ -130,6 +130,11 @@ public class DescriptorParserImpl implements DescriptorParser {
} else if (firstLines.startsWith("@type torperf 1.")) {
return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
sourceFile);
} else if (firstLines.startsWith("@type snowflake-stats 1.")
|| firstLines.startsWith(Key.SNOWFLAKE_STATS_END.keyword + SP)
|| firstLines.contains(NL + Key.SNOWFLAKE_STATS_END.keyword + SP)) {
return this.parseOneOrMoreDescriptors(rawDescriptorBytes, sourceFile,
Key.SNOWFLAKE_STATS_END, SnowflakeStatsImpl.class);
} else if (fileName.contains(LogDescriptorImpl.MARKER)) {
return LogDescriptorImpl.parse(rawDescriptorBytes, sourceFile, fileName);
} else if (firstLines.startsWith("@type bandwidth-file 1.")
......
......@@ -29,6 +29,8 @@ public enum Key {
CELL_QUEUED_CELLS("cell-queued-cells"),
CELL_STATS_END("cell-stats-end"),
CELL_TIME_IN_QUEUE("cell-time-in-queue"),
CLIENT_DENIED_COUNT("client-denied-count"),
CLIENT_SNOWFLAKE_MATCH_COUNT("client-snowflake-match-count"),
CLIENT_VERSIONS("client-versions"),
CONN_BI_DIRECT("conn-bi-direct"),
CONSENSUS_METHOD("consensus-method"),
......@@ -132,6 +134,10 @@ public enum Key {
SHARED_RAND_PREVIOUS_VALUE("shared-rand-previous-value"),
SIGNED_DIRECTORY("signed-directory"),
SIGNING_KEY("signing-key"),
SNOWFLAKE_IDLE_COUNT("snowflake-idle-count"),
SNOWFLAKE_IPS("snowflake-ips"),
SNOWFLAKE_IPS_TOTAL("snowflake-ips-total"),
SNOWFLAKE_STATS_END("snowflake-stats-end"),
TRANSPORT("transport"),
TUNNELLED_DIR_SERVER("tunnelled-dir-server"),
UPTIME("uptime"),
......
......@@ -11,6 +11,10 @@ import org.apache.commons.codec.binary.Hex;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
......@@ -101,6 +105,16 @@ public class ParseHelper {
}
}
static Duration parseDuration(String line, String secondsString)
throws DescriptorParseException {
long parsedSeconds = parseSeconds(line, secondsString);
if (parsedSeconds <= 0L) {
throw new DescriptorParseException("Duration must be positive in line '"
+ line + "'.");
}
return Duration.ofSeconds(parsedSeconds);
}
protected static String parseExitPattern(String line, String exitPattern)
throws DescriptorParseException {
if (!exitPattern.contains(":")) {
......@@ -188,6 +202,13 @@ public class ParseHelper {
return result;
}
static LocalDateTime parseLocalDateTime(String line, String[] parts,
int dateIndex, int timeIndex) throws DescriptorParseException {
return LocalDateTime.ofInstant(Instant.ofEpochMilli(
parseTimestampAtIndex(line, parts, dateIndex, timeIndex)),
ZoneOffset.UTC);
}
protected static long parseDateAtIndex(String line, String[] parts,
int dateIndex) throws DescriptorParseException {
if (dateIndex >= parts.length) {
......
/* Copyright 2019 The Tor Project
* See LICENSE for licensing information */
package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.SnowflakeStats;
import java.io.File;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.Optional;
import java.util.Scanner;
import java.util.Set;
import java.util.SortedMap;
public class SnowflakeStatsImpl extends DescriptorImpl
implements SnowflakeStats {
private static final Set<Key> atMostOnce = EnumSet.of(
Key.SNOWFLAKE_IPS, Key.SNOWFLAKE_IPS_TOTAL, Key.SNOWFLAKE_IDLE_COUNT,
Key.CLIENT_DENIED_COUNT, Key.CLIENT_SNOWFLAKE_MATCH_COUNT);
private static final Set<Key> exactlyOnce = EnumSet.of(
Key.SNOWFLAKE_STATS_END);
SnowflakeStatsImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
File descriptorFile) throws DescriptorParseException {
super(rawDescriptorBytes, offsetAndLength, descriptorFile, false);
this.parseDescriptorBytes();
this.checkExactlyOnceKeys(exactlyOnce);
this.checkAtMostOnceKeys(atMostOnce);
this.checkFirstKey(Key.SNOWFLAKE_STATS_END);
this.clearParsedKeys();
}
SnowflakeStatsImpl(byte[] rawDescriptorBytes, File descriptorFile)
throws DescriptorParseException {
this(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
descriptorFile);
}
private void parseDescriptorBytes() throws DescriptorParseException {
Scanner scanner = this.newScanner().useDelimiter(NL);
while (scanner.hasNext()) {
String line = scanner.next();
if (line.startsWith("@")) {
continue;
}
String[] parts = line.split("[ \t]+");
Key key = Key.get(parts[0]);
switch (key) {
case SNOWFLAKE_STATS_END:
this.parseSnowflakeStatsEnd(line, parts);
break;
case SNOWFLAKE_IPS:
this.parseSnowflakeIps(line, parts);
break;
case SNOWFLAKE_IPS_TOTAL:
this.parseSnowflakeIpsTotal(line, parts);
break;
case SNOWFLAKE_IDLE_COUNT:
this.parseSnowflakeIdleCount(line, parts);
break;
case CLIENT_DENIED_COUNT:
this.parseClientDeniedCount(line, parts);
break;
case CLIENT_SNOWFLAKE_MATCH_COUNT:
this.parseClientSnowflakeMatchCount(line, parts);
break;
case INVALID:
default:
ParseHelper.parseKeyword(line, parts[0]);
if (this.unrecognizedLines == null) {
this.unrecognizedLines = new ArrayList<>();
}
this.unrecognizedLines.add(line);
}
}
}
private void parseSnowflakeStatsEnd(String line, String[] parts)
throws DescriptorParseException {
if (parts.length < 5 || parts[3].length() < 2 || !parts[3].startsWith("(")
|| !parts[4].equals("s)")) {
throw new DescriptorParseException("Illegal line '" + line + "'.");
}
this.snowflakeStatsEnd = ParseHelper.parseLocalDateTime(line, parts,
1, 2);
this.snowflakeStatsIntervalLength = ParseHelper.parseDuration(line,
parts[3].substring(1));
}
private void parseSnowflakeIps(String line, String[] parts)
throws DescriptorParseException {
this.snowflakeIps = ParseHelper.parseCommaSeparatedKeyLongValueList(line,
parts, 1, 2);
}
private void parseSnowflakeIpsTotal(String line, String[] parts)
throws DescriptorParseException {
this.snowflakeIpsTotal = parseLong(line, parts, 1);
}
private void parseSnowflakeIdleCount(String line, String[] parts)
throws DescriptorParseException {
this.snowflakeIdleCount = parseLong(line, parts, 1);
}
private void parseClientDeniedCount(String line, String[] parts)
throws DescriptorParseException {
this.clientDeniedCount = parseLong(line, parts, 1);
}
private void parseClientSnowflakeMatchCount(String line, String[] parts)
throws DescriptorParseException {
this.clientSnowflakeMatchCount = parseLong(line, parts, 1);
}
private static Long parseLong(String line, String[] parts, int index)
throws DescriptorParseException {
if (index >= parts.length) {
throw new DescriptorParseException(String.format(
"Line '%s' does not contain a long value at index %d.", line, index));
}
try {
return Long.parseLong(parts[index]);
} catch (NumberFormatException e) {
throw new DescriptorParseException(String.format(
"Unable to parse long value '%s' in line '%s'.", parts[index], line));
}
}
private LocalDateTime snowflakeStatsEnd;
@Override
public LocalDateTime snowflakeStatsEnd() {
return this.snowflakeStatsEnd;
}
private Duration snowflakeStatsIntervalLength;
@Override
public Duration snowflakeStatsIntervalLength() {
return this.snowflakeStatsIntervalLength;
}
private SortedMap<String, Long> snowflakeIps;
@Override
public Optional<SortedMap<String, Long>> snowflakeIps() {
return Optional.ofNullable(this.snowflakeIps);
}
private Long snowflakeIpsTotal;
@Override
public Optional<Long> snowflakeIpsTotal() {
return Optional.ofNullable(this.snowflakeIpsTotal);
}
private Long snowflakeIdleCount;
@Override
public Optional<Long> snowflakeIdleCount() {
return Optional.ofNullable(this.snowflakeIdleCount);
}
private Long clientDeniedCount;
@Override
public Optional<Long> clientDeniedCount() {
return Optional.ofNullable(this.clientDeniedCount);
}
private Long clientSnowflakeMatchCount;
@Override
public Optional<Long> clientSnowflakeMatchCount() {
return Optional.ofNullable(this.clientSnowflakeMatchCount);
}
}
......@@ -80,7 +80,12 @@ public class DescriptorTest {
{"bridge/2017-07-17-17-09-00-server-descriptors",
BridgeServerDescriptor.class,
new String[] {"@type bridge-server-descriptor 1.2"},
13}
13},
{"snowflake/example_metrics.log",
SnowflakeStats.class,
new String[0],
2}
});
}
......
/* Copyright 2019 The Tor Project
* See LICENSE for licensing information */
package org.torproject.descriptor.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.SnowflakeStats;
import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.time.Duration;
import java.time.LocalDateTime;
public class SnowflakeStatsImplTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
/**
* Example from example_metrics.log attached to #29461.
*/
private static final String[] exampleMetricsLog = new String[] {
"snowflake-stats-end 2019-08-07 19:52:11 (86400 s)",
"snowflake-ips VN=5,NL=26,AU=30,GT=2,NO=5,EG=3,NI=1,AT=22,FR=42,CA=44,"
+ "ZA=3,PL=20,RU=10,HR=1,CN=1,RO=4,??=7,TH=7,UA=5,DZ=5,HU=5,CH=15,"
+ "AE=1,PH=6,RS=3,BR=20,IT=8,KR=13,HK=7,GR=5,GB=41,DK=4,CZ=7,IE=4,"
+ "PT=7,TR=2,NP=2,BA=1,BE=2,IN=45,SE=23,CL=3,IL=3,FI=7,MX=6,CO=1,"
+ "PK=4,ID=9,IR=7,JO=2,CR=2,US=265,DE=92,LV=1,MY=8,AR=5,NZ=10,BG=2,"
+ "UY=1,TW=5,SI=3,LU=2,GE=2,BN=1,JP=15,ES=9,SG=7,EC=1",
"snowflake-ips-total 937",
"snowflake-idle-count 660976",
"client-denied-count 0",
"client-snowflake-match-count 864" };
@Test
public void testExampleMetricsLog() throws DescriptorParseException {
SnowflakeStats snowflakeStats = new SnowflakeStatsImpl(
new TestDescriptorBuilder(exampleMetricsLog).build(), null);
assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11),
snowflakeStats.snowflakeStatsEnd());
assertEquals(Duration.ofDays(1L),
snowflakeStats.snowflakeStatsIntervalLength());
assertTrue(snowflakeStats.snowflakeIps().isPresent());
assertEquals(68, snowflakeStats.snowflakeIps().get().size());
assertTrue(snowflakeStats.snowflakeIpsTotal().isPresent());
assertEquals((Long) 937L, snowflakeStats.snowflakeIpsTotal().get());
assertTrue(snowflakeStats.snowflakeIdleCount().isPresent());
assertEquals((Long) 660976L, snowflakeStats.snowflakeIdleCount().get());
assertTrue(snowflakeStats.clientDeniedCount().isPresent());
assertEquals((Long) 0L, snowflakeStats.clientDeniedCount().get());
assertTrue(snowflakeStats.clientSnowflakeMatchCount().isPresent());
assertEquals((Long) 864L, snowflakeStats.clientSnowflakeMatchCount().get());
}
@Test
public void testMinimalSnowflakeStats() throws DescriptorParseException {
SnowflakeStats snowflakeStats = new SnowflakeStatsImpl(
new TestDescriptorBuilder(exampleMetricsLog[0]).build(), null);
assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11),
snowflakeStats.snowflakeStatsEnd());
assertEquals(Duration.ofDays(1L),
snowflakeStats.snowflakeStatsIntervalLength());
assertFalse(snowflakeStats.snowflakeIps().isPresent());
assertFalse(snowflakeStats.snowflakeIpsTotal().isPresent());
assertFalse(snowflakeStats.snowflakeIdleCount().isPresent());
assertFalse(snowflakeStats.clientDeniedCount().isPresent());
assertFalse(snowflakeStats.clientSnowflakeMatchCount().isPresent());
}
@Test
public void testEmptyLine() throws DescriptorParseException {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage(Matchers.containsString(
"Blank lines are not allowed."));
new SnowflakeStatsImpl(new TestDescriptorBuilder(exampleMetricsLog)
.appendLines("")
.build(), null);
}
@Test
public void testDuplicateLine() throws DescriptorParseException {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage(Matchers.containsString(
"must be contained at most once."));
new SnowflakeStatsImpl(new TestDescriptorBuilder(
exampleMetricsLog[0], exampleMetricsLog[1], exampleMetricsLog[1])
.build(), null);
}
@Test
public void testEmptyList() throws DescriptorParseException {
SnowflakeStats snowflakeStats = new SnowflakeStatsImpl(
new TestDescriptorBuilder(exampleMetricsLog[0], "snowflake-ips ")
.build(), null);
assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11),
snowflakeStats.snowflakeStatsEnd());
assertEquals(Duration.ofDays(1L),
snowflakeStats.snowflakeStatsIntervalLength());
assertTrue(snowflakeStats.snowflakeIps().isPresent());
assertTrue(snowflakeStats.snowflakeIps().get().isEmpty());
}
@Test
public void testNoValue() throws DescriptorParseException {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage(Matchers.containsString(
"does not contain a long value at index 1."));
new SnowflakeStatsImpl(new TestDescriptorBuilder(
exampleMetricsLog[0], "snowflake-ips-total").build(), null);
}
@Test
public void testNotANumber() throws DescriptorParseException {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage(Matchers.containsString(
"Unable to parse long value"));
new SnowflakeStatsImpl(new TestDescriptorBuilder(
exampleMetricsLog[0], "snowflake-ips-total NaN").build(), null);
}
@Test
public void testNonPositiveIntervalLength() throws DescriptorParseException {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage(Matchers.containsString(
"Duration must be positive"));
new SnowflakeStatsImpl(new TestDescriptorBuilder(
"snowflake-stats-end 2019-08-07 19:52:11 (0 s)").build(), null);
}
}
snowflake-stats-end 2019-08-07 19:52:11 (86400 s)
snowflake-ips VN=5,NL=26,AU=30,GT=2,NO=5,EG=3,NI=1,AT=22,FR=42,CA=44,ZA=3,PL=20,RU=10,HR=1,CN=1,RO=4,??=7,TH=7,UA=5,DZ=5,HU=5,CH=15,AE=1,PH=6,RS=3,BR=20,IT=8,KR=13,HK=7,GR=5,GB=41,DK=4,CZ=7,IE=4,PT=7,TR=2,NP=2,BA=1,BE=2,IN=45,SE=23,CL=3,IL=3,FI=7,MX=6,CO=1,PK=4,ID=9,IR=7,JO=2,CR=2,US=265,DE=92,LV=1,MY=8,AR=5,NZ=10,BG=2,UY=1,TW=5,SI=3,LU=2,GE=2,BN=1,JP=15,ES=9,SG=7,EC=1
snowflake-ips-total 937
snowflake-idle-count 660976
client-denied-count 0
client-snowflake-match-count 864
snowflake-stats-end 2019-08-08 19:52:11 (86400 s)
snowflake-ips IE=7,IN=31,ES=17,MY=7,NO=7,IR=6,NL=36,ZA=2,GT=1,PK=5,US=284,SE=21,UY=1,AR=8,VN=6,RS=3,GB=37,CZ=12,NZ=7,CO=2,PH=6,RO=5,AT=24,GE=1,BE=5,EE=1,TR=5,CL=5,CA=59,PT=9,MX=76,IL=3,BG=2,BA=1,HU=9,JO=2,PL=16,GR=5,KR=13,EG=2,TW=10,ID=14,FI=9,DK=3,IT=11,TH=3,DE=118,SI=4,CH=19,UA=5,AU=32,NG=3,AE=1,RU=17,NI=1,JP=18,SD=1,LU=2,FR=80,BR=12,CR=6,CN=16,DZ=4,SG=13,NP=2,??=7,HK=12,HR=5,BN=1
snowflake-ips-total 1178
snowflake-idle-count 979344
client-denied-count 0
client-snowflake-match-count 392
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment