Skip to content
GitLab
Commit b23ea70a authored by Hiro's avatar Hiro 🏄
Browse files

This commit fixes test but more test are needed to make sure we are not changing functionality 

nor messing too much with the protocol.
parent 863fdf74
Pipeline #32857 failed with stage
in 1 minute and 36 seconds
Hide whitespace changes
Inline Side-by-side
src/main/java/org/torproject/metrics/onionoo/docs/UptimeDocument.java
View file @ b23ea70a
......@@ -15,6 +15,13 @@ public class UptimeDocument extends Document {
this.fingerprint = fingerprint;
}
private String nickname;
public void setNickname(String nickname) {
this.nickname = nickname;
}
private Map<String, GraphHistory> uptime;
public void setUptime(Map<String, GraphHistory> uptime) {
......@@ -36,4 +43,3 @@ public class UptimeDocument extends Document {
return this.flags;
}
}
src/main/java/org/torproject/metrics/onionoo/updater/UptimeStatusUpdater.java
View file @ b23ea70a
......@@ -29,7 +29,7 @@ public class UptimeStatusUpdater implements DescriptorListener,
private static final Logger logger = LoggerFactory.getLogger(
UptimeStatusUpdater.class);
private DescriptorSource descriptorSource;
private DocumentStore documentStore;
......@@ -109,8 +109,12 @@ public class UptimeStatusUpdater implements DescriptorListener,
for (NetworkStatusEntry entry :
consensus.getStatusEntries().values()) {
String fingerprint = entry.getFingerprint();
String nickname = entry.getNickname();
String nodeId = String.format("%s:%s", fingerprint, nickname);
/*
* We do not have the nickname, so to keep the consistency
* with the fingerprint/nickname folder adn file status,
* we set the nickname as status.
*/
String nodeId = String.format("%s:%s", fingerprint, "status");
this.newRunningRelays.putIfAbsent(nodeId, new TreeMap<>());
this.newRunningRelays.get(nodeId).put(dateHourMillis,
new Flags(entry.getFlags()));
......@@ -124,8 +128,12 @@ public class UptimeStatusUpdater implements DescriptorListener,
for (NetworkStatusEntry entry :
status.getStatusEntries().values()) {
String fingerprint = entry.getFingerprint();
String nickname = entry.getNickname();
String nodeId = String.format("%s:%s", fingerprint, nickname);
/*
* We do not have the nickname, so to keep the consistency
* with the fingerprint/nickname folder adn file status,
* we set the nickname as status.
*/
String nodeId = String.format("%s:%s", fingerprint, "status");
nodes.add(nodeId);
}
if (!nodes.isEmpty()) {
......@@ -141,7 +149,6 @@ public class UptimeStatusUpdater implements DescriptorListener,
@Override
public void updateStatuses() {
logger.warn("updating statuses");
for (Map.Entry<String, SortedMap<Long, Flags>> e :
this.newRunningRelays.entrySet()) {
String[] parts = e.getKey().split(":");
......@@ -172,7 +179,6 @@ public class UptimeStatusUpdater implements DescriptorListener,
SortedMap<Long, Flags> dateHourMillisFlags) {
UptimeStatus uptimeStatus = new UptimeStatus();
if (fingerprint == null && nickname == null) {
logger.warn("updatating status with null nickname and figerprint");
uptimeStatus = this.documentStore.retrieve(UptimeStatus.class, true);
} else {
uptimeStatus = this.documentStore.retrieve(UptimeStatus.class, true,
......
src/main/java/org/torproject/metrics/onionoo/writer/DetailsDocumentWriter.java
View file @ b23ea70a
......@@ -47,9 +47,12 @@ public class DetailsDocumentWriter implements DocumentWriter {
? updateStatus.getUpdatedMillis() : 0L;
SortedSet<String> updatedDetailsStatuses = this.documentStore.list(
DetailsStatus.class, updatedMillis);
for (String fingerprint : updatedDetailsStatuses) {
for (String nodeId : updatedDetailsStatuses) {
String[] parts = nodeId.split(":");
String fingerprint = parts[0];
String nickname = parts[1];
DetailsStatus detailsStatus = this.documentStore.retrieve(
DetailsStatus.class, true, fingerprint);
DetailsStatus.class, true, fingerprint, nickname);
if (detailsStatus.isRelay()) {
this.updateRelayDetailsFile(fingerprint, detailsStatus);
} else {
......@@ -68,7 +71,8 @@ public class DetailsDocumentWriter implements DocumentWriter {
private void updateRelayDetailsFile(String fingerprint,
DetailsStatus detailsStatus) {
DetailsDocument detailsDocument = new DetailsDocument();
detailsDocument.setNickname(detailsStatus.getNickname());
String nickname = detailsStatus.getNickname();
detailsDocument.setNickname(nickname);
detailsDocument.setFingerprint(fingerprint);
List<String> orAddresses = new ArrayList<>();
orAddresses.add(detailsStatus.getAddress() + ":"
......@@ -174,7 +178,8 @@ public class DetailsDocumentWriter implements DocumentWriter {
}
detailsDocument.setVersion(detailsStatus.getVersion());
detailsDocument.setVersionStatus(detailsStatus.getVersionStatus());
this.documentStore.store(detailsDocument, fingerprint);
this.documentStore.store(detailsDocument, fingerprint,
nickname);
}
/**
......@@ -186,7 +191,8 @@ public class DetailsDocumentWriter implements DocumentWriter {
private void updateBridgeDetailsFile(String fingerprint,
DetailsStatus detailsStatus) {
DetailsDocument detailsDocument = new DetailsDocument();
detailsDocument.setNickname(detailsStatus.getNickname());
String nickname = detailsStatus.getNickname();
detailsDocument.setNickname(nickname);
detailsDocument.setHashedFingerprint(fingerprint);
String address = detailsStatus.getAddress();
......@@ -221,7 +227,8 @@ public class DetailsDocumentWriter implements DocumentWriter {
detailsDocument.setVersionStatus(detailsStatus.getVersionStatus());
detailsDocument.setBridgedbDistributor(
detailsStatus.getBridgedbDistributor());
this.documentStore.store(detailsDocument, fingerprint);
this.documentStore.store(detailsDocument, fingerprint,
nickname);
}
@Override
......
src/main/java/org/torproject/metrics/onionoo/writer/UptimeDocumentWriter.java
View file @ b23ea70a
......@@ -49,7 +49,9 @@ public class UptimeDocumentWriter implements DocumentWriter {
? updateStatus.getUpdatedMillis() : 0L;
SortedSet<String> updatedUptimeStatuses = this.documentStore.list(
UptimeStatus.class, updatedMillis);
for (String fingerprint : updatedUptimeStatuses) {
for (String node : updatedUptimeStatuses) {
String[] parts = node.split(":");
String fingerprint = parts[0];
this.updateDocument(fingerprint, mostRecentStatusMillis, uptimeStatus);
}
logger.info("Wrote uptime document files");
......@@ -60,7 +62,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
private void updateDocument(String fingerprint, long mostRecentStatusMillis,
UptimeStatus knownStatuses) {
UptimeStatus uptimeStatus = this.documentStore.retrieve(
UptimeStatus.class, true, fingerprint);
UptimeStatus.class, true, fingerprint, "status");
if (null != uptimeStatus) {
boolean relay = uptimeStatus.getBridgeHistory().isEmpty();
SortedSet<UptimeHistory> history = relay
......@@ -71,7 +73,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
: knownStatuses.getBridgeHistory();
UptimeDocument uptimeDocument = this.compileUptimeDocument(relay,
fingerprint, history, knownStatusesHistory, mostRecentStatusMillis);
this.documentStore.store(uptimeDocument, fingerprint);
this.documentStore.store(uptimeDocument, fingerprint, "status");
this.writtenDocuments++;
}
}
......@@ -239,4 +241,3 @@ public class UptimeDocumentWriter implements DocumentWriter {
FormattingUtils.formatDecimalNumber(this.writtenDocuments));
}
}
src/test/java/org/torproject/metrics/onionoo/docs/DummyDocumentStore.java
View file @ b23ea70a
......@@ -44,7 +44,6 @@ public class DummyDocumentStore extends DocumentStore {
fingerprint = (fingerprint == null) ? FINGERPRINT_NULL : fingerprint;
nickname = (nickname == null) ? NICKNAME_NULL : nickname;
String nodeId = String.format("%s:%s", fingerprint, nickname);
logger.warn("Trying to retrieve document with nodeId {}", nodeId);
return documentType.cast(this.getStoredDocumentsByClass(documentType)
.get(nodeId));
}
......
src/test/java/org/torproject/metrics/onionoo/updater/UptimeStatusUpdaterTest.java
View file @ b23ea70a
......@@ -13,11 +13,17 @@ import org.torproject.metrics.onionoo.docs.DummyDocumentStore;
import org.torproject.metrics.onionoo.docs.UptimeHistory;
import org.torproject.metrics.onionoo.docs.UptimeStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Before;
import org.junit.Test;
public class UptimeStatusUpdaterTest {
private static final Logger logger = LoggerFactory.getLogger(
UptimeStatusUpdaterTest.class);
private DummyDescriptorSource descriptorSource;
@Before
......@@ -70,15 +76,19 @@ public class UptimeStatusUpdaterTest {
updater.updateStatuses();
assertEquals("Two status files should have been written to disk.",
2, this.documentStore.getPerformedStoreOperations());
for (String fingerprint : new String[] { GABELMOO_FINGERPRINT,
for (String node : new String[] { GABELMOO_FINGERPRINT,
null }) {
UptimeStatus status = new UptimeStatus();
if (fingerprint == null){
String nickname = null;
String fingerprint = null;
if (node == null) {
status = this.documentStore.getDocument(
UptimeStatus.class, null, null);
} else {
nickname = "status";
fingerprint = node;
status = this.documentStore.getDocument(
UptimeStatus.class, fingerprint, "gabelmoo");
UptimeStatus.class, fingerprint, nickname);
}
UptimeHistory history = status.getRelayHistory().first();
assertEquals("History must contain one entry.", 1,
......@@ -155,10 +165,20 @@ public class UptimeStatusUpdaterTest {
updater.updateStatuses();
assertEquals("Two status files should have been written to disk.",
2, this.documentStore.getPerformedStoreOperations());
for (String fingerprint : new String[] { NDNOP2_FINGERPRINT,
for (String node : new String[] { NDNOP2_FINGERPRINT,
null }) {
UptimeStatus status = this.documentStore.getDocument(
UptimeStatus.class, fingerprint, "ndnop2");
UptimeStatus status = new UptimeStatus();
String nickname = null;
String fingerprint = null;
if (node == null) {
status = this.documentStore.getDocument(
UptimeStatus.class, null, null);
} else {
nickname = "status";
fingerprint = node;
status = this.documentStore.getDocument(
UptimeStatus.class, fingerprint, nickname);
}
UptimeHistory history = status.getBridgeHistory().first();
assertEquals("Bridge history must contain one entry.", 1,
status.getBridgeHistory().size());
......
src/test/java/org/torproject/metrics/onionoo/writer/BandwidthDocumentWriterTest.java
View file @ b23ea70a
......@@ -44,14 +44,14 @@ public class BandwidthDocumentWriterTest {
+ "r 2021-08-11 13:31:45 2021-08-12 13:31:45 0\n";
BandwidthStatus status = new BandwidthStatus();
status.setFromDocumentString(documentString);
this.documentStore.addDocument(status, ibibUnc0Fingerprint);
this.documentStore.addDocument(status, ibibUnc0Fingerprint, "ibibUnc0");
BandwidthDocumentWriter writer = new BandwidthDocumentWriter();
writer.writeDocuments(Instant.parse("2020-05-15T12:00:00Z").toEpochMilli());
assertEquals(1, this.documentStore.getPerformedListOperations());
assertEquals(2, this.documentStore.getPerformedRetrieveOperations());
assertEquals(1, this.documentStore.getPerformedStoreOperations());
BandwidthDocument document = this.documentStore.getDocument(
BandwidthDocument.class, ibibUnc0Fingerprint);
BandwidthDocument.class, ibibUnc0Fingerprint, "ibibUnc0");
assertEquals(1, document.getReadHistory().size());
assertTrue(document.getReadHistory().containsKey("6_months"));
GraphHistory history = document.getReadHistory().get("6_months");
......@@ -65,4 +65,3 @@ public class BandwidthDocumentWriterTest {
assertEquals(5, history.getValues().size());
}
}
src/test/java/org/torproject/metrics/onionoo/writer/DetailsDocumentWriterTest.java
View file @ b23ea70a
......@@ -9,6 +9,9 @@ import org.torproject.metrics.onionoo.docs.DetailsStatus;
import org.torproject.metrics.onionoo.docs.DocumentStoreFactory;
import org.torproject.metrics.onionoo.docs.DummyDocumentStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Before;
import org.junit.Test;
......@@ -17,6 +20,9 @@ import java.util.TreeSet;
public class DetailsDocumentWriterTest {
private static final Logger logger = LoggerFactory.getLogger(
DetailsDocumentWriterTest.class);
private DummyDocumentStore documentStore;
@Before
......@@ -43,16 +49,17 @@ public class DetailsDocumentWriterTest {
public void testAdvertisedAndReachableOrAddress() {
DetailsStatus status = new DetailsStatus();
status.setRelay(true);
status.setNickname("gabelmoo");
status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS));
status.setOrAddressesAndPorts(new TreeSet<>(Arrays.asList(
GABELMOO_OR_ADDRESS)));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT, "gabelmoo");
DetailsDocumentWriter writer = new DetailsDocumentWriter();
writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
DetailsDocument.class, GABELMOO_FINGERPRINT);
DetailsDocument.class, GABELMOO_FINGERPRINT, "gabelmoo");
assertNotNull("There should be a document for the given fingerprint.",
document);
assertNull("Document should not contain any unreachable OR addresses.",
......@@ -63,15 +70,16 @@ public class DetailsDocumentWriterTest {
public void testUnadvertisedButSomehowReachableOrAddress() {
DetailsStatus status = new DetailsStatus();
status.setRelay(true);
status.setNickname("gabelmoo");
status.setOrAddressesAndPorts(new TreeSet<>(Arrays.asList(
GABELMOO_OR_ADDRESS)));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT, "gabelmoo");
DetailsDocumentWriter writer = new DetailsDocumentWriter();
writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
DetailsDocument.class, GABELMOO_FINGERPRINT);
DetailsDocument.class, GABELMOO_FINGERPRINT, "gabelmoo");
assertNotNull("There should be a document for the given fingerprint.",
document);
assertNull("Document should not contain unreachable OR addresses.",
......@@ -82,14 +90,15 @@ public class DetailsDocumentWriterTest {
public void testAdvertisedButUnreachableOrAddress() {
DetailsStatus status = new DetailsStatus();
status.setRelay(true);
status.setNickname("gabelmoo");
status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT, "gabelmoo");
DetailsDocumentWriter writer = new DetailsDocumentWriter();
writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
DetailsDocument.class, GABELMOO_FINGERPRINT);
DetailsDocument.class, GABELMOO_FINGERPRINT, "gabelmoo");
assertNotNull("There should be a document for the given fingerprint.",
document);
assertEquals("Document should contain one unreachable OR address.",
......@@ -97,4 +106,3 @@ public class DetailsDocumentWriterTest {
document.getUnreachableOrAddresses());
}
}
src/test/java/org/torproject/metrics/onionoo/writer/UptimeDocumentWriterTest.java
View file @ b23ea70a
......@@ -51,10 +51,10 @@ public class UptimeDocumentWriterTest {
String gabelmooUptime) {
UptimeStatus status = new UptimeStatus();
status.setFromDocumentString(allRelaysUptime);
this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT);
this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT, null);
status = new UptimeStatus();
status.setFromDocumentString(gabelmooUptime);
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT, "status");
}
private static final long ONE_SECOND = 1000L;
......@@ -113,7 +113,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
assertEquals("Should not contain any graph.", 0,
document.getUptime().size());
}
......@@ -128,7 +128,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-23 06:00:00",
"2014-03-23 10:00:00", 2, null);
}
......@@ -143,7 +143,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
assertEquals("Should not contain any graph.", 0,
document.getUptime().size());
}
......@@ -158,7 +158,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-23 02:00:00",
"2014-03-23 10:00:00", 3,
Arrays.asList(999, 0, 999));
......@@ -174,7 +174,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-23 02:00:00",
"2014-03-23 10:00:00", 3,
Arrays.asList(999, 999, 0));
......@@ -190,7 +190,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-16 14:00:00",
"2014-03-23 10:00:00", 42, null);
}
......@@ -205,7 +205,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-16 14:00:00",
"2014-03-23 10:00:00", 42, null);
}
......@@ -220,7 +220,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-16 10:00:00",
"2014-03-16 14:00:00", 2, null);
}
......@@ -235,7 +235,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertOneMonthGraph(document, 1, "2014-03-16 10:00:00",
"2014-03-16 14:00:00", 2,
Arrays.asList(999, 499));
......@@ -254,7 +254,7 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
assertEquals("Should not contain any graph.", 0,
document.getUptime().size());
}
......@@ -272,9 +272,8 @@ public class UptimeDocumentWriterTest {
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
UptimeDocument.class, GABELMOO_FINGERPRINT, "status");
this.assertFiveYearGraph(document, 1, "2012-03-06 00:00:00",
"2012-03-16 00:00:00", 2, Arrays.asList(999, 999));
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment