Commit 73f8dedf authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Replace Gson with Jackson.

Update metrics-lib dependency to 2.4.0.

Adapt touched logging statements to our standards.

Implements #25848.
parent 960ded01
# Changes in version 6.0-1.??.? - 2018-05-??
* Medium changes
- Replace Gson with Jackson.
# Changes in version 6.0-1.13.0 - 2018-04-17
* Medium changes
......
......@@ -11,7 +11,7 @@
<property name="onionoo.protocol.version" value="6.0"/>
<property name="release.version"
value="${onionoo.protocol.version}-1.13.0-dev"/>
<property name="metricslibversion" value="2.2.0"/>
<property name="metricslibversion" value="2.4.0"/>
<property name="jetty.version" value="-9.2.21.v20170120" />
<property name="warfile"
value="onionoo-${release.version}.war"/>
......@@ -47,7 +47,9 @@
<include name="commons-codec-1.10.jar"/>
<include name="commons-compress-1.13.jar"/>
<include name="commons-lang3-3.5.jar"/>
<include name="gson-2.4.jar"/>
<include name="jackson-annotations-2.8.6.jar"/>
<include name="jackson-core-2.8.6.jar"/>
<include name="jackson-databind-2.8.6.jar"/>
<include name="logback-classic-1.1.9.jar"/>
<include name="logback-core-1.1.9.jar"/>
<include name="slf4j-api-1.7.22.jar"/>
......
......@@ -34,12 +34,12 @@ public class DetailsStatus extends Document {
private String desc_published;
public void setDescPublished(Long descPublished) {
this.desc_published = DateTimeHelper.format(descPublished);
this.desc_published = null == descPublished ? null
: DateTimeHelper.format(descPublished);
}
public Long getDescPublished() {
return this.desc_published == null ? null :
DateTimeHelper.parse(this.desc_published);
return DateTimeHelper.parse(this.desc_published);
}
private String last_restarted;
......
......@@ -5,9 +5,12 @@ package org.torproject.onionoo.docs;
import org.torproject.onionoo.util.FormattingUtils;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonParseException;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -41,6 +44,12 @@ public class DocumentStore {
private static Logger log = LoggerFactory.getLogger(
DocumentStore.class);
private static ObjectMapper objectMapper = new ObjectMapper()
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
private final File statusDir = new File("status");
private File outDir = null;
......@@ -162,12 +171,11 @@ public class DocumentStore {
String line = null;
try (BufferedReader br = new BufferedReader(new FileReader(
summaryFile))) {
Gson gson = new Gson();
while ((line = br.readLine()) != null) {
if (line.length() == 0) {
continue;
}
SummaryDocument summaryDocument = gson.fromJson(line,
SummaryDocument summaryDocument = objectMapper.readValue(line,
SummaryDocument.class);
if (summaryDocument != null) {
parsedSummaryDocuments.put(summaryDocument.getFingerprint(),
......@@ -178,11 +186,8 @@ public class DocumentStore {
this.listedFiles += parsedSummaryDocuments.size();
this.listOperations++;
} catch (IOException e) {
log.error("Could not read file '"
+ summaryFile.getAbsolutePath() + "'.", e);
} catch (JsonParseException e) {
log.error("Could not parse summary document '" + line
+ "' in file '" + summaryFile.getAbsolutePath() + "'.", e);
log.error("Could not parse summary document '{}' from file '{}'.",
line, summaryFile.getAbsolutePath(), e);
}
}
}
......@@ -303,13 +308,15 @@ public class DocumentStore {
|| document instanceof WeightsDocument
|| document instanceof ClientsDocument
|| document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(document);
try {
documentString = objectMapper.writeValueAsString(document);
} catch (JsonProcessingException e) {
log.error("Serializing failed for type {}.",
document.getClass().getName(), e);
return false;
}
} else if (document instanceof DetailsStatus
|| document instanceof DetailsDocument) {
/* Don't escape HTML characters, like < and >, contained in
* strings. */
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
/* We must ensure that details files only contain ASCII characters
* and no UTF-8 characters. While UTF-8 characters are perfectly
* valid in JSON, this would break compatibility with existing files
......@@ -317,7 +324,14 @@ public class DocumentStore {
* objects are escaped JSON, e.g., \u00F2. When Gson serlializes
* this string, it escapes the \ to \\, hence writes \\u00F2. We
* need to undo this and change \\u00F2 back to \u00F2. */
documentString = FormattingUtils.replaceValidUtf(gson.toJson(document));
try {
documentString = FormattingUtils.replaceValidUtf(
objectMapper.writeValueAsString(document));
} catch (JsonProcessingException e) {
log.error("Serializing failed for type {}.",
document.getClass().getName(), e);
return false;
}
/* Existing details statuses don't contain opening and closing curly
* brackets, so we should remove them from new details statuses,
* too. */
......@@ -536,9 +550,8 @@ public class DocumentStore {
private <T extends Document> T retrieveParsedDocumentFile(
Class<T> documentType, String documentString) {
T result = null;
Gson gson = new Gson();
try {
result = gson.fromJson(documentString, documentType);
result = objectMapper.readValue(documentString, documentType);
} catch (Throwable e) {
/* Handle below. */
log.error(documentString);
......@@ -777,10 +790,14 @@ public class DocumentStore {
return;
}
StringBuilder sb = new StringBuilder();
Gson gson = new Gson();
for (SummaryDocument summaryDocument :
this.cachedSummaryDocuments.values()) {
String line = gson.toJson(summaryDocument);
String line = null;
try {
line = objectMapper.writeValueAsString(summaryDocument);
} catch (JsonProcessingException e) {
line = null;
}
if (line != null) {
sb.append(line + "\n");
} else {
......
......@@ -3,8 +3,8 @@
package org.torproject.onionoo.docs;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
......@@ -20,8 +20,7 @@ import java.util.regex.Pattern;
public class SummaryDocument extends Document {
@Expose
@SerializedName("t")
@JsonProperty("t")
private boolean isRelay;
public void setRelay(boolean isRelay) {
......@@ -32,8 +31,7 @@ public class SummaryDocument extends Document {
return this.isRelay;
}
@Expose
@SerializedName("f")
@JsonProperty("f")
private String fingerprint;
/** Sets the fingerprint to the given 40 hex characters and clears
......@@ -57,6 +55,7 @@ public class SummaryDocument extends Document {
return this.fingerprint;
}
@JsonIgnore
private transient String hashedFingerprint = null;
/** Returns the SHA1-hashed fingerprint, or <code>null</code> if no
......@@ -73,6 +72,7 @@ public class SummaryDocument extends Document {
return this.hashedFingerprint;
}
@JsonIgnore
private transient String base64Fingerprint = null;
/** Returns the base64-encoded fingerprint, or <code>null</code> if no
......@@ -89,6 +89,7 @@ public class SummaryDocument extends Document {
return this.base64Fingerprint;
}
@JsonIgnore
private transient String[] fingerprintSortedHexBlocks = null;
/** Returns a sorted array containing blocks of 4 upper-case hex
......@@ -109,8 +110,7 @@ public class SummaryDocument extends Document {
return this.fingerprintSortedHexBlocks;
}
@Expose
@SerializedName("n")
@JsonProperty("n")
private String nickname;
@SuppressWarnings("checkstyle:javadocmethod")
......@@ -126,8 +126,7 @@ public class SummaryDocument extends Document {
return this.nickname == null ? "Unnamed" : this.nickname;
}
@Expose
@SerializedName("ad")
@JsonProperty("ad")
private String[] addresses;
public void setAddresses(List<String> addresses) {
......@@ -169,8 +168,7 @@ public class SummaryDocument extends Document {
return sortedSet;
}
@Expose
@SerializedName("cc")
@JsonProperty("cc")
private String countryCode;
public void setCountryCode(String countryCode) {
......@@ -181,8 +179,7 @@ public class SummaryDocument extends Document {
return this.countryCode;
}
@Expose
@SerializedName("as")
@JsonProperty("as")
private String asNumber;
public void setAsNumber(String asNumber) {
......@@ -193,8 +190,7 @@ public class SummaryDocument extends Document {
return this.asNumber;
}
@Expose
@SerializedName("fs")
@JsonProperty("fs")
private String firstSeenMillis;
public void setFirstSeenMillis(long firstSeenMillis) {
......@@ -205,8 +201,7 @@ public class SummaryDocument extends Document {
return DateTimeHelper.parse(this.firstSeenMillis);
}
@Expose
@SerializedName("ls")
@JsonProperty("ls")
private String lastSeenMillis;
public void setLastSeenMillis(long lastSeenMillis) {
......@@ -217,8 +212,7 @@ public class SummaryDocument extends Document {
return DateTimeHelper.parse(this.lastSeenMillis);
}
@Expose
@SerializedName("rf")
@JsonProperty("rf")
private String[] relayFlags;
public void setRelayFlags(SortedSet<String> relayFlags) {
......@@ -229,8 +223,7 @@ public class SummaryDocument extends Document {
return this.stringArrayToSortedSet(this.relayFlags);
}
@Expose
@SerializedName("cw")
@JsonProperty("cw")
private long consensusWeight;
public void setConsensusWeight(long consensusWeight) {
......@@ -241,8 +234,7 @@ public class SummaryDocument extends Document {
return this.consensusWeight;
}
@Expose
@SerializedName("r")
@JsonProperty("r")
private boolean running;
public void setRunning(boolean isRunning) {
......@@ -253,8 +245,7 @@ public class SummaryDocument extends Document {
return this.running;
}
@Expose
@SerializedName("c")
@JsonProperty("c")
private String contact;
@SuppressWarnings("checkstyle:javadocmethod")
......@@ -273,8 +264,7 @@ public class SummaryDocument extends Document {
/* This attribute can go away once all Onionoo services had their hourly
* updater write effective families to summary documents at least once.
* Remove this code after September 8, 2015. */
@Expose
@SerializedName("ff")
@JsonProperty("ff")
private String[] familyFingerprints;
public void setFamilyFingerprints(
......@@ -286,8 +276,7 @@ public class SummaryDocument extends Document {
return this.stringArrayToSortedSet(this.familyFingerprints);
}
@Expose
@SerializedName("ef")
@JsonProperty("ef")
private String[] effectiveFamily;
public void setEffectiveFamily(SortedSet<String> effectiveFamily) {
......@@ -298,8 +287,7 @@ public class SummaryDocument extends Document {
return this.stringArrayToSortedSet(this.effectiveFamily);
}
@Expose
@SerializedName("v")
@JsonProperty("v")
private String version;
public void setVersion(String version) {
......@@ -310,8 +298,7 @@ public class SummaryDocument extends Document {
return this.version;
}
@Expose
@SerializedName("h")
@JsonProperty("h")
private String hostName;
public void setHostName(String hostName) {
......@@ -322,8 +309,7 @@ public class SummaryDocument extends Document {
return this.hostName;
}
@Expose
@SerializedName("rv")
@JsonProperty("rv")
private Boolean recommendedVersion;
public void setRecommendedVersion(Boolean recommendedVersion) {
......@@ -334,6 +320,11 @@ public class SummaryDocument extends Document {
return this.recommendedVersion;
}
/** Instantiate an empty summary document. */
public SummaryDocument() {
/* empty */
}
/* The familyFingerprints parameter can go away after September 8, 2015.
* See above. */
/** Instantiates a summary document with all given properties. */
......
......@@ -14,8 +14,12 @@ import org.torproject.onionoo.docs.UptimeDocument;
import org.torproject.onionoo.docs.WeightsDocument;
import org.torproject.onionoo.util.FormattingUtils;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -31,6 +35,12 @@ public class ResponseBuilder {
private static final Logger logger =
LoggerFactory.getLogger(ResponseBuilder.class);
private static ObjectMapper objectMapper = new ObjectMapper()
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
private DocumentStore documentStore;
private String buildRevision;
......@@ -343,13 +353,15 @@ public class ResponseBuilder {
dd.setVersionStatus(detailsDocument.getVersionStatus());
}
}
/* Don't escape HTML characters, like < and >, contained in
* strings. */
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
/* Whenever we provide Gson with a string containing an escaped
/* Whenever we provide Jackson with a string containing an escaped
* non-ASCII character like \u00F2, it escapes the \ to \\, which
* we need to undo before including the string in a response. */
return FormattingUtils.replaceValidUtf(gson.toJson(dd));
try {
return FormattingUtils.replaceValidUtf(
objectMapper.writeValueAsString(dd));
} catch (JsonProcessingException e) {
return "";
}
} else {
// TODO We should probably log that we didn't find a details
// document that we expected to exist.
......
......@@ -13,7 +13,11 @@ import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.DummyDocumentStore;
import org.torproject.onionoo.docs.UpdateStatus;
import com.google.gson.Gson;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import org.junit.Before;
import org.junit.Test;
......@@ -36,6 +40,12 @@ import java.util.TreeSet;
* which tests servlet specifics. */
public class ResourceServletTest {
private static ObjectMapper objectMapper = new ObjectMapper()
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument> relays;
private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument>
bridges;
......@@ -263,11 +273,10 @@ public class ResourceServletTest {
rs.doGet(this.request, this.response, TEST_TIME);
}
private void parseResponse() {
private void parseResponse() throws IOException {
this.responseString = this.response.getWrittenContent();
if (this.responseString != null) {
Gson gson = new Gson();
this.summaryDocument = gson.fromJson(this.responseString,
this.summaryDocument = objectMapper.readValue(this.responseString,
SummaryDocument.class);
}
}
......@@ -344,6 +353,9 @@ public class ResourceServletTest {
@SuppressWarnings("MemberName")
private static class SummaryDocument {
private String version;
private String next_major_version_scheduled;
private String build_revision;
private String relays_published;
private int relays_skipped;
private int relays_truncated;
......
......@@ -9,7 +9,11 @@ import static org.junit.Assert.assertNotNull;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.GraphHistory;
import com.google.gson.Gson;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -17,6 +21,7 @@ import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
import java.io.IOException;
import java.time.Period;
import java.util.Arrays;
import java.util.Collection;
......@@ -25,6 +30,12 @@ import java.util.Map;
@RunWith(Parameterized.class)
public class GraphHistoryCompilerTest {
private static ObjectMapper objectMapper = new ObjectMapper()
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
/** Provide test data. */
@Parameters
public static Collection<Object[]> data() {
......@@ -156,7 +167,7 @@ public class GraphHistoryCompilerTest {
DateTimeHelper.TEN_DAYS };
@Test
public void test() {
public void test() throws IOException {
GraphHistoryCompiler ghc = new GraphHistoryCompiler(DateTimeHelper.parse(
"2018-01-01 00:00:00"));
ghc.setDivisible(this.divisible);
......@@ -172,7 +183,7 @@ public class GraphHistoryCompilerTest {
Map<String, GraphHistory> compiledGraphHistories =
ghc.compileGraphHistories();
String message = this.testDescription + "; "
+ new Gson().toJson(compiledGraphHistories);
+ objectMapper.writeValueAsString(compiledGraphHistories);
assertEquals(message, this.expectedGraphs, compiledGraphHistories.size());
if (null != this.expectedGraphName) {
GraphHistory gh = compiledGraphHistories.get(this.expectedGraphName);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment