Commit ad9425fd authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Temp commit towards switching from Gson to Jackson.

parent 1a62407a

build.xml

+3 −0
Original line number Diff line number Diff line
@@ -48,6 +48,9 @@ 
    <include name="commons-compress-1.13.jar"/>

    <include name="commons-lang3-3.5.jar"/>

    <include name="gson-2.4.jar"/>

    <include name="jackson-annotations-2.8.6.jar"/>

    <include name="jackson-core-2.8.6.jar"/>

    <include name="jackson-databind-2.8.6.jar"/>

    <include name="logback-classic-1.1.9.jar"/>

    <include name="logback-core-1.1.9.jar"/>

    <include name="slf4j-api-1.7.22.jar"/>

src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java

+1 −1
Original line number Diff line number Diff line
@@ -116,7 +116,7 @@ public class DateTimeHelper { 
   * string cannot be parsed. */

  public static long parse(String string, String format) {

    if (null == string) {

      log.warn("Date String was null.");

      //log.warn("Date String was null."); // this is much too loud!

      return NO_TIME_AVAILABLE;

    }

    try {

src/main/java/org/torproject/onionoo/docs/DetailsStatus.java

+3 −3
Original line number Diff line number Diff line
@@ -34,12 +34,12 @@ public class DetailsStatus extends Document { 
  private String desc_published;



  public void setDescPublished(Long descPublished) {

    this.desc_published = DateTimeHelper.format(descPublished);

    this.desc_published = null == descPublished ? null

        : DateTimeHelper.format(descPublished);

  }



  public Long getDescPublished() {

    return this.desc_published == null ? null :

        DateTimeHelper.parse(this.desc_published);

    return DateTimeHelper.parse(this.desc_published);

  }



  private String last_restarted;

src/main/java/org/torproject/onionoo/docs/DocumentStore.java

+43 −20
Original line number Diff line number Diff line
@@ -5,9 +5,12 @@ package org.torproject.onionoo.docs; 


import org.torproject.onionoo.util.FormattingUtils;



import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import com.google.gson.JsonParseException;

import com.fasterxml.jackson.annotation.JsonAutoDetect;

import com.fasterxml.jackson.annotation.JsonInclude;

import com.fasterxml.jackson.annotation.PropertyAccessor;

import com.fasterxml.jackson.core.JsonProcessingException;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.fasterxml.jackson.databind.PropertyNamingStrategy;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;
@@ -41,6 +44,15 @@ public class DocumentStore { 
  private static Logger log = LoggerFactory.getLogger(

      DocumentStore.class);



  private ObjectMapper objectMapper = new ObjectMapper()

      /* Use snake_case rather than CamelCase. */

      .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)

      /* Skip fields that are null or empty. */

      .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)

      /* Ignore getters and setters and only use fields. */

      .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)

      .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);



  private final File statusDir = new File("status");



  private File outDir = null;
@@ -174,12 +186,11 @@ public class DocumentStore { 
        String line = null;

        try (BufferedReader br = new BufferedReader(new FileReader(

            summaryFile))) {

          Gson gson = new Gson();

          while ((line = br.readLine()) != null) {

            if (line.length() == 0) {

              continue;

            }

            SummaryDocument summaryDocument = gson.fromJson(line,

            SummaryDocument summaryDocument = this.objectMapper.readValue(line,

                SummaryDocument.class);

            if (summaryDocument != null) {

              parsedSummaryDocuments.put(summaryDocument.getFingerprint(),
@@ -192,9 +203,6 @@ public class DocumentStore { 
        } catch (IOException e) {

          log.error("Could not read file '"

              + summaryFile.getAbsolutePath() + "'.", e);

        } catch (JsonParseException e) {

          log.error("Could not parse summary document '" + line

              + "' in file '" + summaryFile.getAbsolutePath() + "'.", e);

        }

      }

    }
@@ -342,13 +350,17 @@ public class DocumentStore { 
        || document instanceof WeightsDocument

        || document instanceof ClientsDocument

        || document instanceof UptimeDocument) {

      Gson gson = new Gson();

      documentString = gson.toJson(document);

      try {

        documentString = this.objectMapper.writeValueAsString(document);

      } catch (JsonProcessingException e) {

        e.printStackTrace();

        return null;

      }

    } else if (document instanceof DetailsStatus

        || document instanceof DetailsDocument) {

      /* Don't escape HTML characters, like < and >, contained in

       * strings. */

      Gson gson = new GsonBuilder().disableHtmlEscaping().create();

      //Gson gson = new GsonBuilder().disableHtmlEscaping().create();

      /* We must ensure that details files only contain ASCII characters

       * and no UTF-8 characters.  While UTF-8 characters are perfectly

       * valid in JSON, this would break compatibility with existing files
@@ -356,7 +368,14 @@ public class DocumentStore { 
       * objects are escaped JSON, e.g., \u00F2.  When Gson serlializes

       * this string, it escapes the \ to \\, hence writes \\u00F2.  We

       * need to undo this and change \\u00F2 back to \u00F2. */

      documentString = FormattingUtils.replaceValidUtf(gson.toJson(document));

      try {

        documentString = FormattingUtils.replaceValidUtf(

            this.objectMapper.writeValueAsString(document));

      } catch (JsonProcessingException e) {

        log.error("Serializing failed for type "

            + document.getClass().getName() + ".");

        return null;

      }

      /* Existing details statuses don't contain opening and closing curly

       * brackets, so we should remove them from new details statuses,

       * too. */
@@ -375,8 +394,8 @@ public class DocumentStore { 
    return documentString;

  }



  private void writeDocumentStringToFile(File documentFile, String documentString)

      throws IOException {

  private void writeDocumentStringToFile(File documentFile,

      String documentString) throws IOException {

    if (documentString.length() > ONE_MIBIBYTE) {

      log.warn("Attempting to store very large document file: path='"

          + documentFile.getAbsolutePath() + "', bytes="
@@ -534,8 +553,9 @@ public class DocumentStore { 
    return result;

  }



  private String readDocumentStringFromFile(File documentFile) throws IOException {

    String documentString = null;

  private String readDocumentStringFromFile(File documentFile)

      throws IOException {

    String documentString;

    try (ByteArrayOutputStream baos = new ByteArrayOutputStream();

         BufferedInputStream bis = new BufferedInputStream(

             new FileInputStream(documentFile))) {
@@ -608,9 +628,8 @@ public class DocumentStore { 
  private <T extends Document> T retrieveParsedDocumentFile(

      Class<T> documentType, String documentString) {

    T result = null;

    Gson gson = new Gson();

    try {

      result = gson.fromJson(documentString, documentType);

      result = objectMapper.readValue(documentString, documentType);

    } catch (Throwable e) {

      /* Handle below. */

      log.error(documentString);
@@ -854,10 +873,14 @@ public class DocumentStore { 
      return;

    }

    StringBuilder sb = new StringBuilder();

    Gson gson = new Gson();

    for (SummaryDocument summaryDocument :

        this.cachedSummaryDocuments.values()) {

      String line = gson.toJson(summaryDocument);

      String line = null;

      try {

        line = objectMapper.writeValueAsString(summaryDocument);

      } catch (JsonProcessingException e) {

        e.printStackTrace();

      }

      if (line != null) {

        sb.append(line + "\n");

      } else {

src/main/java/org/torproject/onionoo/docs/SummaryDocument.java

+27 −36
Original line number Diff line number Diff line
@@ -3,8 +3,8 @@ 


package org.torproject.onionoo.docs;



import com.google.gson.annotations.Expose;

import com.google.gson.annotations.SerializedName;

import com.fasterxml.jackson.annotation.JsonIgnore;

import com.fasterxml.jackson.annotation.JsonProperty;

import org.apache.commons.codec.DecoderException;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.codec.binary.Hex;
@@ -20,8 +20,7 @@ import java.util.regex.Pattern; 


public class SummaryDocument extends Document {



  @Expose

  @SerializedName("t")

  @JsonProperty("t")

  private boolean isRelay;



  public void setRelay(boolean isRelay) {
@@ -32,8 +31,7 @@ public class SummaryDocument extends Document { 
    return this.isRelay;

  }



  @Expose

  @SerializedName("f")

  @JsonProperty("f")

  private String fingerprint;



  /** Sets the fingerprint to the given 40 hex characters and clears
@@ -57,6 +55,7 @@ public class SummaryDocument extends Document { 
    return this.fingerprint;

  }



  @JsonIgnore

  private transient String hashedFingerprint = null;



  /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no
@@ -73,6 +72,7 @@ public class SummaryDocument extends Document { 
    return this.hashedFingerprint;

  }



  @JsonIgnore

  private transient String base64Fingerprint = null;



  /** Returns the base64-encoded fingerprint, or <code>null</code> if no
@@ -89,6 +89,7 @@ public class SummaryDocument extends Document { 
    return this.base64Fingerprint;

  }



  @JsonIgnore

  private transient String[] fingerprintSortedHexBlocks = null;



  /** Returns a sorted array containing blocks of 4 upper-case hex
@@ -109,8 +110,7 @@ public class SummaryDocument extends Document { 
    return this.fingerprintSortedHexBlocks;

  }



  @Expose

  @SerializedName("n")

  @JsonProperty("n")

  private String nickname;



  @SuppressWarnings("checkstyle:javadocmethod")
@@ -126,8 +126,7 @@ public class SummaryDocument extends Document { 
    return this.nickname == null ? "Unnamed" : this.nickname;

  }



  @Expose

  @SerializedName("ad")

  @JsonProperty("ad")

  private String[] addresses;



  public void setAddresses(List<String> addresses) {
@@ -169,8 +168,7 @@ public class SummaryDocument extends Document { 
    return sortedSet;

  }



  @Expose

  @SerializedName("cc")

  @JsonProperty("cc")

  private String countryCode;



  public void setCountryCode(String countryCode) {
@@ -181,8 +179,7 @@ public class SummaryDocument extends Document { 
    return this.countryCode;

  }



  @Expose

  @SerializedName("as")

  @JsonProperty("as")

  private String asNumber;



  public void setAsNumber(String asNumber) {
@@ -193,8 +190,7 @@ public class SummaryDocument extends Document { 
    return this.asNumber;

  }



  @Expose

  @SerializedName("fs")

  @JsonProperty("fs")

  private String firstSeenMillis;



  public void setFirstSeenMillis(long firstSeenMillis) {
@@ -205,8 +201,7 @@ public class SummaryDocument extends Document { 
    return DateTimeHelper.parse(this.firstSeenMillis);

  }



  @Expose

  @SerializedName("ls")

  @JsonProperty("ls")

  private String lastSeenMillis;



  public void setLastSeenMillis(long lastSeenMillis) {
@@ -217,8 +212,7 @@ public class SummaryDocument extends Document { 
    return DateTimeHelper.parse(this.lastSeenMillis);

  }



  @Expose

  @SerializedName("rf")

  @JsonProperty("rf")

  private String[] relayFlags;



  public void setRelayFlags(SortedSet<String> relayFlags) {
@@ -229,8 +223,7 @@ public class SummaryDocument extends Document { 
    return this.stringArrayToSortedSet(this.relayFlags);

  }



  @Expose

  @SerializedName("cw")

  @JsonProperty("cw")

  private long consensusWeight;



  public void setConsensusWeight(long consensusWeight) {
@@ -241,8 +234,7 @@ public class SummaryDocument extends Document { 
    return this.consensusWeight;

  }



  @Expose

  @SerializedName("r")

  @JsonProperty("r")

  private boolean running;



  public void setRunning(boolean isRunning) {
@@ -253,8 +245,7 @@ public class SummaryDocument extends Document { 
    return this.running;

  }



  @Expose

  @SerializedName("c")

  @JsonProperty("c")

  private String contact;



  @SuppressWarnings("checkstyle:javadocmethod")
@@ -273,8 +264,7 @@ public class SummaryDocument extends Document { 
  /* This attribute can go away once all Onionoo services had their hourly

   * updater write effective families to summary documents at least once.

   * Remove this code after September 8, 2015. */

  @Expose

  @SerializedName("ff")

  @JsonProperty("ff")

  private String[] familyFingerprints;



  public void setFamilyFingerprints(
@@ -286,8 +276,7 @@ public class SummaryDocument extends Document { 
    return this.stringArrayToSortedSet(this.familyFingerprints);

  }



  @Expose

  @SerializedName("ef")

  @JsonProperty("ef")

  private String[] effectiveFamily;



  public void setEffectiveFamily(SortedSet<String> effectiveFamily) {
@@ -298,8 +287,7 @@ public class SummaryDocument extends Document { 
    return this.stringArrayToSortedSet(this.effectiveFamily);

  }



  @Expose

  @SerializedName("v")

  @JsonProperty("v")

  private String version;



  public void setVersion(String version) {
@@ -310,8 +298,7 @@ public class SummaryDocument extends Document { 
    return this.version;

  }



  @Expose

  @SerializedName("h")

  @JsonProperty("h")

  private String hostName;



  public void setHostName(String hostName) {
@@ -322,8 +309,7 @@ public class SummaryDocument extends Document { 
    return this.hostName;

  }



  @Expose

  @SerializedName("rv")

  @JsonProperty("rv")

  private Boolean recommendedVersion;



  public void setRecommendedVersion(Boolean recommendedVersion) {
@@ -334,6 +320,11 @@ public class SummaryDocument extends Document { 
    return this.recommendedVersion;

  }



  /** Instantiate an empty summary document. */

  public SummaryDocument() {

    /* empty */

  }



  /* The familyFingerprints parameter can go away after September 8, 2015.

   * See above. */

  /** Instantiates a summary document with all given properties. */