Commit 4b7b13d3 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Only write changed status and document files.

Fixes #32660.
parent d5413820
# Changes in version 7.0-1.2?.? - 2019-1?-?? # Changes in version 7.0-1.2?.? - 2019-1?-??
* Medium changes
- Only write status files and document files if their content has
changed.
# Changes in version 7.0-1.22.0 - 2019-11-28 # Changes in version 7.0-1.22.0 - 2019-11-28
......
...@@ -8,6 +8,7 @@ import org.torproject.metrics.onionoo.util.FormattingUtils; ...@@ -8,6 +8,7 @@ import org.torproject.metrics.onionoo.util.FormattingUtils;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -20,7 +21,9 @@ import java.io.FileInputStream; ...@@ -20,7 +21,9 @@ import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader; import java.io.FileReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
...@@ -59,6 +62,10 @@ public class DocumentStore { ...@@ -59,6 +62,10 @@ public class DocumentStore {
private long storedBytes = 0L; private long storedBytes = 0L;
private long unchangedFiles = 0L;
private long unchangedBytes = 0L;
private long retrievedFiles = 0L; private long retrievedFiles = 0L;
private long retrievedBytes = 0L; private long retrievedBytes = 0L;
...@@ -351,6 +358,17 @@ public class DocumentStore { ...@@ -351,6 +358,17 @@ public class DocumentStore {
documentString.length()); documentString.length());
} }
documentFile.getParentFile().mkdirs(); documentFile.getParentFile().mkdirs();
if (documentFile.exists()) {
try (InputStream stream = Files.newInputStream(documentFile.toPath())) {
String existingFileDigest = DigestUtils.sha256Hex(stream);
String newFileDigest = DigestUtils.sha256Hex(documentString);
if (existingFileDigest.equals(newFileDigest)) {
this.unchangedFiles++;
this.unchangedBytes += documentString.length();
return true;
}
}
}
File documentTempFile = new File( File documentTempFile = new File(
documentFile.getAbsolutePath() + ".tmp"); documentFile.getAbsolutePath() + ".tmp");
writeToFile(documentTempFile, documentString); writeToFile(documentTempFile, documentString);
...@@ -820,6 +838,8 @@ public class DocumentStore { ...@@ -820,6 +838,8 @@ public class DocumentStore {
+ " %s files listed\n" + " %s files listed\n"
+ " %s files stored\n" + " %s files stored\n"
+ " %s stored\n" + " %s stored\n"
+ " %s files not rewritten\n"
+ " %s not rewritten\n"
+ " %s files retrieved\n" + " %s files retrieved\n"
+ " %s retrieved\n" + " %s retrieved\n"
+ " %s files removed\n", + " %s files removed\n",
...@@ -827,6 +847,8 @@ public class DocumentStore { ...@@ -827,6 +847,8 @@ public class DocumentStore {
FormattingUtils.formatDecimalNumber(listedFiles), FormattingUtils.formatDecimalNumber(listedFiles),
FormattingUtils.formatDecimalNumber(storedFiles), FormattingUtils.formatDecimalNumber(storedFiles),
FormattingUtils.formatBytes(storedBytes), FormattingUtils.formatBytes(storedBytes),
FormattingUtils.formatDecimalNumber(unchangedFiles),
FormattingUtils.formatBytes(unchangedBytes),
FormattingUtils.formatDecimalNumber(retrievedFiles), FormattingUtils.formatDecimalNumber(retrievedFiles),
FormattingUtils.formatBytes(retrievedBytes), FormattingUtils.formatBytes(retrievedBytes),
FormattingUtils.formatDecimalNumber(removedFiles)); FormattingUtils.formatDecimalNumber(removedFiles));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment