Remove dependency on metrics-lib's log package (2/4). (ea1b1b4f) · Commits · Legacy / gitolite / user / iwakeh / Collector

src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java

+0 −5

Original line number	Diff line number	Diff line
		@@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog;
		import org.torproject.metrics.collector.webstats.FileType;
		import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import java.nio.file.Paths;
		import java.nio.file.StandardOpenOption;
		import java.time.format.DateTimeFormatter;
		@@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence

		public static final String SEP = InternalWebServerAccessLog.SEP;
		public static final FileType COMPRESSION = FileType.XZ;
		private static final Logger log
		= LoggerFactory.getLogger(WebServerAccessLogPersistence.class);

		private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy");
		private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM");

src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java

+0 −35

Original line number	Diff line number	Diff line
		@@ -3,7 +3,6 @@

		package org.torproject.metrics.collector.webstats;

		import org.torproject.descriptor.DescriptorParseException;
		import org.torproject.descriptor.LogDescriptor;

		/**
		@@ -16,23 +15,6 @@ public interface InternalLogDescriptor extends LogDescriptor {
		/** Logfile name parts separator. */
		String SEP = "_";

		/**
		* Validate log lines.
		*
		* @since 2.2.0
		*/
		void validate() throws DescriptorParseException;

		/**
		* Set the {@code Validator} that will perform the validation on log
		* lines.
		*
		* <p>Usually set by the implementing class.</p>
		*
		* @since 2.2.0
		*/
		void setValidator(Validator validator);

		/**
		* Set the descriptor's bytes.
		*
		@@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor {

		/** Return the descriptor's preferred compression. */
		String getCompressionType();

		/**
		* Provides a single function for validating a single log line.
		*
		* @since 2.2.0
		*/
		interface Validator {

		/**
		* Verifies a log line.
		*
		* @since 2.2.0
		*/
		boolean validate(String line);

		}

		}

src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java

+2 −51

Original line number	Diff line number	Diff line
		@@ -3,25 +3,17 @@

		package org.torproject.metrics.collector.webstats;

		import org.torproject.descriptor.Descriptor;
		import org.torproject.descriptor.DescriptorParseException;
		import org.torproject.descriptor.LogDescriptor;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import java.io.BufferedReader;
		import java.io.ByteArrayInputStream;
		import java.io.File;
		import java.io.InputStream;
		import java.io.InputStreamReader;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collections;
		import java.util.List;
		import java.util.regex.Matcher;
		import java.util.regex.Pattern;
		import java.util.stream.Collectors;

		/**
		* Base class for log descriptors.
		@@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl
		/** The log's file name should contain this string. */
		public static final String MARKER = ".log";

		private static final int unrecognizedLinesLimit = 3;

		private static final Logger log
		= LoggerFactory.getLogger(LogDescriptorImpl.class);

		private static Pattern filenamePattern = Pattern.compile(
		"(?:\\S)" + MARKER + SEP + "(?:[0-9a-zA-Z])(?:\\.?)([a-zA-Z2]*)");

		@@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl

		private List<String> unrecognizedLines = new ArrayList<>();

		private Validator validator = (String line) -> true;

		/**
		* This constructor performs basic operations on the given bytes.
		*
		@@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl
		* @since 2.2.0
		*/
		protected LogDescriptorImpl(byte[] logBytes, File descriptorFile,
		String logName, FileType defaultCompression)
		throws DescriptorParseException {
		String logName) throws DescriptorParseException {
		this.logBytes = logBytes;
		this.descriptorFile = descriptorFile;
		try {
		@@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl
		}
		this.fileType = FileType.findType(mat.group(1).toUpperCase());
		if (FileType.PLAIN == this.fileType) {
		this.fileType = defaultCompression;
		this.fileType = FileType.XZ;
		this.logBytes = this.fileType.compress(this.logBytes);
		}
		} catch (Exception ex) {
		@@ -94,39 +78,6 @@ public abstract class LogDescriptorImpl
		}
		}

		@Override
		public void validate() throws DescriptorParseException {
		try (BufferedReader br = new BufferedReader(
		new InputStreamReader(decompressedByteStream()))) {
		this.unrecognizedLines.addAll(br.lines().parallel().filter((line)
		-> null != line && !line.isEmpty() && !validator.validate(line))
		.limit(unrecognizedLinesLimit).collect(Collectors.toList()));
		} catch (Exception ex) {
		throw new DescriptorParseException("Cannot validate log lines.", ex);
		}
		}

		/**
		* Assemble a LogDescriptor.
		*
		* @since 2.2.0
		*/
		public static List<Descriptor> parse(byte[] logBytes,
		File descriptorFile, String logName) throws DescriptorParseException {
		if (logName.contains(InternalWebServerAccessLog.MARKER)) {
		return Arrays.asList(new Descriptor[]{
		new WebServerAccessLogImpl(logBytes, descriptorFile, logName)});
		} else {
		throw new DescriptorParseException("Cannot parse file " + logName
		+ " from file " + descriptorFile.getName());
		}
		}

		@Override
		public void setValidator(Validator validator) {
		this.validator = validator;
		}

		@Override
		public String getCompressionType() {
		return this.fileType.name().toLowerCase();

src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java

+0 −26

Original line number	Diff line number	Diff line
		@@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes;
		import java.time.LocalDate;
		import java.util.Optional;
		import java.util.TreeMap;
		import java.util.stream.Stream;

		/**
		* Processes the given path and stores metadata for log files.
		@@ -74,30 +73,5 @@ public class LogFileMap
		k -> new TreeMap<>());
		physicalHosts.put(metadata.date, metadata);
		}

		/**
		* Takes the given metadata and returns the LogMetadata for the entry
		* of the next day.
		*/
		public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) {
		TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts
		= this.get(metadata.virtualHost);
		if (null == virtualHosts) {
		return Optional.empty();
		}
		TreeMap<LocalDate, LogMetadata> physicalHosts
		= virtualHosts.get(metadata.physicalHost);
		if (null == physicalHosts) {
		return Optional.empty();
		}
		return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1)));
		}

		/** Returns a stream of all contained log metadata. */
		public Stream<LogMetadata> metadataStream() {
		return this.values().stream()
		.flatMap((virtualHosts) -> virtualHosts.values().stream())
		.flatMap((physicalHosts) -> physicalHosts.values().stream());
		}
		}

src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java

+1 −1

Original line number	Diff line number	Diff line
		@@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain {
		WebServerAccessLogPersistence walp
		= new WebServerAccessLogPersistence(
		new WebServerAccessLogImpl(toCompressedBytes(retainedLines),
		new File(name), name, false));
		new File(name), name));
		log.debug("Storing {}.", name);
		walp.storeOut(this.outputPathName);
		walp.storeRecent(this.recentPathName);