Commit 7b859f8d authored by iwakeh's avatar iwakeh Committed by Karsten Loesing
Browse files

Use the descriptor file name for determining descriptor type.

Initial solution suggested by Karsten.
In addition, add a test that would fail without this patch
and adapt old test to expect the correct behavior.

Also provide the origin of the descriptor with getDescriptorFile.
Include a test for this functionality in log descriptors.
Provide proper file name to log descriptor classes, because this
filename contains log metadata for storing log files properly.

Solves task-25332.
parent 42daf8e5
......@@ -4,6 +4,9 @@
- Replace ServerDescriptor#getHiddenServiceDirVersions with
ServerDescriptor#isHiddenServiceDir, because Tor has never
supported versions in the hidden-service-dir descriptor line.
- Add support for reading web server logs contained in tarballs by
providing file names of both the tarball and of contained files
to WebServerAccessLog.
* Minor changes
- Override logLines() method from LogDescriptor in
......
......@@ -130,8 +130,8 @@ public class DescriptorParserImpl implements DescriptorParser {
} else if (firstLines.startsWith("@type torperf 1.")) {
return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
sourceFile);
} else if (sourceFile.getName().contains(LogDescriptorImpl.MARKER)) {
return LogDescriptorImpl.parse(rawDescriptorBytes, sourceFile);
} else if (fileName.contains(LogDescriptorImpl.MARKER)) {
return LogDescriptorImpl.parse(rawDescriptorBytes, sourceFile, fileName);
} else {
throw new DescriptorParseException("Could not detect descriptor "
+ "type in descriptor starting with '" + firstLines + "'.");
......
......@@ -64,14 +64,15 @@ public abstract class LogDescriptorImpl
* @since 2.2.0
*/
protected LogDescriptorImpl(byte[] logBytes, File descriptorFile,
FileType defaultCompression) throws DescriptorParseException {
String logName, FileType defaultCompression)
throws DescriptorParseException {
this.logBytes = logBytes;
this.descriptorFile = descriptorFile;
try {
Matcher mat = filenamePattern.matcher(descriptorFile.getName());
Matcher mat = filenamePattern.matcher(logName);
if (!mat.find()) {
throw new DescriptorParseException(
"Log file name doesn't comply to standard: " + descriptorFile);
"Log file name doesn't comply to standard: " + logName);
}
this.fileType = FileType.findType(mat.group(1).toUpperCase());
if (FileType.PLAIN == this.fileType) {
......@@ -80,7 +81,7 @@ public abstract class LogDescriptorImpl
}
} catch (Exception ex) {
throw new DescriptorParseException("Cannot parse file "
+ descriptorFile.getName(), ex);
+ logName + " from file " + descriptorFile.getName(), ex);
}
}
......@@ -112,13 +113,13 @@ public abstract class LogDescriptorImpl
* @since 2.2.0
*/
public static List<Descriptor> parse(byte[] logBytes,
File descriptorFile) throws DescriptorParseException {
if (descriptorFile.getName().contains(InternalWebServerAccessLog.MARKER)) {
File descriptorFile, String logName) throws DescriptorParseException {
if (logName.contains(InternalWebServerAccessLog.MARKER)) {
return Arrays.asList(new Descriptor[]{
new WebServerAccessLogImpl(logBytes, descriptorFile)});
new WebServerAccessLogImpl(logBytes, descriptorFile, logName)});
} else {
throw new DescriptorParseException("Cannot parse file "
+ descriptorFile.getName());
throw new DescriptorParseException("Cannot parse file " + logName
+ " from file " + descriptorFile.getName());
}
}
......
......@@ -13,6 +13,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
......@@ -65,28 +66,34 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
* The immediate parent name is taken to be the physical host collecting the
* logs.</p>
*/
protected WebServerAccessLogImpl(byte[] logBytes, File file)
protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName)
throws DescriptorParseException {
this(logBytes, file, FileType.XZ);
this(logBytes, file, logName, FileType.XZ);
}
/** For internal use only. */
public WebServerAccessLogImpl(byte[] bytes, String filename,
boolean validate) throws DescriptorParseException {
this(bytes, new File(filename), FileType.XZ, validate);
this(bytes, null, filename, FileType.XZ, validate);
}
private WebServerAccessLogImpl(byte[] logBytes, File file,
/** For internal use only. */
public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename,
boolean validate) throws DescriptorParseException {
this(bytes, sourceFile, filename, FileType.XZ, validate);
}
private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
FileType defaultCompression) throws DescriptorParseException {
this(logBytes, file, defaultCompression, true);
this(logBytes, file, logName, defaultCompression, true);
}
private WebServerAccessLogImpl(byte[] logBytes, File file,
private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
FileType defaultCompression, boolean validate)
throws DescriptorParseException {
super(logBytes, file, defaultCompression);
super(logBytes, file, logName, defaultCompression);
try {
String fn = file.toPath().getFileName().toString();
String fn = Paths.get(logName).getFileName().toString();
Matcher mat = filenamePattern.matcher(fn);
if (!mat.find()) {
throw new DescriptorParseException(
......@@ -110,7 +117,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
throw dpe; // escalate
} catch (Exception pe) {
throw new DescriptorParseException(
"Cannot parse WebServerAccessLog file: " + file, pe);
"Cannot parse WebServerAccessLog file: " + logName, pe);
}
}
......
......@@ -4,9 +4,13 @@
package org.torproject.descriptor.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.WebServerAccessLog;
import org.junit.Rule;
import org.junit.Test;
......@@ -47,6 +51,20 @@ public class DescriptorParserImplTest {
assertEquals(1, parsedDescriptors);
}
@Test
public void testParseDescriptorTar() throws Exception {
DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
int count = 0;
for (Descriptor desc : dr.readDescriptors(new File(getClass()
.getClassLoader().getResource("webstats-2015-02.tar").toURI()))) {
assertTrue(desc instanceof WebServerAccessLog);
assertTrue(desc.getDescriptorFile().toString()
.endsWith("webstats-2015-02.tar"));
count++;
}
assertEquals(7, count);
}
@Test
public void testParseDescriptor() throws DescriptorParseException {
Constructor<? extends DescriptorImpl> constructor;
......
......@@ -149,8 +149,8 @@ public class LogDescriptorTest {
File invalidFile = new File(this.reader.getParsedFiles().firstKey()
.replace("access", "-"));
List<Descriptor> descs = new ArrayList<>();
for (Descriptor desc // note: only 'invalidFile' is used by LogDescriptor
: dp.parseDescriptors(new byte[]{}, invalidFile, logFile.getName())) {
for (Descriptor desc // note: only 'invalidFile' is used by DescriptorReader
: dp.parseDescriptors(new byte[]{}, logFile, invalidFile.getName())) {
descs.add(desc);
}
assertTrue(dataUsed() + "\nWrong type: "
......
......@@ -79,7 +79,7 @@ public class WebServerAccessLogTest {
@Test
public void testValidation() throws Exception {
WebServerAccessLogImpl wsal
= new WebServerAccessLogImpl(real.getBytes(), file);
= new WebServerAccessLogImpl(real.getBytes(), file, fn);
wsal.validate();
if (valid) {
assertEquals(0, wsal.getUnrecognizedLines().size());
......
......@@ -14,7 +14,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.File;
import java.util.Arrays;
import java.util.stream.Collectors;
......@@ -31,8 +30,7 @@ public class WebServerModuleTest {
thrown.expectMessage(Matchers
.containsString("Cannot parse WebServerAccessLog file: "
+ filename));
new WebServerAccessLogImpl(new byte[0],
new File(filename));
new WebServerAccessLogImpl(new byte[0], null, filename);
}
@Test
......@@ -42,8 +40,7 @@ public class WebServerModuleTest {
thrown.expectMessage(Matchers
.containsString("Cannot parse WebServerAccessLog file: "
+ filename));
new WebServerAccessLogImpl(new byte[0],
new File(filename));
new WebServerAccessLogImpl(new byte[0], null, filename);
}
@Test
......@@ -53,8 +50,7 @@ public class WebServerModuleTest {
thrown.expectMessage(Matchers
.containsString("WebServerAccessLog "
+ "file name doesn't comply to standard: " + filename));
new WebServerAccessLogImpl(new byte[0],
new File(filename));
new WebServerAccessLogImpl(new byte[0], null, filename);
}
@Test
......@@ -64,8 +60,7 @@ public class WebServerModuleTest {
thrown.expectMessage(Matchers
.containsString("WebServerAccessLog "
+ "file name doesn't comply to standard: " + filename));
new WebServerAccessLogImpl(new byte[0],
new File(filename));
new WebServerAccessLogImpl(new byte[0], null, filename);
}
@Test
......@@ -75,8 +70,7 @@ public class WebServerModuleTest {
thrown.expectMessage(Matchers
.containsString("WebServerAccessLog "
+ "file name doesn't comply to standard: " + filename));
new WebServerAccessLogImpl(new byte[0],
new File(filename));
new WebServerAccessLogImpl(new byte[0], null, filename);
}
private static String[] logLines = {
......@@ -103,7 +97,7 @@ public class WebServerModuleTest {
@Test
public void testBasics() throws Exception {
WebServerAccessLogImpl wsal = new WebServerAccessLogImpl(logText.getBytes(),
new File("vhost_host7_access.log_20170530"));
null, "vhost_host7_access.log_20170530");
assertEquals(wsal.getAnnotations().size(), 0);
assertEquals(logText,
new String(FileType.XZ.decompress(wsal.getRawDescriptorBytes())));
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment