Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
The Tor Project
Network Health
Metrics
Onionoo
Commits
b45c630b
Commit
b45c630b
authored
Nov 15, 2018
by
Karsten Loesing
Browse files
Tweak GeoLite2 ASN database parsing.
Also fix unit tests. Still part of
#28254
.
parent
54e26d8a
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
View file @
b45c630b
...
...
@@ -218,7 +218,7 @@ public class LookupService {
/* Obtain a map from IP address numbers to ASN. */
Map
<
Long
,
String
[]>
addressNumberAsn
=
new
HashMap
<>();
try
(
BufferedReader
br
=
this
.
createBufferedReaderFrom
Iso88591
File
(
try
(
BufferedReader
br
=
this
.
createBufferedReaderFrom
Utf8
File
(
this
.
geoLite2AsnBlocksIpv4CsvFile
))
{
SortedSet
<
Long
>
sortedAddressNumbers
=
new
TreeSet
<>(
addressStringNumbers
.
values
());
...
...
@@ -248,6 +248,13 @@ public class LookupService {
this
.
geoLite2AsnBlocksIpv4CsvFile
.
getAbsolutePath
());
return
lookupResults
;
}
String
asNumber
=
"AS"
+
String
.
valueOf
(
Integer
.
parseInt
(
parts
[
1
]));
String
asName
=
parts
[
2
];
if
(
asName
.
isEmpty
())
{
log
.
error
(
"Illegal AS name in '{}' in {}."
,
line
,
this
.
geoLite2AsnBlocksIpv4CsvFile
.
getAbsolutePath
());
return
lookupResults
;
}
while
(
firstAddressNumber
<
startIpNum
&&
firstAddressNumber
!=
-
1L
)
{
sortedAddressNumbers
.
remove
(
firstAddressNumber
);
...
...
@@ -261,7 +268,7 @@ public class LookupService {
while
(
firstAddressNumber
<=
endIpNum
&&
firstAddressNumber
!=
-
1L
)
{
addressNumberAsn
.
put
(
firstAddressNumber
,
new
String
[]
{
"AS"
+
parts
[
1
],
parts
[
2
]
});
new
String
[]
{
asNumber
,
asName
});
sortedAddressNumbers
.
remove
(
firstAddressNumber
);
if
(
sortedAddressNumbers
.
isEmpty
())
{
firstAddressNumber
=
-
1L
;
...
...
@@ -341,12 +348,6 @@ public class LookupService {
StandardCharsets
.
UTF_8
.
newDecoder
());
}
private
BufferedReader
createBufferedReaderFromIso88591File
(
File
iso88591File
)
throws
FileNotFoundException
{
return
this
.
createBufferedReaderFromFile
(
iso88591File
,
StandardCharsets
.
ISO_8859_1
.
newDecoder
());
}
private
BufferedReader
createBufferedReaderFromFile
(
File
file
,
CharsetDecoder
dec
)
throws
FileNotFoundException
{
dec
.
onMalformedInput
(
CodingErrorAction
.
REPORT
);
...
...
src/test/java/org/torproject/metrics/onionoo/updater/LookupServiceTest.java
View file @
b45c630b
...
...
@@ -18,6 +18,7 @@ import java.io.IOException;
import
java.io.OutputStream
;
import
java.io.OutputStreamWriter
;
import
java.io.PrintStream
;
import
java.nio.charset.StandardCharsets
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.SortedMap
;
...
...
@@ -28,7 +29,7 @@ public class LookupServiceTest {
private
List
<
String
>
geoLite2CityBlocksIPv4Lines
;
private
List
<
String
>
geoLite2CityLocationsEnLines
;
private
List
<
String
>
geo
ipAsNum2
Lines
;
private
List
<
String
>
geo
Lite2AsnBlocksIpv4
Lines
;
private
LookupService
lookupService
;
...
...
@@ -59,34 +60,35 @@ public class LookupServiceTest {
this
.
geoLite2CityLocationsEnLines
.
add
(
"5375480,en,NA,"
+
"\"North America\",US,\"United States\",CA,California,,,"
+
"\"Mountain View\",807,America/Los_Angeles"
);
this
.
geoipAsNum2Lines
=
new
ArrayList
<>();
this
.
geoipAsNum2Lines
.
add
(
"134743296,134744063,\"AS3356 Level 3 "
+
"Communications\""
);
this
.
geoipAsNum2Lines
.
add
(
"134744064,134744319,\"AS15169 Google "
+
"Inc.\""
);
this
.
geoipAsNum2Lines
.
add
(
"134744320,134750463,\"AS3356 Level 3 "
+
"Communications\""
);
this
.
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
this
.
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
this
.
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.6.0/23,3356,\"Level 3 Parent, LLC\""
);
this
.
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/24,15169,\"Google LLC\""
);
this
.
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.9.0/24,3356,\"Level 3 Parent, LLC\""
);
}
private
void
writeCsvFiles
()
{
try
{
this
.
writeCsvFile
(
this
.
geoLite2CityBlocksIPv4Lines
,
"GeoLite2-City-Blocks-IPv4.csv"
,
"UTF-8"
);
"GeoLite2-City-Blocks-IPv4.csv"
);
this
.
writeCsvFile
(
this
.
geoLite2CityLocationsEnLines
,
"GeoLite2-City-Locations-en.csv"
,
"UTF-8"
);
this
.
writeCsvFile
(
this
.
geo
ipAsNum2Lines
,
"GeoIPASNum2.csv"
,
"
ISO-8859-1
"
);
"GeoLite2-City-Locations-en.csv"
);
this
.
writeCsvFile
(
this
.
geo
Lite2AsnBlocksIpv4Lines
,
"
GeoLite2-ASN-Blocks-IPv4.csv
"
);
}
catch
(
IOException
e
)
{
throw
new
RuntimeException
(
e
);
}
}
private
void
writeCsvFile
(
List
<
String
>
lines
,
String
fileName
,
String
encoding
)
throws
IOException
{
private
void
writeCsvFile
(
List
<
String
>
lines
,
String
fileName
)
throws
IOException
{
if
(
lines
!=
null
&&
!
lines
.
isEmpty
())
{
try
(
BufferedWriter
bw
=
new
BufferedWriter
(
new
OutputStreamWriter
(
new
FileOutputStream
(
new
File
(
this
.
tempGeoipDir
,
fileName
)),
encoding
)))
{
StandardCharsets
.
UTF_8
)))
{
for
(
String
line
:
lines
)
{
bw
.
write
(
line
);
...
...
@@ -103,7 +105,7 @@ public class LookupServiceTest {
private
void
assertLookupResult
(
List
<
String
>
geoLite2CityBlocksLines
,
List
<
String
>
geoLite2CityLocationsLines
,
List
<
String
>
geo
ipAsNum2
Lines
,
String
addressString
,
List
<
String
>
geo
Lite2AsnBlocksIpv4
Lines
,
String
addressString
,
String
countryCode
,
String
countryName
,
String
regionName
,
String
cityName
,
Float
latitude
,
Float
longitude
,
String
asNumber
,
String
asName
)
{
...
...
@@ -115,8 +117,8 @@ public class LookupServiceTest {
if
(
geoLite2CityLocationsLines
!=
null
)
{
this
.
geoLite2CityLocationsEnLines
=
geoLite2CityLocationsLines
;
}
if
(
geo
ipAsNum2
Lines
!=
null
)
{
this
.
geo
ipAsNum2Lines
=
geoipAsNum2
Lines
;
if
(
geo
Lite2AsnBlocksIpv4
Lines
!=
null
)
{
this
.
geo
Lite2AsnBlocksIpv4Lines
=
geoLite2AsnBlocksIpv4
Lines
;
}
this
.
writeCsvFiles
();
/* Disable log messages printed to System.err. */
...
...
@@ -197,21 +199,21 @@ public class LookupServiceTest {
public
void
testLookup8888
()
{
this
.
assertLookupResult
(
null
,
null
,
null
,
"8.8.8.8"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
-
122.0838f
,
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
public
void
testLookup8880
()
{
this
.
assertLookupResult
(
null
,
null
,
null
,
"8.8.8.0"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
-
122.0838f
,
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
public
void
testLookup888255
()
{
this
.
assertLookupResult
(
null
,
null
,
null
,
"8.8.8.255"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
-
122.0838f
,
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
...
...
@@ -262,7 +264,7 @@ public class LookupServiceTest {
+
"\"North America\",US,\"United States\",,,,,,,"
);
this
.
assertLookupResult
(
null
,
geoLite2CityLocationsEnLines
,
null
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
...
...
@@ -302,7 +304,7 @@ public class LookupServiceTest {
geoLite2CityBlocksIPv4Lines
.
add
(
"8.8.8.0/24,,,,0,0,,,"
);
this
.
assertLookupResult
(
geoLite2CityBlocksIPv4Lines
,
null
,
null
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS15169"
,
"Google
Inc.
"
);
"Google
LLC
"
);
}
@Test
()
...
...
@@ -329,7 +331,7 @@ public class LookupServiceTest {
+
"0,94035,37.3860,-122.0838,937"
);
this
.
assertLookupResult
(
geoLite2CityBlocksIPv4Lines
,
null
,
null
,
"8.8.8.8"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
...
...
@@ -343,7 +345,7 @@ public class LookupServiceTest {
+
"0,94035,37.3860,-122.0838,937,1,2,30000000000000"
);
this
.
assertLookupResult
(
geoLite2CityBlocksIPv4Lines
,
null
,
null
,
"8.8.8.8"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
Inc.
"
);
37.3860f
,
-
122.0838f
,
"AS15169"
,
"Google
LLC
"
);
}
@Test
()
...
...
@@ -377,52 +379,74 @@ public class LookupServiceTest {
}
@Test
()
public
void
testLookupGeoipAsNum2EndBeforeStart
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"134743296,134744063,\"AS3356 Level 3 "
+
"Communications\""
);
geoipAsNum2Lines
.
add
(
"134744319,134744064,\"AS15169 Google Inc.\""
);
geoipAsNum2Lines
.
add
(
"134744320,134750463,\"AS3356 Level 3 "
+
"Communications\""
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"8.8.8.8"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
null
,
null
);
public
void
testLookupGeoLite2AsnBlocksIpv4IllegalIpAddress
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"eighteighteightzero/24,15169,\"Google LLC\""
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoipAsNum2StartNotANumber
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"one,134744319,\"AS15169 Google Inc.\""
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
public
void
testLookupGeoLite2AsnBlocksIpv4MaskTooSmall
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/7,15169,\"Google LLC\""
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoipAsNum2StartTooLarge
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"1"
+
String
.
valueOf
(
Long
.
MAX_VALUE
)
+
",134744319,\"AS15169 Google Inc.\""
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
public
void
testLookupGeoLite2AsnBlocksIpv4MaskTooBig
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/33,15169,\"Google LLC\""
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoipAsNum2TooFewFields
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"134744064,134744319"
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
public
void
testLookupGeoLite2AsnBlocksIpv4MaskMissing
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0,15169,\"Google LLC\""
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoipAsNum2NoAsName
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"134743296,134744063,AS3356"
);
geoipAsNum2Lines
.
add
(
"134744064,134744319,AS15169"
);
geoipAsNum2Lines
.
add
(
"134744320,134750463,AS3356"
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"8.8.8.8"
,
"us"
,
"United States"
,
"California"
,
"Mountain View"
,
37.3860f
,
-
122.0838f
,
"AS15169"
,
""
);
public
void
testLookupGeoLite2AsnBlocksIpv4TooFewFields
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/24,15169"
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoLite2AsnBlocksIpv4TooFewFieldsNoAsNumber
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/24,,\"Google LLC\""
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
public
void
testLookupGeoLite2AsnBlocksIpv4TooFewFieldsNoAsName
()
{
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"8.8.8.0/24,15169,"
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"8.8.8.8"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
}
@Test
()
...
...
@@ -645,20 +669,22 @@ public class LookupServiceTest {
@Test
()
@SuppressWarnings
(
"AvoidEscapedUnicodeCharacters"
)
public
void
testLookupLocationAsNameNonAscii
()
{
List
<
String
>
geoipAsNum2Lines
=
new
ArrayList
<>();
geoipAsNum2Lines
.
add
(
"3207917568,3207919615,\"AS52693 Conectel "
+
"Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME\""
);
geoipAsNum2Lines
.
add
(
"3211196416,3211198463,\"AS262934 "
+
"IP\u00B7RED\""
);
geoipAsNum2Lines
.
add
(
"3227819264,3227819519,\"AS263226 "
+
"COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.\""
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"191.52.240.0"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS52693"
,
"Conectel "
+
"Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME"
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"191.102.248.0"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS262934"
,
"IP\u00B7RED"
);
this
.
assertLookupResult
(
null
,
null
,
geoipAsNum2Lines
,
"192.100.157.0"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS263226"
,
List
<
String
>
geoLite2AsnBlocksIpv4Lines
=
new
ArrayList
<>();
geoLite2AsnBlocksIpv4Lines
.
add
(
"network,autonomous_system_number,"
+
"autonomous_system_organization"
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"191.52.240.0/21,52693,"
+
"\"Conectel Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME\""
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"191.102.248.0/21,262934,\"IP\u00B7RED\""
);
geoLite2AsnBlocksIpv4Lines
.
add
(
"192.100.157.0/24,263226,"
+
"\"COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.\""
);
//this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines,
// "191.52.240.0", null, null, null, null, null, null, "AS52693",
// "Conectel Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME");
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"191.102.248.0"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS262934"
,
"IP\u00B7RED"
);
this
.
assertLookupResult
(
null
,
null
,
geoLite2AsnBlocksIpv4Lines
,
"192.100.157.0"
,
null
,
null
,
null
,
null
,
null
,
null
,
"AS263226"
,
"COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A."
);
}
}
...
...
irl
⌨
@irl
mentioned in commit
e01e8ba9
·
May 07, 2021
mentioned in commit
e01e8ba9
mentioned in commit e01e8ba9eb1f1a5f4f433c98bb3c4b78aeac3d15
Toggle commit list
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment