commit 2d8eb09d1e37af4efd1776acf73ff94b98ca8bbd
Author: Karsten Loesing <karsten.loes...@gmx.net>
Date: Tue Feb 24 10:50:54 2015 +0100
Fix character encoding of ASN database file.
Looks like MaxMind's ASN database files are ISO-8859-1 encoded, not UTF-8
encoded...
---
.../torproject/onionoo/updater/LookupService.java | 2 +-
.../onionoo/updater/LookupServiceTest.java | 32 ++++++++++++++++----
2 files changed, 27 insertions(+), 7 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java
b/src/main/java/org/torproject/onionoo/updater/LookupService.java
index f88e752..a48e721 100644
--- a/src/main/java/org/torproject/onionoo/updater/LookupService.java
+++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java
@@ -228,7 +228,7 @@ public class LookupService {
addressStringNumbers.values());
long firstAddressNumber = sortedAddressNumbers.first();
BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(geoIPASNum2CsvFile), "UTF-8"));
+ new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1"));
String line;
long previousStartIpNum = -1L;
while ((line = br.readLine()) != null) {
diff --git
a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
index 9275fbf..5673d9a 100644
--- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
+++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
@@ -72,21 +72,22 @@ public class LookupServiceTest {
private void writeCsvFiles() {
try {
this.writeCsvFile(this.geoLite2CityBlocksIPv4Lines,
- "GeoLite2-City-Blocks-IPv4.csv");
+ "GeoLite2-City-Blocks-IPv4.csv", "UTF-8");
this.writeCsvFile(this.geoLite2CityLocationsEnLines,
- "GeoLite2-City-Locations-en.csv");
- this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv");
+ "GeoLite2-City-Locations-en.csv", "UTF-8");
+ this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv",
+ "ISO-8859-1");
} catch (IOException e) {
throw new RuntimeException(e);
}
}
- private void writeCsvFile(List<String> lines, String fileName)
- throws IOException {
+ private void writeCsvFile(List<String> lines, String fileName,
+ String encoding) throws IOException {
if (lines != null && !lines.isEmpty()) {
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(new File(this.tempGeoipDir, fileName)),
- "UTF-8"));
+ encoding));
for (String line : lines) {
bw.write(line + "\n");
}
@@ -592,5 +593,24 @@ public class LookupServiceTest {
"Pomeranian Voivodeship", "Gda\u0144sk", 54.3608f, 18.6583f, null,
null);
}
+
+ @Test()
+ public void testLookupLocationASNameNonAscii() {
+ List<String> geoipASNum2Lines = new ArrayList<String>();
+ geoipASNum2Lines.add("3207917568,3207919615,\"AS52693 Conectel "
+ + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME\"");
+ geoipASNum2Lines.add("3211196416,3211198463,\"AS262934 "
+ + "IP\u00B7RED\"");
+ geoipASNum2Lines.add("3227819264,3227819519,\"AS263226 "
+ + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.\"");
+ this.assertLookupResult(null, null, geoipASNum2Lines, "191.52.240.0",
+ null, null, null, null, null, null, "AS52693", "Conectel "
+ + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME");
+ this.assertLookupResult(null, null, geoipASNum2Lines, "191.102.248.0",
+ null, null, null, null, null, null, "AS262934", "IP\u00B7RED");
+ this.assertLookupResult(null, null, geoipASNum2Lines, "192.100.157.0",
+ null, null, null, null, null, null, "AS263226",
+ "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.");
+ }
}
_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
