Github user cestella commented on a diff in the pull request: https://github.com/apache/metron/pull/1011#discussion_r187759693 --- Diff: metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java --- @@ -167,26 +184,38 @@ protected void loadGeoIpDatabase(CommandLine cli) throws IOException { System.out.println("Successfully created and updated new GeoIP information"); } - protected File downloadGeoFile(String urlStr, String tmpDir) { + protected File downloadGeoFile(String urlStr, String tmpDir, int numRetries) { File localFile = null; - try { - URL url = new URL(urlStr); - localFile = new File(tmpDir + new File(url.getPath()).getName()); + int attempts = 0; + boolean valid = false; + while (attempts <= numRetries) { + try { + URL url = new URL(urlStr); + localFile = new File(tmpDir + new File(url.getPath()).getName()); - System.out.println("Downloading " + url.toString() + " to " + localFile.getAbsolutePath()); - if (localFile.exists() && !localFile.delete()) { - System.err.println("File already exists locally and can't be deleted. Please delete before continuing"); - System.exit(3); + System.out.println("Downloading " + url.toString() + " to " + localFile.getAbsolutePath()); + if (localFile.exists() && !localFile.delete()) { + System.err.println( + "File already exists locally and can't be deleted. Please delete before continuing"); + System.exit(3); + } + FileUtils.copyURLToFile(url, localFile, 5000, 10000); + if (!CompressionStrategies.GZIP.test(localFile)) { + throw new IOException("Invalid Gzip file"); + } + } catch (MalformedURLException e) { + System.err.println("Malformed URL - aborting: " + e); + e.printStackTrace(); + System.exit(4); + } catch (IOException e) { + System.err.println("Warning: Unable to copy remote GeoIP database to local file, attempt " + attempts + ": " + e); + e.printStackTrace(); --- End diff -- D'oh, I retract the comment, this is to stderr. My bad
---