Github user mmiklavc commented on a diff in the pull request: https://github.com/apache/metron/pull/1011#discussion_r188093107 --- Diff: metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java --- @@ -167,26 +184,38 @@ protected void loadGeoIpDatabase(CommandLine cli) throws IOException { System.out.println("Successfully created and updated new GeoIP information"); } - protected File downloadGeoFile(String urlStr, String tmpDir) { + protected File downloadGeoFile(String urlStr, String tmpDir, int numRetries) { File localFile = null; - try { - URL url = new URL(urlStr); - localFile = new File(tmpDir + new File(url.getPath()).getName()); + int attempts = 0; + boolean valid = false; + while (attempts <= numRetries) { + try { + URL url = new URL(urlStr); + localFile = new File(tmpDir + new File(url.getPath()).getName()); - System.out.println("Downloading " + url.toString() + " to " + localFile.getAbsolutePath()); - if (localFile.exists() && !localFile.delete()) { - System.err.println("File already exists locally and can't be deleted. Please delete before continuing"); - System.exit(3); + System.out.println("Downloading " + url.toString() + " to " + localFile.getAbsolutePath()); + if (localFile.exists() && !localFile.delete()) { + System.err.println( + "File already exists locally and can't be deleted. Please delete before continuing"); + System.exit(3); + } + FileUtils.copyURLToFile(url, localFile, 5000, 10000); + if (!CompressionStrategies.GZIP.test(localFile)) { + throw new IOException("Invalid Gzip file"); + } + } catch (MalformedURLException e) { + System.err.println("Malformed URL - aborting: " + e); + e.printStackTrace(); + System.exit(4); --- End diff -- The existing exit call seemed reasonable to me if only because I'd expect this to be user error, not due to issues with an external system.
---