[
https://issues.apache.org/jira/browse/METRON-1552?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16472828#comment-16472828
]
ASF GitHub Bot commented on METRON-1552:
----------------------------------------
Github user cestella commented on a diff in the pull request:
https://github.com/apache/metron/pull/1011#discussion_r187757704
--- Diff:
metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
---
@@ -167,26 +184,38 @@ protected void loadGeoIpDatabase(CommandLine cli)
throws IOException {
System.out.println("Successfully created and updated new GeoIP
information");
}
- protected File downloadGeoFile(String urlStr, String tmpDir) {
+ protected File downloadGeoFile(String urlStr, String tmpDir, int
numRetries) {
File localFile = null;
- try {
- URL url = new URL(urlStr);
- localFile = new File(tmpDir + new File(url.getPath()).getName());
+ int attempts = 0;
+ boolean valid = false;
+ while (attempts <= numRetries) {
+ try {
+ URL url = new URL(urlStr);
+ localFile = new File(tmpDir + new File(url.getPath()).getName());
- System.out.println("Downloading " + url.toString() + " to " +
localFile.getAbsolutePath());
- if (localFile.exists() && !localFile.delete()) {
- System.err.println("File already exists locally and can't be
deleted. Please delete before continuing");
- System.exit(3);
+ System.out.println("Downloading " + url.toString() + " to " +
localFile.getAbsolutePath());
+ if (localFile.exists() && !localFile.delete()) {
+ System.err.println(
+ "File already exists locally and can't be deleted. Please
delete before continuing");
+ System.exit(3);
+ }
+ FileUtils.copyURLToFile(url, localFile, 5000, 10000);
+ if (!CompressionStrategies.GZIP.test(localFile)) {
+ throw new IOException("Invalid Gzip file");
+ }
+ } catch (MalformedURLException e) {
+ System.err.println("Malformed URL - aborting: " + e);
+ e.printStackTrace();
+ System.exit(4);
+ } catch (IOException e) {
+ System.err.println("Warning: Unable to copy remote GeoIP database
to local file, attempt " + attempts + ": " + e);
+ e.printStackTrace();
--- End diff --
same question, print to stderr?
> Add gzip file validation check to the geo loader
> ------------------------------------------------
>
> Key: METRON-1552
> URL: https://issues.apache.org/jira/browse/METRON-1552
> Project: Metron
> Issue Type: Improvement
> Reporter: Michael Miklavcic
> Assignee: Michael Miklavcic
> Priority: Major
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
