>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14583 )
Change subject: [NO ISSUE][EXT] Azure Datalake ext dataset: Skip instead of
failing when json file not found
......................................................................
[NO ISSUE][EXT] Azure Datalake ext dataset: Skip instead of failing when json
file not found
Details:
- When a JSON file is not found while reading external azure
datalake dataset, skip the file and continue reading, do
not fail.
Change-Id: Ic9b04e418483cc245379e35c9a20f1a4c4389e87
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
1 file changed, 2 insertions(+), 3 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/83/14583/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
index f0c185e..b7d142f 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
@@ -35,10 +35,10 @@
import org.apache.hyracks.util.LogRedactionUtil;
import com.azure.storage.blob.models.BlobErrorCode;
+import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.file.datalake.DataLakeFileClient;
import com.azure.storage.file.datalake.DataLakeFileSystemClient;
import com.azure.storage.file.datalake.DataLakeServiceClient;
-import com.azure.storage.file.datalake.models.DataLakeStorageException;
public class AzureDataLakeInputStream extends AbstractExternalInputStream {
@@ -67,8 +67,7 @@
if (lowerCaseFileName.endsWith(".gz") ||
lowerCaseFileName.endsWith(".gzip")) {
in = new GZIPInputStream(in,
ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
- } catch (DataLakeStorageException ex) {
- // TODO(htowaileb): need to find the right error for Azure Data
Lake
+ } catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
LOGGER.debug(() -> "Key " +
LogRedactionUtil.userData(filePaths.get(nextFileIndex)) + " was not "
+ "found in container " + container);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14583
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ic9b04e418483cc245379e35c9a20f1a4c4389e87
Gerrit-Change-Number: 14583
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
