SWJTU-ZhangLei commented on code in PR #48073:
URL: https://github.com/apache/doris/pull/48073#discussion_r1971524197
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java:
##########
@@ -78,21 +81,56 @@ protected void setProperties(Map<String, String>
newProperties) throws DdlExcept
this.properties = newProperties;
}
- private static void pingAzure(String bucketName, String rootPath,
+ protected static void pingAzure(String bucketName, String rootPath,
Map<String, String> newProperties) throws DdlException {
- if (FeConstants.runningUnitTest) {
- return;
+
+ String testFile = "azure://" + bucketName + "/" + rootPath + "/"
+ + UUID.randomUUID().toString() + "/test-object-valid.txt";
Review Comment:
> use timestamp in milliseconds instead e.g.
doris-test-object-valid-${timstamp}.txt
done
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java:
##########
@@ -102,45 +105,57 @@ protected void setProperties(Map<String, String>
properties) throws DdlException
}
String region = S3Properties.getRegionOfEndpoint(pingEndpoint);
properties.putIfAbsent(S3Properties.REGION, region);
- String ak = properties.get(S3Properties.ACCESS_KEY);
- String sk = properties.get(S3Properties.SECRET_KEY);
- String token = properties.get(S3Properties.SESSION_TOKEN);
- CloudCredentialWithEndpoint credential = new
CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk, token);
if (needCheck) {
String bucketName = properties.get(S3Properties.BUCKET);
String rootPath = properties.get(S3Properties.ROOT_PATH);
- pingS3(credential, bucketName, rootPath, properties);
+ pingS3(bucketName, rootPath, properties);
}
// optional
S3Properties.optionalS3Property(properties);
this.properties = properties;
}
- private static void pingS3(CloudCredentialWithEndpoint credential, String
bucketName, String rootPath,
- Map<String, String> properties) throws DdlException {
- S3FileSystem fileSystem = new S3FileSystem(properties);
- String testFile = "s3://" + bucketName + "/" + rootPath +
"/test-object-valid.txt";
- String content = "doris will be better";
- if (FeConstants.runningUnitTest) {
- return;
+ protected static void pingS3(String bucketName, String rootPath,
Map<String, String> newProperties)
+ throws DdlException {
+
+ String prefix = "s3://" + bucketName + "/" + rootPath + "/" +
UUID.randomUUID().toString();
+ String testObj = prefix + "/test-object-valid.txt";
+
+ byte[] contentData = new byte[2 * ObjStorage.CHUNK_SIZE];
+ Arrays.fill(contentData, (byte) 'A');
+ S3ObjStorage s3ObjStorage = new S3ObjStorage(newProperties);
+
+ Status status = s3ObjStorage.putObject(testObj, new
ByteArrayInputStream(contentData), contentData.length);
+ if (!Status.OK.equals(status)) {
+ throw new DdlException(
+ "pingS3 failed(put), status: " + status + ", properties: "
+ new PrintableMap<>(
+ newProperties, "=", true, false, true, false));
}
- Status status = Status.OK;
- try {
- status = fileSystem.directUpload(content, testFile);
- if (status != Status.OK) {
- throw new DdlException(
- "ping s3 failed(upload), status: " + status + ",
properties: " + new PrintableMap<>(
- properties, "=", true, false, true, false));
- }
- } finally {
- if (status.ok()) {
- Status delete = fileSystem.delete(testFile);
- if (delete != Status.OK) {
- LOG.warn("delete test file failed, status: {}, properties:
{}", delete, new PrintableMap<>(
- properties, "=", true, false, true, false));
- }
- }
+
+ status = s3ObjStorage.headObject(testObj);
+ if (!Status.OK.equals(status)) {
+ throw new DdlException(
+ "pingS3 failed(head), status: " + status + ", properties:
" + new PrintableMap<>(
+ newProperties, "=", true, false, true, false));
+ }
+
+ RemoteObjects remoteObjects = s3ObjStorage.listObjects(prefix, null);
+ LOG.info("remoteObjects: {}", remoteObjects);
+ Preconditions.checkArgument(remoteObjects.getObjectList().size() == 1,
"remoteObjects.size() must equal 1");
+
+ status = s3ObjStorage.multiPartPutObject(testObj, new
ByteArrayInputStream(contentData), contentData.length);
Review Comment:
> abort should be also considered
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
