steveloughran commented on code in PR #6069:
URL: https://github.com/apache/hadoop/pull/6069#discussion_r1350485508
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java:
##########
@@ -1412,6 +1444,102 @@ private void appendIfNotEmpty(StringBuilder sb, String
regEx,
}
}
+ /**
+ * Add MD5 hash as request header to the append request
+ * @param requestHeaders
+ * @param reqParams
+ * @param buffer
+ */
+ private void addCheckSumHeaderForWrite(List<AbfsHttpHeader> requestHeaders,
+ final AppendRequestParameters reqParams, final byte[] buffer)
+ throws AbfsRestOperationException {
+ try {
+ MessageDigest md5Digest = MessageDigest.getInstance(MD5);
+ byte[] dataToBeWritten = new byte[reqParams.getLength()];
+
+ if (reqParams.getoffset() == 0 && reqParams.getLength() ==
buffer.length) {
+ dataToBeWritten = buffer;
+ } else {
+ System.arraycopy(buffer, reqParams.getoffset(), dataToBeWritten, 0,
+ reqParams.getLength());
+ }
+
+ byte[] md5Bytes = md5Digest.digest(dataToBeWritten);
+ String md5Hash = Base64.getEncoder().encodeToString(md5Bytes);
+ requestHeaders.add(new AbfsHttpHeader(CONTENT_MD5, md5Hash));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new AbfsRuntimeException(ex);
+ }
+ }
+
+ /**
+ * To verify the checksum information received from server for the data read
+ * @param buffer stores the data received from server
+ * @param result HTTP Operation Result
+ * @param bufferOffset Position where data returned by server is saved in
buffer
+ * @throws AbfsRestOperationException
+ */
+ private void verifyCheckSumForRead(final byte[] buffer,
+ final AbfsHttpOperation result, final int bufferOffset)
+ throws AbfsRestOperationException {
+ // Number of bytes returned by server could be less than or equal to what
+ // caller requests. In case it is less, extra bytes will be initialized to 0
+ // Server returned MD5 Hash will be computed on what server returned.
+ // We need to get exact data that server returned and compute its md5 hash
+ // Computed hash should be equal to what server returned
+ int numberOfBytesRead = (int)result.getBytesReceived();
+ if (numberOfBytesRead == 0) {
+ return;
+ }
+ byte[] dataRead = new byte[numberOfBytesRead];
+
+ if (bufferOffset == 0 && numberOfBytesRead == buffer.length) {
+ dataRead = buffer;
+ } else {
+ System.arraycopy(buffer, bufferOffset, dataRead, 0, numberOfBytesRead);
+ }
+
+ try {
+ MessageDigest md5Digest = MessageDigest.getInstance(MD5);
Review Comment:
if its tested its fine.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
