OliLay commented on code in PR #43096:
URL: https://github.com/apache/arrow/pull/43096#discussion_r1683926128
##########
cpp/src/arrow/filesystem/azurefs_test.cc:
##########
@@ -1471,6 +1473,93 @@ class TestAzureFileSystem : public ::testing::Test {
arrow::fs::AssertFileInfo(fs(), data.Path("dir/file0"), FileType::File);
}
+ void AssertObjectContents(AzureFileSystem* fs, std::string_view path,
+ std::string_view expected) {
+ ASSERT_OK_AND_ASSIGN(auto input, fs->OpenInputStream(std::string{path}));
+ std::string contents;
+ std::shared_ptr<Buffer> buffer;
+ do {
+ ASSERT_OK_AND_ASSIGN(buffer, input->Read(128 * 1024));
+ ASSERT_TRUE(buffer);
+ contents.append(buffer->ToString());
+ } while (buffer->size() != 0);
+
+ EXPECT_EQ(expected, contents);
+ }
+
+ void TestOpenOutputStreamSmall() {
+ ASSERT_OK_AND_ASSIGN(auto fs, AzureFileSystem::Make(options_));
+
+ auto data = SetUpPreexistingData();
+ const auto path = data.ContainerPath("test-write-object");
+ ASSERT_OK_AND_ASSIGN(auto output, fs->OpenOutputStream(path, {}));
+ const std::string_view expected(PreexistingData::kLoremIpsum);
+ ASSERT_OK(output->Write(expected));
+ ASSERT_OK(output->Close());
+
+ // Verify we can read the object back.
+ AssertObjectContents(fs.get(), path, expected);
+ }
+
+ void TestOpenOutputStreamLarge() {
+ ASSERT_OK_AND_ASSIGN(auto fs, AzureFileSystem::Make(options_));
+
+ auto data = SetUpPreexistingData();
+ const auto path = data.ContainerPath("test-write-object");
+ ASSERT_OK_AND_ASSIGN(auto output, fs->OpenOutputStream(path, {}));
+ std::array<std::int64_t, 3> sizes{2570 * 1024, 258 * 1024, 259 * 1024};
+ std::array<std::string, 3> buffers{
+ std::string(sizes[0], 'A'),
+ std::string(sizes[1], 'B'),
+ std::string(sizes[2], 'C'),
+ };
+ auto expected = std::int64_t{0};
+ for (auto i = 0; i != 3; ++i) {
+ ASSERT_OK(output->Write(buffers[i]));
+ expected += sizes[i];
+ ASSERT_EQ(expected, output->Tell());
+ }
+ ASSERT_OK(output->Close());
+
+ AssertObjectContents(fs.get(), path, buffers[0] + buffers[1] + buffers[2]);
+ }
+
+ void TestOpenOutputStreamCloseAsyncDestructor() {
+ ASSERT_OK_AND_ASSIGN(auto fs, AzureFileSystem::Make(options_));
+ std::shared_ptr<io::OutputStream> stream;
+ auto data = SetUpPreexistingData();
+ const std::string path = data.ContainerPath("test-write-object");
+ constexpr auto payload = PreexistingData::kLoremIpsum;
+
+ ASSERT_OK_AND_ASSIGN(stream, fs->OpenOutputStream(path));
+ ASSERT_OK(stream->Write(payload));
+ // Destructor implicitly closes stream and completes the multipart upload.
+ // GH-37670: Testing it doesn't matter whether flush is triggered
asynchronously
Review Comment:
Yes, not the exact wording, as we don't use a "multipart upload" but we also
trigger the upload of the last data, if any, and then commit it. So there is
also some I/O when the stream is closed which we need to make sure we wait on.
I'll clarified the comment and removed the ref to the GitHub issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
