martin-g commented on code in PR #2217: URL: https://github.com/apache/datafusion-ballista/pull/2217#discussion_r4102902809
########## integrations/iceberg/tests/fixture/mod.rs: ########## @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Docker fixture for the Iceberg integration tests and the +//! `standalone-iceberg-write` / `cluster-iceberg-write` examples: an Iceberg +//! REST catalog backed by MinIO. +//! +//! Each [`IcebergFixture`] is a private catalog on its own docker network, so +//! tests holding one share no namespace, table name, or catalog state. The +//! examples include this module by path, so it must not depend on anything +//! test-only. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; +use iceberg_catalog_rest::RestCatalogBuilder; +use iceberg_storage_opendal::OpenDalStorageFactory; +use testcontainers_modules::minio::MinIO; +use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; +use testcontainers_modules::testcontainers::core::{ + CmdWaitFor, ExecCommand, IntoContainerPort, WaitFor, +}; +use testcontainers_modules::testcontainers::runners::AsyncRunner; +use testcontainers_modules::testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +const MINIO_TAG: &str = "RELEASE.2025-05-24T17-08-30Z"; +const REST_IMAGE: &str = "apache/iceberg-rest-fixture"; +const REST_TAG: &str = "1.10.1"; + +const MINIO_PORT: u16 = 9000; +const REST_PORT: u16 = 8181; + +const BUCKET: &str = "icebergdata"; +const ACCESS_KEY: &str = "admin"; +const SECRET_KEY: &str = "password"; +const REGION: &str = "us-east-1"; + +/// A running Iceberg REST catalog + MinIO pair, removed on drop. +pub struct IcebergFixture { + props: HashMap<String, String>, + _minio: ContainerAsync<MinIO>, + _rest: ContainerAsync<GenericImage>, +} + +impl IcebergFixture { + /// Starts MinIO and an Iceberg REST catalog in front of it, panicking on any + /// docker failure. + pub async fn start() -> Self { + // Container and network names are global to the docker daemon, and + // several fixtures run at once. + let suffix = unique_suffix(); + let network = format!("iceberg-ballista-{suffix}"); + // Also the hostname the REST catalog reaches MinIO on. + let minio_host = format!("iceberg-ballista-minio-{suffix}"); + + let minio = MinIO::default() Review Comment: Time to move to RustFS :-) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
