luoyuxia commented on code in PR #10: URL: https://github.com/apache/fluss-rust/pull/10#discussion_r2438018237
########## bindings/python/python/fluss_python/__init__.pyi: ########## Review Comment: should put it to `bindings/python/fluss`? ########## bindings/python/example/example.py: ########## @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import asyncio +import time + +import pandas as pd +import pyarrow as pa + +import fluss + + +async def main(): + # Create connection configuration + config_spec = { + "bootstrap.servers": "127.0.0.1:9123", + # Add other configuration options as needed + "request.max.size": "10485760", # 10 MB + "writer.acks": "all", # Wait for all replicas to acknowledge + "writer.retries": "3", # Retry up to 3 times on failure + "writer.batch.size": "1000", # Batch size for writes + } + config = fluss.Config(config_spec) + + # Create connection using the static connect method + conn = await fluss.FlussConnection.connect(config) + + # Define fields for PyArrow + fields = [ + pa.field("id", pa.int32()), + pa.field("name", pa.string()), + pa.field("score", pa.float32()), + pa.field("age", pa.int32()), + ] + + # Create a PyArrow schema + schema = pa.schema(fields) + + # Create a Fluss Schema first (this is what TableDescriptor expects) + fluss_schema = fluss.Schema(schema) + + # Create a Fluss TableDescriptor + table_descriptor = fluss.TableDescriptor(fluss_schema) + + # Get the admin for Fluss + admin = await conn.get_admin() + + # Create a Fluss table + table_path = fluss.TablePath("fluss", "sample_table") + + try: + await admin.create_table(table_path, table_descriptor, True) + print(f"Created table: {table_path}") + except Exception as e: + print(f"Table creation failed: {e}") + + # Get table information via admin + try: + table_info = await admin.get_table(table_path) + print(f"Table info: {table_info}") + print(f"Table ID: {table_info.table_id}") + print(f"Schema ID: {table_info.schema_id}") + print(f"Created time: {table_info.created_time}") + print(f"Primary keys: {table_info.get_primary_keys()}") + except Exception as e: + print(f"Failed to get table info: {e}") + + # Get the table instance + table = await conn.get_table(table_path) + print(f"Got table: {table}") + + # Create a writer for the table + append_writer = await table.new_append_writer() + print(f"Created append writer: {append_writer}") + + try: + # Test 1: Write PyArrow Table + print("\n--- Testing PyArrow Table write ---") + pa_table = pa.Table.from_arrays( + [ + pa.array([1, 2, 3], type=pa.int32()), + pa.array(["Alice", "Bob", "Charlie"], type=pa.string()), + pa.array([95.2, 87.2, 92.1], type=pa.float32()), + pa.array([25, 30, 35], type=pa.int32()), + ], + schema=schema, + ) + + append_writer.write_arrow(pa_table) + print("Successfully wrote PyArrow Table") + + # Test 2: Write PyArrow RecordBatch + print("\n--- Testing PyArrow RecordBatch write ---") + pa_record_batch = pa.RecordBatch.from_arrays( + [ + pa.array([4, 5], type=pa.int32()), + pa.array(["David", "Eve"], type=pa.string()), + pa.array([88.5, 91.0], type=pa.float32()), + pa.array([28, 32], type=pa.int32()), + ], + schema=schema, + ) + + append_writer.write_arrow_batch(pa_record_batch) + print("Successfully wrote PyArrow RecordBatch") + + # Test 3: Write Pandas DataFrame + print("\n--- Testing Pandas DataFrame write ---") + df = pd.DataFrame( + { + "id": [6, 7], + "name": ["Frank", "Grace"], + "score": [89.3, 94.7], + "age": [29, 27], + } + ) + + append_writer.write_pandas(df) + print("Successfully wrote Pandas DataFrame") + + # Flush all pending data + print("\n--- Flushing data ---") + append_writer.flush() + print("Successfully flushed data") + + except Exception as e: + print(f"Error during writing: {e}") + + # Now scan the table to verify data was written + print("\n--- Scanning table ---") + try: + log_scanner = await table.new_log_scanner() + print(f"Created log scanner: {log_scanner}") + + # Subscribe to scan from earliest to current timestamp + # current timestamp in microseconds + cur_timestamp = time.time_ns() // 1_000 Review Comment: remove `cur_timestamp` ########## bindings/python/example/example.py: ########## @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import asyncio +import time + +import pandas as pd +import pyarrow as pa + +import fluss + + +async def main(): + # Create connection configuration + config_spec = { + "bootstrap.servers": "127.0.0.1:9123", + # Add other configuration options as needed + "request.max.size": "10485760", # 10 MB + "writer.acks": "all", # Wait for all replicas to acknowledge + "writer.retries": "3", # Retry up to 3 times on failure + "writer.batch.size": "1000", # Batch size for writes + } + config = fluss.Config(config_spec) + + # Create connection using the static connect method + conn = await fluss.FlussConnection.connect(config) + + # Define fields for PyArrow + fields = [ + pa.field("id", pa.int32()), + pa.field("name", pa.string()), + pa.field("score", pa.float32()), + pa.field("age", pa.int32()), + ] + + # Create a PyArrow schema + schema = pa.schema(fields) + + # Create a Fluss Schema first (this is what TableDescriptor expects) + fluss_schema = fluss.Schema(schema) + + # Create a Fluss TableDescriptor + table_descriptor = fluss.TableDescriptor(fluss_schema) + + # Get the admin for Fluss + admin = await conn.get_admin() + + # Create a Fluss table + table_path = fluss.TablePath("fluss", "sample_table") + + try: + await admin.create_table(table_path, table_descriptor, True) + print(f"Created table: {table_path}") + except Exception as e: + print(f"Table creation failed: {e}") + + # Get table information via admin + try: + table_info = await admin.get_table(table_path) + print(f"Table info: {table_info}") + print(f"Table ID: {table_info.table_id}") + print(f"Schema ID: {table_info.schema_id}") + print(f"Created time: {table_info.created_time}") + print(f"Primary keys: {table_info.get_primary_keys()}") + except Exception as e: + print(f"Failed to get table info: {e}") + + # Get the table instance + table = await conn.get_table(table_path) + print(f"Got table: {table}") + + # Create a writer for the table + append_writer = await table.new_append_writer() + print(f"Created append writer: {append_writer}") + + try: + # Test 1: Write PyArrow Table + print("\n--- Testing PyArrow Table write ---") + pa_table = pa.Table.from_arrays( + [ + pa.array([1, 2, 3], type=pa.int32()), + pa.array(["Alice", "Bob", "Charlie"], type=pa.string()), + pa.array([95.2, 87.2, 92.1], type=pa.float32()), + pa.array([25, 30, 35], type=pa.int32()), + ], + schema=schema, + ) + + append_writer.write_arrow(pa_table) + print("Successfully wrote PyArrow Table") + + # Test 2: Write PyArrow RecordBatch + print("\n--- Testing PyArrow RecordBatch write ---") + pa_record_batch = pa.RecordBatch.from_arrays( + [ + pa.array([4, 5], type=pa.int32()), + pa.array(["David", "Eve"], type=pa.string()), + pa.array([88.5, 91.0], type=pa.float32()), + pa.array([28, 32], type=pa.int32()), + ], + schema=schema, + ) + + append_writer.write_arrow_batch(pa_record_batch) + print("Successfully wrote PyArrow RecordBatch") + + # Test 3: Write Pandas DataFrame + print("\n--- Testing Pandas DataFrame write ---") + df = pd.DataFrame( + { + "id": [6, 7], + "name": ["Frank", "Grace"], + "score": [89.3, 94.7], + "age": [29, 27], + } + ) + + append_writer.write_pandas(df) + print("Successfully wrote Pandas DataFrame") + + # Flush all pending data + print("\n--- Flushing data ---") + append_writer.flush() + print("Successfully flushed data") + + except Exception as e: + print(f"Error during writing: {e}") + + # Now scan the table to verify data was written + print("\n--- Scanning table ---") + try: + log_scanner = await table.new_log_scanner() + print(f"Created log scanner: {log_scanner}") + + # Subscribe to scan from earliest to current timestamp + # current timestamp in microseconds + cur_timestamp = time.time_ns() // 1_000 + # start_timestamp=None (earliest), end_timestamp=current Review Comment: ```suggestion # start_timestamp=None (earliest), end_timestamp=None (latest) ``` ########## bindings/python/python/fluss_python/__init__.pyi: ########## @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Type stubs for Fluss Python bindings.""" + +from typing import Dict, List, Optional, Tuple + +import pandas as pd +import pyarrow as pa + +class Config: + def __init__(self, config: Dict[str, str]) -> None: ... + +class FlussConnection: + @staticmethod + async def connect(config: Config) -> FlussConnection: ... + async def get_admin(self) -> FlussAdmin: ... + async def get_table(self, table_path: TablePath) -> FlussTable: ... + async def close(self) -> None: ... + +class FlussAdmin: + async def create_table( + self, + table_path: TablePath, + table_descriptor: TableDescriptor, + ignore_if_exists: bool = False, + ) -> None: ... + async def get_table(self, table_path: TablePath) -> TableInfo: ... + async def get_latest_lake_snapshot(self, table_path: TablePath) -> LakeSnapshot: ... + +class FlussTable: + async def new_append_writer(self) -> AppendWriter: ... + def new_log_scanner(self) -> LogScanner: ... + def new_log_scanner_sync(self) -> LogScanner: ... Review Comment: remove this ########## bindings/python/python/fluss_python/__init__.pyi: ########## @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Type stubs for Fluss Python bindings.""" + +from typing import Dict, List, Optional, Tuple + +import pandas as pd +import pyarrow as pa + +class Config: + def __init__(self, config: Dict[str, str]) -> None: ... + +class FlussConnection: + @staticmethod + async def connect(config: Config) -> FlussConnection: ... + async def get_admin(self) -> FlussAdmin: ... + async def get_table(self, table_path: TablePath) -> FlussTable: ... + async def close(self) -> None: ... + +class FlussAdmin: + async def create_table( + self, + table_path: TablePath, + table_descriptor: TableDescriptor, + ignore_if_exists: bool = False, + ) -> None: ... + async def get_table(self, table_path: TablePath) -> TableInfo: ... + async def get_latest_lake_snapshot(self, table_path: TablePath) -> LakeSnapshot: ... + +class FlussTable: + async def new_append_writer(self) -> AppendWriter: ... + def new_log_scanner(self) -> LogScanner: ... + def new_log_scanner_sync(self) -> LogScanner: ... + +class AppendWriter: + def write_arrow(self, table: pa.Table) -> None: ... + def write_arrow_batch(self, batch: pa.RecordBatch) -> None: ... + def write_pandas(self, df: pd.DataFrame) -> None: ... + def flush(self) -> None: ... + def close(self) -> None: ... + +class LogScanner: + def subscribe( + self, start_timestamp: Optional[int], end_timestamp: Optional[int] + ) -> None: ... + def to_pandas(self) -> pd.DataFrame: ... + def to_arrow(self) -> pa.Table: ... + def __iter__(self) -> LogScannerIterator: ... + +class LogScannerIterator: + def __iter__(self) -> LogScannerIterator: ... + def __next__(self) -> pa.RecordBatch: ... + +class Schema: + def __init__(self, schema: pa.Schema) -> None: ... + def get_column_names(self) -> List[str]: ... + def get_column_types(self) -> List[str]: ... + def get_columns(self) -> List[Tuple[str,str]]: ... + +class TableDescriptor: + def __init__( + self, schema: Schema, properties: Optional[Dict[str, str]] = None + ) -> None: ... + +class TablePath: + def __init__(self, database: str, table: str) -> None: ... + def database(self) -> str: ... + def table(self) -> str: ... Review Comment: also add `__hash__` and `__eq__` ########## bindings/python/python/fluss_python/__init__.pyi: ########## @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Type stubs for Fluss Python bindings.""" + +from typing import Dict, List, Optional, Tuple + +import pandas as pd +import pyarrow as pa + +class Config: + def __init__(self, config: Dict[str, str]) -> None: ... + +class FlussConnection: + @staticmethod + async def connect(config: Config) -> FlussConnection: ... + async def get_admin(self) -> FlussAdmin: ... + async def get_table(self, table_path: TablePath) -> FlussTable: ... + async def close(self) -> None: ... + +class FlussAdmin: + async def create_table( + self, + table_path: TablePath, + table_descriptor: TableDescriptor, + ignore_if_exists: bool = False, + ) -> None: ... + async def get_table(self, table_path: TablePath) -> TableInfo: ... + async def get_latest_lake_snapshot(self, table_path: TablePath) -> LakeSnapshot: ... + +class FlussTable: + async def new_append_writer(self) -> AppendWriter: ... + def new_log_scanner(self) -> LogScanner: ... + def new_log_scanner_sync(self) -> LogScanner: ... + +class AppendWriter: + def write_arrow(self, table: pa.Table) -> None: ... + def write_arrow_batch(self, batch: pa.RecordBatch) -> None: ... + def write_pandas(self, df: pd.DataFrame) -> None: ... + def flush(self) -> None: ... + def close(self) -> None: ... + +class LogScanner: + def subscribe( + self, start_timestamp: Optional[int], end_timestamp: Optional[int] + ) -> None: ... + def to_pandas(self) -> pd.DataFrame: ... + def to_arrow(self) -> pa.Table: ... + def __iter__(self) -> LogScannerIterator: ... + +class LogScannerIterator: + def __iter__(self) -> LogScannerIterator: ... + def __next__(self) -> pa.RecordBatch: ... + +class Schema: + def __init__(self, schema: pa.Schema) -> None: ... + def get_column_names(self) -> List[str]: ... + def get_column_types(self) -> List[str]: ... + def get_columns(self) -> List[Tuple[str,str]]: ... + +class TableDescriptor: + def __init__( + self, schema: Schema, properties: Optional[Dict[str, str]] = None + ) -> None: ... + +class TablePath: + def __init__(self, database: str, table: str) -> None: ... + def database(self) -> str: ... + def table(self) -> str: ... + +class TableInfo: + @property + def table_id(self) -> int: ... + @property + def schema_id(self) -> int: ... + @property + def created_time(self) -> int: ... + def get_primary_keys(self) -> List[str]: ... Review Comment: should add other methods like `num_buckets`, `has_primary_key`, `get_properties`? ########## bindings/python/example/lance_on_minio.py: ########## Review Comment: we can remove this example for simplicity ########## bindings/python/python/fluss_python/__init__.pyi: ########## @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Type stubs for Fluss Python bindings.""" + +from typing import Dict, List, Optional, Tuple + +import pandas as pd +import pyarrow as pa + +class Config: + def __init__(self, config: Dict[str, str]) -> None: ... + +class FlussConnection: + @staticmethod + async def connect(config: Config) -> FlussConnection: ... + async def get_admin(self) -> FlussAdmin: ... + async def get_table(self, table_path: TablePath) -> FlussTable: ... + async def close(self) -> None: ... + +class FlussAdmin: + async def create_table( + self, + table_path: TablePath, + table_descriptor: TableDescriptor, + ignore_if_exists: bool = False, + ) -> None: ... + async def get_table(self, table_path: TablePath) -> TableInfo: ... + async def get_latest_lake_snapshot(self, table_path: TablePath) -> LakeSnapshot: ... + +class FlussTable: + async def new_append_writer(self) -> AppendWriter: ... + def new_log_scanner(self) -> LogScanner: ... + def new_log_scanner_sync(self) -> LogScanner: ... + +class AppendWriter: + def write_arrow(self, table: pa.Table) -> None: ... + def write_arrow_batch(self, batch: pa.RecordBatch) -> None: ... + def write_pandas(self, df: pd.DataFrame) -> None: ... + def flush(self) -> None: ... + def close(self) -> None: ... + +class LogScanner: + def subscribe( + self, start_timestamp: Optional[int], end_timestamp: Optional[int] + ) -> None: ... + def to_pandas(self) -> pd.DataFrame: ... + def to_arrow(self) -> pa.Table: ... + def __iter__(self) -> LogScannerIterator: ... + +class LogScannerIterator: + def __iter__(self) -> LogScannerIterator: ... + def __next__(self) -> pa.RecordBatch: ... + +class Schema: + def __init__(self, schema: pa.Schema) -> None: ... + def get_column_names(self) -> List[str]: ... + def get_column_types(self) -> List[str]: ... + def get_columns(self) -> List[Tuple[str,str]]: ... + +class TableDescriptor: + def __init__( + self, schema: Schema, properties: Optional[Dict[str, str]] = None + ) -> None: ... + +class TablePath: + def __init__(self, database: str, table: str) -> None: ... + def database(self) -> str: ... + def table(self) -> str: ... + +class TableInfo: + @property + def table_id(self) -> int: ... + @property + def schema_id(self) -> int: ... + @property + def created_time(self) -> int: ... + def get_primary_keys(self) -> List[str]: ... + +class DatabaseInfo: Review Comment: why can't I find `#[pyclass]` `DatabaseInfo` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
