This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7e99c594b2 [rest] Add TableSnapshot including basic statistics to
SupportsSnapshots (#5249)
7e99c594b2 is described below
commit 7e99c594b26df1ca1fdc566456a48af27f9cae1b
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Mar 10 20:39:16 2025 +0800
[rest] Add TableSnapshot including basic statistics to SupportsSnapshots
(#5249)
---
.../apache/paimon/catalog/SupportsSnapshots.java | 4 +-
.../java/org/apache/paimon/rest/RESTCatalog.java | 4 +-
.../rest/responses/GetTableSnapshotResponse.java | 8 +-
.../org/apache/paimon/table/TableSnapshot.java | 136 +++++++++++++++++++++
.../org/apache/paimon/tag/SnapshotLoaderImpl.java | 5 +-
.../apache/paimon/rest/MockRESTCatalogTest.java | 16 ++-
.../org/apache/paimon/rest/RESTCatalogServer.java | 53 +++++++-
.../apache/paimon/rest/RESTCatalogTestBase.java | 21 +++-
paimon-open-api/rest-catalog-open-api.yaml | 19 ++-
9 files changed, 245 insertions(+), 21 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/SupportsSnapshots.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/SupportsSnapshots.java
index 1e292b9d31..733146c68c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/SupportsSnapshots.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/SupportsSnapshots.java
@@ -20,6 +20,7 @@ package org.apache.paimon.catalog;
import org.apache.paimon.Snapshot;
import org.apache.paimon.partition.Partition;
+import org.apache.paimon.table.TableSnapshot;
import java.util.List;
import java.util.Optional;
@@ -46,5 +47,6 @@ public interface SupportsSnapshots extends Catalog {
* @return The requested snapshot of the table
* @throws Catalog.TableNotExistException if the target does not exist
*/
- Optional<Snapshot> loadSnapshot(Identifier identifier) throws
Catalog.TableNotExistException;
+ Optional<TableSnapshot> loadSnapshot(Identifier identifier)
+ throws Catalog.TableNotExistException;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
index d28a6cd670..d7a4d281c1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
@@ -78,6 +78,7 @@ import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.Table;
+import org.apache.paimon.table.TableSnapshot;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.view.View;
@@ -349,7 +350,8 @@ public class RESTCatalog implements Catalog,
SupportsSnapshots, SupportsBranches
}
@Override
- public Optional<Snapshot> loadSnapshot(Identifier identifier) throws
TableNotExistException {
+ public Optional<TableSnapshot> loadSnapshot(Identifier identifier)
+ throws TableNotExistException {
GetTableSnapshotResponse response;
try {
response =
diff --git
a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableSnapshotResponse.java
b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableSnapshotResponse.java
index c1d25d0b40..47678c10d0 100644
---
a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableSnapshotResponse.java
+++
b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableSnapshotResponse.java
@@ -18,8 +18,8 @@
package org.apache.paimon.rest.responses;
-import org.apache.paimon.Snapshot;
import org.apache.paimon.rest.RESTResponse;
+import org.apache.paimon.table.TableSnapshot;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
@@ -33,15 +33,15 @@ public class GetTableSnapshotResponse implements
RESTResponse {
private static final String FIELD_SNAPSHOT = "snapshot";
@JsonProperty(FIELD_SNAPSHOT)
- private final Snapshot snapshot;
+ private final TableSnapshot snapshot;
@JsonCreator
- public GetTableSnapshotResponse(@JsonProperty(FIELD_SNAPSHOT) Snapshot
snapshot) {
+ public GetTableSnapshotResponse(@JsonProperty(FIELD_SNAPSHOT)
TableSnapshot snapshot) {
this.snapshot = snapshot;
}
@JsonGetter(FIELD_SNAPSHOT)
- public Snapshot getSnapshot() {
+ public TableSnapshot getSnapshot() {
return snapshot;
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/TableSnapshot.java
b/paimon-core/src/main/java/org/apache/paimon/table/TableSnapshot.java
new file mode 100644
index 0000000000..b275ddd80b
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/TableSnapshot.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.annotation.Public;
+
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/** Snapshot of a table, including basic statistics of this table. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Public
+public class TableSnapshot implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String FIELD_SNAPSHOT = "snapshot";
+ public static final String FIELD_RECORD_COUNT = "recordCount";
+ public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes";
+ public static final String FIELD_FILE_COUNT = "fileCount";
+ public static final String FIELD_LAST_FILE_CREATION_TIME =
"lastFileCreationTime";
+
+ @JsonProperty(FIELD_SNAPSHOT)
+ private final Snapshot snapshot;
+
+ @JsonProperty(FIELD_RECORD_COUNT)
+ private final long recordCount;
+
+ @JsonProperty(FIELD_FILE_SIZE_IN_BYTES)
+ private final long fileSizeInBytes;
+
+ @JsonProperty(FIELD_FILE_COUNT)
+ private final long fileCount;
+
+ @JsonProperty(FIELD_LAST_FILE_CREATION_TIME)
+ private final long lastFileCreationTime;
+
+ @JsonCreator
+ public TableSnapshot(
+ @JsonProperty(FIELD_SNAPSHOT) Snapshot snapshot,
+ @JsonProperty(FIELD_RECORD_COUNT) long recordCount,
+ @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes,
+ @JsonProperty(FIELD_FILE_COUNT) long fileCount,
+ @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long
lastFileCreationTime) {
+ this.snapshot = snapshot;
+ this.recordCount = recordCount;
+ this.fileSizeInBytes = fileSizeInBytes;
+ this.fileCount = fileCount;
+ this.lastFileCreationTime = lastFileCreationTime;
+ }
+
+ @JsonGetter(FIELD_SNAPSHOT)
+ public Snapshot snapshot() {
+ return snapshot;
+ }
+
+ @JsonGetter(FIELD_RECORD_COUNT)
+ public long recordCount() {
+ return recordCount;
+ }
+
+ @JsonGetter(FIELD_FILE_SIZE_IN_BYTES)
+ public long fileSizeInBytes() {
+ return fileSizeInBytes;
+ }
+
+ @JsonGetter(FIELD_FILE_COUNT)
+ public long fileCount() {
+ return fileCount;
+ }
+
+ @JsonGetter(FIELD_LAST_FILE_CREATION_TIME)
+ public long lastFileCreationTime() {
+ return lastFileCreationTime;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableSnapshot that = (TableSnapshot) o;
+ return recordCount == that.recordCount
+ && fileSizeInBytes == that.fileSizeInBytes
+ && fileCount == that.fileCount
+ && lastFileCreationTime == that.lastFileCreationTime
+ && Objects.equals(snapshot, that.snapshot);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ snapshot, recordCount, fileSizeInBytes, fileCount,
lastFileCreationTime);
+ }
+
+ @Override
+ public String toString() {
+ return "{"
+ + "snapshot="
+ + snapshot
+ + ", recordCount="
+ + recordCount
+ + ", fileSizeInBytes="
+ + fileSizeInBytes
+ + ", fileCount="
+ + fileCount
+ + ", lastFileCreationTime="
+ + lastFileCreationTime
+ + '}';
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/tag/SnapshotLoaderImpl.java
b/paimon-core/src/main/java/org/apache/paimon/tag/SnapshotLoaderImpl.java
index bd275ee29c..c9987af039 100644
--- a/paimon-core/src/main/java/org/apache/paimon/tag/SnapshotLoaderImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/tag/SnapshotLoaderImpl.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogLoader;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.catalog.SupportsSnapshots;
+import org.apache.paimon.table.TableSnapshot;
import org.apache.paimon.utils.SnapshotLoader;
import java.io.IOException;
@@ -42,7 +43,9 @@ public class SnapshotLoaderImpl implements SnapshotLoader {
@Override
public Optional<Snapshot> load() throws IOException {
try (Catalog catalog = catalogLoader.load()) {
- return ((SupportsSnapshots) catalog).loadSnapshot(identifier);
+ return ((SupportsSnapshots) catalog)
+ .loadSnapshot(identifier)
+ .map(TableSnapshot::snapshot);
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index 1426c56f16..a58d3fcb83 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -211,7 +211,19 @@ class MockRESTCatalogTest extends RESTCatalogTestBase {
}
@Override
- protected void updateSnapshotOnRestServer(Identifier identifier, Snapshot
snapshot) {
- restCatalogServer.setTableSnapshot(identifier, snapshot);
+ protected void updateSnapshotOnRestServer(
+ Identifier identifier,
+ Snapshot snapshot,
+ long recordCount,
+ long fileSizeInBytes,
+ long fileCount,
+ long lastFileCreationTime) {
+ restCatalogServer.setTableSnapshot(
+ identifier,
+ snapshot,
+ recordCount,
+ fileSizeInBytes,
+ fileCount,
+ lastFileCreationTime);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index 7ed259aad8..ed112e3843 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -77,6 +77,7 @@ import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.CatalogEnvironment;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.TableSnapshot;
import org.apache.paimon.utils.BranchManager;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.view.View;
@@ -133,7 +134,7 @@ public class RESTCatalogServer {
private final Map<String, TableMetadata> tableMetadataStore = new
HashMap<>();
private final Map<String, List<Partition>> tablePartitionsStore = new
HashMap<>();
private final Map<String, View> viewStore = new HashMap<>();
- private final Map<String, Snapshot> tableSnapshotStore = new HashMap<>();
+ private final Map<String, TableSnapshot> tableSnapshotStore = new
HashMap<>();
private final List<String> noPermissionDatabases = new ArrayList<>();
private final List<String> noPermissionTables = new ArrayList<>();
public final ConfigResponse configResponse;
@@ -150,7 +151,7 @@ public class RESTCatalogServer {
ResourcePaths resourcePaths = new ResourcePaths(prefix);
this.databaseUri = resourcePaths.databases();
Options conf = new Options();
- this.configResponse.getDefaults().forEach((k, v) -> conf.setString(k,
v));
+ this.configResponse.getDefaults().forEach(conf::setString);
conf.setString(CatalogOptions.WAREHOUSE.key(), dataPath);
CatalogContext context = CatalogContext.create(conf);
Path warehousePath = new Path(dataPath);
@@ -180,8 +181,17 @@ public class RESTCatalogServer {
server.shutdown();
}
- public void setTableSnapshot(Identifier identifier, Snapshot snapshot) {
- tableSnapshotStore.put(identifier.getFullName(), snapshot);
+ public void setTableSnapshot(
+ Identifier identifier,
+ Snapshot snapshot,
+ long recordCount,
+ long fileSizeInBytes,
+ long fileCount,
+ long lastFileCreationTime) {
+ tableSnapshotStore.put(
+ identifier.getFullName(),
+ new TableSnapshot(
+ snapshot, recordCount, fileSizeInBytes, fileCount,
lastFileCreationTime));
}
public void setDataToken(Identifier identifier, RESTToken token) {
@@ -539,7 +549,7 @@ public class RESTCatalogServer {
private MockResponse snapshotHandle(Identifier identifier) throws
Exception {
RESTResponse response;
- Optional<Snapshot> snapshotOptional =
+ Optional<TableSnapshot> snapshotOptional =
Optional.ofNullable(tableSnapshotStore.get(identifier.getFullName()));
if (!snapshotOptional.isPresent()) {
response =
@@ -1354,7 +1364,38 @@ public class RESTCatalogServer {
}
try {
boolean success = commit.commit(snapshot, branchName,
Collections.emptyList());
- tableSnapshotStore.put(identifier.getFullName(), snapshot);
+ tableSnapshotStore.compute(
+ identifier.getFullName(),
+ (k, old) -> {
+ long recordCount = 0;
+ long fileSizeInBytes = 0;
+ long fileCount = 0;
+ long lastFileCreationTime = 0;
+ if (statistics != null) {
+ for (Partition partition : statistics) {
+ recordCount += partition.recordCount();
+ fileSizeInBytes += partition.fileSizeInBytes();
+ fileCount += partition.fileCount();
+ if (partition.lastFileCreationTime() >
lastFileCreationTime) {
+ lastFileCreationTime =
partition.lastFileCreationTime();
+ }
+ }
+ }
+ if (old != null) {
+ recordCount += old.recordCount();
+ fileSizeInBytes += old.fileSizeInBytes();
+ fileCount += old.fileCount();
+ if (old.lastFileCreationTime() >
lastFileCreationTime) {
+ lastFileCreationTime =
old.lastFileCreationTime();
+ }
+ }
+ return new TableSnapshot(
+ snapshot,
+ recordCount,
+ fileCount,
+ lastFileCreationTime,
+ fileSizeInBytes);
+ });
return success;
} catch (Exception e) {
throw new RuntimeException(e);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTestBase.java
index 298e81a6b2..99b065bcbc 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTestBase.java
@@ -36,6 +36,7 @@ import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
+import org.apache.paimon.table.TableSnapshot;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
@@ -728,11 +729,15 @@ public abstract class RESTCatalogTestBase extends
CatalogTestBase {
long id = 10086;
long millis = System.currentTimeMillis();
updateSnapshotOnRestServer(
- hasSnapshotTableIdentifier, createSnapshotWithMillis(id,
millis));
- Optional<Snapshot> snapshot =
catalog.loadSnapshot(hasSnapshotTableIdentifier);
+ hasSnapshotTableIdentifier, createSnapshotWithMillis(id,
millis), 1, 2, 3, 4);
+ Optional<TableSnapshot> snapshot =
catalog.loadSnapshot(hasSnapshotTableIdentifier);
assertThat(snapshot).isPresent();
- assertThat(snapshot.get().id()).isEqualTo(id);
- assertThat(snapshot.get().timeMillis()).isEqualTo(millis);
+ assertThat(snapshot.get().snapshot().id()).isEqualTo(id);
+ assertThat(snapshot.get().snapshot().timeMillis()).isEqualTo(millis);
+ assertThat(snapshot.get().recordCount()).isEqualTo(1);
+ assertThat(snapshot.get().fileSizeInBytes()).isEqualTo(2);
+ assertThat(snapshot.get().fileCount()).isEqualTo(3);
+ assertThat(snapshot.get().lastFileCreationTime()).isEqualTo(4);
Identifier noSnapshotTableIdentifier =
Identifier.create("test_db_a_1", "unknown");
createTable(noSnapshotTableIdentifier, Maps.newHashMap(),
Lists.newArrayList("col1"));
snapshot = catalog.loadSnapshot(noSnapshotTableIdentifier);
@@ -947,7 +952,13 @@ public abstract class RESTCatalogTestBase extends
CatalogTestBase {
protected abstract void resetDataTokenOnRestServer(Identifier identifier);
- protected abstract void updateSnapshotOnRestServer(Identifier identifier,
Snapshot snapshot);
+ protected abstract void updateSnapshotOnRestServer(
+ Identifier identifier,
+ Snapshot snapshot,
+ long recordCount,
+ long fileSizeInBytes,
+ long fileCount,
+ long lastFileCreationTime);
protected void batchWrite(FileStoreTable tableTestWrite, List<Integer>
data) throws Exception {
BatchWriteBuilder writeBuilder = tableTestWrite.newBatchWriteBuilder();
diff --git a/paimon-open-api/rest-catalog-open-api.yaml
b/paimon-open-api/rest-catalog-open-api.yaml
index 982f91bf85..688655901e 100644
--- a/paimon-open-api/rest-catalog-open-api.yaml
+++ b/paimon-open-api/rest-catalog-open-api.yaml
@@ -1516,6 +1516,23 @@ components:
format: int64
statistics:
type: string
+ TableSnapshot:
+ type: object
+ properties:
+ snapshot:
+ $ref: '#/components/schemas/Snapshot'
+ recordCount:
+ type: integer
+ format: int64
+ fileSizeInBytes:
+ type: integer
+ format: int64
+ fileCount:
+ type: integer
+ format: int64
+ lastFileCreationTime:
+ type: integer
+ format: int64
CommitTableResponse:
type: object
properties:
@@ -1535,7 +1552,7 @@ components:
type: object
properties:
snapshot:
- $ref: '#/components/schemas/Snapshot'
+ $ref: '#/components/schemas/TableSnapshot'
AlterDatabaseRequest:
type: object
properties:
