This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-paimon-webui.git
The following commit(s) were added to refs/heads/main by this push:
new 55c6b90 [Improvement] Unify builder of VOs via lombok.Builder (#160)
55c6b90 is described below
commit 55c6b9081f271099608a282eee0d92f63f000e44
Author: s7monk <[email protected]>
AuthorDate: Thu Feb 1 14:30:58 2024 +0800
[Improvement] Unify builder of VOs via lombok.Builder (#160)
---
.../web/server/data/model/MetadataFieldsModel.java | 4 +
.../paimon/web/server/data/vo/DataFileVO.java | 4 +-
.../paimon/web/server/data/vo/ManifestsVO.java | 60 ++------
.../apache/paimon/web/server/data/vo/SchemaVO.java | 117 ++--------------
.../server/service/impl/MetadataServiceImpl.java | 44 +++---
.../server/controller/MetadataControllerTest.java | 155 ++++++++++++++++++++-
6 files changed, 209 insertions(+), 175 deletions(-)
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/model/MetadataFieldsModel.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/model/MetadataFieldsModel.java
index f442fcf..43fdf46 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/model/MetadataFieldsModel.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/model/MetadataFieldsModel.java
@@ -18,10 +18,14 @@
package org.apache.paimon.web.server.data.model;
+import lombok.AllArgsConstructor;
import lombok.Data;
+import lombok.NoArgsConstructor;
/** Model of metadata fields. */
@Data
+@NoArgsConstructor
+@AllArgsConstructor
public class MetadataFieldsModel {
private int id;
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
index 33387eb..5017325 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
@@ -19,18 +19,20 @@
package org.apache.paimon.web.server.data.vo;
import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/** VO of metadata data file. */
@Data
+@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DataFileVO {
private String partition;
- private long bucket;
+ private Integer bucket;
private String filePath;
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
index 639bbcb..bfab103 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
@@ -18,57 +18,21 @@
package org.apache.paimon.web.server.data.vo;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
/** VO of metadata manifest. */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
public class ManifestsVO {
- private final String fileName;
- private final Long fileSize;
- private final Long numAddedFiles;
-
- public ManifestsVO(String fileName, Long fileSize, Long numAddedFiles) {
- this.fileName = fileName;
- this.fileSize = fileSize;
- this.numAddedFiles = numAddedFiles;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public Long getFileSize() {
- return fileSize;
- }
-
- public Long getNumAddedFiles() {
- return numAddedFiles;
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- /** ManifestsInfoVo Builder. */
- public static class Builder {
- private String fileName;
- private Long fileSize;
- private Long numAddedFiles;
-
- public Builder setFileName(String fileName) {
- this.fileName = fileName;
- return this;
- }
- public Builder setFileSize(Long fileSize) {
- this.fileSize = fileSize;
- return this;
- }
+ private String fileName;
- public Builder setNumAddedFiles(Long numAddedFiles) {
- this.numAddedFiles = numAddedFiles;
- return this;
- }
+ private Long fileSize;
- public ManifestsVO build() {
- return new ManifestsVO(fileName, fileSize, numAddedFiles);
- }
- }
+ private Long numAddedFiles;
}
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SchemaVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SchemaVO.java
index 15b5e63..5e8d3f1 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SchemaVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SchemaVO.java
@@ -21,119 +21,32 @@ package org.apache.paimon.web.server.data.vo;
import org.apache.paimon.web.server.data.model.MetadataFieldsModel;
import org.apache.paimon.web.server.data.model.MetadataOptionModel;
-import javax.annotation.Nullable;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
import java.util.List;
/** VO of metadata schema. */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
public class SchemaVO {
- private final Long schemaId;
- private final List<MetadataFieldsModel> fields;
- private final String partitionKeys;
- private final String primaryKeys;
- private final String comment;
- private final List<MetadataOptionModel> option;
- private final LocalDateTime updateTime;
+ private Long schemaId;
- public SchemaVO(
- Long schemaId,
- List<MetadataFieldsModel> fields,
- String partitionKeys,
- String primaryKeys,
- String comment,
- List<MetadataOptionModel> option,
- LocalDateTime updateTime) {
- this.schemaId = schemaId;
- this.fields = fields;
- this.partitionKeys = partitionKeys;
- this.primaryKeys = primaryKeys;
- this.comment = comment;
- this.option = option;
- this.updateTime = updateTime;
- }
+ private List<MetadataFieldsModel> fields;
- public Long getSchemaId() {
- return schemaId;
- }
+ private String partitionKeys;
- public List<MetadataFieldsModel> getFields() {
- return fields;
- }
+ private String primaryKeys;
- public String getPartitionKeys() {
- return partitionKeys;
- }
+ private String comment;
- public String getPrimaryKeys() {
- return primaryKeys;
- }
+ private List<MetadataOptionModel> option;
- public String getComment() {
- return comment;
- }
-
- public LocalDateTime getUpdateTime() {
- return updateTime;
- }
-
- public List<MetadataOptionModel> getOption() {
- return option;
- }
-
- public static SchemaVO.Builder builder() {
- return new Builder();
- }
-
- /** Builder for SchemaInfoVo. */
- public static class Builder {
- private Long schemaId;
- private List<MetadataFieldsModel> fields;
- private String partitionKeys;
- private String primaryKeys;
- @Nullable private String comment;
- @Nullable private List<MetadataOptionModel> option;
- private LocalDateTime updateTime;
-
- public Builder setSchemaId(Long schemaId) {
- this.schemaId = schemaId;
- return this;
- }
-
- public Builder setFields(List<MetadataFieldsModel> fields) {
- this.fields = fields;
- return this;
- }
-
- public Builder setPartitionKeys(String partitionKeys) {
- this.partitionKeys = partitionKeys;
- return this;
- }
-
- public Builder setPrimaryKeys(String primaryKeys) {
- this.primaryKeys = primaryKeys;
- return this;
- }
-
- public Builder setComment(String comment) {
- this.comment = comment;
- return this;
- }
-
- public Builder setOption(List<MetadataOptionModel> option) {
- this.option = option;
- return this;
- }
-
- public Builder setUpdateTime(LocalDateTime updateTime) {
- this.updateTime = updateTime;
- return this;
- }
-
- public SchemaVO build() {
- return new SchemaVO(
- schemaId, fields, partitionKeys, primaryKeys, comment,
option, updateTime);
- }
- }
+ private LocalDateTime updateTime;
}
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
index 36b1f80..68faba0 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
@@ -44,6 +44,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
+import java.time.LocalDateTime;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -72,21 +73,19 @@ public class MetadataServiceImpl implements MetadataService
{
internalRow -> {
SchemaVO schemaVo =
SchemaVO.builder()
- .setSchemaId(internalRow.getLong(0))
- .setFields(
+ .schemaId(internalRow.getLong(0))
+ .fields(
new Gson()
.fromJson(
internalRow.getString(1).toString(),
new TypeToken<
LinkedList<
MetadataFieldsModel>>() {}))
-
.setPartitionKeys(internalRow.getString(2).toString())
-
.setPrimaryKeys(internalRow.getString(3).toString())
- .setOption(
-
formatOptions(internalRow.getString(4).toString()))
-
.setComment(internalRow.getString(5).toString())
- .setUpdateTime(
- internalRow.getTimestamp(6,
3).toLocalDateTime())
+
.partitionKeys(getSafeString(internalRow, 2))
+
.primaryKeys(getSafeString(internalRow, 3))
+
.option(formatOptions(getSafeString(internalRow, 4)))
+ .comment(getSafeString(internalRow, 5))
+
.updateTime(getSafeLocalDateTime(internalRow, 6))
.build();
result.add(schemaVo);
});
@@ -112,9 +111,8 @@ public class MetadataServiceImpl implements MetadataService
{
.snapshotId(internalRow.getLong(0))
.schemaId(internalRow.getLong(1))
.commitIdentifier(internalRow.getLong(3))
-
.commitKind(internalRow.getString(4).toString())
- .commitTime(
- internalRow.getTimestamp(5,
3).toLocalDateTime())
+ .commitKind(getSafeString(internalRow,
4))
+
.commitTime(getSafeLocalDateTime(internalRow, 5))
.build();
result.add(build);
});
@@ -136,9 +134,9 @@ public class MetadataServiceImpl implements MetadataService
{
internalRow -> {
ManifestsVO manifestsVo =
ManifestsVO.builder()
-
.setFileName(internalRow.getString(0).toString())
- .setFileSize(internalRow.getLong(1))
-
.setNumAddedFiles(internalRow.getLong(2))
+ .fileName(getSafeString(internalRow,
0))
+ .fileSize(internalRow.getLong(1))
+ .numAddedFiles(internalRow.getLong(2))
.build();
result.add(manifestsVo);
});
@@ -160,10 +158,10 @@ public class MetadataServiceImpl implements
MetadataService {
reader.forEachRemaining(
internalRow -> {
DataFileVO dataFileVo = new DataFileVO();
-
dataFileVo.setPartition(internalRow.getString(0).toString());
+ dataFileVo.setPartition(getSafeString(internalRow, 0));
dataFileVo.setBucket(internalRow.getInt(1));
-
dataFileVo.setFilePath(internalRow.getString(2).toString());
-
dataFileVo.setFileFormat(internalRow.getString(3).toString());
+ dataFileVo.setFilePath(getSafeString(internalRow, 2));
+ dataFileVo.setFileFormat(getSafeString(internalRow,
3));
result.add(dataFileVo);
});
} catch (IOException e) {
@@ -228,4 +226,14 @@ public class MetadataServiceImpl implements
MetadataService {
throw new RuntimeException(e);
}
}
+
+ private String getSafeString(InternalRow internalRow, int index) {
+ return internalRow.isNullAt(index) ? "" :
internalRow.getString(index).toString();
+ }
+
+ private LocalDateTime getSafeLocalDateTime(InternalRow internalRow, int
index) {
+ return internalRow.isNullAt(index)
+ ? null
+ : internalRow.getTimestamp(index, 3).toLocalDateTime();
+ }
}
diff --git
a/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
b/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
index f15f3e4..e07132e 100644
---
a/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
+++
b/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
@@ -18,16 +18,32 @@
package org.apache.paimon.web.server.controller;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.web.api.catalog.PaimonService;
import org.apache.paimon.web.server.data.dto.CatalogDTO;
import org.apache.paimon.web.server.data.dto.DatabaseDTO;
import org.apache.paimon.web.server.data.dto.MetadataDTO;
import org.apache.paimon.web.server.data.dto.TableDTO;
import org.apache.paimon.web.server.data.model.CatalogInfo;
+import org.apache.paimon.web.server.data.model.MetadataFieldsModel;
+import org.apache.paimon.web.server.data.model.MetadataOptionModel;
import org.apache.paimon.web.server.data.model.TableColumn;
import org.apache.paimon.web.server.data.result.R;
+import org.apache.paimon.web.server.data.vo.DataFileVO;
+import org.apache.paimon.web.server.data.vo.ManifestsVO;
import org.apache.paimon.web.server.data.vo.OptionVO;
+import org.apache.paimon.web.server.data.vo.SchemaVO;
+import org.apache.paimon.web.server.data.vo.SnapshotVO;
import org.apache.paimon.web.server.util.ObjectMapperUtils;
import org.apache.paimon.web.server.util.PaimonDataType;
+import org.apache.paimon.web.server.util.PaimonServiceUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.ImmutableMap;
@@ -42,12 +58,16 @@ import
org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultHandlers;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/** Tests for {@link MetadataController}. */
@SpringBootTest
@@ -68,6 +88,8 @@ public class MetadataControllerTest extends
ControllerTestBase {
private Integer catalogId;
+ private List<CommitMessage> messages;
+
@BeforeEach
public void setup() throws Exception {
CatalogDTO catalog = new CatalogDTO();
@@ -159,7 +181,7 @@ public class MetadataControllerTest extends
ControllerTestBase {
List<String> partitionKey = Lists.newArrayList("create_time");
Map<String, String> tableOptions = ImmutableMap.of("bucket", "2");
- TableDTO table =
+ TableDTO tableDTO =
TableDTO.builder()
.catalogName(catalogName)
.databaseName(databaseName)
@@ -172,10 +194,52 @@ public class MetadataControllerTest extends
ControllerTestBase {
mockMvc.perform(
MockMvcRequestBuilders.post(tablePath + "/create")
.cookie(cookie)
- .content(ObjectMapperUtils.toJSON(table))
+ .content(ObjectMapperUtils.toJSON(tableDTO))
.contentType(MediaType.APPLICATION_JSON_VALUE)
.accept(MediaType.APPLICATION_JSON_VALUE))
.andExpect(MockMvcResultMatchers.status().isOk());
+
+ // insert
+ CatalogInfo catalogInfo =
+ CatalogInfo.builder()
+ .catalogName(catalog.getName())
+ .catalogType(catalog.getType())
+ .warehouse(catalog.getWarehouse())
+ .build();
+ PaimonService paimonService =
PaimonServiceUtils.getPaimonService(catalogInfo);
+ Table table = paimonService.getTable(databaseName, tableName);
+ BatchWriteBuilder writeBuilder =
table.newBatchWriteBuilder().withOverwrite();
+ BatchTableWrite write = writeBuilder.newWrite();
+
+ GenericRow record1 =
+ GenericRow.of(
+ 1,
+ BinaryString.fromString("Alice"),
+ 24,
+ BinaryString.fromString("2023-12-04 00:00:00"));
+ GenericRow record2 =
+ GenericRow.of(
+ 2,
+ BinaryString.fromString("Bob"),
+ 28,
+ BinaryString.fromString("2023-10-11 00:00:00"));
+ GenericRow record3 =
+ GenericRow.of(
+ 3,
+ BinaryString.fromString("Emily"),
+ 32,
+ BinaryString.fromString("2023-10-04 00:00:00"));
+
+ write.write(record1);
+ write.write(record2);
+ write.write(record3);
+
+ messages = write.prepareCommit();
+ BatchTableCommit commit = writeBuilder.newCommit();
+ commit.commit(messages);
+
+ write.close();
+ commit.close();
}
@AfterEach
@@ -225,8 +289,38 @@ public class MetadataControllerTest extends
ControllerTestBase {
.getResponse()
.getContentAsString();
- R<Void> result = ObjectMapperUtils.fromJSON(response, new
TypeReference<R<Void>>() {});
+ R<List<SchemaVO>> result =
+ ObjectMapperUtils.fromJSON(response, new
TypeReference<R<List<SchemaVO>>>() {});
+ List<SchemaVO> schemaVOS = result.getData();
assertEquals(200, result.getCode());
+ assertFalse(schemaVOS.isEmpty());
+ assertEquals(1, schemaVOS.size());
+
+ // Make assertions on each field of the SchemaVO class.
+ SchemaVO schemaVO = schemaVOS.get(0);
+ assertNotNull(schemaVO.getUpdateTime());
+
+ assertEquals(0, schemaVO.getSchemaId());
+ assertEquals("[\"id\",\"create_time\"]", schemaVO.getPrimaryKeys());
+ assertEquals("[\"create_time\"]", schemaVO.getPartitionKeys());
+ assertEquals("", schemaVO.getComment());
+
+ assertEquals(4, schemaVO.getFields().size());
+ ArrayList<MetadataFieldsModel> expectedFields =
+ Lists.newArrayList(
+ new MetadataFieldsModel(0, "id", "INT NOT NULL", null),
+ new MetadataFieldsModel(1, "name", "STRING NOT NULL",
null),
+ new MetadataFieldsModel(2, "age", "INT NOT NULL",
null),
+ new MetadataFieldsModel(3, "create_time", "STRING NOT
NULL", null));
+ assertEquals(expectedFields, schemaVO.getFields());
+
+ assertEquals(3, schemaVO.getOption().size());
+ ArrayList<MetadataOptionModel> expectedOptions =
+ Lists.newArrayList(
+ new MetadataOptionModel("bucket", "2"),
+ new
MetadataOptionModel("FIELDS.create_time.default-value", "0"),
+ new MetadataOptionModel("FIELDS.age.default-value",
"0"));
+ assertEquals(expectedOptions, schemaVO.getOption());
}
@Test
@@ -249,8 +343,17 @@ public class MetadataControllerTest extends
ControllerTestBase {
.getResponse()
.getContentAsString();
- R<Void> result = ObjectMapperUtils.fromJSON(response, new
TypeReference<R<Void>>() {});
+ R<List<ManifestsVO>> result =
+ ObjectMapperUtils.fromJSON(response, new
TypeReference<R<List<ManifestsVO>>>() {});
assertEquals(200, result.getCode());
+ List<ManifestsVO> manifestsVOS = result.getData();
+ assertFalse(manifestsVOS.isEmpty());
+
+ // Make assertions on each field of the ManifestsVO class.
+ ManifestsVO manifestsVO = manifestsVOS.get(0);
+ assertNotNull(manifestsVO.getFileName());
+ assertTrue(manifestsVO.getFileSize() > 0);
+ assertEquals(3, manifestsVO.getNumAddedFiles());
}
@Test
@@ -273,8 +376,36 @@ public class MetadataControllerTest extends
ControllerTestBase {
.getResponse()
.getContentAsString();
- R<Void> result = ObjectMapperUtils.fromJSON(response, new
TypeReference<R<Void>>() {});
+ R<List<DataFileVO>> result =
+ ObjectMapperUtils.fromJSON(response, new
TypeReference<R<List<DataFileVO>>>() {});
assertEquals(200, result.getCode());
+ List<DataFileVO> dataFileVOS = result.getData();
+ assertFalse(dataFileVOS.isEmpty());
+ assertEquals(3, dataFileVOS.size());
+
+ // Make assertions on each field of the DataFileVO class.
+ DataFileVO dataFileVO = dataFileVOS.get(0);
+ assertEquals("[2023-12-04 00:00:00]", dataFileVO.getPartition());
+ assertEquals(0, dataFileVO.getBucket());
+ assertNotNull(dataFileVO.getFilePath());
+ assertEquals("orc", dataFileVO.getFileFormat());
+
+ List<String> actualPartitions = new ArrayList<>();
+ List<String> expectedPartitions =
+ Lists.newArrayList(
+ "[2023-12-04 00:00:00]", "[2023-10-11 00:00:00]",
"[2023-10-04 00:00:00]");
+ dataFileVOS.forEach(item -> actualPartitions.add(item.getPartition()));
+ assertEquals(actualPartitions, expectedPartitions);
+
+ List<String> actualFilePaths = new ArrayList<>();
+ dataFileVOS.forEach(item -> actualFilePaths.add(item.getFilePath()));
+ List<String> expectedFilePaths = new ArrayList<>();
+ messages.forEach(
+ item -> {
+ CommitMessageImpl message = (CommitMessageImpl) item;
+
expectedFilePaths.add(message.newFilesIncrement().newFiles().get(0).fileName());
+ });
+ assertEquals(actualFilePaths, expectedFilePaths);
}
@Test
@@ -297,8 +428,20 @@ public class MetadataControllerTest extends
ControllerTestBase {
.getResponse()
.getContentAsString();
- R<Void> result = ObjectMapperUtils.fromJSON(response, new
TypeReference<R<Void>>() {});
+ R<List<SnapshotVO>> result =
+ ObjectMapperUtils.fromJSON(response, new
TypeReference<R<List<SnapshotVO>>>() {});
assertEquals(200, result.getCode());
+ List<SnapshotVO> snapshotVOS = result.getData();
+ assertFalse(snapshotVOS.isEmpty());
+ assertEquals(1, snapshotVOS.size());
+
+ // Make assertions on each field of the SnapshotVO class.
+ SnapshotVO snapshotVO = snapshotVOS.get(0);
+ assertEquals(1, snapshotVO.getSnapshotId());
+ assertEquals(0, snapshotVO.getSchemaId());
+ assertEquals("OVERWRITE", snapshotVO.getCommitKind());
+ assertTrue(snapshotVO.getCommitIdentifier() > 0);
+ assertTrue(snapshotVO.getCommitTime().isBefore(LocalDateTime.now()));
}
@Test
