pratyakshsharma commented on a change in pull request #1150: [HUDI-288]: Add
support for ingesting multiple kafka streams in a single DeltaStreamer
deployment
URL: https://github.com/apache/incubator-hudi/pull/1150#discussion_r399494359
##########
File path:
hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
##########
@@ -286,24 +350,46 @@ public static void createSavepointFile(String basePath,
String commitTime, Confi
* @return List of {@link HoodieRecord}s
*/
public List<HoodieRecord> generateInserts(String commitTime, Integer n,
boolean isFlattened) {
- return generateInsertsStream(commitTime, n,
isFlattened).collect(Collectors.toList());
+ return generateInsertsStream(commitTime, n, isFlattened,
TRIP_EXAMPLE_SCHEMA).collect(Collectors.toList());
+ }
+
+ /**
+ * Generates new inserts, uniformly across the partition paths above. It
also updates the list of existing keys.
+ */
+ public Stream<HoodieRecord> generateInsertsStream(String commitTime, Integer
n, String schemaStr) {
+ int currSize = getNumExistingKeys(schemaStr);
+
+ return IntStream.range(0, n).boxed().map(i -> {
+ String partitionPath =
partitionPaths[rand.nextInt(partitionPaths.length)];
+ HoodieKey key = new HoodieKey(UUID.randomUUID().toString(),
partitionPath);
+ KeyPartition kp = new KeyPartition();
+ kp.key = key;
+ kp.partitionPath = partitionPath;
+ populateKeysBySchema(schemaStr, currSize + i, kp);
+ incrementNumExistingKeysBySchema(schemaStr);
+ try {
+ return new HoodieRecord(key, generateRandomValueAsPerSchema(schemaStr,
key, commitTime));
+ } catch (IOException e) {
+ throw new HoodieIOException(e.getMessage(), e);
+ }
+ });
}
/**
* Generates new inserts, uniformly across the partition paths above. It
also updates the list of existing keys.
*/
public Stream<HoodieRecord> generateInsertsStream(
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
