wgtmac commented on code in PR #1026: URL: https://github.com/apache/parquet-mr/pull/1026#discussion_r1106565210
########## parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java: ########## @@ -183,12 +186,61 @@ public ParquetRewriter(TransParquetFileReader reader, } } + // Open all input files to validate their schemas are compatible to merge + private void openInputFiles(List<Path> inputFiles, Configuration conf) { + Preconditions.checkArgument(inputFiles != null && !inputFiles.isEmpty(), "No input files"); + + for (Path inputFile : inputFiles) { + try { + TransParquetFileReader reader = new TransParquetFileReader( + HadoopInputFile.fromPath(inputFile, conf), HadoopReadOptions.builder(conf).build()); + MessageType inputFileSchema = reader.getFooter().getFileMetaData().getSchema(); + if (this.schema == null) { + this.schema = inputFileSchema; + } else { + // Now we enforce equality of schemas from input files for simplicity. + if (!this.schema.equals(inputFileSchema)) { + throw new InvalidSchemaException("Input files have different schemas"); Review Comment: Fixed ########## parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java: ########## @@ -183,12 +186,61 @@ public ParquetRewriter(TransParquetFileReader reader, } } + // Open all input files to validate their schemas are compatible to merge + private void openInputFiles(List<Path> inputFiles, Configuration conf) { + Preconditions.checkArgument(inputFiles != null && !inputFiles.isEmpty(), "No input files"); + + for (Path inputFile : inputFiles) { + try { + TransParquetFileReader reader = new TransParquetFileReader( + HadoopInputFile.fromPath(inputFile, conf), HadoopReadOptions.builder(conf).build()); + MessageType inputFileSchema = reader.getFooter().getFileMetaData().getSchema(); + if (this.schema == null) { + this.schema = inputFileSchema; + } else { + // Now we enforce equality of schemas from input files for simplicity. + if (!this.schema.equals(inputFileSchema)) { + throw new InvalidSchemaException("Input files have different schemas"); + } + } + this.allOriginalCreatedBys.add(reader.getFooter().getFileMetaData().getCreatedBy()); + this.inputFiles.add(reader); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to open input file: " + inputFile, e); + } + } + + extraMetaData.put(ORIGINAL_CREATED_BY_KEY, String.join("\n", allOriginalCreatedBys)); + } + + // Routines to get reader of next input file. + // Returns true if there is a next file to read, false otherwise. Review Comment: Fixed ########## parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/ParquetRewriterTest.java: ########## @@ -484,15 +673,22 @@ private List<Long> getOffsets(TransParquetFileReader reader, ColumnChunkMetaData } private void validateCreatedBy() throws Exception { - FileMetaData inFMD = getFileMetaData(inputFile.getFileName(), null).getFileMetaData(); - FileMetaData outFMD = getFileMetaData(outputFile, null).getFileMetaData(); + Set<String> createdBySet = new HashSet<>(); + for (EncryptionTestFile inputFile : inputFiles) { + ParquetMetadata pmd = getFileMetaData(inputFile.getFileName(), null); + createdBySet.add(pmd.getFileMetaData().getCreatedBy()); + assertNull(pmd.getFileMetaData().getKeyValueMetaData().get(ParquetRewriter.ORIGINAL_CREATED_BY_KEY)); + } + Object[] inputCreatedBys = createdBySet.toArray(); + assertEquals(1, inputCreatedBys.length); + String inputCreatedBy = (String) inputCreatedBys[0]; - assertEquals(inFMD.getCreatedBy(), outFMD.getCreatedBy()); - assertNull(inFMD.getKeyValueMetaData().get(ParquetRewriter.ORIGINAL_CREATED_BY_KEY)); + FileMetaData outFMD = getFileMetaData(outputFile, null).getFileMetaData(); + assertEquals(inputCreatedBy, outFMD.getCreatedBy()); String originalCreatedBy = outFMD.getKeyValueMetaData().get(ParquetRewriter.ORIGINAL_CREATED_BY_KEY); assertNotNull(originalCreatedBy); - assertEquals(inFMD.getCreatedBy(), originalCreatedBy); + assertEquals(inputCreatedBy, originalCreatedBy); } Review Comment: Fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: dev-unsubscr...@parquet.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org