mihaibudiu commented on code in PR #3145: URL: https://github.com/apache/calcite/pull/3145#discussion_r1266049814
########## plus/src/test/java/org/apache/calcite/slt/SqlLogicTestsForCalciteTests.java: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.slt; + +import org.apache.calcite.slt.executors.CalciteExecutor; + +import net.hydromatic.sqllogictest.OptionsParser; +import net.hydromatic.sqllogictest.TestStatistics; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.function.Executable; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; + +/** + * Tests using sql-logic-test suite. + */ +public class SqlLogicTestsForCalciteTests { + /** + * Short summary of the results of a test execution. + */ + static class TestSummary { + /** + * File containing tests. + */ + final String file; + /** + * Number of tests that have passed. + */ + final int passed; + /** + * Number of tests that have failed. + */ + final int failed; + + TestSummary(String file, int passed, int failed) { + this.file = file; + this.passed = passed; + this.failed = failed; + } + + /** + * Parse a TestSummary from a string. + * The inverse of 'toString'. + * + * @return The parsed TestSummary or null on failure. + */ + static TestSummary parse(String line) { + String[] parts = line.split(":"); + if (parts.length != 3) { + return null; + } + try { + int passed = Integer.parseInt(parts[1]); + int failed = Integer.parseInt(parts[2]); + return new TestSummary(parts[0], passed, failed); + } catch (NumberFormatException ex) { + return null; + } + } + + @Override public String toString() { + return this.file + ":" + this.passed + ":" + this.failed; + } + + /** + * Check if the 'other' TestSummaries are a regressions + * when compared to 'this'. + * + * @param other TestSummary to compare against. + * @return 'true' if 'other' is a regression from 'this'. + */ + boolean regression(TestSummary other) { + return other.failed > this.failed; + } + } + + /** + * Summary for all tests executed. + */ + static class AllTestSummaries { + /** + * Map test summary name to test summary. + */ + final Map<String, TestSummary> testResults; + + AllTestSummaries() { + this.testResults = new HashMap<>(); + } + + void add(TestSummary summary) { + this.testResults.put(summary.file, summary); + } + + void read(InputStream stream) throws IOException { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + reader.lines().forEach(line -> { + TestSummary summary = TestSummary.parse(line); + if (summary != null) { + this.add(summary); + } else { + System.err.println("Could not parse line " + line); + } + }); + } + } + + /** + * Check if 'other' summaries have regressions compared to `this`. + * + * @return 'true' if other contains regressions. + * @param other Test results to compare with. + * 'other' can contain only a subset of the tests. + */ + boolean regression(AllTestSummaries other) { + boolean regression = false; + for (TestSummary summary: other.testResults.values()) { + TestSummary original = this.testResults.get(summary.file); + if (original == null) { + System.err.println("No historical data for test " + summary.file); Review Comment: I will remove the prints, but the hydromatic project also prints stuff, and we currently pass System.out and System.err for the PrintStream values it expects in `launchSqlLogicTest`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
