(flink) branch master updated (9d21878befa -> 84eab0405a0)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 9d21878befa [FLINK-36287] Disallow UC for inner sink channels
add 363d8bdfa99 [FLINK-36369][table] Move deprecated user-visible classes
in table-common to the legacy package
add 2836d250262 [FLINK-36369][table] Move deprecated user-visible classes
in table-api-java to the legacy package
add 10037e39b5c [FLINK-36369][table] Move deprecated user-visible classes
in table-api-java-bridge to the legacy package
add 84eab0405a0 [FLINK-36166][table-planner] Temporarily ignore
testJoinDisorderChangeLog
No new revisions were added by this update.
Summary of changes:
docs/content.zh/docs/dev/table/functions/udfs.md | 6 ++--
docs/content.zh/docs/dev/table/sourcesSinks.md | 2 +-
docs/content.zh/docs/dev/table/tableApi.md | 2 +-
docs/content/docs/dev/table/sourcesSinks.md| 2 +-
docs/content/docs/dev/table/tableApi.md| 2 +-
.../file/table/FileSystemTableFactory.java | 2 +-
flink-connectors/flink-connector-hive/pom.xml | 4 +--
.../connectors/hive/HiveLookupTableSource.java | 2 +-
.../flink/connectors/hive/HiveTableFactory.java| 8 ++---
.../flink/table/catalog/hive/HiveCatalog.java | 4 +--
.../hive/client/HiveMetastoreClientWrapper.java| 2 +-
.../flink/table/catalog/hive/client/HiveShim.java | 2 +-
.../table/catalog/hive/client/HiveShimV100.java| 2 +-
.../table/catalog/hive/client/HiveShimV210.java| 2 +-
.../table/catalog/hive/client/HiveShimV310.java| 2 +-
.../table/catalog/hive/util/HiveTableUtil.java | 2 +-
.../hive/parse/HiveParserDDLSemanticAnalyzer.java | 2 +-
.../connectors/hive/HiveTableFactoryTest.java | 10 +++---
.../TestScanTableSourceWithWatermarkPushDown.java | 2 +-
.../flink/sql/tests/StreamSQLTestProgram.java | 12 +++
.../avro/typeutils/AvroSchemaConverter.java| 2 +-
.../avro/typeutils/AvroSchemaConverterTest.java| 2 +-
.../protobuf/table/TestProtobufTableSource.java| 2 +-
flink-python/pyflink/java_gateway.py | 6
flink-python/pyflink/table/descriptors.py | 2 +-
flink-python/pyflink/table/table_result.py | 2 +-
.../pyflink/table/tests/test_descriptor.py | 8 ++---
.../runtime/arrow/sources/ArrowTableSource.java| 4 +--
.../chain/PythonOperatorChainingOptimizerTest.java | 2 +-
.../table/utils/TestCollectionTableFactory.java| 2 +-
.../flink/table/utils/TestingDescriptors.java | 2 +-
.../AbstractStreamTableEnvironmentImpl.java| 2 +-
.../table/operations/DataStreamQueryOperation.java | 2 +-
.../datagen/table/DataGenTableSource.java | 4 +--
.../connector/source/SourceFunctionProvider.java | 3 +-
.../table/descriptors/RowtimeValidator.java| 41 --
.../table/descriptors/SchemaValidator.java | 33 +
.../table/factories/StreamTableSinkFactory.java| 8 +++--
.../table/factories/StreamTableSourceFactory.java | 8 +++--
.../table/sinks/AppendStreamTableSink.java | 3 +-
.../table/sinks/OutputFormatTableSink.java | 3 +-
.../table/sinks/RetractStreamTableSink.java| 3 +-
.../{ => legacy}/table/sinks/StreamTableSink.java | 3 +-
.../table/sinks/UpsertStreamTableSink.java | 3 +-
.../table/sources/InputFormatTableSource.java | 3 +-
.../table/sources/StreamTableSource.java | 3 +-
.../java/internal/StreamTableEnvironmentImpl.java | 2 +-
.../connector/sink/DataStreamSinkProvider.java | 1 +
.../connector/source/DataStreamScanProvider.java | 1 +
.../table/sinks/CsvAppendTableSinkFactory.java | 3 +-
.../table/sinks/CsvBatchTableSinkFactory.java | 3 +-
.../org/apache/flink/table/sinks/CsvTableSink.java | 4 ++-
.../flink/table/sinks/CsvTableSinkFactoryBase.java | 8 ++---
.../table/sources/CsvAppendTableSourceFactory.java | 3 +-
.../table/sources/CsvBatchTableSourceFactory.java | 3 +-
.../apache/flink/table/sources/CsvTableSource.java | 6 +++-
.../table/sources/CsvTableSourceFactoryBase.java | 8 ++---
.../sources/wmstrategies/AscendingTimestamps.java | 2 +-
.../wmstrategies/BoundedOutOfOrderTimestamps.java | 2 +-
...ache.flink.table.legacy.factories.TableFactory} | 0
.../table/factories/CsvTableSinkFactoryTest.java | 10 +++---
.../factories/DataGenTableSourceFactoryTest.java | 2 +-
.../flink/table/sources/CsvTableSourceTest.java| 3 +-
.../java/org/apache/flink/table/api/Table.java | 1 +
.../org/apache/flink/table/api/TableResult.java| 1 +
.../table/api/internal/TableEnvironmentImpl.java | 4 +--
.../api/internal/TableEnvironmentInternal.java | 4 +--
.../flink/table/catalog/AbstractCatalogTable.java | 2 +-
.../flink/table/catalog/AbstractCatalogView.java | 2 +-
.../flink/ta
(flink) 04/07: [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdUniqueKeys
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e81b1089d17fe4271d8c34a7cd9d0b61072a29bb
Author: lincoln lee
AuthorDate: Tue Sep 24 22:47:24 2024 +0800
[FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from
FlinkRelMdUniqueKeys
---
.../plan/metadata/FlinkRelMdUniqueKeys.scala | 23 +++---
1 file changed, 3 insertions(+), 20 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
index e171925351d..492f3c6fb93 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
@@ -27,7 +27,7 @@ import
org.apache.flink.table.planner.plan.nodes.physical.stream._
import org.apache.flink.table.planner.plan.schema.{FlinkPreparingTableBase,
TableSourceTable}
import org.apache.flink.table.planner.plan.utils.{FlinkRelMdUtil, RankUtil}
import org.apache.flink.table.runtime.groupwindow.NamedWindowProperty
-import org.apache.flink.table.runtime.operators.rank.{ConstantRankRange,
RankType}
+import org.apache.flink.table.runtime.operators.rank.RankType
import org.apache.flink.table.types.logical.utils.LogicalTypeCasts
import com.google.common.collect.ImmutableSet
@@ -290,19 +290,9 @@ class FlinkRelMdUniqueKeys private extends
MetadataHandler[BuiltInMetadata.Uniqu
def getRankUniqueKeys(rel: Rank, inputKeys: JSet[ImmutableBitSet]):
JSet[ImmutableBitSet] = {
val rankFunColumnIndex =
RankUtil.getRankNumberColumnIndex(rel).getOrElse(-1)
-// for Rank node that can convert to Deduplicate, unique key is partition
key
-val canConvertToDeduplicate: Boolean = {
- val rankRange = rel.rankRange
- val isRowNumberType = rel.rankType == RankType.ROW_NUMBER
- val isLimit1 = rankRange match {
-case rankRange: ConstantRankRange =>
- rankRange.getRankStart == 1 && rankRange.getRankEnd == 1
-case _ => false
- }
- isRowNumberType && isLimit1
-}
-if (canConvertToDeduplicate) {
+if (RankUtil.isDeduplication(rel)) {
+ // for Rank node that can convert to Deduplicate, unique key is
partition key
val retSet = new JHashSet[ImmutableBitSet]
retSet.add(rel.partitionKey)
retSet
@@ -325,13 +315,6 @@ class FlinkRelMdUniqueKeys private extends
MetadataHandler[BuiltInMetadata.Uniqu
def getUniqueKeys(rel: Sort, mq: RelMetadataQuery, ignoreNulls: Boolean):
JSet[ImmutableBitSet] =
mq.getUniqueKeys(rel.getInput, ignoreNulls)
- def getUniqueKeys(
- rel: StreamPhysicalDeduplicate,
- mq: RelMetadataQuery,
- ignoreNulls: Boolean): JSet[ImmutableBitSet] = {
-
ImmutableSet.of(ImmutableBitSet.of(rel.getUniqueKeys.map(Integer.valueOf).toList))
- }
-
def getUniqueKeys(
rel: StreamPhysicalChangelogNormalize,
mq: RelMetadataQuery,
(flink) 01/07: [FLINK-34702][table-planner] Avoid using StreamPhysicalDeduplicate and instead decide to perform exec node transformations in StreamPhysicalRank
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 1543c72455629842a9424aa11ea7641fb5d13760
Author: lincoln lee
AuthorDate: Tue Sep 24 10:24:42 2024 +0800
[FLINK-34702][table-planner] Avoid using StreamPhysicalDeduplicate and
instead decide to perform exec node transformations in StreamPhysicalRank
---
.../stream/StreamPhysicalDeduplicate.scala | 7 +-
.../nodes/physical/stream/StreamPhysicalRank.scala | 71 +
.../planner/plan/rules/FlinkStreamRuleSets.scala | 1 -
.../stream/StreamPhysicalDeduplicateRule.scala | 99 -
.../physical/stream/StreamPhysicalRankRule.scala | 12 +--
.../flink/table/planner/plan/utils/RankUtil.scala | 54 --
.../nodes/exec/operator/StreamOperatorNameTest.xml | 4 +-
.../physical/stream/ChangelogModeInferenceTest.xml | 2 +-
.../planner/plan/stream/sql/DeduplicateTest.xml| 33 +-
.../plan/stream/sql/NonDeterministicDagTest.xml| 105 ++
.../table/planner/plan/stream/sql/RankTest.xml | 8 +-
.../plan/stream/sql/agg/GroupWindowTest.xml| 2 +-
.../plan/stream/sql/agg/WindowAggregateTest.xml| 8 +-
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 12 ++-
.../planner/plan/stream/sql/DeduplicateTest.scala | 26 +
.../plan/stream/sql/NonDeterministicDagTest.scala | 118 -
.../table/planner/plan/stream/sql/RankTest.scala | 17 +--
.../plan/stream/sql/agg/WindowAggregateTest.scala | 2 +-
18 files changed, 361 insertions(+), 220 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
index d732e21d0c0..dbc5e60043d 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
@@ -30,9 +30,10 @@ import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
import java.util
/**
- * Stream physical RelNode which deduplicate on keys and keeps only first row
or last row. This node
- * is an optimization of [[StreamPhysicalRank]] for some special cases.
Compared to
- * [[StreamPhysicalRank]], this node could use mini-batch and access less
state.
+ * TODO to be removed after FLINK-34702 is fixed. Stream physical RelNode
which deduplicate on keys
+ * and keeps only first row or last row. This node is an optimization of
[[StreamPhysicalRank]] for
+ * some special cases. Compared to [[StreamPhysicalRank]], this node could use
mini-batch and access
+ * less state.
*/
class StreamPhysicalDeduplicate(
cluster: RelOptCluster,
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalRank.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalRank.scala
index 534c9dda8d2..084d59d9893 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalRank.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalRank.scala
@@ -21,7 +21,7 @@ import org.apache.flink.table.planner.calcite.FlinkTypeFactory
import org.apache.flink.table.planner.plan.nodes.calcite.Rank
import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
import org.apache.flink.table.planner.plan.nodes.exec.spec.PartitionSpec
-import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecRank
+import
org.apache.flink.table.planner.plan.nodes.exec.stream.{StreamExecDeduplicate,
StreamExecRank}
import org.apache.flink.table.planner.plan.utils._
import org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTableConfig
import org.apache.flink.table.runtime.operators.rank._
@@ -46,8 +46,9 @@ class StreamPhysicalRank(
rankRange: RankRange,
rankNumberType: RelDataTypeField,
outputRankNumber: Boolean,
-val rankStrategy: RankProcessStrategy)
- extends Rank(
+val rankStrategy: RankProcessStrategy,
+val sortOnRowTime: Boolean
+) extends Rank(
cluster,
traitSet,
inputRel,
@@ -59,7 +60,7 @@ class StreamPhysicalRank(
outputRankNumber)
with StreamPhysicalRel {
- override def requireWatermark: Boolean = false
+ override def requireWatermark: Boolean = sortOnRowTime
override def copy(traitSet: RelTraitSet, inputs: util.List[RelNode]):
RelNode = {
new StreamPhysicalRank(
@@ -72,7 +73,9 @@ class
(flink) 06/07: [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdUpsertKeys
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a134711aeca205aabfe3ce04c97e1f931c5a5a64
Author: lincoln lee
AuthorDate: Tue Sep 24 23:36:04 2024 +0800
[FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from
FlinkRelMdUpsertKeys
---
.../plan/metadata/FlinkRelMdUpsertKeys.scala | 23 +++---
.../table/planner/plan/stream/sql/RankTest.xml | 2 +-
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
index 358862826a2..df151c8a607 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
@@ -24,7 +24,7 @@ import
org.apache.flink.table.planner.plan.nodes.physical.batch.{BatchPhysicalGr
import
org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalLookupJoin
import org.apache.flink.table.planner.plan.nodes.physical.stream._
import org.apache.flink.table.planner.plan.schema.IntermediateRelTable
-import org.apache.flink.table.planner.plan.utils.FlinkRexUtil
+import org.apache.flink.table.planner.plan.utils.{FlinkRexUtil, RankUtil}
import com.google.common.collect.ImmutableSet
import org.apache.calcite.plan.hep.HepRelVertex
@@ -91,12 +91,17 @@ class FlinkRelMdUpsertKeys private extends
MetadataHandler[UpsertKeys] {
}
def getUpsertKeys(rel: Rank, mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
-val inputKeys = filterKeys(
- FlinkRelMetadataQuery
-.reuseOrCreate(mq)
-.getUpsertKeys(rel.getInput),
- rel.partitionKey)
-FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(rel, inputKeys)
+rel match {
+ case rank: StreamPhysicalRank if RankUtil.isDeduplication(rel) =>
+
ImmutableSet.of(ImmutableBitSet.of(rank.partitionKey.toArray.map(Integer.valueOf).toList))
+ case _ =>
+val inputKeys = filterKeys(
+ FlinkRelMetadataQuery
+.reuseOrCreate(mq)
+.getUpsertKeys(rel.getInput),
+ rel.partitionKey)
+FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(rel, inputKeys)
+}
}
def getUpsertKeys(rel: Sort, mq: RelMetadataQuery): JSet[ImmutableBitSet] =
@@ -104,10 +109,6 @@ class FlinkRelMdUpsertKeys private extends
MetadataHandler[UpsertKeys] {
FlinkRelMetadataQuery.reuseOrCreate(mq).getUpsertKeys(rel.getInput),
ImmutableBitSet.of(rel.getCollation.getKeys))
- def getUpsertKeys(rel: StreamPhysicalDeduplicate, mq: RelMetadataQuery):
JSet[ImmutableBitSet] = {
-
ImmutableSet.of(ImmutableBitSet.of(rel.getUniqueKeys.map(Integer.valueOf).toList))
- }
-
def getUpsertKeys(
rel: StreamPhysicalChangelogNormalize,
mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
diff --git
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/RankTest.xml
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/RankTest.xml
index 6ea1d56dc73..f6abe90d2d0 100644
---
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/RankTest.xml
+++
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/RankTest.xml
@@ -1176,7 +1176,7 @@ LogicalProject(c=[$0], b=[$1], d=[$2])
(flink) 07/07: [FLINK-34702][table-planner] Remove physical node StreamPhysicalDeduplicate
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2f8b2d81a97a681d67964ef17932ea0abf35730f
Author: lincoln lee
AuthorDate: Tue Sep 24 23:43:25 2024 +0800
[FLINK-34702][table-planner] Remove physical node StreamPhysicalDeduplicate
---
.../TemporalJoinRewriteWithUniqueKeyRule.java | 6 +-
.../stream/StreamPhysicalDeduplicate.scala | 87 --
.../physical/stream/StreamPhysicalRankRule.scala | 7 +-
.../flink/table/planner/plan/utils/RankUtil.scala | 6 +-
4 files changed, 8 insertions(+), 98 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/TemporalJoinRewriteWithUniqueKeyRule.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/TemporalJoinRewriteWithUniqueKeyRule.java
index 342cafd2526..b6e3f1eaf83 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/TemporalJoinRewriteWithUniqueKeyRule.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/TemporalJoinRewriteWithUniqueKeyRule.java
@@ -20,10 +20,10 @@ package org.apache.flink.table.planner.plan.rules.logical;
import org.apache.flink.table.api.ValidationException;
import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery;
+import
org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecDeduplicate;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalRel;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalSnapshot;
-import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDeduplicate;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalRank;
import
org.apache.flink.table.planner.plan.rules.common.CommonTemporalTableJoinRule;
import org.apache.flink.table.planner.plan.utils.TemporalJoinUtil;
@@ -57,8 +57,8 @@ import scala.collection.JavaConverters;
* a table source or a view only if it contains the unique key and time
attribute.
*
* Flink supports extract the primary key and row time attribute from the
view if the view comes
- * from {@link StreamPhysicalRank} node which can convert to a {@link
StreamPhysicalDeduplicate}
- * node.
+ * from {@link StreamPhysicalRank} node which can convert to a {@link
StreamExecDeduplicate} node
+ * finally.
*/
@Value.Enclosing
public class TemporalJoinRewriteWithUniqueKeyRule
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
deleted file mode 100644
index dbc5e60043d..000
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.planner.plan.nodes.physical.stream
-
-import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
-import
org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecDeduplicate
-import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils
-import org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTableConfig
-
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-
-import java.util
-
-/**
- * TODO to be removed after FLINK-34702 is fixed. Stream physical RelNode
which deduplicate on keys
- * and keeps only first row or last row. This node is an optimization of
[[StreamPhysicalRank]] for
- * some special cases. Compared to [[StreamPhysicalRank]], this
(flink) 05/07: [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdModifiedMonotonicity
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 026f233ef039be6d1739bf01daf2470a39c4f51d
Author: lincoln lee
AuthorDate: Tue Sep 24 23:27:55 2024 +0800
[FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from
FlinkRelMdModifiedMonotonicity
---
.../metadata/FlinkRelMdModifiedMonotonicity.scala | 111 +++--
.../nodes/physical/stream/StreamPhysicalRank.scala | 2 +-
.../flink/table/planner/plan/utils/RankUtil.scala | 6 +-
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 2 +-
4 files changed, 66 insertions(+), 55 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdModifiedMonotonicity.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdModifiedMonotonicity.scala
index f25e2cebdd2..0f8b5e83ad8 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdModifiedMonotonicity.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdModifiedMonotonicity.scala
@@ -29,6 +29,7 @@ import
org.apache.flink.table.planner.plan.nodes.physical.batch.{BatchPhysicalCo
import org.apache.flink.table.planner.plan.nodes.physical.stream._
import org.apache.flink.table.planner.plan.schema.{FlinkPreparingTableBase,
IntermediateRelTable, TableSourceTable}
import org.apache.flink.table.planner.plan.stats.{WithLower, WithUpper}
+import org.apache.flink.table.planner.plan.utils.RankUtil
import org.apache.flink.types.RowKind
import org.apache.calcite.plan.hep.HepRelVertex
@@ -186,70 +187,78 @@ class FlinkRelMdModifiedMonotonicity private extends
MetadataHandler[ModifiedMon
}
def getRelModifiedMonotonicity(rel: Rank, mq: RelMetadataQuery):
RelModifiedMonotonicity = {
-val fmq = FlinkRelMetadataQuery.reuseOrCreate(mq)
-val inputMonotonicity = fmq.getRelModifiedMonotonicity(rel.getInput)
+rel match {
+ case physicalRank: StreamPhysicalRank if RankUtil.isDeduplication(rel) =>
+getPhysicalRankModifiedMonotonicity(physicalRank, mq)
-// If child monotonicity is null, we should return early.
-if (inputMonotonicity == null) {
- return null
-}
+ case _ =>
+val fmq = FlinkRelMetadataQuery.reuseOrCreate(mq)
+val inputMonotonicity = fmq.getRelModifiedMonotonicity(rel.getInput)
-// if partitionBy a update field or partitionBy a field whose mono is
null, just return null
-if (rel.partitionKey.exists(e => inputMonotonicity.fieldMonotonicities(e)
!= CONSTANT)) {
- return null
-}
+// If child monotonicity is null, we should return early.
+if (inputMonotonicity == null) {
+ return null
+}
-val fieldCount = rel.getRowType.getFieldCount
+// if partitionBy a update field or partitionBy a field whose mono is
null, just return null
+if (rel.partitionKey.exists(e =>
inputMonotonicity.fieldMonotonicities(e) != CONSTANT)) {
+ return null
+}
-// init current mono
-val currentMonotonicity = notMonotonic(fieldCount)
-// 1. partitionBy field is CONSTANT
-rel.partitionKey.foreach(e => currentMonotonicity.fieldMonotonicities(e) =
CONSTANT)
-// 2. row number filed is CONSTANT
-if (rel.outputRankNumber) {
- currentMonotonicity.fieldMonotonicities(fieldCount - 1) = CONSTANT
-}
-// 3. time attribute field is increasing
-(0 until fieldCount).foreach(
- e => {
-if
(FlinkTypeFactory.isTimeIndicatorType(rel.getRowType.getFieldList.get(e).getType))
{
- inputMonotonicity.fieldMonotonicities(e) = INCREASING
+val fieldCount = rel.getRowType.getFieldCount
+
+// init current mono
+val currentMonotonicity = notMonotonic(fieldCount)
+// 1. partitionBy field is CONSTANT
+rel.partitionKey.foreach(e =>
currentMonotonicity.fieldMonotonicities(e) = CONSTANT)
+// 2. row number filed is CONSTANT
+if (rel.outputRankNumber) {
+ currentMonotonicity.fieldMonotonicities(fieldCount - 1) = CONSTANT
+}
+// 3. time attribute field is increasing
+(0 until fieldCount).foreach(
+ e => {
+if
(FlinkTypeFactory.isTimeIndicatorType(rel.getRowType.getFieldList.get(e).getType))
{
+ inputMonotonicity.fieldMonotonicities(e) = INCREASING
+}
+ })
+val fieldCollations = rel.orderKey.getFieldCollations
+if (fieldCollations.nonEmpty) {
+ // 4. process the first collation field, we can only deduce the
first collation field
+ val firstCollation = fieldCollations.get(0)
+ // Collation field index in child node will be same with Rank
(flink) 03/07: [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdColumnUniqueness
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9df9dce57e6331d4b7bfd130d7f637d9db1cebfd
Author: lincoln lee
AuthorDate: Tue Sep 24 22:40:12 2024 +0800
[FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from
FlinkRelMdColumnUniqueness
---
.../plan/metadata/FlinkRelMdColumnUniqueness.scala | 39 ++
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 39 +++---
2 files changed, 45 insertions(+), 33 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdColumnUniqueness.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdColumnUniqueness.scala
index 619d5c70940..d6340831dac 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdColumnUniqueness.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdColumnUniqueness.scala
@@ -251,21 +251,26 @@ class FlinkRelMdColumnUniqueness private extends
MetadataHandler[BuiltInMetadata
mq: RelMetadataQuery,
columns: ImmutableBitSet,
ignoreNulls: Boolean): JBoolean = {
-val input = rank.getInput
-val rankFunColumnIndex =
RankUtil.getRankNumberColumnIndex(rank).getOrElse(-1)
-if (rankFunColumnIndex < 0) {
- mq.areColumnsUnique(input, columns, ignoreNulls)
+if (RankUtil.isDeduplication(rank)) {
+ columns != null && util.Arrays.equals(columns.toArray,
rank.partitionKey.toArray)
} else {
- val childColumns = columns.clear(rankFunColumnIndex)
- val isChildColumnsUnique = mq.areColumnsUnique(input, childColumns,
ignoreNulls)
- if (isChildColumnsUnique != null && isChildColumnsUnique) {
-true
+ val input = rank.getInput
+
+ val rankFunColumnIndex =
RankUtil.getRankNumberColumnIndex(rank).getOrElse(-1)
+ if (rankFunColumnIndex < 0) {
+mq.areColumnsUnique(input, columns, ignoreNulls)
} else {
-rank.rankType match {
- case RankType.ROW_NUMBER =>
-val fields = columns.toArray
-(rank.partitionKey.toArray :+
rankFunColumnIndex).forall(fields.contains(_))
- case _ => false
+val childColumns = columns.clear(rankFunColumnIndex)
+val isChildColumnsUnique = mq.areColumnsUnique(input, childColumns,
ignoreNulls)
+if (isChildColumnsUnique != null && isChildColumnsUnique) {
+ true
+} else {
+ rank.rankType match {
+case RankType.ROW_NUMBER =>
+ val fields = columns.toArray
+ (rank.partitionKey.toArray :+
rankFunColumnIndex).forall(fields.contains(_))
+case _ => false
+ }
}
}
}
@@ -277,14 +282,6 @@ class FlinkRelMdColumnUniqueness private extends
MetadataHandler[BuiltInMetadata
columns: ImmutableBitSet,
ignoreNulls: Boolean): JBoolean = mq.areColumnsUnique(rel.getInput,
columns, ignoreNulls)
- def areColumnsUnique(
- rel: StreamPhysicalDeduplicate,
- mq: RelMetadataQuery,
- columns: ImmutableBitSet,
- ignoreNulls: Boolean): JBoolean = {
-columns != null && util.Arrays.equals(columns.toArray, rel.getUniqueKeys)
- }
-
def areColumnsUnique(
rel: StreamPhysicalChangelogNormalize,
mq: RelMetadataQuery,
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
index 83cd4f011b3..e9a224da2d3 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
@@ -666,7 +666,7 @@ class FlinkRelMdHandlerTestBase {
new RelDataTypeFieldImpl("rk", 7, longType),
outputRankNumber = true,
RankProcessStrategy.UNDEFINED_STRATEGY,
- sortOnRowtime = false
+ sortOnRowTime = false
)
(logicalRank, flinkLogicalRank, batchLocalRank, batchGlobalRank,
streamRank)
@@ -755,7 +755,7 @@ class FlinkRelMdHandlerTestBase {
new RelDataTypeFieldImpl("rk", 7, longType),
outputRankNumber = true,
RankProcessStrategy.UNDEFINED_STRATEGY,
- sortOnRowtime = false
+ sortOnRowTime = false
)
(logicalRank, flinkLogicalRank, batchLocalRank, batchGlobalRank,
streamRank)
@@ -808,7 +808,7 @@ class FlinkRelMdHandlerTestBase {
new RelDataTypeFieldImpl("rn", 7, longType),
outputRankNu
(flink) 02/07: [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkChangelogModeInferenceProgram and StreamNonDeterministicUpdatePlanVisitor
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 00625b0019e492eab9e62d953f915c5afcc06a0a
Author: lincoln lee
AuthorDate: Tue Sep 24 22:39:33 2024 +0800
[FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from
FlinkChangelogModeInferenceProgram and StreamNonDeterministicUpdatePlanVisitor
---
.../StreamNonDeterministicUpdatePlanVisitor.java | 19 ---
.../program/FlinkChangelogModeInferenceProgram.scala | 14 +-
2 files changed, 1 insertion(+), 32 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
index 14e8b72c540..cf96ce7aab6 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
@@ -34,7 +34,6 @@ import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalC
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCorrelateBase;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDataStreamScan;
-import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDeduplicate;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDropUpdateBefore;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange;
import
org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExpand;
@@ -184,8 +183,6 @@ public class StreamNonDeterministicUpdatePlanVisitor {
return visitOverAggregate((StreamPhysicalOverAggregateBase) rel,
requireDeterminism);
} else if (rel instanceof StreamPhysicalRank) {
return visitRank((StreamPhysicalRank) rel, requireDeterminism);
-} else if (rel instanceof StreamPhysicalDeduplicate) {
-return visitDeduplicate((StreamPhysicalDeduplicate) rel,
requireDeterminism);
} else if (rel instanceof StreamPhysicalWindowDeduplicate) {
return visitWindowDeduplicate(
(StreamPhysicalWindowDeduplicate) rel, requireDeterminism);
@@ -677,22 +674,6 @@ public class StreamNonDeterministicUpdatePlanVisitor {
}
}
-private StreamPhysicalRel visitDeduplicate(
-final StreamPhysicalDeduplicate dedup, final ImmutableBitSet
requireDeterminism) {
-// output row type same as input and does not change output columns'
order
-if (inputInsertOnly(dedup)) {
-// similar to rank, output is deterministic when input is insert
only, so required
-// determinism always be satisfied here.
-return transmitDeterminismRequirement(dedup,
NO_REQUIRED_DETERMINISM);
-} else {
-// Deduplicate always has unique key currently(exec node has null
check and inner
-// state only support data with keys), so only pass the left
columns of required
-// determinism to input.
-return transmitDeterminismRequirement(
-dedup,
requireDeterminism.except(ImmutableBitSet.of(dedup.getUniqueKeys(;
-}
-}
-
private StreamPhysicalRel visitWindowDeduplicate(
final StreamPhysicalWindowDeduplicate winDedup,
final ImmutableBitSet requireDeterminism) {
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala
index ddebacaf842..2843fc5e8d9 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala
@@ -155,18 +155,6 @@ class FlinkChangelogModeInferenceProgram extends
FlinkOptimizeProgram[StreamOpti
// ignore required trait from context, because sink is the true root
sink.copy(sinkTrait, children).asInstanceOf[StreamPhysicalRel]
- case deduplicate: StreamPhysicalDeduplicate =>
-// deduplicate only support insert only as input
-val children = visitChildren(deduplicate,
ModifyKindSetTrait.INSERT_ONLY)
-val providedTr
(flink) branch master updated (f33abffe1da -> 2f8b2d81a97)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from f33abffe1da [FLINK-34975][state/forst] Support multiget for forstdb (#25363) new 1543c724556 [FLINK-34702][table-planner] Avoid using StreamPhysicalDeduplicate and instead decide to perform exec node transformations in StreamPhysicalRank new 00625b0019e [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkChangelogModeInferenceProgram and StreamNonDeterministicUpdatePlanVisitor new 9df9dce57e6 [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdColumnUniqueness new e81b1089d17 [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdUniqueKeys new 026f233ef03 [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdModifiedMonotonicity new a134711aeca [FLINK-34702][table-planner] Remove StreamPhysicalDeduplicate from FlinkRelMdUpsertKeys new 2f8b2d81a97 [FLINK-34702][table-planner] Remove physical node StreamPhysicalDeduplicate The 7 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../StreamNonDeterministicUpdatePlanVisitor.java | 19 .../TemporalJoinRewriteWithUniqueKeyRule.java | 6 +- .../plan/metadata/FlinkRelMdColumnUniqueness.scala | 39 --- .../metadata/FlinkRelMdModifiedMonotonicity.scala | 111 ++- .../plan/metadata/FlinkRelMdUniqueKeys.scala | 23 +--- .../plan/metadata/FlinkRelMdUpsertKeys.scala | 23 ++-- .../stream/StreamPhysicalDeduplicate.scala | 86 --- .../nodes/physical/stream/StreamPhysicalRank.scala | 71 + .../FlinkChangelogModeInferenceProgram.scala | 14 +-- .../planner/plan/rules/FlinkStreamRuleSets.scala | 1 - .../stream/StreamPhysicalDeduplicateRule.scala | 99 - .../physical/stream/StreamPhysicalRankRule.scala | 19 ++-- .../flink/table/planner/plan/utils/RankUtil.scala | 60 --- .../nodes/exec/operator/StreamOperatorNameTest.xml | 4 +- .../physical/stream/ChangelogModeInferenceTest.xml | 2 +- .../planner/plan/stream/sql/DeduplicateTest.xml| 33 +- .../plan/stream/sql/NonDeterministicDagTest.xml| 105 ++ .../table/planner/plan/stream/sql/RankTest.xml | 6 +- .../plan/stream/sql/agg/GroupWindowTest.xml| 2 +- .../plan/stream/sql/agg/WindowAggregateTest.xml| 8 +- .../plan/metadata/FlinkRelMdHandlerTestBase.scala | 43 +--- .../planner/plan/stream/sql/DeduplicateTest.scala | 26 + .../plan/stream/sql/NonDeterministicDagTest.scala | 118 - .../table/planner/plan/stream/sql/RankTest.scala | 17 +-- .../plan/stream/sql/agg/WindowAggregateTest.scala | 2 +- 25 files changed, 482 insertions(+), 455 deletions(-) delete mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalDeduplicate.scala delete mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalDeduplicateRule.scala
(flink) 04/05: [FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0 in flink-table-api module
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 499df42b99d07f2f3c90df51d5533a74ce3990e8
Author: xuyang
AuthorDate: Mon Sep 23 11:52:37 2024 +0800
[FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0
in flink-table-api module
---
flink-end-to-end-tests/flink-tpcds-test/pom.xml| 6 ++
.../apache/flink/table/tpcds/TpcdsTestProgram.java | 7 --
.../table/api/config/ExecutionConfigOptions.java | 57 -
.../table/api/config/OptimizerConfigOptions.java | 1 -
.../planner/plan/nodes/exec/ExecNodeConfig.java| 3 +-
.../nodes/exec/stream/StreamExecIntervalJoin.java | 3 +-
.../planner/utils/StreamExchangeModeUtils.java | 33 --
.../plan/nodes/exec/TransformationsTest.java | 8 ---
.../planner/utils/StreamExchangeModeUtilsTest.java | 74 --
9 files changed, 8 insertions(+), 184 deletions(-)
diff --git a/flink-end-to-end-tests/flink-tpcds-test/pom.xml
b/flink-end-to-end-tests/flink-tpcds-test/pom.xml
index f8ecd8076da..1a231b9e839 100644
--- a/flink-end-to-end-tests/flink-tpcds-test/pom.xml
+++ b/flink-end-to-end-tests/flink-tpcds-test/pom.xml
@@ -29,6 +29,12 @@ under the License.
Flink : E2E Tests : TPCDS
+
+ org.apache.flink
+ flink-core
+ ${project.version}
+ provided
+
org.apache.flink
flink-java
diff --git
a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
index 7c40ae2b1c7..8ebf202afcb 100644
---
a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
+++
b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
@@ -20,12 +20,10 @@ package org.apache.flink.table.tpcds;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.streaming.api.graph.GlobalStreamExchangeMode;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.TableResult;
-import org.apache.flink.table.api.config.ExecutionConfigOptions;
import org.apache.flink.table.api.config.OptimizerConfigOptions;
import org.apache.flink.table.api.internal.TableEnvironmentInternal;
import org.apache.flink.table.catalog.ConnectorCatalogTable;
@@ -138,11 +136,6 @@ public class TpcdsTestProgram {
TableEnvironment tEnv = TableEnvironment.create(environmentSettings);
// config Optimizer parameters
-// TODO use the default shuffle mode of batch runtime mode once
FLINK-23470 is implemented
-tEnv.getConfig()
-.set(
-ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE,
-
GlobalStreamExchangeMode.POINTWISE_EDGES_PIPELINED.toString());
tEnv.getConfig()
.set(
OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD,
diff --git
a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/ExecutionConfigOptions.java
b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/ExecutionConfigOptions.java
index c79567341db..97fe37b8d5c 100644
---
a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/ExecutionConfigOptions.java
+++
b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/ExecutionConfigOptions.java
@@ -24,7 +24,6 @@ import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.DescribedEnum;
-import org.apache.flink.configuration.ExecutionOptions;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.configuration.description.Description;
import org.apache.flink.configuration.description.InlineElement;
@@ -32,7 +31,6 @@ import
org.apache.flink.configuration.description.InlineElement;
import java.time.Duration;
import static org.apache.flink.configuration.ConfigOptions.key;
-import static org.apache.flink.configuration.description.TextElement.code;
import static org.apache.flink.configuration.description.TextElement.text;
/**
@@ -499,43 +497,6 @@ public class ExecutionConfigOptions {
.withDescription(
"If true, multiple physical operators will be
compiled into a single operator by planner whic
(flink) 03/05: [FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0 in flink-sql-client module
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git commit 2ca6c2f1f72ddb6499b8147545c44bb65e98c239 Author: xuyang AuthorDate: Mon Sep 23 10:56:47 2024 +0800 [FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0 in flink-sql-client module --- docs/content.zh/docs/dev/table/sqlClient.md| 27 -- docs/content/docs/dev/table/sqlClient.md | 27 -- .../test-scripts/test_pyflink.sh | 11 - flink-end-to-end-tests/test-scripts/test_tpch.sh | 17 +++--- .../org/apache/flink/table/client/SqlClient.java | 18 ++- .../apache/flink/table/client/cli/CliOptions.java | 27 ++ .../flink/table/client/cli/CliOptionsParser.java | 21 - .../src/test/resources/cli/all-mode-help.out | 27 -- .../src/test/resources/cli/embedded-mode-help.out | 15 .../src/test/resources/cli/gateway-mode-help.out | 12 -- 10 files changed, 23 insertions(+), 179 deletions(-) diff --git a/docs/content.zh/docs/dev/table/sqlClient.md b/docs/content.zh/docs/dev/table/sqlClient.md index 32fb4fe215b..9fc664fbc34 100644 --- a/docs/content.zh/docs/dev/table/sqlClient.md +++ b/docs/content.zh/docs/dev/table/sqlClient.md @@ -360,21 +360,6 @@ Mode "embedded" (default) submits Flink jobs from the local machine. -s,--session The identifier for a session. 'default' is the default identifier. - -u,--update Deprecated Experimental (for -testing only!) feature: -Instructs the SQL Client to -immediately execute the given -update statement after starting -up. The process is shut down -after the statement has been -submitted to the cluster and -returns an appropriate return -code. Currently, this feature is -only supported for INSERT INTO -statements that declare the -target sink table.Please use -option -f to submit update -statement. Mode "gateway" mode connects to the SQL gateway for submission. @@ -401,18 +386,6 @@ Mode "gateway" mode connects to the SQL gateway for submission. or insert into the init file. -s,--session The identifier for a session. 'default' is the default identifier. - -u,--update Deprecated Experimental (for testing - only!) feature: Instructs the SQL - Client to immediately execute the - given update statement after starting - up. The process is shut down after - the statement has been submitted to - the cluster and returns an - appropriate return code. Currently, - this feature is only supported for - INSERT INTO statements that declare - the target sink table.Please use - option -f to submit update statement. ``` ### SQL Client Configuration diff --git a/docs/content/docs/dev/table/sqlClient.md b/docs/content/docs/dev/table/sqlClient.md index 40255f47ef3..55405d049f0 100644 --- a/docs/content/docs/dev/table/sqlClient.md +++ b/docs/content/docs/dev/table/sqlClient.md @@ -298,21 +298,6 @@ Mode "embedded" (default) submits Flink jobs from the local machine. -s,--session The identifier for a session. 'default' is the default identifier. - -u,--update Deprecated Experimental (for -testing only!) feature: -Instructs the SQL Client to -
(flink) 05/05: [FLINK-36273][table] Remove all deprecated configuration in 2.0 in flink-connector-hive module
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 733a56682bcceabab6625b621547538c6d993976
Author: xuyang
AuthorDate: Mon Sep 23 12:06:49 2024 +0800
[FLINK-36273][table] Remove all deprecated configuration in 2.0 in
flink-connector-hive module
---
.../docs/connectors/table/hive/hive_read_write.md | 2 +-
.../docs/connectors/table/hive/hive_read_write.md | 2 +-
.../HiveDynamicParallelismInferenceFactory.java| 8 +--
.../apache/flink/connectors/hive/HiveOptions.java | 17 +-
.../connectors/hive/HiveParallelismInference.java | 1 -
.../flink/connectors/hive/HiveSourceBuilder.java | 3 ---
.../HiveStaticParallelismInferenceFactory.java | 7 ++
.../hive/HiveDynamicPartitionPruningITCase.java| 10 ++--
.../hive/HiveDynamicTableFactoryTest.java | 17 --
.../connectors/hive/HiveSinkCompactionITCase.java | 5 +++-
.../flink/connectors/hive/HiveTableSinkITCase.java | 4 ++--
.../connectors/hive/HiveTableSourceITCase.java | 27 +-
.../connectors/hive/PartitionMonitorTest.java | 2 +-
13 files changed, 42 insertions(+), 63 deletions(-)
diff --git a/docs/content.zh/docs/connectors/table/hive/hive_read_write.md
b/docs/content.zh/docs/connectors/table/hive/hive_read_write.md
index 64f9f7dc8f0..ad986a47ad5 100644
--- a/docs/content.zh/docs/connectors/table/hive/hive_read_write.md
+++ b/docs/content.zh/docs/connectors/table/hive/hive_read_write.md
@@ -69,7 +69,7 @@ Flink 支持以批和流两种模式从 Hive 表中读取数据。批读的时
streaming-source.partition-order
partition-name
String
-streaming source 分区排序，支持 create-time， partition-time 和
partition-name。 create-time 比较分区/文件创建时间， 这不是 Hive metastore
中创建分区的时间，而是文件夹/文件在文件系统的修改时间，如果分区文件夹以某种方式更新，比如添加在文件夹里新增了一个文件，它会影响到数据的使用。partition-time
从分区名称中抽取时间进行比较。partition-name 会比较分区名称的字典顺序。对于非分区的表，总是会比较
'create-time'。对于分区表默认值是 'partition-name'。该选项与已经弃用的
'streaming-source.consume-order' 的选项相同
+streaming source 分区排序，支持 create-time， partition-time 和
partition-name。 create-time 比较分区/文件创建时间， 这不是 Hive metastore
中创建分区的时间，而是文件夹/文件在文件系统的修改时间，如果分区文件夹以某种方式更新，比如添加在文件夹里新增了一个文件，它会影响到数据的使用。partition-time
从分区名称中抽取时间进行比较。partition-name 会比较分区名称的字典顺序。对于非分区的表，总是会比较
'create-time'。对于分区表默认值是 'partition-name'。
streaming-source.consume-start-offset
diff --git a/docs/content/docs/connectors/table/hive/hive_read_write.md
b/docs/content/docs/connectors/table/hive/hive_read_write.md
index 9a9834b6579..2f8252a6565 100644
--- a/docs/content/docs/connectors/table/hive/hive_read_write.md
+++ b/docs/content/docs/connectors/table/hive/hive_read_write.md
@@ -79,7 +79,7 @@ of new files in the folder and read new files incrementally.
streaming-source.partition-order
partition-name
String
-The partition order of streaming source, support create-time,
partition-time and partition-name. create-time compares partition/file creation
time, this is not the partition create time in Hive metaStore, but the
folder/file modification time in filesystem, if the partition folder somehow
gets updated, e.g. add new file into folder, it can affect how the data is
consumed. partition-time compares the time extracted from partition name.
partition-name compares partition name's [...]
+The partition order of streaming source, support create-time,
partition-time and partition-name. create-time compares partition/file creation
time, this is not the partition create time in Hive metaStore, but the
folder/file modification time in filesystem, if the partition folder somehow
gets updated, e.g. add new file into folder, it can affect how the data is
consumed. partition-time compares the time extracted from partition name.
partition-name compares partition name's [...]
streaming-source.consume-start-offset
diff --git
a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveDynamicParallelismInferenceFactory.java
b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveDynamicParallelismInferenceFactory.java
index 4932bbee8b4..36e655ec5fa 100644
---
a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveDynamicParallelismInferenceFactory.java
+++
b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveDynamicParallelismInferenceFactory.java
@@ -42,17 +42,11 @@ class HiveDynamicParallelismInferenceFactory implements
HiveParallelismInference
@Override
public HiveParallelismInference create() {
-boolean inferEnabled =
-jobConf.getBoolean(
-
HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM.key(),
-
HiveO
(flink) 01/05: [FLINK-36273][table-planner] Add the config in values source to let it can disable agg push down
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 987cf6f6f1eb7770e62cb23573ca43baa107d163
Author: xuyang
AuthorDate: Fri Sep 13 14:29:53 2024 +0800
[FLINK-36273][table-planner] Add the config in values source to let it can
disable agg push down
---
.../planner/factories/TestValuesTableFactory.java | 55 --
.../plan/batch/sql/ForwardHashExchangeTest.java| 16 +++
.../DynamicPartitionPruningProgramTest.java| 37 ---
.../PushLocalAggIntoTableSourceScanRuleTest.java | 32 ++---
.../sql/agg/LocalAggregatePushDownITCase.java | 32 ++---
.../program/DynamicPartitionPruningProgramTest.xml | 6 +--
.../PushLocalAggIntoTableSourceScanRuleTest.xml| 6 +--
7 files changed, 111 insertions(+), 73 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index 1054d52ed36..5c1b249a52a 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -395,6 +395,9 @@ public final class TestValuesTableFactory
private static final ConfigOption ENABLE_PROJECTION_PUSH_DOWN =
ConfigOptions.key("enable-projection-push-down").booleanType().defaultValue(true);
+private static final ConfigOption ENABLE_AGGREGATE_PUSH_DOWN =
+
ConfigOptions.key("enable-aggregate-push-down").booleanType().defaultValue(true);
+
private static final ConfigOption NESTED_PROJECTION_SUPPORTED =
ConfigOptions.key("nested-projection-supported").booleanType().defaultValue(false);
@@ -501,6 +504,7 @@ public final class TestValuesTableFactory
String lookupFunctionClass =
helper.getOptions().get(LOOKUP_FUNCTION_CLASS);
boolean disableLookup = helper.getOptions().get(DISABLE_LOOKUP);
boolean enableProjectionPushDown =
helper.getOptions().get(ENABLE_PROJECTION_PUSH_DOWN);
+boolean enableAggregatePushDown =
helper.getOptions().get(ENABLE_AGGREGATE_PUSH_DOWN);
boolean nestedProjectionSupported =
helper.getOptions().get(NESTED_PROJECTION_SUPPORTED);
boolean enableWatermarkPushDown =
helper.getOptions().get(ENABLE_WATERMARK_PUSH_DOWN);
boolean failingSource = helper.getOptions().get(FAILING_SOURCE);
@@ -585,7 +589,8 @@ public final class TestValuesTableFactory
partitions,
readableMetadata,
null,
-parallelism);
+parallelism,
+enableAggregatePushDown);
}
if (disableLookup) {
@@ -607,7 +612,8 @@ public final class TestValuesTableFactory
Long.MAX_VALUE,
partitions,
readableMetadata,
-null);
+null,
+enableAggregatePushDown);
} else {
return new TestValuesScanTableSource(
producedDataType,
@@ -626,7 +632,8 @@ public final class TestValuesTableFactory
Long.MAX_VALUE,
partitions,
readableMetadata,
-null);
+null,
+enableAggregatePushDown);
}
} else {
return new TestValuesScanLookupTableSource(
@@ -652,7 +659,8 @@ public final class TestValuesTableFactory
null,
cache,
reloadTrigger,
-lookupThreshold);
+lookupThreshold,
+enableAggregatePushDown);
}
} else {
try {
@@ -749,6 +757,7 @@ public final class TestValuesTableFactory
RUNTIME_SINK,
SINK_EXPECTED_MESSAGES_NUM,
ENABLE_PROJECTION_PUSH_DOWN,
+ENABLE_AGGREGATE_PUSH_DOWN,
NESTED_PROJECTION_SUPPORTED,
FILTERABLE_FIELDS,
DYNAMIC_FILTERING_FIELDS,
@@ -930,6 +939,7 @@ public final class TestValuesTableFactory
protected List> allPartitions;
protected final Map readableMetadata;
protected @Nullable int[] projectedMetadataFields;
+protected final boolean enableAggregatePushDown;
(flink) branch master updated (0578c8e17dd -> 733a56682bc)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 0578c8e17dd [FLINK-36361][table] Do not use StringBuilder for EquivalentExprShuttle new 987cf6f6f1e [FLINK-36273][table-planner] Add the config in values source to let it can disable agg push down new e60278d7684 [FLINK-36273][table] Remove deprecated Table/SQL configuration related FLINK-35473 in 2.0 new 2ca6c2f1f72 [FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0 in flink-sql-client module new 499df42b99d [FLINK-36273][table] Remove all deprecated Table/SQL configuration in 2.0 in flink-table-api module new 733a56682bc [FLINK-36273][table] Remove all deprecated configuration in 2.0 in flink-connector-hive module The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../docs/connectors/table/hive/hive_read_write.md | 2 +- docs/content.zh/docs/dev/table/sqlClient.md| 27 --- .../docs/connectors/table/hive/hive_read_write.md | 2 +- docs/content/docs/dev/table/sqlClient.md | 27 --- .../HiveDynamicParallelismInferenceFactory.java| 8 +- .../apache/flink/connectors/hive/HiveOptions.java | 17 +- .../connectors/hive/HiveParallelismInference.java | 1 - .../flink/connectors/hive/HiveSourceBuilder.java | 3 - .../HiveStaticParallelismInferenceFactory.java | 7 +- .../hive/HiveDynamicPartitionPruningITCase.java| 10 +- .../hive/HiveDynamicTableFactoryTest.java | 17 -- .../connectors/hive/HiveSinkCompactionITCase.java | 5 +- .../flink/connectors/hive/HiveTableSinkITCase.java | 4 +- .../connectors/hive/HiveTableSourceITCase.java | 27 ++- .../connectors/hive/PartitionMonitorTest.java | 2 +- flink-end-to-end-tests/flink-tpcds-test/pom.xml| 6 + .../apache/flink/table/tpcds/TpcdsTestProgram.java | 7 - .../test-scripts/test_pyflink.sh | 11 +- flink-end-to-end-tests/test-scripts/test_tpch.sh | 17 +- .../org/apache/flink/table/client/SqlClient.java | 18 +- .../apache/flink/table/client/cli/CliOptions.java | 27 +-- .../flink/table/client/cli/CliOptionsParser.java | 21 --- .../table/client/config/SqlClientOptions.java | 13 -- .../table/client/gateway/ResultDescriptor.java | 4 +- .../table/client/cli/CliTableauResultViewTest.java | 6 +- .../src/test/resources/cli/all-mode-help.out | 27 --- .../src/test/resources/cli/embedded-mode-help.out | 15 -- .../src/test/resources/cli/gateway-mode-help.out | 12 -- .../src/test/resources/sql/select.q| 16 +- .../src/test/resources/sql/select_batch.q | 8 +- .../src/test/resources/sql/table.q | 4 +- .../table/api/config/ExecutionConfigOptions.java | 57 -- .../table/api/config/OptimizerConfigOptions.java | 23 --- .../table/planner/hint/LookupJoinHintOptions.java | 203 - .../planner/plan/nodes/exec/ExecNodeConfig.java| 3 +- .../nodes/exec/stream/StreamExecIntervalJoin.java | 3 +- .../PushFilterIntoLegacyTableSourceScanRule.java | 12 -- .../logical/PushFilterIntoSourceScanRuleBase.java | 11 -- .../PushPartitionIntoTableSourceScanRule.java | 2 - .../plan/rules/logical/SubQueryDecorrelator.java | 25 +-- .../batch/PushLocalAggIntoScanRuleBase.java| 10 - .../batch/PushLocalHashAggIntoScanRule.java| 6 +- .../PushLocalHashAggWithCalcIntoScanRule.java | 6 +- .../batch/PushLocalSortAggIntoScanRule.java| 6 +- .../PushLocalSortAggWithCalcIntoScanRule.java | 6 +- ...ushLocalSortAggWithSortAndCalcIntoScanRule.java | 6 +- .../PushLocalSortAggWithSortIntoScanRule.java | 6 +- .../planner/utils/StreamExchangeModeUtils.java | 33 .../codegen/agg/batch/HashAggCodeGenerator.scala | 83 - .../plan/metadata/FlinkRelMdColumnNullCount.scala | 5 +- .../plan/metadata/FlinkRelMdDistinctRowCount.scala | 5 +- .../plan/metadata/FlinkRelMdDistribution.scala | 15 +- .../planner/plan/metadata/FlinkRelMdRowCount.scala | 23 +-- .../plan/metadata/SelectivityEstimator.scala | 5 +- .../plan/nodes/logical/FlinkLogicalSort.scala | 7 +- .../batch/BatchPhysicalHashAggregate.scala | 7 +- .../batch/BatchPhysicalOverAggregateBase.scala | 12 +- .../batch/BatchPhysicalPythonGroupAggregate.scala | 7 +- .../nodes/physical/batch/BatchPhysicalRank.scala | 7 +- .../batch/BatchPhysicalSortAggregate.scala | 7 +- .../table/planner/plan/optimize/RelNodeBlock.scala | 45 + .../rules/logical/JoinDeriveNullFilterRule.scala | 27 +-- ...ushPartitionIntoLegacyTableSource
(flink) branch master updated (7a06c3ba00e -> 74b9a480317)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 7a06c3ba00e [FLINK-36327][table] Remove dependency about flink-scala and flink-streaming-scala in table module add 74b9a480317 [FLINK-36358][table] Fix incorrect precision retention when converting timestamps to strings No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/flink/table/utils/DateTimeUtils.java | 4 ++-- .../apache/flink/table/planner/expressions/TemporalTypesTest.scala | 7 +++ 2 files changed, 9 insertions(+), 2 deletions(-)
(flink) branch master updated (c5ed471414c -> 7a06c3ba00e)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from c5ed471414c [FLINK-34511][Runtime/State] Remove deprecated checkpointing related APIs new ed39f0acdce [FLINK-36327][table-api-scala] Copy utils and tests from flink-scala new 7a06c3ba00e [FLINK-36327][table] Remove dependency about flink-scala and flink-streaming-scala in table module The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../e5126cae-f3fe-48aa-b6fb-60ae6cc3fcd5 | 1 - .../typeutils/runtime/TupleSerializerBase.java | 2 - .../typeutils/runtime/kryo/KryoSerializer.java | 46 +- .../formats/parquet/ParquetTimestampITCase.java| 50 +- .../chain/PythonOperatorChainingOptimizerTest.java | 2 +- flink-table/flink-table-api-scala-bridge/pom.xml | 11 - .../api/bridge/scala/DataStreamConversions.scala | 10 +- .../api/bridge/scala/StreamStatementSet.scala | 3 +- .../api/bridge/scala/StreamTableEnvironment.scala | 5 +- .../table/api/bridge/scala/TableConversions.scala | 4 +- .../internal/StreamTableEnvironmentImpl.scala | 47 +- .../flink/table/api/bridge/scala/package.scala | 2 +- .../internal/StreamTableEnvironmentImplTest.scala | 7 +- .../types/TypeInfoDataTypeConverterScalaTest.scala | 3 +- flink-table/flink-table-api-scala/pom.xml | 109 .../ScalaCaseClassSerializerSnapshot.java | 98 .../typeutils/ScalaEitherSerializerSnapshot.java | 68 +++ .../typeutils/ScalaOptionSerializerSnapshot.java | 59 ++ .../api/typeutils/ScalaTrySerializerSnapshot.java | 71 +++ .../typeutils/TraversableSerializerSnapshot.java | 110 .../Tuple2CaseClassSerializerSnapshot.java | 100 .../table/api/ImplicitExpressionConversions.scala | 5 +- .../flink/table/api/ImplicitTypeConversions.scala | 73 +++ .../apache/flink/table/api/codegen/Counter.scala} | 31 +- .../table/api/codegen/MacroContextHolder.scala}| 32 +- .../apache/flink/table/api/codegen/TreeGen.scala | 268 + .../flink/table/api/codegen/TypeAnalyzer.scala | 531 + .../flink/table/api/codegen/TypeDescriptors.scala | 176 ++ .../table/api/codegen/TypeInformationGen.scala | 472 .../scala/org/apache/flink/table/api/package.scala | 17 +- .../runtime/types/FlinkScalaKryoInstantiator.scala | 191 +++ .../types/JavaIterableWrapperSerializer.scala | 75 +++ .../table/api/typeutils/CaseClassComparator.scala | 166 ++ .../table/api/typeutils/CaseClassSerializer.scala | 134 + .../table/api/typeutils/CaseClassTypeInfo.scala| 288 ++ .../table/api/typeutils/EitherSerializer.scala | 133 + .../flink/table/api/typeutils/EitherTypeInfo.scala | 97 .../table/api/typeutils/EnumValueComparator.scala | 105 .../table/api/typeutils/EnumValueSerializer.scala | 72 +++ .../table/api/typeutils/EnumValueTypeInfo.scala| 90 +++ .../table/api/typeutils/NothingSerializer.scala| 79 +++ .../table/api/typeutils/OptionSerializer.scala | 100 .../table/api/typeutils/OptionTypeComparator.scala | 155 + .../flink/table/api/typeutils/OptionTypeInfo.scala | 104 .../api/typeutils/ScalaCaseClassSerializer.scala | 93 +++ .../typeutils/ScalaEnumSerializerSnapshot.scala| 115 .../table/api/typeutils/ScalaNothingTypeInfo.scala | 65 +++ .../api/typeutils/TraversableSerializer.scala | 238 .../table/api/typeutils/TraversableTypeInfo.scala | 82 +++ .../flink/table/api/typeutils/TrySerializer.scala | 117 .../flink/table/api/typeutils/TryTypeInfo.scala| 86 +++ .../api/typeutils/Tuple2CaseClassSerializer.scala} | 36 +- .../flink/table/api/typeutils/TypeUtils.scala} | 29 +- .../apache/flink/table/api/typeutils/Types.scala | 411 ++ .../flink/table/api/typeutils/UnitSerializer.scala | 79 +++ .../flink/table/api/typeutils/UnitTypeInfo.scala | 64 +++ .../misc/KryoSerializerRegistrationsTest.java | 87 +++ .../api/typeutils/OptionSerializerUpgradeTest.java | 109 .../ScalaEitherSerializerUpgradeTest.java | 108 .../typeutils/ScalaTrySerializerUpgradeTest.java | 133 + .../src/test/resources/flink_11-kryo_registrations | 0 .../src/test/resources/log4j2-test.properties | 0 .../api/typeutils/CaseClassTypeInfoTest.scala | 43 ++ .../table/api/typeutils/EitherSerializerTest.scala | 84 +++ .../table/api/typeutils/EitherTypeInfoTest.scala} | 41 +- .../EnumValueSerializerCompatibilityTest.scala | 193 +++ .../api/typeutils/EnumValueSerializerTest.scala} | 38 +- .../typeutils/EnumValueSeriali
(flink) branch master updated: [FLINK-36248][table] Introduce new Join Operator with Async State API
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5a63c56ca0b [FLINK-36248][table] Introduce new Join Operator with
Async State API
5a63c56ca0b is described below
commit 5a63c56ca0b38ebae9fc475708917c35a4168844
Author: Xuyang
AuthorDate: Tue Sep 24 18:47:31 2024 +0800
[FLINK-36248][table] Introduce new Join Operator with Async State API
This closes #25320
---
.../generated/execution_config_configuration.html | 6 +
.../util/AbstractStreamOperatorTestHarness.java| 11 -
.../table/api/config/ExecutionConfigOptions.java | 9 +
.../plan/nodes/exec/stream/StreamExecJoin.java | 47 +++-
.../StreamNonDeterministicUpdatePlanVisitor.java | 2 +-
.../flink/table/planner/plan/utils/JoinUtil.scala | 2 +-
.../planner/runtime/stream/sql/JoinITCase.scala| 29 +-
.../planner/runtime/stream/table/JoinITCase.scala | 29 +-
.../runtime/utils/StreamingWithStateTestBase.scala | 1 +
.../join/stream/AbstractStreamingJoinOperator.java | 139 +-
.../stream/MiniBatchStreamingJoinOperator.java | 2 +-
.../join/stream/StreamingJoinOperator.java | 240 +++-
.../join/stream/StreamingSemiAntiJoinOperator.java | 10 +-
.../AbstractAsyncStateStreamingJoinOperator.java | 98 +++
.../AsyncStateStreamingJoinOperator.java | 209 ++
.../state/JoinRecordAsyncStateView.java| 51
.../state/JoinRecordAsyncStateViews.java | 254 +
.../state/OuterJoinRecordAsyncStateView.java | 76 +
.../state/OuterJoinRecordAsyncStateViews.java | 307 +
.../operators/join/stream/bundle/BufferBundle.java | 2 +-
.../join/stream/state/JoinRecordStateView.java | 1 +
.../join/stream/state/JoinRecordStateViews.java| 1 +
.../stream/state/OuterJoinRecordStateViews.java| 1 +
.../join/stream/utils/AssociatedRecords.java | 176
.../JoinHelper.java} | 216 ++-
.../stream/{state => utils}/JoinInputSideSpec.java | 2 +-
.../operators/join/stream/utils/OuterRecord.java | 55
.../operators/join/stream/BufferBundleTest.java| 2 +-
.../join/stream/StreamingJoinOperatorTest.java | 92 --
.../join/stream/StreamingJoinOperatorTestBase.java | 6 +-
.../stream/StreamingMiniBatchJoinOperatorTest.java | 2 +-
31 files changed, 1541 insertions(+), 537 deletions(-)
diff --git
a/docs/layouts/shortcodes/generated/execution_config_configuration.html
b/docs/layouts/shortcodes/generated/execution_config_configuration.html
index 078b434fade..0eccee0eb40 100644
--- a/docs/layouts/shortcodes/generated/execution_config_configuration.html
+++ b/docs/layouts/shortcodes/generated/execution_config_configuration.html
@@ -56,6 +56,12 @@
Duration
The async timeout for the asynchronous operation to
complete.
+
+table.exec.async-state.enabled Streaming
+false
+Boolean
+Set whether to use the SQL/Table operators based on the
asynchronous state api. Default value is false.
+
table.exec.deduplicate.insert-update-after-sensitive-enabled
Streaming
true
diff --git
a/flink-runtime/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
b/flink-runtime/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
index d689f009842..f21cf61719f 100644
---
a/flink-runtime/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
+++
b/flink-runtime/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
@@ -50,9 +50,7 @@ import
org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider;
import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateHandle;
import org.apache.flink.runtime.state.StateBackend;
-import org.apache.flink.runtime.state.StateBackendTestUtils;
import org.apache.flink.runtime.state.TestTaskStateManager;
-import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.runtime.state.ttl.MockTtlTimeProvider;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
@@ -75,7 +73,6 @@ import
org.apache.flink.streaming.api.operators.StreamOperatorFactoryUtil;
import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer;
import org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl;
import org.apache.flink.streaming.api.watermark.Watermark;
-import
org.apache.flink.streaming.runtime.operators.asyncprocessing.AsyncStateProcessing;
imp
(flink) branch master updated: [FLINK-36284][format-avro] Support extracting Avro type with new DataType stack
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 1cd9110f858 [FLINK-36284][format-avro] Support extracting Avro type
with new DataType stack
1cd9110f858 is described below
commit 1cd9110f858a809ea640a23e872303be02015db9
Author: Xuyang
AuthorDate: Fri Sep 20 18:23:05 2024 +0800
[FLINK-36284][format-avro] Support extracting Avro type with new DataType
stack
This closes #25336
---
.../apache/flink/api/java/typeutils/AvroUtils.java | 12 +++--
.../flink/api/java/typeutils/TypeExtractor.java| 6 +--
.../flink/table/runtime/batch/AvroTypesITCase.java | 54 +-
.../table/types/extraction/DataTypeExtractor.java | 18
4 files changed, 49 insertions(+), 41 deletions(-)
diff --git
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/AvroUtils.java
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/AvroUtils.java
index c10fccf3266..23b3a4c83d3 100644
---
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/AvroUtils.java
+++
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/AvroUtils.java
@@ -39,6 +39,9 @@ public abstract class AvroUtils {
private static final String AVRO_KRYO_UTILS =
"org.apache.flink.formats.avro.utils.AvroKryoSerializerUtils";
+protected static final String AVRO_SPECIFIC_RECORD_BASE_CLASS =
+"org.apache.avro.specific.SpecificRecordBase";
+
/**
* Returns either the default {@link AvroUtils} which throw an exception
in cases where Avro
* would be needed or loads the specific utils for Avro from flink-avro.
@@ -77,6 +80,10 @@ public abstract class AvroUtils {
}
}
+public static boolean isAvroSpecificRecord(Class clazz) {
+return hasSuperclass(clazz, AVRO_SPECIFIC_RECORD_BASE_CLASS);
+}
+
//
/**
@@ -102,9 +109,6 @@ public abstract class AvroUtils {
/** A default implementation of the AvroUtils used in the absence of Avro.
*/
private static class DefaultAvroUtils extends AvroUtils {
-private static final String AVRO_SPECIFIC_RECORD_BASE =
-"org.apache.avro.specific.SpecificRecordBase";
-
private static final String AVRO_GENERIC_RECORD =
"org.apache.avro.generic.GenericData$Record";
@@ -113,7 +117,7 @@ public abstract class AvroUtils {
@Override
public void addAvroSerializersIfRequired(SerializerConfig reg,
Class type) {
-if (hasSuperclass(type, AVRO_SPECIFIC_RECORD_BASE)
+if (hasSuperclass(type, AVRO_SPECIFIC_RECORD_BASE_CLASS)
|| hasSuperclass(type, AVRO_GENERIC_RECORD)) {
throw new RuntimeException(
diff --git
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 5b4fce8b221..9bf0fd1bcfe 100644
---
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -75,7 +75,6 @@ import java.util.Map;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.checkAndExtractLambda;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.getAllDeclaredMethods;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.getTypeHierarchy;
-import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.hasSuperclass;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.sameTypeVars;
import static
org.apache.flink.api.java.typeutils.TypeExtractionUtils.typeToClass;
@@ -121,9 +120,6 @@ public class TypeExtractor {
private static final String HADOOP_WRITABLE_TYPEINFO_CLASS =
"org.apache.flink.api.java.typeutils.WritableTypeInfo";
-private static final String AVRO_SPECIFIC_RECORD_BASE_CLASS =
-"org.apache.avro.specific.SpecificRecordBase";
-
private static final Logger LOG =
LoggerFactory.getLogger(TypeExtractor.class);
private static final String GENERIC_TYPE_DOC_HINT =
@@ -1971,7 +1967,7 @@ public class TypeExtractor {
}
// special case for POJOs generated by Avro.
-if (hasSuperclass(clazz, AVRO_SPECIFIC_RECORD_BASE_CLASS)) {
+if (AvroUtils.isAvroSpecificRecord(clazz)) {
return AvroUtils.getAvroUtils().createAvroTypeInfo(clazz);
}
diff --git
a/flink-formats/flink-avro/src/test/java/org/apache/flink/table/runtime/batch/AvroTypesITCase.java
b/flink-for
(flink) branch master updated: [FLINK-36283][table-planner] Fix udaf incorrectly accessing the built-in function PERCENTILE's implementation with the same name
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new cfd532d5520 [FLINK-36283][table-planner] Fix udaf incorrectly
accessing the built-in function PERCENTILE's implementation with the same name
cfd532d5520 is described below
commit cfd532d552008fb10d158707da9e5538d5938a05
Author: dylanhz <53137516+dyla...@users.noreply.github.com>
AuthorDate: Wed Sep 18 11:11:36 2024 +0800
[FLINK-36283][table-planner] Fix udaf incorrectly accessing the built-in
function PERCENTILE's implementation with the same name
This closes #25335
---
.../planner/plan/utils/AggFunctionFactory.scala| 8 ++-
.../runtime/stream/sql/AggregateITCase.scala | 28 ++
.../utils/UserDefinedFunctionTestUtils.scala | 17 +
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggFunctionFactory.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggFunctionFactory.scala
index 5284aafd395..32ae59c513f 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggFunctionFactory.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggFunctionFactory.scala
@@ -62,10 +62,6 @@ class AggFunctionFactory(
/** The entry point to create an aggregate function from the given
[[AggregateCall]]. */
def createAggFunction(call: AggregateCall, index: Int): UserDefinedFunction
= {
-object BuiltInAggFunctionName {
- val PERCENTILE: String = BuiltInFunctionDefinitions.PERCENTILE.getName
-}
-
val argTypes: Array[LogicalType] = call.getArgList
.map(inputRowType.getChildren.get(_))
.toArray
@@ -170,9 +166,9 @@ class AggFunctionFactory(
udagg.makeFunction(constants.toArray, argTypes)
case bridge: BridgingSqlAggFunction =>
-bridge.getName match {
+bridge.getDefinition match {
// built-in imperativeFunction
- case BuiltInAggFunctionName.PERCENTILE =>
+ case BuiltInFunctionDefinitions.PERCENTILE =>
createPercentileAggFunction(argTypes)
// DeclarativeAggregateFunction & UDF
case _ =>
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala
index 9b95408eba3..647ca88f9ce 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala
@@ -2049,4 +2049,32 @@ class AggregateITCase(aggMode: AggMode, miniBatch:
MiniBatchMode, backend: State
}
}
}
+
+ @TestTemplate
+ def testAggFunctionPriority(): Unit = {
+// reported in FLINK-36283
+val sql =
+ """
+|SELECT
+| c,
+| PERCENTILE(b, 0.5) AS `swo`
+|FROM MyTable
+|GROUP BY c
+ """.stripMargin
+
+val t = failingDataSource(data).toTable(tEnv, 'a, 'b, 'c)
+tEnv.createTemporaryView("MyTable", t)
+
+// create a UDAF to cover built-in agg function with the same name
+tEnv.createTemporarySystemFunction("PERCENTILE", classOf[FakePercentile])
+
+val sink = new TestingRetractSink
+tEnv.sqlQuery(sql).toRetractStream[Row].addSink(sink).setParallelism(1)
+env.execute()
+
+val expected = List("Hello,21.0", "Hello World,35.0")
+assertThat(sink.getRetractResults.sorted).isEqualTo(expected.sorted)
+
+tEnv.dropTemporarySystemFunction("PERCENTILE")
+ }
}
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/UserDefinedFunctionTestUtils.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/UserDefinedFunctionTestUtils.scala
index bac0e9edfad..96e49926ff4 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/UserDefinedFunctionTestUtils.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/utils/UserDefinedFunctionTestUtils.scala
@@ -190,6 +190,23 @@ object UserDefinedFunctionTestUtils {
}
+ class FakePercentile extends AggregateFunction[Double, Tuple1[Double]] {
+def accumulate(acc: Tuple1[Double], value: Int, percentage: Double): Unit
= acc.f0 += value
+
+override def creat
(flink) branch master updated: [FLINK-36230][pyflink-table] Fix the incorrect calling of REGEXP_EXTRACT Python Table API
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 8c6f13cead9 [FLINK-36230][pyflink-table] Fix the incorrect calling of
REGEXP_EXTRACT Python Table API
8c6f13cead9 is described below
commit 8c6f13cead9e982f568c36135a350c45baf90d6c
Author: dylanhz <53137516+dyla...@users.noreply.github.com>
AuthorDate: Thu Sep 12 09:14:06 2024 +0800
[FLINK-36230][pyflink-table] Fix the incorrect calling of REGEXP_EXTRACT
Python Table API
This closes #25309
---
flink-python/pyflink/table/expression.py| 2 +-
flink-python/pyflink/table/tests/test_expression.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index ae1923556d9..d53f8bb12fe 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1286,7 +1286,7 @@ class Expression(Generic[T]):
group index.
"""
if extract_index is None:
-return _ternary_op("regexpExtract")(self, regex)
+return _binary_op("regexpExtract")(self, regex)
else:
return _ternary_op("regexpExtract")(self, regex, extract_index)
diff --git a/flink-python/pyflink/table/tests/test_expression.py
b/flink-python/pyflink/table/tests/test_expression.py
index 178570d3fa4..2ea4bbdffbd 100644
--- a/flink-python/pyflink/table/tests/test_expression.py
+++ b/flink-python/pyflink/table/tests/test_expression.py
@@ -185,6 +185,7 @@ class PyFlinkBatchExpressionTests(PyFlinkTestCase):
# regexp functions
self.assertEqual("regexp(a, b)", str(expr1.regexp(expr2)))
self.assertEqual("REGEXP_COUNT(a, b)", str(expr1.regexp_count(expr2)))
+self.assertEqual('regexpExtract(a, b)',
str(expr1.regexp_extract(expr2)))
self.assertEqual('regexpExtract(a, b, 3)',
str(expr1.regexp_extract(expr2, 3)))
self.assertEqual('REGEXP_EXTRACT_ALL(a, b)',
str(expr1.regexp_extract_all(expr2)))
self.assertEqual('REGEXP_EXTRACT_ALL(a, b, 3)',
str(expr1.regexp_extract_all(expr2, 3)))
(flink) branch master updated: [FLINK-35827][table-planner] Fix equivalence comparison between row type fields and constants
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 54b0d6a858c [FLINK-35827][table-planner] Fix equivalence comparison
between row type fields and constants
54b0d6a858c is described below
commit 54b0d6a858cd18e57fde60966a01cd673cb6da7a
Author: Xuyang
AuthorDate: Wed Sep 11 16:58:20 2024 +0800
[FLINK-35827][table-planner] Fix equivalence comparison between row type
fields and constants
This closes #25229
---
.../planner/codegen/EqualiserCodeGenerator.scala | 103 ++---
.../planner/codegen/calls/ScalarOperatorGens.scala | 26 +-
.../table/planner/plan/stream/sql/CalcTest.xml | 81 +---
.../table/planner/plan/stream/table/CalcTest.xml | 14 +++
.../table/planner/expressions/RowTypeTest.scala| 7 ++
.../table/planner/plan/stream/sql/CalcTest.scala | 17 +++-
.../table/planner/plan/stream/table/CalcTest.scala | 18
7 files changed, 202 insertions(+), 64 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala
index b1f2adc267e..4ae1a443336 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala
@@ -19,6 +19,7 @@ package org.apache.flink.table.planner.codegen
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.planner.codegen.CodeGenUtils._
+import
org.apache.flink.table.planner.codegen.EqualiserCodeGenerator.generateRecordEqualiserCode
import org.apache.flink.table.planner.codegen.Indenter.toISC
import
org.apache.flink.table.planner.codegen.calls.ScalarOperatorGens.generateEquals
import org.apache.flink.table.runtime.generated.{GeneratedRecordEqualiser,
RecordEqualiser}
@@ -30,14 +31,21 @@ import
org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldTyp
import scala.annotation.tailrec
import scala.collection.JavaConverters._
-class EqualiserCodeGenerator(fieldTypes: Array[LogicalType], classLoader:
ClassLoader) {
+class EqualiserCodeGenerator(
+leftFieldTypes: Array[LogicalType],
+rightFieldTypes: Array[LogicalType],
+classLoader: ClassLoader) {
private val RECORD_EQUALISER = className[RecordEqualiser]
private val LEFT_INPUT = "left"
private val RIGHT_INPUT = "right"
def this(rowType: RowType, classLoader: ClassLoader) = {
-this(rowType.getChildren.asScala.toArray, classLoader)
+this(rowType.getChildren.asScala.toArray,
rowType.getChildren.asScala.toArray, classLoader)
+ }
+
+ def this(fieldTypes: Array[LogicalType], classLoader: ClassLoader) = {
+this(fieldTypes, fieldTypes, classLoader)
}
def generateRecordEqualiser(name: String): GeneratedRecordEqualiser = {
@@ -45,8 +53,8 @@ class EqualiserCodeGenerator(fieldTypes: Array[LogicalType],
classLoader: ClassL
val ctx = new CodeGeneratorContext(new Configuration, classLoader)
val className = newName(ctx, name)
-val equalsMethodCodes = for (idx <- fieldTypes.indices) yield
generateEqualsMethod(ctx, idx)
-val equalsMethodCalls = for (idx <- fieldTypes.indices) yield {
+val equalsMethodCodes = for (idx <- leftFieldTypes.indices) yield
generateEqualsMethod(ctx, idx)
+val equalsMethodCalls = for (idx <- leftFieldTypes.indices) yield {
val methodName = getEqualsMethodName(idx)
s"""result = result && $methodName($LEFT_INPUT, $RIGHT_INPUT);"""
}
@@ -93,18 +101,28 @@ class EqualiserCodeGenerator(fieldTypes:
Array[LogicalType], classLoader: ClassL
("boolean", "isNullRight")
)
-val fieldType = fieldTypes(idx)
-val fieldTypeTerm = primitiveTypeTermForType(fieldType)
+val leftFieldType = leftFieldTypes(idx)
+val leftFieldTypeTerm = primitiveTypeTermForType(leftFieldType)
+val rightFieldType = rightFieldTypes(idx)
+val rightFieldTypeTerm = primitiveTypeTermForType(rightFieldType)
+
val Seq(leftFieldTerm, rightFieldTerm) = ctx.addReusableLocalVariables(
- (fieldTypeTerm, "leftField"),
- (fieldTypeTerm, "rightField")
+ (leftFieldTypeTerm, "leftField"),
+ (rightFieldTypeTerm, "rightField")
)
-val leftReadCode = rowFieldReadAccess(idx, LEFT_INPUT, fieldType)
-val rightReadCode = rowFieldReadAccess(idx, RIGHT_INPUT, fieldType)
+val leftReadCode = rowFieldReadAccess(idx, LEFT_INPUT, leftFieldType)
+val rightReadCode = rowFieldReadAccess
(flink) 04/04: [FLINK-36123][table] Add the built-in function PERCENTILE
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git commit c749a1b61c9c154f32cb877b96274804a950bfd0 Author: dylanhz AuthorDate: Wed Sep 4 18:08:44 2024 +0800 [FLINK-36123][table] Add the built-in function PERCENTILE --- docs/data/sql_functions.yml| 20 ++ docs/data/sql_functions_zh.yml | 20 ++ .../docs/reference/pyflink.table/expressions.rst | 1 + flink-python/pyflink/table/expression.py | 27 ++ .../pyflink/table/tests/test_expression.py | 6 + .../flink/table/api/internal/BaseExpressions.java | 40 +++ .../functions/BuiltInFunctionDefinitions.java | 23 ++ .../strategies/SpecificTypeStrategies.java | 13 + .../table/types/inference/TypeStrategiesTest.java | 11 +- .../planner/plan/utils/AggFunctionFactory.scala| 30 +- .../functions/PercentileAggFunctionITCase.java | 279 +++ .../runtime/batch/sql/OverAggregateITCase.scala| 30 ++ .../runtime/stream/sql/AggregateITCase.scala | 56 ++- .../runtime/stream/sql/OverAggregateITCase.scala | 62 .../runtime/stream/sql/WindowAggregateITCase.scala | 68 +++- .../runtime/stream/table/AggregateITCase.scala | 35 +- .../functions/aggregate/PercentileAggFunction.java | 389 + 17 files changed, 1104 insertions(+), 6 deletions(-) diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml index 32a092dbcf1..dad1dfac25f 100644 --- a/docs/data/sql_functions.yml +++ b/docs/data/sql_functions.yml @@ -251,6 +251,26 @@ arithmetic: - sql: TRUNCATE(numeric1, integer2) table: NUMERIC1.truncate(INTEGER2) description: Returns a numeric of truncated to integer2 decimal places. Returns NULL if numeric1 or integer2 is NULL. If integer2 is 0, the result has no decimal point or fractional part. integer2 can be negative to cause integer2 digits left of the decimal point of the value to become zero. This function can also pass in only one numeric1 parameter and not set integer2 to use. If integer2 is not set, the function truncates as if integer2 were 0. E.g. 42.324.truncate(2) to 42.32. and [...] + - sql: PERCENTILE(expr, percentage[, frequency]) +table: expr.percentile(percentage[, frequency]) +description: | + Returns the exact percentile value of expr at the specified percentage in a group. + + percentage must be a literal numeric value between `[0.0, 1.0]` or an array of such values. + If a variable expression is passed to this function, the result will be calculated using any one of them. + frequency describes how many times expr should be counted, the default value is 1. + + If no expr lies exactly at the desired percentile, the result is calculated using linear interpolation of the two nearest exprs. + If expr or frequency is `NULL`, or frequency is not positive, the input row will be ignored. + + NOTE: It is recommended to use this function in a window scenario, as it typically offers better performance. + In a regular group aggregation scenario, users should be aware of the performance overhead caused by a full sort triggered by each record. + + `value , percentage [ | NOT NULL>], frequency ` + `(INTEGER_NUMERIC: TINYINT, SMALLINT, INTEGER, BIGINT)` + `(NUMERIC: INTEGER_NUMERIC, FLOAT, DOUBLE, DECIMAL)` + + Returns a `DOUBLE` if percentage is numeric, or an `ARRAY` if percentage is an array. `NULL` if percentage is an empty array. string: - sql: string1 || string2 diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml index b269b12f156..d057df792aa 100644 --- a/docs/data/sql_functions_zh.yml +++ b/docs/data/sql_functions_zh.yml @@ -313,6 +313,26 @@ arithmetic: 如果 integer2 为 0，则结果没有小数点或小数部分。integer2 可以为负数，使值的小数点左边的 integer2 位变为零。 此函数也可以只传入一个参数 numeric1 且不设置参数 integer2 来使用。如果未设置 integer2 则 integer2 默认为 0。 例如 42.324.truncate(2) 为 42.32，42.324.truncate() 为 42.0。 + - sql: PERCENTILE(expr, percentage[, frequency]) +table: expr.percentile(percentage[, frequency]) +description: | + 返回 expr 的 percentage 百分位值。 + + percentage 必须是一个在 `[0.0, 1.0]` 之间的字面数值，或者是一个该范围的数组。 + 如果传递了一个可变表达式给这个函数，结果将根据其中的任意一个值进行计算。 + frequency 描述了 expr 的统计次数，默认值为 1。 + + 如果没有 expr 恰好位于指定的百分位上，则结果通过对两个最接近的 expr 进行线性插值来计算。 + 如果 expr 或 frequency 为 `NULL`，或者 frequency 不是正数，则将忽略该输入行。 + + 注意: 建议在窗口场景中使用此函数，因为它通常能提供更好的性能。 + 在常规的分组聚合场景中，用户应注意每条记录引发的全面排序带来的性能开销。 + + `value , percentage [ | NOT NULL>], frequency ` + `(INTEGER_NUMERIC: TINYINT, SMALLINT, INTEGER, BIGINT)` + `(NUMERIC: INTEGER_NUMERIC, FLOAT, DOUBLE, DECIMAL)` + + 如果 percentage 是数字，返回类型为 `DOUBLE`；如果 percentage 是数组，返回类型为 `ARRAY`。 如果 percentage 是空数组，则返回 `NULL`。 string: - sql: string1 || string2 diff --git a
(flink) 03/04: [FLINK-36123][table-planner] Support creating AggregateInfo through BuiltInFunctionDefinition directly to avoid an extra SqlAggFunction conversion
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit b723e0df79662a3f040eb08cacb88516d656e657
Author: dylanhz
AuthorDate: Wed Sep 4 09:50:04 2024 +0800
[FLINK-36123][table-planner] Support creating AggregateInfo through
BuiltInFunctionDefinition directly to avoid an extra SqlAggFunction conversion
---
.../flink/table/planner/plan/utils/AggregateUtil.scala | 14 +-
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggregateUtil.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggregateUtil.scala
index 66850f7f9d0..a6ffe0ec44b 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggregateUtil.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/AggregateUtil.scala
@@ -17,10 +17,7 @@
*/
package org.apache.flink.table.planner.plan.utils
-import org.apache.flink.configuration.ReadableConfig
import org.apache.flink.table.api.TableException
-import org.apache.flink.table.api.config.ExecutionConfigOptions
-import org.apache.flink.table.data.RowData
import org.apache.flink.table.expressions._
import org.apache.flink.table.expressions.ExpressionUtils.extractValue
import org.apache.flink.table.functions._
@@ -46,7 +43,6 @@ import
org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTypeFactory
import org.apache.flink.table.runtime.dataview.DataViewSpec
import
org.apache.flink.table.runtime.functions.aggregate.BuiltInAggregateFunction
import org.apache.flink.table.runtime.groupwindow._
-import
org.apache.flink.table.runtime.operators.bundle.trigger.CountBundleTrigger
import
org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromDataTypeToLogicalType
import org.apache.flink.table.types.DataType
import org.apache.flink.table.types.inference.TypeInferenceUtil
@@ -501,9 +497,17 @@ object AggregateUtil extends Enumeration {
hasStateBackedDataViews: Boolean,
needsRetraction: Boolean): AggregateInfo =
call.getAggregation match {
+ // In the new function stack, for imperativeFunction, the conversion from
+ // BuiltInFunctionDefinition to SqlAggFunction is unnecessary, we can
simply create
+ // AggregateInfo through BuiltInFunctionDefinition and runtime
implementation (obtained from
+ // AggFunctionFactory) directly.
+ // NOTE: make sure to use .runtimeProvided() in
BuiltInFunctionDefinition in this case.
case bridging: BridgingSqlAggFunction =>
// The FunctionDefinition maybe also instance of
DeclarativeAggregateFunction
-if (bridging.getDefinition.isInstanceOf[DeclarativeAggregateFunction])
{
+if (
+ bridging.getDefinition.isInstanceOf[BuiltInFunctionDefinition] ||
bridging.getDefinition
+.isInstanceOf[DeclarativeAggregateFunction]
+) {
createAggregateInfoFromInternalFunction(
call,
udf,
(flink) 02/04: [FLINK-36123][table-planner] Refactor error test for built-in agg functions
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4f01a10c624755632d7eb933bc6e5256e6801848
Author: dylanhz
AuthorDate: Mon Sep 2 18:16:41 2024 +0800
[FLINK-36123][table-planner] Refactor error test for built-in agg functions
---
.../BuiltInAggregateFunctionTestBase.java | 225 +++--
1 file changed, 160 insertions(+), 65 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/BuiltInAggregateFunctionTestBase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/BuiltInAggregateFunctionTestBase.java
index 58ec16dac50..b59e7a5a6df 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/BuiltInAggregateFunctionTestBase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/BuiltInAggregateFunctionTestBase.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableDescriptor;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.api.ValidationException;
import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.SourceFunctionProvider;
@@ -42,7 +43,7 @@ import org.apache.flink.test.junit5.MiniClusterExtension;
import org.apache.flink.types.Row;
import org.apache.flink.types.RowKind;
import org.apache.flink.util.CloseableIterator;
-import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.Preconditions;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -59,17 +60,21 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
+import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches;
import static
org.apache.flink.runtime.state.StateBackendLoader.HASHMAP_STATE_BACKEND_NAME;
import static
org.apache.flink.runtime.state.StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME;
import static org.apache.flink.table.test.TableAssertions.assertThat;
import static org.apache.flink.table.types.DataType.getFieldDataTypes;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
/** Test base for testing aggregate {@link BuiltInFunctionDefinition built-in
functions}. */
@Execution(ExecutionMode.CONCURRENT)
@@ -214,17 +219,6 @@ abstract class BuiltInAggregateFunctionTestBase {
return this;
}
-TestSpec testSqlRuntimeError(
-Function sqlSpec,
-DataType expectedRowType,
-Class exceptionClass,
-String exceptionMessage) {
-this.testItems.add(
-new SqlRuntimeErrorItem(
-sqlSpec, expectedRowType, exceptionClass,
exceptionMessage));
-return this;
-}
-
TestSpec testApiResult(
List selectExpr,
List groupByExpr,
@@ -275,6 +269,45 @@ abstract class BuiltInAggregateFunctionTestBase {
return this;
}
+TestSpec testSqlValidationError(Function sqlSpec,
String errorMessage) {
+this.testItems.add(
+new SqlErrorTestItem(
+sqlSpec, null, ValidationException.class,
errorMessage, true));
+return this;
+}
+
+TestSpec testTableApiValidationError(TableApiAggSpec tableApiSpec,
String errorMessage) {
+this.testItems.add(
+new TableApiErrorTestItem(
+tableApiSpec.selectExpr,
+tableApiSpec.groupByExpr,
+null,
+ValidationException.class,
+errorMessage,
+true));
+return this;
+}
+
+TestSpec testValidationError(
+Function sqlSpec,
+TableApiAggSpec tableApiSpec,
+String errorMessage) {
+testSqlValidationError(sqlSpec, errorMessage);
+testTableApiValidationError(tableApiSpec, errorMessage);
+return this;
+}
+
+TestSpec testSqlRuntimeError(
+Function sqlSpec,
+DataType expectedRowType
(flink) 01/04: [FLINK-36123][table-common] Add ArgumentTypeStrategy for percentage (array) value
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 6404214eeac97510556e4aaf880e36481936ee21
Author: dylanhz
AuthorDate: Wed Aug 28 11:39:33 2024 +0800
[FLINK-36123][table-common] Add ArgumentTypeStrategy for percentage (array)
value
---
.../strategies/PercentageArgumentTypeStrategy.java | 80
.../PercentageArrayArgumentTypeStrategy.java | 103 +
.../strategies/SpecificInputTypeStrategies.java| 13 +++
.../types/inference/InputTypeStrategiesTest.java | 102 +++-
4 files changed, 297 insertions(+), 1 deletion(-)
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArgumentTypeStrategy.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArgumentTypeStrategy.java
new file mode 100644
index 000..b04d4ef4fa5
--- /dev/null
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArgumentTypeStrategy.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.functions.FunctionDefinition;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.ArgumentTypeStrategy;
+import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.types.inference.Signature;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+
+import java.util.Optional;
+
+/** An {@link ArgumentTypeStrategy} that expects a percentage value between
[0.0, 1.0]. */
+@Internal
+public final class PercentageArgumentTypeStrategy implements
ArgumentTypeStrategy {
+
+private final boolean expectedNullability;
+
+public PercentageArgumentTypeStrategy(boolean nullable) {
+this.expectedNullability = nullable;
+}
+
+@Override
+public Optional inferArgumentType(
+CallContext callContext, int argumentPos, boolean throwOnFailure) {
+final LogicalType actualType =
+
callContext.getArgumentDataTypes().get(argumentPos).getLogicalType();
+
+if (!actualType.is(LogicalTypeFamily.NUMERIC)) {
+return callContext.fail(throwOnFailure, "Percentage must be of
NUMERIC type.");
+}
+if (!expectedNullability && actualType.isNullable()) {
+return callContext.fail(throwOnFailure, "Percentage must be of NOT
NULL type.");
+}
+
+if (callContext.isArgumentLiteral(argumentPos)) {
+Optional literalVal =
callContext.getArgumentValue(argumentPos, Number.class);
+
+Double val = null;
+if (literalVal.isPresent()) {
+val = literalVal.get().doubleValue();
+}
+
+if (val == null || val < 0.0 || val > 1.0) {
+return callContext.fail(
+throwOnFailure,
+"Percentage must be between [0.0, 1.0], but was '%s'.",
+val);
+}
+}
+
+return Optional.of(expectedNullability ? DataTypes.DOUBLE() :
DataTypes.DOUBLE().notNull());
+}
+
+@Override
+public Signature.Argument getExpectedArgument(
+FunctionDefinition functionDefinition, int argumentPos) {
+return Signature.Argument.of(expectedNullability ? "" :
"");
+}
+}
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArrayArgumentTypeStrategy.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArrayArgumentTypeStrategy.java
new file mode 100644
index 000..1d5560bfcc0
--- /dev/null
+++
b/
(flink) branch master updated (db682b9869d -> c749a1b61c9)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from db682b9869d [FLINK-36014] [runtime] Align the desired and sufficient resources definiton in Executing and WaitForResources states new 6404214eeac [FLINK-36123][table-common] Add ArgumentTypeStrategy for percentage (array) value new 4f01a10c624 [FLINK-36123][table-planner] Refactor error test for built-in agg functions new b723e0df796 [FLINK-36123][table-planner] Support creating AggregateInfo through BuiltInFunctionDefinition directly to avoid an extra SqlAggFunction conversion new c749a1b61c9 [FLINK-36123][table] Add the built-in function PERCENTILE The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/data/sql_functions.yml| 20 ++ docs/data/sql_functions_zh.yml | 20 ++ .../docs/reference/pyflink.table/expressions.rst | 1 + flink-python/pyflink/table/expression.py | 27 ++ .../pyflink/table/tests/test_expression.py | 6 + .../flink/table/api/internal/BaseExpressions.java | 40 +++ .../functions/BuiltInFunctionDefinitions.java | 23 ++ .../strategies/PercentageArgumentTypeStrategy.java | 80 + .../PercentageArrayArgumentTypeStrategy.java | 103 ++ .../strategies/SpecificInputTypeStrategies.java| 13 + .../strategies/SpecificTypeStrategies.java | 13 + .../types/inference/InputTypeStrategiesTest.java | 102 +- .../table/types/inference/TypeStrategiesTest.java | 11 +- .../planner/plan/utils/AggFunctionFactory.scala| 30 +- .../table/planner/plan/utils/AggregateUtil.scala | 14 +- .../BuiltInAggregateFunctionTestBase.java | 225 .../functions/PercentileAggFunctionITCase.java | 279 +++ .../runtime/batch/sql/OverAggregateITCase.scala| 30 ++ .../runtime/stream/sql/AggregateITCase.scala | 56 ++- .../runtime/stream/sql/OverAggregateITCase.scala | 62 .../runtime/stream/sql/WindowAggregateITCase.scala | 68 +++- .../runtime/stream/table/AggregateITCase.scala | 35 +- .../functions/aggregate/PercentileAggFunction.java | 389 + 23 files changed, 1570 insertions(+), 77 deletions(-) create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArgumentTypeStrategy.java create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/PercentageArrayArgumentTypeStrategy.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/PercentileAggFunctionITCase.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/aggregate/PercentileAggFunction.java
(flink) branch master updated: [FLINK-35988][table-runtime] Reduce the number of state lookups in AppendOnlyFirstNFunction
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new d9931c8af05 [FLINK-35988][table-runtime] Reduce the number of state
lookups in AppendOnlyFirstNFunction
d9931c8af05 is described below
commit d9931c8af05d0f1f721be9fe920690fe122507ad
Author: lexluo09 <39718951+lexlu...@users.noreply.github.com>
AuthorDate: Thu Aug 29 20:56:38 2024 +0800
[FLINK-35988][table-runtime] Reduce the number of state lookups in
AppendOnlyFirstNFunction
This closes #25159
---
.../runtime/operators/rank/AppendOnlyFirstNFunction.java | 15 +++
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/AppendOnlyFirstNFunction.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/AppendOnlyFirstNFunction.java
index e93c3ea66a5..a597a61d85a 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/AppendOnlyFirstNFunction.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/AppendOnlyFirstNFunction.java
@@ -31,6 +31,8 @@ import org.apache.flink.types.RowKind;
import org.apache.flink.util.Collector;
import org.apache.flink.util.Preconditions;
+import java.io.IOException;
+
/**
* A variant of {@link AppendOnlyTopNFunction} to handle first-n case.
*
@@ -79,14 +81,14 @@ public class AppendOnlyFirstNFunction extends
AbstractTopNFunction {
throws Exception {
initRankEnd(input);
-// check message should be insert only.
+// Ensure the message is an insert-only operation.
Preconditions.checkArgument(input.getRowKind() == RowKind.INSERT);
-int currentRank = state.value() == null ? 0 : state.value();
-// ignore record if it does not belong to the first-n rows
+int currentRank = getCurrentRank();
+// Ignore record if it does not belong to the first-n rows
if (currentRank >= rankEnd) {
return;
}
-currentRank += 1;
+currentRank++;
state.update(currentRank);
if (outputRankNumber || hasOffset()) {
@@ -95,4 +97,9 @@ public class AppendOnlyFirstNFunction extends
AbstractTopNFunction {
collectInsert(out, input);
}
}
+
+private int getCurrentRank() throws IOException {
+Integer value = state.value();
+return value == null ? 0 : value;
+}
}
(flink) branch master updated (ae4eb7dc3ae -> 5bbdbb28892)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from ae4eb7dc3ae [FLINK-36100][table] Support ESCAPE in built-in function
LIKE
new 4c14c763ca0 [FLINK-35964][table] Add the built-in function STARTSWITH
new 5bbdbb28892 [FLINK-35965][table] Add the built-in function ENDSWITH
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/data/sql_functions.yml| 24 ++
docs/data/sql_functions_zh.yml | 24 ++
.../docs/reference/pyflink.table/expressions.rst | 2 +
flink-python/pyflink/table/expression.py | 20 ++
.../pyflink/table/tests/test_expression.py | 2 +
.../flink/table/api/internal/BaseExpressions.java | 28 +++
.../functions/BuiltInFunctionDefinitions.java | 42
.../planner/functions/StringFunctionsITCase.java | 280 -
.../{BTrimFunction.java => EndsWithFunction.java} | 39 ++-
...{BTrimFunction.java => StartsWithFunction.java} | 38 ++-
10 files changed, 478 insertions(+), 21 deletions(-)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{BTrimFunction.java
=> EndsWithFunction.java} (53%)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{BTrimFunction.java
=> StartsWithFunction.java} (53%)
(flink) 01/02: [FLINK-35964][table] Add the built-in function STARTSWITH
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4c14c763ca0f9c71d642953228b364dce3eb3bf4
Author: dylanhz
AuthorDate: Tue Aug 6 10:33:01 2024 +0800
[FLINK-35964][table] Add the built-in function STARTSWITH
---
docs/data/sql_functions.yml| 12 ++
docs/data/sql_functions_zh.yml | 12 ++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 10 ++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 14 ++
.../functions/BuiltInFunctionDefinitions.java | 21 +++
.../planner/functions/StringFunctionsITCase.java | 143 -
.../functions/scalar/StartsWithFunction.java | 69 ++
9 files changed, 282 insertions(+), 1 deletion(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 43c37f58281..355f8c532d2 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -329,6 +329,18 @@ string:
STRING1.overlay(STRING2, INT1)
STRING1.overlay(STRING2, INT1, INT2)
description: Returns a string that replaces INT2 (STRING2's length by
default) characters of STRING1 with STRING2 from position INT1. E.g.,
'xtest'.overlay('', 6) returns "x"; 'xtest'.overlay('',
6, 2) returns "xst".
+ - sql: STARTSWITH(expr, startExpr)
+table: expr.startsWith(startExpr)
+description: |
+ Returns whether expr starts with startExpr. If startExpr is empty, the
result is true.
+
+ expr and startExpr should have same type.
+
+ `expr , startExpr `
+
+ `expr , startExpr `
+
+ Returns a `BOOLEAN`. `NULL` if any of the arguments are `NULL`.
- sql: SUBSTRING(string FROM integer1 [ FOR integer2 ])
table: |
STRING.substring(INT1)
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index e604c2ca7c9..2f199613c56 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -398,6 +398,18 @@ string:
将字符串 STRING1 从位置 INT1 开始替换 INT2（默认为 STRING2 的长度）个来自字符串 STRING2 的字符并返回。
例如 `'xtest'.overlay('', 6)` 返回 `"x"`；
`'xtest'.overlay('', 6, 2)` 返回 `"xst"`。
+ - sql: STARTSWITH(expr, startExpr)
+table: expr.startsWith(startExpr)
+description: |
+ 判断 expr 是否以 startExpr 开头。如果 startExpr 为空，则结果为 true。
+
+ expr 和 startExpr 应具有相同的类型。
+
+ `expr , startExpr `
+
+ `expr , startExpr `
+
+ 返回一个 `BOOLEAN`。如果任意参数为 `NULL`，则返回 `NULL`。
- sql: SUBSTRING(string FROM integer1 [ FOR integer2 ])
table: |
STRING.substring(INT1)
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index 3f494f18438..6f498eb1936 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -160,6 +160,7 @@ string functions
.. autosummary::
:toctree: api/
+Expression.starts_with
Expression.substring
Expression.substr
Expression.trim_leading
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index 1e125de501c..474c60a3041 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1028,6 +1028,16 @@ class Expression(Generic[T]):
# string functions
--
+def starts_with(self, start_expr) -> 'Expression':
+"""
+Returns whether expr starts with start_expr. If start_expr is empty,
the result is true.
+expr and start_expr should have same type.
+
+:param start_expr: A STRING or BINARY expression.
+:return: A BOOLEAN.
+"""
+return _binary_op("startsWith")(self, start_expr)
+
def substring(self,
begin_index: Union[int, 'Expression[int]'],
length: Union[int, 'Expression[int]'] = None) ->
'Expression[str]':
diff --git a/flink-python/pyflink/table/tests/test_expression.py
b/flink-python/pyflink/table/tests/test_expression.py
index 2337d33c62e..f4e60be1812 100644
--- a/flink-python/pyflink/table/tests/test_expression.py
+++ b/flink-python/pyflink/table/tests/test_expression.py
@@ -173,6 +173,7 @@ class PyFlinkBatchExpressionTests(PyFlinkTestCase):
self.assertEqual("ELT(1, a)", str(lit(1).elt(expr1)))
self.assertEqual('ELT(3, a, b, c)', str(li
(flink) 02/02: [FLINK-35965][table] Add the built-in function ENDSWITH
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5bbdbb2889268e879b6aa06d2cdab02fd6154cb6
Author: dylanhz
AuthorDate: Tue Aug 6 10:33:01 2024 +0800
[FLINK-35965][table] Add the built-in function ENDSWITH
---
docs/data/sql_functions.yml| 12 ++
docs/data/sql_functions_zh.yml | 12 ++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 10 ++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 14 +++
.../functions/BuiltInFunctionDefinitions.java | 21
.../planner/functions/StringFunctionsITCase.java | 137 +
.../runtime/functions/scalar/EndsWithFunction.java | 70 +++
9 files changed, 278 insertions(+)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 355f8c532d2..32a092dbcf1 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -340,6 +340,18 @@ string:
`expr , startExpr `
+ Returns a `BOOLEAN`. `NULL` if any of the arguments are `NULL`.
+ - sql: ENDSWITH(expr, endExpr)
+table: expr.endsWith(endExpr)
+description: |
+ Returns whether expr ends with endExpr. If endExpr is empty, the result
is true.
+
+ expr and endExpr should have same type.
+
+ `expr , endExpr `
+
+ `expr , endExpr `
+
Returns a `BOOLEAN`. `NULL` if any of the arguments are `NULL`.
- sql: SUBSTRING(string FROM integer1 [ FOR integer2 ])
table: |
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index 2f199613c56..b269b12f156 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -409,6 +409,18 @@ string:
`expr , startExpr `
+ 返回一个 `BOOLEAN`。如果任意参数为 `NULL`，则返回 `NULL`。
+ - sql: ENDSWITH(expr, endExpr)
+table: expr.endsWith(endExpr)
+description: |
+ 判断 expr 是否以 endExpr 结尾。如果 endExpr 为空，则结果为 true。
+
+ expr 和 endExpr 应具有相同的类型。
+
+ `expr , endExpr `
+
+ `expr , endExpr `
+
返回一个 `BOOLEAN`。如果任意参数为 `NULL`，则返回 `NULL`。
- sql: SUBSTRING(string FROM integer1 [ FOR integer2 ])
table: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index 6f498eb1936..e2761cd488c 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -161,6 +161,7 @@ string functions
:toctree: api/
Expression.starts_with
+Expression.ends_with
Expression.substring
Expression.substr
Expression.trim_leading
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index 474c60a3041..092fba6cd9c 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1038,6 +1038,16 @@ class Expression(Generic[T]):
"""
return _binary_op("startsWith")(self, start_expr)
+def ends_with(self, end_expr) -> 'Expression':
+"""
+Returns whether expr ends with end_expr. If end_expr is empty, the
result is true.
+expr and end_expr should have same type.
+
+:param end_expr: A STRING or BINARY expression.
+:return: A BOOLEAN.
+"""
+return _binary_op("endsWith")(self, end_expr)
+
def substring(self,
begin_index: Union[int, 'Expression[int]'],
length: Union[int, 'Expression[int]'] = None) ->
'Expression[str]':
diff --git a/flink-python/pyflink/table/tests/test_expression.py
b/flink-python/pyflink/table/tests/test_expression.py
index f4e60be1812..77cdfc457d1 100644
--- a/flink-python/pyflink/table/tests/test_expression.py
+++ b/flink-python/pyflink/table/tests/test_expression.py
@@ -174,6 +174,7 @@ class PyFlinkBatchExpressionTests(PyFlinkTestCase):
self.assertEqual('ELT(3, a, b, c)', str(lit(3).elt(expr1, expr2,
expr3)))
self.assertEqual("PRINTF('%d %s', a, b)", str(lit("%d
%s").printf(expr1, expr2)))
self.assertEqual("STARTSWITH(a, b)", str(expr1.starts_with(expr2)))
+self.assertEqual("ENDSWITH(a, b)", str(expr1.ends_with(expr2)))
# regexp functions
self.assertEqual("regexp(a, b)", str(expr1.regexp(expr2)))
diff --git
a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
index 4df16
(flink) branch master updated (9bcd8f4b8f4 -> ae4eb7dc3ae)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 9bcd8f4b8f4 [FLINK-36140] Log a warning when pods are terminated by kubernetes add ae4eb7dc3ae [FLINK-36100][table] Support ESCAPE in built-in function LIKE No new revisions were added by this update. Summary of changes: docs/data/sql_functions.yml| 6 +- docs/data/sql_functions_zh.yml | 6 +- flink-python/pyflink/table/expression.py | 15 ++- .../pyflink/table/tests/test_expression.py | 3 +- .../flink/table/api/internal/BaseExpressions.java | 15 ++- .../apache/flink/table/functions/SqlLikeUtils.java | 16 ++- .../functions/BuiltInFunctionDefinitions.java | 15 ++- .../flink/table/functions/SqlCallSyntax.java | 23 .../table/planner/codegen/calls/LikeCallGen.scala | 10 +- .../planner/calcite/FlinkSqlLikeUtilsTest.java | 30 + .../planner/expressions/ScalarFunctionsTest.scala | 150 + 11 files changed, 238 insertions(+), 51 deletions(-)
(flink) 05/05: [FLINK-35963][table] Add the built-in function REGEXP_SUBSTR
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e8c1d1b9d411e17130a57851c0bbaecb2e816048
Author: dylanhz
AuthorDate: Fri Aug 16 14:19:57 2024 +0800
[FLINK-35963][table] Add the built-in function REGEXP_SUBSTR
---
docs/data/sql_functions.yml| 8 +++
docs/data/sql_functions_zh.yml | 8 +++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 10 +++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 14
.../functions/BuiltInFunctionDefinitions.java | 15 +
.../planner/functions/RegexpFunctionsITCase.java | 77 +-
.../table/runtime/functions/SqlFunctionUtils.java | 21 +-
.../functions/scalar/RegexpCountFunction.java | 14 +---
.../functions/scalar/RegexpExtractAllFunction.java | 16 ++---
.../functions/scalar/RegexpInstrFunction.java | 14 +---
...nstrFunction.java => RegexpSubstrFunction.java} | 30 -
13 files changed, 176 insertions(+), 53 deletions(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index e6f768ca4c4..08f0b860238 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -376,6 +376,14 @@ string:
`str , regex `
Returns an `INTEGER` representation of the first matched substring
index. `NULL` if any of the arguments are `NULL` or regex is invalid.
+ - sql: REGEXP_SUBSTR(str, regex)
+table: str.regexpSubstr(regex)
+description: |
+ Returns the first substring in str that matches regex.
+
+ `str , regex `
+
+ Returns an `STRING` representation of the first matched substring.
`NULL` if any of the arguments are `NULL` or regex if invalid or pattern is not
found.
- sql: TRANSLATE(expr, fromStr, toStr)
table: expr.translate(fromStr, toStr)
description: |
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index e50040608c8..c7c332ed007 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -443,6 +443,14 @@ string:
`str , regex `
返回一个 `INTEGER` 表示 str 中第一个匹配 regex 的子字符串索引。如果任何参数为 `NULL` 或 regex 非法，则返回
`NULL`。
+ - sql: REGEXP_SUBSTR(str, regex)
+table: str.regexpSubStr(regex)
+description: |
+ 返回 str 中第一个匹配 regex 的子字符串。
+
+ `str , regex `
+
+ 返回一个 `STRING` 表示 str 中第一个匹配 regex 的子字符串。如果任何参数为 `NULL` 或 regex
非法或匹配失败，则返回 `NULL`。
- sql: TRANSLATE(expr, fromStr, toStr)
table: expr.translate(fromStr, toStr)
description: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index 23796fbaca3..3f494f18438 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -183,6 +183,7 @@ string functions
Expression.regexp_extract
Expression.regexp_extract_all
Expression.regexp_instr
+Expression.regexp_substr
Expression.from_base64
Expression.to_base64
Expression.ascii
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index 6eabd528e02..54806af9498 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1265,6 +1265,16 @@ class Expression(Generic[T]):
"""
return _binary_op("regexpInstr")(self, regex)
+def regexp_substr(self, regex) -> 'Expression':
+"""
+Returns the first substring in str that matches regex.
+null if any of the arguments are null or regex is invalid or pattern
is not found.
+
+:param regex: A STRING expression with a matching pattern.
+:return: A STRING representation of the first matched substring.
+"""
+return _binary_op("regexpSubstr")(self, regex)
+
@property
def from_base64(self) -> 'Expression[str]':
"""
diff --git a/flink-python/pyflink/table/tests/test_expression.py
b/flink-python/pyflink/table/tests/test_expression.py
index 3aa101b685e..9ae5e22dc5e 100644
--- a/flink-python/pyflink/table/tests/test_expression.py
+++ b/flink-python/pyflink/table/tests/test_expression.py
@@ -181,6 +181,7 @@ class PyFlinkBatchExpressionTests(PyFlinkTestCase):
self.assertEqual('REGEXP_EXTRACT_ALL(a, b, 3)',
str(expr1.regexp_extract_all(expr2, 3)))
self.assertEqual("regexpReplace(a, b, 'abc')",
str(expr1.regexp_replace(expr2, 'abc')))
self.assertEqual("REGEXP_INSTR(a, b)", str(expr1.regexp_instr(expr2)))
+self.assertEqual("REGEXP_SUBST
(flink) 01/05: [FLINK-35931][table-planner] Use a separate test class for the regexp family of functions
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 697b4b26010c6e7014bc48f8b1bf1ad5ee1ca55e
Author: dylanhz
AuthorDate: Fri Aug 16 10:38:09 2024 +0800
[FLINK-35931][table-planner] Use a separate test class for the regexp
family of functions
---
.../planner/functions/RegexpFunctionsITCase.java | 53 ++
.../planner/functions/StringFunctionsITCase.java | 25 +-
2 files changed, 54 insertions(+), 24 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/RegexpFunctionsITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/RegexpFunctionsITCase.java
new file mode 100644
index 000..ab66e6e4030
--- /dev/null
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/RegexpFunctionsITCase.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.functions;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
+
+import java.util.stream.Stream;
+
+import static org.apache.flink.table.api.Expressions.$;
+import static org.apache.flink.table.api.Expressions.call;
+
+/** Test regex functions correct behaviour. */
+class RegexpFunctionsITCase extends BuiltInFunctionTestBase {
+
+@Override
+Stream getTestSetSpecs() {
+return Stream.of(regexpExtractTestCases()).flatMap(s -> s);
+}
+
+private Stream regexpExtractTestCases() {
+return Stream.of(
+TestSetSpec.forFunction(
+BuiltInFunctionDefinitions.REGEXP_EXTRACT,
"Check return type")
+.onFieldsWithData("22", "ABC")
+.testResult(
+call("regexpExtract", $("f0"), "[A-Z]+"),
+"REGEXP_EXTRACT(f0,'[A-Z]+')",
+null,
+DataTypes.STRING().nullable())
+.testResult(
+call("regexpExtract", $("f1"), "[A-Z]+"),
+"REGEXP_EXTRACT(f1, '[A-Z]+')",
+"ABC",
+DataTypes.STRING().nullable()));
+}
+}
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/StringFunctionsITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/StringFunctionsITCase.java
index d1baff469e4..08a390b0897 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/StringFunctionsITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/StringFunctionsITCase.java
@@ -31,7 +31,6 @@ import java.time.temporal.ChronoUnit;
import java.util.stream.Stream;
import static org.apache.flink.table.api.Expressions.$;
-import static org.apache.flink.table.api.Expressions.call;
import static org.apache.flink.table.api.Expressions.lit;
/** Test String functions correct behaviour. */
@@ -39,12 +38,7 @@ class StringFunctionsITCase extends BuiltInFunctionTestBase {
@Override
Stream getTestSetSpecs() {
-return Stream.of(
-bTrimTestCases(),
-eltTestCases(),
-printfTestCases(),
-regexpExtractTestCases(),
-translateTestCases())
+return Stream.of(bTrimTestCases(), eltTestCases(), printfTestCases(),
translateTestCases())
.flatMap(s -> s);
}
@@ -316,23 +310,6 @@ class StringFunctionsITCase extends
BuiltInFunctionTestBase {
+ "PRINTF(format ,
obj ...)"));
}
-private S
(flink) branch master updated (9399e669e5a -> e8c1d1b9d41)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 9399e669e5a [hotfix][API/DataStream] Log the right syncDurationMillis
time
new 697b4b26010 [FLINK-35931][table-planner] Use a separate test class for
the regexp family of functions
new 7fa4ebad057 [FLINK-35931][table] Add the built-in function
REGEXP_EXTRACT_ALL
new 8189bba1953 [FLINK-35932][table] Add the built-in function REGEXP_COUNT
new cbbb1cd178a [FLINK-35962][table] Add the built-in function REGEXP_INSTR
new e8c1d1b9d41 [FLINK-35963][table] Add the built-in function
REGEXP_SUBSTR
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/data/sql_functions.yml| 49 ++-
docs/data/sql_functions_zh.yml | 50 ++-
.../docs/reference/pyflink.table/expressions.rst | 4 +
flink-python/pyflink/table/expression.py | 50 +++
.../pyflink/table/tests/test_expression.py | 13 +-
.../flink/table/api/internal/BaseExpressions.java | 70
.../functions/BuiltInFunctionDefinitions.java | 68
.../planner/functions/RegexpFunctionsITCase.java | 390 +
.../planner/functions/StringFunctionsITCase.java | 25 +-
.../table/runtime/functions/SqlFunctionUtils.java | 19 +
...ncodeFunction.java => RegexpCountFunction.java} | 30 +-
...Function.java => RegexpExtractAllFunction.java} | 58 +--
...UnhexFunction.java => RegexpInstrFunction.java} | 20 +-
...TrimFunction.java => RegexpSubstrFunction.java} | 30 +-
14 files changed, 769 insertions(+), 107 deletions(-)
create mode 100644
flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/RegexpFunctionsITCase.java
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{UrlEncodeFunction.java
=> RegexpCountFunction.java} (61%)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{SplitFunction.java
=> RegexpExtractAllFunction.java} (50%)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{UnhexFunction.java
=> RegexpInstrFunction.java} (65%)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{BTrimFunction.java
=> RegexpSubstrFunction.java} (63%)
(flink) 02/05: [FLINK-35931][table] Add the built-in function REGEXP_EXTRACT_ALL
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7fa4ebad057bca765b661a553f4ccc2df7861a05
Author: dylanhz
AuthorDate: Fri Aug 16 11:37:31 2024 +0800
[FLINK-35931][table] Add the built-in function REGEXP_EXTRACT_ALL
---
docs/data/sql_functions.yml| 11 +-
docs/data/sql_functions_zh.yml | 10 ++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 18
.../pyflink/table/tests/test_expression.py | 10 +-
.../flink/table/api/internal/BaseExpressions.java | 28 ++
.../functions/BuiltInFunctionDefinitions.java | 23 +
.../planner/functions/RegexpFunctionsITCase.java | 111 -
.../table/runtime/functions/SqlFunctionUtils.java | 2 +-
.../functions/scalar/RegexpExtractAllFunction.java | 75 ++
10 files changed, 282 insertions(+), 7 deletions(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 9d6d8b2acef..70823fc582c 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -358,7 +358,16 @@ string:
not exceed the number of the defined groups.
E.g. REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2)" returns "bar".
-
+ - sql: REGEXP_EXTRACT_ALL(str, regex[, extractIndex])
+table: str.regexpExtractAll(regex[, extractIndex])
+description: |
+ Extracts all the substrings in str that match the regex expression and
correspond to the regex group extractIndex.
+
+ regex may contain multiple groups. extractIndex indicates which regex
group to extract and starts from 1, also the default value if not specified. 0
means matching the entire regular expression.
+
+ `str , regex , extractIndex `
+
+ Returns an `ARRAY` representation of all the matched substrings.
`NULL` if any of the arguments are `NULL` or invalid.
- sql: INITCAP(string)
table: STRING.initCap()
description: Returns a new form of STRING with the first character of each
word converted to uppercase and the rest characters to lowercase. Here a word
means a sequences of alphanumeric characters.
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index e26c3b44267..f2629ca1096 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -425,6 +425,16 @@ string:
将字符串 STRING1 按照 STRING2 正则表达式的规则拆分，返回指定 INTEGER1 处位置的字符串。正则表达式匹配组索引从 1
开始，
0 表示匹配整个正则表达式。此外，正则表达式匹配组索引不应超过定义的组数。
例如 `REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2)` 返回 `"bar"`。
+ - sql: REGEXP_EXTRACT_ALL(str, regex[, extractIndex])
+table: str.regexpExtractAll(regex[, extractIndex])
+description: |
+ 提取字符串 str 中与正则表达式 regex 匹配且与 extractIndex 组对应的所有子串。
+
+ regex 可以包含多个组。extractIndex 用于指示要提取哪个正则组，索引从 1 开始，也作为未指定时的默认值。0
表示匹配整个正则表达式。
+
+ `str , regex , extractIndex `
+
+ 返回一个 `ARRAY`，表示所有匹配的子串。如果任何参数为 `NULL`或非法，则返回 `NULL`。
- sql: INITCAP(string)
table: STRING.initCap()
description: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index 050feb15b30..60a5631e854 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -180,6 +180,7 @@ string functions
Expression.regexp
Expression.regexp_replace
Expression.regexp_extract
+Expression.regexp_extract_all
Expression.from_base64
Expression.to_base64
Expression.ascii
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index d69bcd60277..1e924c508e1 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1225,6 +1225,24 @@ class Expression(Generic[T]):
else:
return _ternary_op("regexpExtract")(self, regex, extract_index)
+def regexp_extract_all(self, regex, extract_index=None) -> 'Expression':
+"""
+Extracts all the substrings in str that match the regex expression and
correspond to the
+regex group extract_index.
+regex may contain multiple groups. extract_index indicates which regex
group to extract and
+starts from 1, also the default value if not specified. 0 means
matching the entire
+regular expression.
+null if any of the arguments are null or invalid.
+
+:param regex: A STRING expression with a matching pattern.
+:param extract_index: An optional INTEGER expression with default 1.
+:return: An ARRAY of all the matched substrings.
+"""
+if extract_index is None:
+
(flink) 03/05: [FLINK-35932][table] Add the built-in function REGEXP_COUNT
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 8189bba195326d899a9830d3a19b50cafa092666
Author: dylanhz
AuthorDate: Thu Aug 1 17:20:52 2024 +0800
[FLINK-35932][table] Add the built-in function REGEXP_COUNT
---
docs/data/sql_functions.yml| 22 --
docs/data/sql_functions_zh.yml | 22 --
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 11 +++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 14
.../functions/BuiltInFunctionDefinitions.java | 15
.../planner/functions/RegexpFunctionsITCase.java | 83 +-
.../functions/scalar/RegexpCountFunction.java | 60
9 files changed, 214 insertions(+), 15 deletions(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 70823fc582c..29b0c27ebcd 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -337,16 +337,14 @@ string:
- sql: REPLACE(string1, string2, string3)
table: STRING1.replace(STRING2, STRING3)
description: Returns a new string which replaces all the occurrences of
STRING2 with STRING3 (non-overlapping) from STRING1. E.g., 'hello
world'.replace('world', 'flink') returns 'hello flink';
'ababab'.replace('abab', 'z') returns 'zab'.
- - sql: TRANSLATE(expr, fromStr, toStr)
-table: expr.translate(fromStr, toStr)
+ - sql: REGEXP_COUNT(str, regex)
+table: str.regexpCount(regex)
description: |
- Translate an expr where all characters in fromStr have been replaced
with those in toStr.
+ Returns the number of times str matches the regex pattern. regex must be
a Java regular expression.
- If toStr has a shorter length than fromStr, unmatched characters are
removed.
+ `str , regex `
- expr , fromStr , toStr
-
- Returns a STRING of translated expr.
+ Returns an `INTEGER` representation of the number of matches. `NULL` if
any of the arguments are `NULL` or regex is invalid.
- sql: REGEXP_EXTRACT(string1, string2[, integer])
table: STRING1.regexpExtract(STRING2[, INTEGER1])
description: |
@@ -368,6 +366,16 @@ string:
`str , regex , extractIndex `
Returns an `ARRAY` representation of all the matched substrings.
`NULL` if any of the arguments are `NULL` or invalid.
+ - sql: TRANSLATE(expr, fromStr, toStr)
+table: expr.translate(fromStr, toStr)
+description: |
+ Translate an expr where all characters in fromStr have been replaced
with those in toStr.
+
+ If toStr has a shorter length than fromStr, unmatched characters are
removed.
+
+ `expr , fromStr , toStr `
+
+ Returns a `STRING` of translated expr.
- sql: INITCAP(string)
table: STRING.initCap()
description: Returns a new form of STRING with the first character of each
word converted to uppercase and the rest characters to lowercase. Here a word
means a sequences of alphanumeric characters.
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index f2629ca1096..76607687f17 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -409,16 +409,14 @@ string:
返回一个新字符串，它用 STRING3（非重叠）替换 STRING1 中所有出现的 STRING2。
例如 `'hello world'.replace('world', 'flink')` 返回 `'hello flink'`；
`'ababab'.replace('abab', 'z')` 返回 `'zab'`。
- - sql: TRANSLATE(expr, fromStr, toStr)
-table: expr.translate(fromStr, toStr)
+ - sql: REGEXP_COUNT(str, regex)
+table: str.regexpCount(regex)
description: |
- 将 expr 中所有出现在 fromStr 之中的字符替换为 toStr 中的相应字符。
+ 返回字符串 str 匹配正则表达式模式 regex 的次数。regex 必须是一个 Java 正则表达式。
- 如果 toStr 的长度短于 fromStr，则未匹配的字符将被移除。
+ `str , regex `
- expr , fromStr , toStr
-
- 返回 STRING 格式的转换结果。
+ 返回一个 `INTEGER` 表示匹配成功的次数。如果任何参数为 `NULL` 或 regex 非法，则返回 `NULL`。
- sql: REGEXP_EXTRACT(string1, string2[, integer])
table: STRING1.regexpExtract(STRING2[, INTEGER1])
description: |
@@ -435,6 +433,16 @@ string:
`str , regex , extractIndex `
返回一个 `ARRAY`，表示所有匹配的子串。如果任何参数为 `NULL`或非法，则返回 `NULL`。
+ - sql: TRANSLATE(expr, fromStr, toStr)
+table: expr.translate(fromStr, toStr)
+description: |
+ 将 expr 中所有出现在 fromStr 之中的字符替换为 toStr 中的相应字符。
+
+ 如果 toStr 的长度短于 fromStr，则未匹配的字符将被移除。
+
+ `expr , fromStr , toStr `
+
+ 返回 `STRING` 格式的转换结果。
- sql: INITCAP(string)
table: STRING.initCap()
description: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyfli
(flink) 04/05: [FLINK-35962][table] Add the built-in function REGEXP_INSTR
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit cbbb1cd178a5b53200ee793f77bac73a0d49ff14
Author: dylanhz
AuthorDate: Fri Aug 16 14:02:40 2024 +0800
[FLINK-35962][table] Add the built-in function REGEXP_INSTR
---
docs/data/sql_functions.yml| 10 +++
docs/data/sql_functions_zh.yml | 10 +++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 11
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 14
.../functions/BuiltInFunctionDefinitions.java | 15 +
.../planner/functions/RegexpFunctionsITCase.java | 76 +-
.../functions/scalar/RegexpInstrFunction.java | 55
9 files changed, 192 insertions(+), 1 deletion(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 29b0c27ebcd..e6f768ca4c4 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -366,6 +366,16 @@ string:
`str , regex , extractIndex `
Returns an `ARRAY` representation of all the matched substrings.
`NULL` if any of the arguments are `NULL` or invalid.
+ - sql: REGEXP_INSTR(str, regex)
+table: str.regexpInstr(regex)
+description: |
+ Returns the position of the first substring in str that matches regex.
+
+ Result indexes begin at 1, 0 if there is no match.
+
+ `str , regex `
+
+ Returns an `INTEGER` representation of the first matched substring
index. `NULL` if any of the arguments are `NULL` or regex is invalid.
- sql: TRANSLATE(expr, fromStr, toStr)
table: expr.translate(fromStr, toStr)
description: |
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index 76607687f17..e50040608c8 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -433,6 +433,16 @@ string:
`str , regex , extractIndex `
返回一个 `ARRAY`，表示所有匹配的子串。如果任何参数为 `NULL`或非法，则返回 `NULL`。
+ - sql: REGEXP_INSTR(str, regex)
+table: str.regexpInstr(regex)
+description: |
+ 返回 str 中第一个匹配 regex 的子字符串的索引。
+
+ 结果索引从 1 开始，如果匹配失败则返回 0。
+
+ `str , regex `
+
+ 返回一个 `INTEGER` 表示 str 中第一个匹配 regex 的子字符串索引。如果任何参数为 `NULL` 或 regex 非法，则返回
`NULL`。
- sql: TRANSLATE(expr, fromStr, toStr)
table: expr.translate(fromStr, toStr)
description: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index 8698e884e1a..23796fbaca3 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -182,6 +182,7 @@ string functions
Expression.regexp_replace
Expression.regexp_extract
Expression.regexp_extract_all
+Expression.regexp_instr
Expression.from_base64
Expression.to_base64
Expression.ascii
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index 5975a79502b..6eabd528e02 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1254,6 +1254,17 @@ class Expression(Generic[T]):
else:
return _ternary_op("regexpExtractAll")(self, regex, extract_index)
+def regexp_instr(self, regex) -> 'Expression':
+"""
+Returns the position of the first substring in str that matches regex.
+Result indexes begin at 1, 0 if there is no match.
+null if any of the arguments are null or regex is invalid.
+
+:param regex: A STRING expression with a matching pattern.
+:return: An INTEGER representation of the first matched substring
index.
+"""
+return _binary_op("regexpInstr")(self, regex)
+
@property
def from_base64(self) -> 'Expression[str]':
"""
diff --git a/flink-python/pyflink/table/tests/test_expression.py
b/flink-python/pyflink/table/tests/test_expression.py
index fb94a02ec7d..3aa101b685e 100644
--- a/flink-python/pyflink/table/tests/test_expression.py
+++ b/flink-python/pyflink/table/tests/test_expression.py
@@ -180,6 +180,7 @@ class PyFlinkBatchExpressionTests(PyFlinkTestCase):
self.assertEqual('REGEXP_EXTRACT_ALL(a, b)',
str(expr1.regexp_extract_all(expr2)))
self.assertEqual('REGEXP_EXTRACT_ALL(a, b, 3)',
str(expr1.regexp_extract_all(expr2, 3)))
self.assertEqual("regexpReplace(a, b, 'abc')",
str(expr1.regexp_replace(expr2, 'abc')))
+self.assertEqual("REGEXP_INSTR(a, b)", str(expr1.regexp_instr(expr2)))
# temporal functions
(flink) 02/02: [FLINK-36095][table-planner] KeyedLookupJoinWrapper should shuffle by input upsertKey instead of join key to avoid changelog disordering
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 624bc5001c831d3d04a5680c03910c634a0c988b
Author: lincoln lee
AuthorDate: Mon Aug 19 20:05:03 2024 +0800
[FLINK-36095][table-planner] KeyedLookupJoinWrapper should shuffle by input
upsertKey instead of join key to avoid changelog disordering
---
.../nodes/exec/stream/StreamExecLookupJoin.java| 58 ++---
.../table/planner/plan/utils/UpsertKeyUtil.java| 15 +-
.../physical/stream/StreamPhysicalLookupJoin.scala | 23 +-
.../nodes/exec/stream/LookupJoinRestoreTest.java | 3 +-
.../nodes/exec/stream/LookupJoinTestPrograms.java | 71 ++
...ggAndAllConstantLookupKeyWithTryResolveMode.out | 4 +-
...nstantLookupKeyWithTryResolveMode_newSource.out | 4 +-
.../analyze/NonDeterministicUpdateAnalyzerTest.xml | 6 +-
.../plan/stream/sql/NonDeterministicDagTest.xml| 42 ++--
.../plan/stream/sql/join/LookupJoinTest.xml| 4 +-
.../plan/lookup-join-with-try-resolve.json | 266 +
.../savepoint/_metadata| Bin 0 -> 13669 bytes
12 files changed, 426 insertions(+), 70 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLookupJoin.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLookupJoin.java
index cd3b46c335d..3cf62046d93 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLookupJoin.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLookupJoin.java
@@ -42,7 +42,6 @@ import
org.apache.flink.table.planner.plan.nodes.exec.spec.TemporalTableSourceSp
import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil;
import org.apache.flink.table.planner.plan.utils.KeySelectorUtil;
import org.apache.flink.table.planner.plan.utils.LookupJoinUtil;
-import org.apache.flink.table.runtime.keyselector.EmptyRowDataKeySelector;
import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
import org.apache.flink.table.runtime.operators.join.FlinkJoinType;
import
org.apache.flink.table.runtime.operators.join.lookup.KeyedLookupJoinWrapper;
@@ -62,10 +61,11 @@ import org.apache.calcite.tools.RelBuilder;
import javax.annotation.Nullable;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import static
org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecSink.PARTITIONER_TRANSFORMATION;
@@ -84,6 +84,7 @@ public class StreamExecLookupJoin extends CommonExecLookupJoin
"lookupKeyContainsPrimaryKey";
public static final String STATE_NAME = "lookupJoinState";
+public static final String FIELD_NAME_INPUT_UPSERT_KEY = "inputUpsertKey";
@JsonProperty(FIELD_NAME_LOOKUP_KEY_CONTAINS_PRIMARY_KEY)
private final boolean lookupKeyContainsPrimaryKey;
@@ -97,6 +98,11 @@ public class StreamExecLookupJoin extends
CommonExecLookupJoin
@JsonInclude(JsonInclude.Include.NON_NULL)
private final List stateMetadataList;
+@Nullable
+@JsonProperty(FIELD_NAME_INPUT_UPSERT_KEY)
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+private final int[] inputUpsertKey;
+
public StreamExecLookupJoin(
ReadableConfig tableConfig,
FlinkJoinType joinType,
@@ -111,6 +117,7 @@ public class StreamExecLookupJoin extends
CommonExecLookupJoin
@Nullable LookupJoinUtil.AsyncLookupOptions asyncLookupOptions,
@Nullable LookupJoinUtil.RetryLookupOptions retryOptions,
ChangelogMode inputChangelogMode,
+@Nullable int[] inputUpsertKey,
InputProperty inputProperty,
RowType outputType,
String description) {
@@ -130,6 +137,7 @@ public class StreamExecLookupJoin extends
CommonExecLookupJoin
asyncLookupOptions,
retryOptions,
inputChangelogMode,
+inputUpsertKey,
// serialize state meta only when upsert materialize is enabled
upsertMaterialize
?
StateMetadata.getOneInputOperatorDefaultMeta(tableConfig, STATE_NAME)
@@ -164,6 +172,7 @@ public class StreamExecLookupJoin extends
CommonExecLookupJoin
LookupJoinUtil.RetryLookupOptions retryOptions,
@JsonProperty(FIELD_NAME_INPUT_CHANGELOG_MODE) @Nullable
ChangelogMode inputChangelogMode,
+@JsonProperty(FIELD_NAME_INPUT_UPSERT_KEY) @Nullable int[]
inputUpsertKey,
@JsonProperty(FIELD_NAME_STATE)
(flink) 01/02: [hotfix][table] Fix KeyedUpsertingSinkFunction's recovery logic and add restore test
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f57ebac39b5b97db0bb41519803a6bde0dfb84db
Author: lincoln lee
AuthorDate: Mon Aug 19 22:26:48 2024 +0800
[hotfix][table] Fix KeyedUpsertingSinkFunction's recovery logic and add
restore test
---
.../factories/TestValuesRuntimeFunctions.java | 16 ++-
.../nodes/exec/common/TableSinkTestPrograms.java | 31 -
.../nodes/exec/stream/TableSinkRestoreTest.java| 3 +-
.../sink-upsert/plan/sink-upsert.json | 141 +
.../sink-upsert/savepoint/_metadata| Bin 0 -> 10027 bytes
5 files changed, 188 insertions(+), 3 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java
index cb12026b4a0..52dd722ee3b 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java
@@ -508,11 +508,25 @@ final class TestValuesRuntimeFunctions {
super.initializeState(context);
synchronized (LOCK) {
-// always store in a single map, global upsert
+// always store in a single map, global upsert similar to
external database
this.localUpsertResult =
globalUpsertResult
.computeIfAbsent(tableName, k -> new
HashMap<>())
.computeIfAbsent(0, k -> new HashMap<>());
+// load all data from global raw result
+globalRawResult.computeIfAbsent(tableName, k -> new
HashMap<>()).values().stream()
+.flatMap(List::stream)
+.forEach(
+row -> {
+boolean isDelete = row.getKind() ==
RowKind.DELETE;
+Row key = Row.project(row, keyIndices);
+key.setKind(RowKind.INSERT);
+if (isDelete) {
+localUpsertResult.remove(key);
+} else {
+localUpsertResult.put(key, row);
+}
+});
}
}
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/TableSinkTestPrograms.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/TableSinkTestPrograms.java
index 3b8c043b4c8..7939a672415 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/TableSinkTestPrograms.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/TableSinkTestPrograms.java
@@ -24,10 +24,11 @@ import org.apache.flink.table.test.program.SinkTestStep;
import org.apache.flink.table.test.program.SourceTestStep;
import org.apache.flink.table.test.program.TableTestProgram;
import org.apache.flink.types.Row;
+import org.apache.flink.types.RowKind;
import java.util.Arrays;
-/** {@link TableTestProgram} definitions for testing {@link
StreamExecDeduplicate}. */
+/** {@link TableTestProgram} definitions for testing {@link StreamExecSink}. */
public class TableSinkTestPrograms {
public static final Row[] BEFORE_DATA = {
@@ -116,6 +117,7 @@ public class TableSinkTestPrograms {
.build())
.runSql("INSERT OVERWRITE sink_t SELECT * FROM source_t")
.build();
+
public static final TableTestProgram SINK_WRITING_METADATA =
TableTestProgram.of("sink-writing-metadata", "validates writing
metadata to sink")
.setupTableSource(
@@ -193,4 +195,31 @@ public class TableSinkTestPrograms {
.build())
.runSql("INSERT INTO sink_t (a, b, c) SELECT a, b, c FROM
source_t")
.build();
+
+public static final TableTestProgram SINK_UPSERT =
+TableTestProgram.of("sink-upsert", "validates sink with primary
key")
+.setupTableSource(
+SourceTestStep.newBuilder("source_t")
+.addOption("changelog-mode", "I,UA,D")
+
(flink) branch master updated (36347e5731b -> 624bc5001c8)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 36347e5731b [FLINK-36085][table] Refactor SHOW operations with
LIKE/ILIKE
new f57ebac39b5 [hotfix][table] Fix KeyedUpsertingSinkFunction's recovery
logic and add restore test
new 624bc5001c8 [FLINK-36095][table-planner] KeyedLookupJoinWrapper should
shuffle by input upsertKey instead of join key to avoid changelog disordering
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../nodes/exec/stream/StreamExecLookupJoin.java| 58 +++-
.../table/planner/plan/utils/UpsertKeyUtil.java| 15 +++-
.../physical/stream/StreamPhysicalLookupJoin.scala | 23 ++-
.../factories/TestValuesRuntimeFunctions.java | 16 -
.../nodes/exec/common/TableSinkTestPrograms.java | 31 -
.../nodes/exec/stream/LookupJoinRestoreTest.java | 3 +-
.../nodes/exec/stream/LookupJoinTestPrograms.java | 71 +++
.../nodes/exec/stream/TableSinkRestoreTest.java| 3 +-
...ggAndAllConstantLookupKeyWithTryResolveMode.out | 4 +-
...nstantLookupKeyWithTryResolveMode_newSource.out | 4 +-
.../analyze/NonDeterministicUpdateAnalyzerTest.xml | 6 +-
.../plan/stream/sql/NonDeterministicDagTest.xml| 42 ++--
.../plan/stream/sql/join/LookupJoinTest.xml| 4 +-
.../plan/lookup-join-with-try-resolve.json}| 76 +
.../savepoint/_metadata| Bin 0 -> 13669 bytes
.../sink-upsert/plan/sink-upsert.json} | 49 +++--
.../sink-upsert/savepoint/_metadata| Bin 0 -> 10027 bytes
17 files changed, 281 insertions(+), 124 deletions(-)
copy
flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lookup-join_1/{lookup-join-project-pushdown/plan/lookup-join-project-pushdown.json
=> lookup-join-with-try-resolve/plan/lookup-join-with-try-resolve.json} (75%)
create mode 100644
flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lookup-join_1/lookup-join-with-try-resolve/savepoint/_metadata
copy
flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-changelog-normalize_1/changelog-normalize-upsert/plan/changelog-normalize-upsert.json
=> stream-exec-sink_1/sink-upsert/plan/sink-upsert.json} (76%)
create mode 100644
flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-sink_1/sink-upsert/savepoint/_metadata
(flink) branch master updated (b460d3d6fe0 -> 500d274265d)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from b460d3d6fe0 [FLINK-36004] Deep copy the BinaryArrayData in ArrayDataSerializer#copy while using custom ArrayData (#25196) add 500d274265d [FLINK-35997][table-api] Align SQL built-in functions LTRIM and RTRIM in table API and python API No new revisions were added by this update. Summary of changes: docs/data/sql_functions.yml| 26 +++- docs/data/sql_functions_zh.yml | 26 +--- flink-python/pyflink/table/expression.py | 24 --- .../pyflink/table/tests/test_expression.py | 6 +++-- .../flink/table/api/internal/BaseExpressions.java | 12 ++ .../planner/expressions/ScalarFunctionsTest.scala | 28 ++ 6 files changed, 81 insertions(+), 41 deletions(-)
(flink) branch master updated (a91085ebdbc -> 215ad96248c)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from a91085ebdbc [FLINK-35920][table] Add the built-in function PRINTF
add 8ab71885994 [FLINK-35993][table] Add the built-in function UNHEX
add 215ad96248c [FLINK-35993][table] Fix doc issue for built-in functions
TRANSLATE & BTRIM & ELT & UNHEX
No new revisions were added by this update.
Summary of changes:
docs/data/sql_functions.yml| 20 +++-
docs/data/sql_functions_zh.yml | 22 -
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 11 +++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 12 +++
.../functions/BuiltInFunctionDefinitions.java | 13 +++
.../apache/flink/table/utils/EncodingUtils.java| 38
.../flink/table/utils/EncodingUtilsTest.java | 16
.../planner/functions/MathFunctionsITCase.java | 102 +++--
.../{EltFunction.java => UnhexFunction.java} | 20 ++--
11 files changed, 236 insertions(+), 20 deletions(-)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{EltFunction.java
=> UnhexFunction.java} (70%)
(flink) branch master updated (729b8b81a77 -> a91085ebdbc)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 729b8b81a77 [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
add a91085ebdbc [FLINK-35920][table] Add the built-in function PRINTF
No new revisions were added by this update.
Summary of changes:
docs/data/sql_functions.yml| 12 ++
docs/data/sql_functions_zh.yml | 12 ++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 14 +++
.../pyflink/table/tests/test_expression.py | 1 +
.../flink/table/api/internal/BaseExpressions.java | 17 +++
.../functions/BuiltInFunctionDefinitions.java | 13 +++
.../planner/functions/StringFunctionsITCase.java | 126 +
...{UrlEncodeFunction.java => PrintfFunction.java} | 30 ++---
9 files changed, 212 insertions(+), 14 deletions(-)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{UrlEncodeFunction.java
=> PrintfFunction.java} (61%)
(flink) branch release-1.19 updated: [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 9c3c8283542 [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
9c3c8283542 is described below
commit 9c3c82835422e68441e936b378045b76aad894f9
Author: Xuyang
AuthorDate: Thu Aug 15 19:22:29 2024 +0800
[FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This closes #25202
---
.../converters/SqlReplaceTableAsConverter.java | 4 +-
.../SqlRTASNodeToOperationConverterTest.java | 10
.../planner/runtime/batch/sql/RTASITCase.java | 22 +
.../planner/runtime/stream/sql/RTASITCase.java | 31 +++-
.../runtime/batch/sql/TableSinkITCase.scala| 55 ++
.../runtime/stream/sql/TableSinkITCase.scala | 24 ++
6 files changed, 114 insertions(+), 32 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
index 396d50bccb5..83b10eb97d5 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
@@ -49,10 +49,10 @@ public class SqlReplaceTableAsConverter implements
SqlNodeConverter context.toQuotedSqlString(asQuerySqlNode));
// get table comment
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
index 5c31f230cd4..d81ed7f0f73 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
@@ -67,6 +67,16 @@ public class SqlRTASNodeToOperationConverterTest extends
SqlNodeToOperationConve
testCommonReplaceTableAs(sql, tableName, null);
}
+@Test
+public void testCreateOrReplaceTableASWithLimit() {
+String tableName = "create_or_replace_table";
+String sql =
+"CREATE OR REPLACE TABLE "
++ tableName
++ " WITH ('k1' = 'v1', 'k2' = 'v2') as (SELECT * FROM
t1 LIMIT 5)";
+testCommonReplaceTableAs(sql, tableName, null);
+}
+
private void testCommonReplaceTableAs(
String sql, String tableName, @Nullable String tableComment) {
ObjectIdentifier expectedIdentifier = ObjectIdentifier.of("builtin",
"default", tableName);
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
index ec077566ccf..5c02656c5c6 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
@@ -78,6 +78,28 @@ class RTASITCase extends BatchTestBase {
verifyCatalogTable(expectCatalogTable, getCatalogTable("target"));
}
+@Test
+void testReplaceTableASWithSortLimit() throws Exception {
+tEnv().executeSql(
+"REPLACE TABLE target WITH ('connector' = 'values',"
++ " 'bounded' = 'true')"
++ " AS (SELECT * FROM source order by `a`
LIMIT 2)")
+.await();
+
+// verify written rows
+
assertThat(TestValuesTableFactory.getResultsAsStrings("target").toString())
+.isEqualTo("[+I[1, 1, Hi], +I[2, 2, Hello]]");
+
+// verify the table after replacing
+CatalogTable expectCatalogTable =
+getExpectCatalogTable(
+new String[] {"a", "b", "c"},
+new AbstractDataType[] {
+DataTypes.INT(), DataTypes.BIGINT(),
DataTypes.STRING()
+});
+verifyCatalogTable(expectCatalogTable, getCatalogTable("target"
(flink) branch release-1.20 updated: [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new c9729dc9201 [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
c9729dc9201 is described below
commit c9729dc9201787b980c1d8b0ad05909fbf716bfb
Author: Xuyang
AuthorDate: Thu Aug 15 19:22:15 2024 +0800
[FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This closes #25201
---
.../converters/SqlReplaceTableAsConverter.java | 4 +-
.../SqlRTASNodeToOperationConverterTest.java | 10
.../planner/runtime/batch/sql/RTASITCase.java | 22 +
.../planner/runtime/stream/sql/RTASITCase.java | 31 +++-
.../runtime/batch/sql/TableSinkITCase.scala| 55 ++
.../runtime/stream/sql/TableSinkITCase.scala | 24 ++
6 files changed, 114 insertions(+), 32 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
index 396d50bccb5..83b10eb97d5 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
@@ -49,10 +49,10 @@ public class SqlReplaceTableAsConverter implements
SqlNodeConverter context.toQuotedSqlString(asQuerySqlNode));
// get table comment
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
index 8a6fc806cab..cc7b5a3f1af 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
@@ -67,6 +67,16 @@ public class SqlRTASNodeToOperationConverterTest extends
SqlNodeToOperationConve
testCommonReplaceTableAs(sql, tableName, null);
}
+@Test
+public void testCreateOrReplaceTableASWithLimit() {
+String tableName = "create_or_replace_table";
+String sql =
+"CREATE OR REPLACE TABLE "
++ tableName
++ " WITH ('k1' = 'v1', 'k2' = 'v2') as (SELECT * FROM
t1 LIMIT 5)";
+testCommonReplaceTableAs(sql, tableName, null);
+}
+
private void testCommonReplaceTableAs(
String sql, String tableName, @Nullable String tableComment) {
ObjectIdentifier expectedIdentifier = ObjectIdentifier.of("builtin",
"default", tableName);
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
index ec077566ccf..5c02656c5c6 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
@@ -78,6 +78,28 @@ class RTASITCase extends BatchTestBase {
verifyCatalogTable(expectCatalogTable, getCatalogTable("target"));
}
+@Test
+void testReplaceTableASWithSortLimit() throws Exception {
+tEnv().executeSql(
+"REPLACE TABLE target WITH ('connector' = 'values',"
++ " 'bounded' = 'true')"
++ " AS (SELECT * FROM source order by `a`
LIMIT 2)")
+.await();
+
+// verify written rows
+
assertThat(TestValuesTableFactory.getResultsAsStrings("target").toString())
+.isEqualTo("[+I[1, 1, Hi], +I[2, 2, Hello]]");
+
+// verify the table after replacing
+CatalogTable expectCatalogTable =
+getExpectCatalogTable(
+new String[] {"a", "b", "c"},
+new AbstractDataType[] {
+DataTypes.INT(), DataTypes.BIGINT(),
DataTypes.STRING()
+});
+verifyCatalogTable(expectCatalogTable, getCatalogTable("target"
(flink) branch master updated: [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 729b8b81a77 [FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
729b8b81a77 is described below
commit 729b8b81a77ba6c32711216b88a1bf57ccddfadc
Author: Xuyang
AuthorDate: Thu Aug 15 09:23:28 2024 +0800
[FLINK-35935][table-planner] Fix RTAS not supporting LIMIT
This closes #25185
---
.../converters/SqlReplaceTableAsConverter.java | 4 +-
.../SqlRTASNodeToOperationConverterTest.java | 10
.../planner/runtime/batch/sql/RTASITCase.java | 22 +
.../planner/runtime/stream/sql/RTASITCase.java | 31 +++-
.../runtime/batch/sql/TableSinkITCase.scala| 55 ++
.../runtime/stream/sql/TableSinkITCase.scala | 24 ++
6 files changed, 114 insertions(+), 32 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
index 396d50bccb5..83b10eb97d5 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlReplaceTableAsConverter.java
@@ -49,10 +49,10 @@ public class SqlReplaceTableAsConverter implements
SqlNodeConverter context.toQuotedSqlString(asQuerySqlNode));
// get table comment
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
index 8a6fc806cab..cc7b5a3f1af 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlRTASNodeToOperationConverterTest.java
@@ -67,6 +67,16 @@ public class SqlRTASNodeToOperationConverterTest extends
SqlNodeToOperationConve
testCommonReplaceTableAs(sql, tableName, null);
}
+@Test
+public void testCreateOrReplaceTableASWithLimit() {
+String tableName = "create_or_replace_table";
+String sql =
+"CREATE OR REPLACE TABLE "
++ tableName
++ " WITH ('k1' = 'v1', 'k2' = 'v2') as (SELECT * FROM
t1 LIMIT 5)";
+testCommonReplaceTableAs(sql, tableName, null);
+}
+
private void testCommonReplaceTableAs(
String sql, String tableName, @Nullable String tableComment) {
ObjectIdentifier expectedIdentifier = ObjectIdentifier.of("builtin",
"default", tableName);
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
index ec077566ccf..5c02656c5c6 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/RTASITCase.java
@@ -78,6 +78,28 @@ class RTASITCase extends BatchTestBase {
verifyCatalogTable(expectCatalogTable, getCatalogTable("target"));
}
+@Test
+void testReplaceTableASWithSortLimit() throws Exception {
+tEnv().executeSql(
+"REPLACE TABLE target WITH ('connector' = 'values',"
++ " 'bounded' = 'true')"
++ " AS (SELECT * FROM source order by `a`
LIMIT 2)")
+.await();
+
+// verify written rows
+
assertThat(TestValuesTableFactory.getResultsAsStrings("target").toString())
+.isEqualTo("[+I[1, 1, Hi], +I[2, 2, Hello]]");
+
+// verify the table after replacing
+CatalogTable expectCatalogTable =
+getExpectCatalogTable(
+new String[] {"a", "b", "c"},
+new AbstractDataType[] {
+DataTypes.INT(), DataTypes.BIGINT(),
DataTypes.STRING()
+});
+verifyCatalogTable(expectCatalogTable, getCatalogTable("target"));
+}
(flink) branch master updated (ff3cfb498e1 -> e33028db6a7)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from ff3cfb498e1 [FLINK-35766][streaming] Fix hang issue in JobGraph
generation when StreamGraph contains many YieldingOperatorFactory.
add 929d96f2ee0 [FLINK-35987][table] Support common type of selected
arguments in CommonTypeStrategy
add 9026a9c6e98 [FLINK-35987][table] Support single named argument in
SubsequenceInputTypeStrategy
add 7715a79097c [FLINK-35987][table] Add IndexArgumentTypeStrategy
add e33028db6a7 [FLINK-35987][table] Add the built-in function ELT
No new revisions were added by this update.
Summary of changes:
docs/data/sql_functions.yml| 7 ++
docs/data/sql_functions_zh.yml | 7 ++
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 16
.../pyflink/table/tests/test_expression.py | 2 +
.../flink/table/api/internal/BaseExpressions.java | 19 +
.../functions/BuiltInFunctionDefinitions.java | 33 +
.../table/types/inference/InputTypeStrategies.java | 10 +++
.../table/types/inference/TypeStrategies.java | 5 ++
.../inference/strategies/CommonTypeStrategy.java | 38 --
...trategy.java => IndexArgumentTypeStrategy.java} | 34 ++---
.../strategies/SpecificInputTypeStrategies.java| 6 ++
.../strategies/SubsequenceInputTypeStrategy.java | 13
.../types/inference/InputTypeStrategiesTest.java | 23 +-
.../SubsequenceInputTypeStrategyTest.java | 32 +++-
.../table/types/inference/TypeStrategiesTest.java | 19 +
.../planner/functions/StringFunctionsITCase.java | 85 +-
.../{CoalesceFunction.java => EltFunction.java}| 22 +++---
18 files changed, 343 insertions(+), 29 deletions(-)
copy
flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/{TypeLiteralArgumentTypeStrategy.java
=> IndexArgumentTypeStrategy.java} (56%)
copy
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/{CoalesceFunction.java
=> EltFunction.java} (68%)
(flink) branch master updated: [FLINK-26840][table-runtime] refactor topN functions adding a general TopNBufferCacheRemovalListener
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 227e61d2e01 [FLINK-26840][table-runtime] refactor topN functions
adding a general TopNBufferCacheRemovalListener
227e61d2e01 is described below
commit 227e61d2e010ec8123dcda78675f18deadf7df1b
Author: lincoln lee
AuthorDate: Tue Aug 13 11:25:24 2024 +0800
[FLINK-26840][table-runtime] refactor topN functions adding a general
TopNBufferCacheRemovalListener
This closes #19220
---
.../runtime/operators/rank/FastTop1Function.java | 31 ++
.../rank/TopNBufferCacheRemovalListener.java | 67 ++
.../operators/rank/UpdatableTopNFunction.java | 45 +++
3 files changed, 80 insertions(+), 63 deletions(-)
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/FastTop1Function.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/FastTop1Function.java
index 7a947b81908..7af4b8cab28 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/FastTop1Function.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/FastTop1Function.java
@@ -35,9 +35,6 @@ import org.apache.flink.util.Collector;
import org.apache.flink.shaded.guava32.com.google.common.cache.Cache;
import org.apache.flink.shaded.guava32.com.google.common.cache.CacheBuilder;
-import org.apache.flink.shaded.guava32.com.google.common.cache.RemovalCause;
-import org.apache.flink.shaded.guava32.com.google.common.cache.RemovalListener;
-import
org.apache.flink.shaded.guava32.com.google.common.cache.RemovalNotification;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,7 +99,9 @@ public class FastTop1Function extends AbstractTopNFunction
implements Checkpoint
kvCache =
cacheBuilder
.maximumSize(lruCacheSize)
-.removalListener(new CacheRemovalListener())
+.removalListener(
+new TopNBufferCacheRemovalListener<>(
+keyContext, this::flushBufferToState))
.build();
LOG.info("Top-1 operator is using LRU caches key-size: {}",
lruCacheSize);
@@ -163,7 +162,7 @@ public class FastTop1Function extends AbstractTopNFunction
implements Checkpoint
public void snapshotState(FunctionSnapshotContext context) throws
Exception {
for (Map.Entry entry : kvCache.asMap().entrySet()) {
keyContext.setCurrentKey(entry.getKey());
-dataState.update(entry.getValue());
+flushBufferToState(entry.getValue());
}
}
@@ -172,25 +171,7 @@ public class FastTop1Function extends AbstractTopNFunction
implements Checkpoint
// nothing to do
}
-private class CacheRemovalListener implements RemovalListener {
-@Override
-public void onRemoval(RemovalNotification
notification) {
-if (notification.getCause() != RemovalCause.SIZE ||
notification.getValue() == null) {
-// Don't flush values to state if cause is ttl expired
-return;
-}
-
-RowData previousKey = (RowData) keyContext.getCurrentKey();
-RowData partitionKey = notification.getKey();
-keyContext.setCurrentKey(partitionKey);
-try {
-dataState.update(notification.getValue());
-} catch (Throwable e) {
-LOG.error("Fail to synchronize state!", e);
-throw new RuntimeException(e);
-} finally {
-keyContext.setCurrentKey(previousKey);
-}
-}
+private void flushBufferToState(RowData rowData) throws Exception {
+dataState.update(rowData);
}
}
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/TopNBufferCacheRemovalListener.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/TopNBufferCacheRemovalListener.java
new file mode 100644
index 000..33eedbe3a12
--- /dev/null
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/TopNBufferCacheRemovalListener.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use th
(flink) branch master updated (dafed7ed282 -> e126f382a1f)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from dafed7ed282 [hotfix] Make use of `CompiledPlan#asJsonString` instead of `getFormattedJson` new fc01ed73764 [refactor][table-planner] Separate specific node logic from visit block for StreamNonDeterministicUpdatePlanVisitor to improve readability new e126f382a1f [hotfix][table-planner] Fix LookupJoinTest incorrectly not enabling test case testAggAndLeftJoinWithTryResolveMode The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../StreamNonDeterministicUpdatePlanVisitor.java | 1089 +++- .../plan/stream/sql/join/LookupJoinTest.scala |1 + 2 files changed, 577 insertions(+), 513 deletions(-)
(flink) 01/02: [refactor][table-planner] Separate specific node logic from visit block for StreamNonDeterministicUpdatePlanVisitor to improve readability
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit fc01ed7376441853f8c0ae2d32a44a8069dbdb1d
Author: lincoln lee
AuthorDate: Fri Aug 9 17:36:33 2024 +0800
[refactor][table-planner] Separate specific node logic from visit block for
StreamNonDeterministicUpdatePlanVisitor to improve readability
---
.../StreamNonDeterministicUpdatePlanVisitor.java | 1089 +++-
1 file changed, 576 insertions(+), 513 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
index f8ccd8fd436..34e5d487a98 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/StreamNonDeterministicUpdatePlanVisitor.java
@@ -155,518 +155,45 @@ public class StreamNonDeterministicUpdatePlanVisitor {
public StreamPhysicalRel visit(
final StreamPhysicalRel rel, final ImmutableBitSet
requireDeterminism) {
if (rel instanceof StreamPhysicalSink) {
-if (inputInsertOnly(rel)) {
-// for append stream, not care about NDU
-return transmitDeterminismRequirement(rel,
NO_REQUIRED_DETERMINISM);
-} else {
-// for update streaming, when
-// 1. sink with pk:
-// upsert sink, update by pk, ideally pk == input.upsertKey,
-// (otherwise upsertMaterialize will handle it)
-
-// 1.1 input.upsertKey nonEmpty -> not care about NDU
-// 1.2 input.upsertKey isEmpty -> retract by complete row,
must not contain NDU
-
-// once sink's requirement on pk was satisfied, no further
request will be transited
-// only when new requirement generated at stateful node which
input has update
-// (e.g., grouping keys)
-
-// 2. sink without pk:
-// retract sink, retract by complete row (all input columns
should be deterministic)
-// whether input.upsertKey is empty or not, must not contain
NDU
-StreamPhysicalSink sink = (StreamPhysicalSink) rel;
-int[] primaryKey =
-
sink.contextResolvedTable().getResolvedSchema().getPrimaryKeyIndexes();
-ImmutableBitSet requireInputDeterminism;
-if (sink.upsertMaterialize() || null == primaryKey ||
primaryKey.length == 0) {
-// SinkUpsertMaterializer only support no upsertKey mode,
it says all input
-// columns should be deterministic (same as no primary key
defined on sink)
-// TODO should optimize it after SinkUpsertMaterializer
support upsertKey
-// FLINK-28569.
-requireInputDeterminism =
-
ImmutableBitSet.range(sink.getInput().getRowType().getFieldCount());
-} else {
-requireInputDeterminism = ImmutableBitSet.of(primaryKey);
-}
-return transmitDeterminismRequirement(sink,
requireInputDeterminism);
-}
+return visitSink((StreamPhysicalSink) rel, requireDeterminism);
} else if (rel instanceof StreamPhysicalLegacySink) {
-if (inputInsertOnly(rel)) {
-// for append stream, not care about NDU
-return transmitDeterminismRequirement(rel,
NO_REQUIRED_DETERMINISM);
-} else {
-StreamPhysicalLegacySink sink =
(StreamPhysicalLegacySink) rel;
-TableSchema tableSchema = sink.sink().getTableSchema();
-Optional primaryKey =
tableSchema.getPrimaryKey();
-List columns =
Arrays.asList(tableSchema.getFieldNames());
-// SinkUpsertMaterializer does not support legacy sink
-ImmutableBitSet requireInputDeterminism;
-if (primaryKey.isPresent()) {
-requireInputDeterminism =
-ImmutableBitSet.of(
-primaryKey.get().getColumns().stream()
-.map(columns::indexOf)
-.collect(Collectors.toList()));
-} else {
-requireInputDeterminism =
ImmutableBitSet.range(columns.size());
-}
-return transmitDeterminismRequirement(rel,
requireInputDeterminism);
-}
+return visi
(flink) 02/02: [hotfix][table-planner] Fix LookupJoinTest incorrectly not enabling test case testAggAndLeftJoinWithTryResolveMode
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e126f382a1f52e228a8a361b9a828a92269b6fa3
Author: lincoln lee
AuthorDate: Mon Aug 12 14:57:19 2024 +0800
[hotfix][table-planner] Fix LookupJoinTest incorrectly not enabling test
case testAggAndLeftJoinWithTryResolveMode
---
.../apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala | 1 +
1 file changed, 1 insertion(+)
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala
index f9e0749a159..3495355fccd 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala
@@ -846,6 +846,7 @@ class LookupJoinTest(legacyTableSource: Boolean) extends
TableTestBase with Seri
util.verifyExplain(stmt, ExplainDetail.JSON_EXECUTION_PLAN)
}
+ @TestTemplate
def testAggAndLeftJoinWithTryResolveMode(): Unit = {
util.tableEnv.getConfig.set(
OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
(flink) branch release-1.19 updated: [FLINK-36000][table-planner] Fix DynamicTableSink#Context's getTargetColumns should return an Optional#empty instead of int[0] for insert stmt without column list
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 81de170ddc9 [FLINK-36000][table-planner] Fix
DynamicTableSink#Context's getTargetColumns should return an Optional#empty
instead of int[0] for insert stmt without column list
81de170ddc9 is described below
commit 81de170ddc9311ef1bc74d10841530a6eb60e5bd
Author: lincoln lee
AuthorDate: Fri Aug 9 16:12:34 2024 +0800
[FLINK-36000][table-planner] Fix DynamicTableSink#Context's
getTargetColumns should return an Optional#empty instead of int[0] for insert
stmt without column list
This closes #25173
---
.../table/planner/operations/SqlNodeToOperationConversion.java | 5 -
.../org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala| 2 +-
.../flink/table/planner/factories/TestValuesTableFactory.java | 6 ++
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
index 63bae2ecd44..cd7fef40274 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
@@ -1358,8 +1358,11 @@ public class SqlNodeToOperationConversion {
private int[][] getTargetColumnIndices(
@Nonnull ContextResolvedTable contextResolvedTable,
@Nullable SqlNodeList targetColumns) {
+if (targetColumns == null) {
+return null;
+}
List allColumns =
contextResolvedTable.getResolvedSchema().getColumnNames();
-return
Optional.ofNullable(targetColumns).orElse(SqlNodeList.EMPTY).stream()
+return targetColumns.stream()
.mapToInt(c -> allColumns.indexOf(((SqlIdentifier)
c).getSimple()))
.mapToObj(idx -> new int[] {idx})
.toArray(int[][]::new);
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
index 1934e980d8a..48356801b3d 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
@@ -73,7 +73,7 @@ abstract class Sink(
.getOrElse(Array.empty[Array[Int]])
.map(_.mkString("[", ",", "]"))
.mkString(","),
-targetColumns != null && targetColumns.length > 0
+targetColumns != null
)
.item("fields", getRowType.getFieldNames.mkString(", "))
.itemIf("hints", RelExplainUtil.hintsToString(hints), !hints.isEmpty)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index 4ed16ba684a..7cea30b132a 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -2088,6 +2088,12 @@ public final class TestValuesTableFactory
} else {
// we don't support OutputFormat for updating query in the
TestValues connector
assertThat(runtimeSink.equals("SinkFunction")).isTrue();
+// check the contract of the context.getTargetColumns method
returns the expected
+// empty Option or non-empty Option with a non-empty array
+assertThat(
+!context.getTargetColumns().isPresent()
+||
context.getTargetColumns().get().length > 0)
+.isTrue();
SinkFunction sinkFunction;
if (primaryKeyIndices.length > 0) {
// TODO FLINK-31301 currently partial-insert composite
columns are not supported
(flink) branch release-1.20 updated: [FLINK-36000][table-planner] Fix DynamicTableSink#Context's getTargetColumns should return an Optional#empty instead of int[0] for insert stmt without column list
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new 42cc2a04459 [FLINK-36000][table-planner] Fix
DynamicTableSink#Context's getTargetColumns should return an Optional#empty
instead of int[0] for insert stmt without column list
42cc2a04459 is described below
commit 42cc2a04459bb4365e6a4b444ccd8e6b328a369f
Author: lincoln lee
AuthorDate: Fri Aug 9 16:11:40 2024 +0800
[FLINK-36000][table-planner] Fix DynamicTableSink#Context's
getTargetColumns should return an Optional#empty instead of int[0] for insert
stmt without column list
This closes #25172
---
.../table/planner/operations/SqlNodeToOperationConversion.java | 5 -
.../org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala| 2 +-
.../flink/table/planner/factories/TestValuesTableFactory.java | 6 ++
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
index 0e3aac1f060..4abf47205a4 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
@@ -1391,8 +1391,11 @@ public class SqlNodeToOperationConversion {
private int[][] getTargetColumnIndices(
@Nonnull ContextResolvedTable contextResolvedTable,
@Nullable SqlNodeList targetColumns) {
+if (targetColumns == null) {
+return null;
+}
List allColumns =
contextResolvedTable.getResolvedSchema().getColumnNames();
-return
Optional.ofNullable(targetColumns).orElse(SqlNodeList.EMPTY).stream()
+return targetColumns.stream()
.mapToInt(c -> allColumns.indexOf(((SqlIdentifier)
c).getSimple()))
.mapToObj(idx -> new int[] {idx})
.toArray(int[][]::new);
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
index 1934e980d8a..48356801b3d 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
@@ -73,7 +73,7 @@ abstract class Sink(
.getOrElse(Array.empty[Array[Int]])
.map(_.mkString("[", ",", "]"))
.mkString(","),
-targetColumns != null && targetColumns.length > 0
+targetColumns != null
)
.item("fields", getRowType.getFieldNames.mkString(", "))
.itemIf("hints", RelExplainUtil.hintsToString(hints), !hints.isEmpty)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index 3ebaa4a3a5c..1054d52ed36 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -2105,6 +2105,12 @@ public final class TestValuesTableFactory
} else {
// we don't support OutputFormat for updating query in the
TestValues connector
assertThat(runtimeSink.equals("SinkFunction")).isTrue();
+// check the contract of the context.getTargetColumns method
returns the expected
+// empty Option or non-empty Option with a non-empty array
+assertThat(
+!context.getTargetColumns().isPresent()
+||
context.getTargetColumns().get().length > 0)
+.isTrue();
SinkFunction sinkFunction;
if (primaryKeyIndices.length > 0) {
// TODO FLINK-31301 currently partial-insert composite
columns are not supported
(flink) branch release-1.19 updated: [FLINK-35885][table] Prohibit advancing the progress of processing time window through watermark
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 08649fd6981 [FLINK-35885][table] Prohibit advancing the progress of
processing time window through watermark
08649fd6981 is described below
commit 08649fd6981655bcd131c76ef36f6dd074566dd0
Author: Xuyang
AuthorDate: Fri Aug 9 16:10:29 2024 +0800
[FLINK-35885][table] Prohibit advancing the progress of processing time
window through watermark
This closes #25180
---
.../nodes/exec/stream/StreamExecWindowRank.java| 1 +
.../harness/WindowAggregateHarnessTest.scala | 46 ++
.../aggregate/window/WindowAggOperatorBuilder.java | 2 +-
.../RowTimeWindowDeduplicateOperatorBuilder.java | 2 +-
.../rank/window/WindowRankOperatorBuilder.java | 11 +-
.../window/tvf/common/WindowAggOperator.java | 27 -
.../rank/window/WindowRankOperatorTest.java| 3 ++
7 files changed, 79 insertions(+), 13 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
index 65471465923..7be4ca61b11 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
@@ -252,6 +252,7 @@ public class StreamExecWindowRank extends
ExecNodeBase
.rankStart(constantRankRange.getRankStart())
.rankEnd(constantRankRange.getRankEnd())
.windowEndIndex(windowEndIndex)
+.withEventTime(windowing.isRowtime())
.build();
OneInputTransformation transform =
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
index b33303ba22a..f46a992592f 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
@@ -18,6 +18,7 @@
package org.apache.flink.table.planner.runtime.harness
import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness
import org.apache.flink.table.api._
@@ -685,6 +686,51 @@ class WindowAggregateHarnessTest(backend:
StateBackendMode, shiftTimeZone: ZoneI
testHarness1.close()
}
+ @TestTemplate
+ def testProcessingTimeTumbleWindowWithFutureWatermark(): Unit = {
+val (testHarness, outputTypes) =
+ createProcessingTimeWindowOperator(TUMBLE, isCDCSource = false)
+val assertor = new RowDataHarnessAssertor(outputTypes)
+
+testHarness.open()
+
+// mock a large watermark arrives before proctime
+testHarness.processWatermark(1L)
+
+testHarness.setProcessingTime(1000L)
+testHarness.processElement(insertRecord("a", 1d, "str1", null))
+testHarness.setProcessingTime(2000L)
+testHarness.processElement(insertRecord("a", 2d, "str2", null))
+testHarness.setProcessingTime(3000L)
+testHarness.processElement(insertRecord("a", 3d, "str2", null))
+
+testHarness.setProcessingTime(6000L)
+testHarness.processElement(insertRecord("a", 4d, "str1", null))
+
+testHarness.setProcessingTime(5L)
+
+val expected = new ConcurrentLinkedQueue[Object]()
+expected.add(new Watermark(1L))
+expected.add(
+ insertRecord(
+"a",
+3L,
+3.0d,
+2L,
+localMills("1970-01-01T00:00:00"),
+localMills("1970-01-01T00:00:05")))
+expected.add(
+ insertRecord(
+"a",
+1L,
+4.0d,
+1L,
+localMills("1970-01-01T00:00:05"),
+localMills("1970-01-01T00:00:10")))
+assertor.assertOutputEqualsSorted("result mismatch", expected,
testHarness.getOutput)
+testHarness.close()
+ }
+
private def createProcessingTimeWindowOperator(testWindow: String,
isCDCSource: Boolean)
: (KeyedOneInputStreamOperatorTestHarness[RowDat
(flink) branch release-1.20 updated: [FLINK-35885][table] Prohibit advancing the progress of processing time window through watermark
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new 01c0a24c915 [FLINK-35885][table] Prohibit advancing the progress of
processing time window through watermark
01c0a24c915 is described below
commit 01c0a24c9152721a6fe976797e197cfa72cea97d
Author: Xuyang
AuthorDate: Fri Aug 9 16:09:26 2024 +0800
[FLINK-35885][table] Prohibit advancing the progress of processing time
window through watermark
This closes #25179
---
.../nodes/exec/stream/StreamExecWindowRank.java| 1 +
.../harness/WindowAggregateHarnessTest.scala | 46 ++
.../aggregate/window/WindowAggOperatorBuilder.java | 2 +-
.../RowTimeWindowDeduplicateOperatorBuilder.java | 2 +-
.../rank/window/WindowRankOperatorBuilder.java | 11 +-
.../window/tvf/common/WindowAggOperator.java | 27 -
.../rank/window/WindowRankOperatorTest.java| 3 ++
7 files changed, 79 insertions(+), 13 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
index 65471465923..7be4ca61b11 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
@@ -252,6 +252,7 @@ public class StreamExecWindowRank extends
ExecNodeBase
.rankStart(constantRankRange.getRankStart())
.rankEnd(constantRankRange.getRankEnd())
.windowEndIndex(windowEndIndex)
+.withEventTime(windowing.isRowtime())
.build();
OneInputTransformation transform =
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
index a4c489cffed..5c5b7100132 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
@@ -18,6 +18,7 @@
package org.apache.flink.table.planner.runtime.harness
import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness
import org.apache.flink.table.api._
@@ -687,6 +688,51 @@ class WindowAggregateHarnessTest(backend:
StateBackendMode, shiftTimeZone: ZoneI
testHarness1.close()
}
+ @TestTemplate
+ def testProcessingTimeTumbleWindowWithFutureWatermark(): Unit = {
+val (testHarness, outputTypes) =
+ createProcessingTimeWindowOperator(TUMBLE, isCDCSource = false)
+val assertor = new RowDataHarnessAssertor(outputTypes)
+
+testHarness.open()
+
+// mock a large watermark arrives before proctime
+testHarness.processWatermark(1L)
+
+testHarness.setProcessingTime(1000L)
+testHarness.processElement(insertRecord("a", 1d, "str1", null))
+testHarness.setProcessingTime(2000L)
+testHarness.processElement(insertRecord("a", 2d, "str2", null))
+testHarness.setProcessingTime(3000L)
+testHarness.processElement(insertRecord("a", 3d, "str2", null))
+
+testHarness.setProcessingTime(6000L)
+testHarness.processElement(insertRecord("a", 4d, "str1", null))
+
+testHarness.setProcessingTime(5L)
+
+val expected = new ConcurrentLinkedQueue[Object]()
+expected.add(new Watermark(1L))
+expected.add(
+ insertRecord(
+"a",
+3L,
+3.0d,
+2L,
+localMills("1970-01-01T00:00:00"),
+localMills("1970-01-01T00:00:05")))
+expected.add(
+ insertRecord(
+"a",
+1L,
+4.0d,
+1L,
+localMills("1970-01-01T00:00:05"),
+localMills("1970-01-01T00:00:10")))
+assertor.assertOutputEqualsSorted("result mismatch", expected,
testHarness.getOutput)
+testHarness.close()
+ }
+
private def createProcessingTimeWindowOperator(testWindow: String,
isCDCSource: Boolean)
: (KeyedOneInputStreamOperatorTestHarness[RowDat
(flink) branch master updated (a885f4ffd08 -> 9457ae8c07b)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from a885f4ffd08 [FLINK-35943][table-planner] Add CompiledPlan annotations to BatchExecHashJoin and BatchExecNestedLoopJoin add 9457ae8c07b [FLINK-35887][core] Fix NPE in TypeExtractor where getSuperclass from an interface returns null No new revisions were added by this update. Summary of changes: .../flink/api/java/typeutils/TypeExtractor.java| 4 +++- .../org/apache/flink/types/PojoTestUtilsTest.java | 24 ++ 2 files changed, 27 insertions(+), 1 deletion(-)
(flink) branch master updated: [FLINK-36000][table-planner] Fix DynamicTableSink#Context's getTargetColumns should return an Optional#empty instead of int[0] for insert stmt without column list
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new fc5f3b5290d [FLINK-36000][table-planner] Fix
DynamicTableSink#Context's getTargetColumns should return an Optional#empty
instead of int[0] for insert stmt without column list
fc5f3b5290d is described below
commit fc5f3b5290dfb0b39682f29d0fc8e851dda5dd31
Author: lincoln lee
AuthorDate: Thu Aug 8 09:25:10 2024 +0800
[FLINK-36000][table-planner] Fix DynamicTableSink#Context's
getTargetColumns should return an Optional#empty instead of int[0] for insert
stmt without column list
This closes #25165
---
.../table/planner/operations/SqlNodeToOperationConversion.java | 5 -
.../org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala| 2 +-
.../flink/table/planner/factories/TestValuesTableFactory.java | 6 ++
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
index f73b314d56a..1a7e4b16317 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/SqlNodeToOperationConversion.java
@@ -1389,8 +1389,11 @@ public class SqlNodeToOperationConversion {
private int[][] getTargetColumnIndices(
@Nonnull ContextResolvedTable contextResolvedTable,
@Nullable SqlNodeList targetColumns) {
+if (targetColumns == null) {
+return null;
+}
List allColumns =
contextResolvedTable.getResolvedSchema().getColumnNames();
-return
Optional.ofNullable(targetColumns).orElse(SqlNodeList.EMPTY).stream()
+return targetColumns.stream()
.mapToInt(c -> allColumns.indexOf(((SqlIdentifier)
c).getSimple()))
.mapToObj(idx -> new int[] {idx})
.toArray(int[][]::new);
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
index 1934e980d8a..48356801b3d 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
@@ -73,7 +73,7 @@ abstract class Sink(
.getOrElse(Array.empty[Array[Int]])
.map(_.mkString("[", ",", "]"))
.mkString(","),
-targetColumns != null && targetColumns.length > 0
+targetColumns != null
)
.item("fields", getRowType.getFieldNames.mkString(", "))
.itemIf("hints", RelExplainUtil.hintsToString(hints), !hints.isEmpty)
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index 3ebaa4a3a5c..1054d52ed36 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -2105,6 +2105,12 @@ public final class TestValuesTableFactory
} else {
// we don't support OutputFormat for updating query in the
TestValues connector
assertThat(runtimeSink.equals("SinkFunction")).isTrue();
+// check the contract of the context.getTargetColumns method
returns the expected
+// empty Option or non-empty Option with a non-empty array
+assertThat(
+!context.getTargetColumns().isPresent()
+||
context.getTargetColumns().get().length > 0)
+.isTrue();
SinkFunction sinkFunction;
if (primaryKeyIndices.length > 0) {
// TODO FLINK-31301 currently partial-insert composite
columns are not supported
(flink) branch master updated: [FLINK-35885][table] Prohibit advancing the progress of processing time window through watermark
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new b3b709f46ca [FLINK-35885][table] Prohibit advancing the progress of
processing time window through watermark
b3b709f46ca is described below
commit b3b709f46ca7b12d9a26192a60cdde790d8523b9
Author: Xuyang
AuthorDate: Wed Aug 7 23:26:11 2024 +0800
[FLINK-35885][table] Prohibit advancing the progress of processing time
window through watermark
This closes #25119
---
.../nodes/exec/stream/StreamExecWindowRank.java| 1 +
.../harness/WindowAggregateHarnessTest.scala | 46 ++
.../aggregate/window/WindowAggOperatorBuilder.java | 2 +-
.../RowTimeWindowDeduplicateOperatorBuilder.java | 2 +-
.../rank/window/WindowRankOperatorBuilder.java | 11 +-
.../window/tvf/common/WindowAggOperator.java | 27 -
.../rank/window/WindowRankOperatorTest.java| 3 ++
7 files changed, 79 insertions(+), 13 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
index 65471465923..7be4ca61b11 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowRank.java
@@ -252,6 +252,7 @@ public class StreamExecWindowRank extends
ExecNodeBase
.rankStart(constantRankRange.getRankStart())
.rankEnd(constantRankRange.getRankEnd())
.windowEndIndex(windowEndIndex)
+.withEventTime(windowing.isRowtime())
.build();
OneInputTransformation transform =
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
index a4c489cffed..5c5b7100132 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/harness/WindowAggregateHarnessTest.scala
@@ -18,6 +18,7 @@
package org.apache.flink.table.planner.runtime.harness
import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness
import org.apache.flink.table.api._
@@ -687,6 +688,51 @@ class WindowAggregateHarnessTest(backend:
StateBackendMode, shiftTimeZone: ZoneI
testHarness1.close()
}
+ @TestTemplate
+ def testProcessingTimeTumbleWindowWithFutureWatermark(): Unit = {
+val (testHarness, outputTypes) =
+ createProcessingTimeWindowOperator(TUMBLE, isCDCSource = false)
+val assertor = new RowDataHarnessAssertor(outputTypes)
+
+testHarness.open()
+
+// mock a large watermark arrives before proctime
+testHarness.processWatermark(1L)
+
+testHarness.setProcessingTime(1000L)
+testHarness.processElement(insertRecord("a", 1d, "str1", null))
+testHarness.setProcessingTime(2000L)
+testHarness.processElement(insertRecord("a", 2d, "str2", null))
+testHarness.setProcessingTime(3000L)
+testHarness.processElement(insertRecord("a", 3d, "str2", null))
+
+testHarness.setProcessingTime(6000L)
+testHarness.processElement(insertRecord("a", 4d, "str1", null))
+
+testHarness.setProcessingTime(5L)
+
+val expected = new ConcurrentLinkedQueue[Object]()
+expected.add(new Watermark(1L))
+expected.add(
+ insertRecord(
+"a",
+3L,
+3.0d,
+2L,
+localMills("1970-01-01T00:00:00"),
+localMills("1970-01-01T00:00:05")))
+expected.add(
+ insertRecord(
+"a",
+1L,
+4.0d,
+1L,
+localMills("1970-01-01T00:00:05"),
+localMills("1970-01-01T00:00:10")))
+assertor.assertOutputEqualsSorted("result mismatch", expected,
testHarness.getOutput)
+testHarness.close()
+ }
+
private def createProcessingTimeWindowOperator(testWindow: String,
isCDCSource: Boolean)
: (KeyedOneInputStreamOperatorTestHarness[RowData, RowData, R
(flink) branch master updated: [FLINK-27519][table-planner] Fix column name conflicts in StreamPhysicalOverAggregate
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 475f45ba0fb [FLINK-27519][table-planner] Fix column name conflicts in
StreamPhysicalOverAggregate
475f45ba0fb is described below
commit 475f45ba0fb78f81adaa627ed2e8fbdcd71b83f6
Author: lincoln lee
AuthorDate: Tue Aug 6 19:30:20 2024 +0800
[FLINK-27519][table-planner] Fix column name conflicts in
StreamPhysicalOverAggregate
This closes #25152
---
.../batch/BatchPhysicalOverAggregateRule.scala | 23 ++
.../stream/StreamPhysicalOverAggregateRule.scala | 12 +--
.../planner/plan/utils/OverAggregateUtil.scala | 23 +-
.../plan/batch/sql/agg/OverAggregateTest.xml | 37 ++
.../plan/stream/sql/agg/OverAggregateTest.xml | 35
.../plan/batch/sql/agg/OverAggregateTest.scala | 28
.../plan/stream/sql/agg/OverAggregateTest.scala| 27
7 files changed, 162 insertions(+), 23 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalOverAggregateRule.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalOverAggregateRule.scala
index 6d20ce229ec..a428d03f95f 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalOverAggregateRule.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalOverAggregateRule.scala
@@ -26,14 +26,13 @@ import
org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggrega
import
org.apache.flink.table.planner.plan.nodes.physical.batch.{BatchPhysicalOverAggregate,
BatchPhysicalOverAggregateBase, BatchPhysicalPythonOverAggregate}
import org.apache.flink.table.planner.plan.utils.{AggregateUtil,
OverAggregateUtil, SortUtil}
import org.apache.flink.table.planner.plan.utils.PythonUtil.isPythonAggregate
-import org.apache.flink.table.planner.typeutils.RowTypeUtils
import org.apache.flink.table.planner.utils.ShortcutUtils
-import org.apache.calcite.plan.{RelOptCluster, RelOptRule, RelOptRuleCall,
RelOptUtil}
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall}
import org.apache.calcite.plan.RelOptRule._
import org.apache.calcite.rel._
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.{AggregateCall, Window}
+import org.apache.calcite.rel.core.Window
import org.apache.calcite.rel.core.Window.Group
import org.apache.calcite.rex.{RexInputRef, RexNode, RexShuttle}
import org.apache.calcite.sql.SqlAggFunction
@@ -107,7 +106,7 @@ class BatchPhysicalOverAggregateRule
(group, aggCallToAggFunction)
}
- val outputRowType = inferOutputRowType(
+ val outputRowType = OverAggregateUtil.inferOutputRowType(
logicWindow.getCluster,
inputRowType,
groupToAggCallToAggFunction.flatMap(_._2).map(_._1))
@@ -198,22 +197,6 @@ class BatchPhysicalOverAggregateRule
isSatisfied
}
- private def inferOutputRowType(
- cluster: RelOptCluster,
- inputType: RelDataType,
- aggCalls: Seq[AggregateCall]): RelDataType = {
-
-val inputNameList = inputType.getFieldNames
-val inputTypeList = inputType.getFieldList.asScala.map(field =>
field.getType)
-
-// we should avoid duplicated names with input column names
-val aggNames = RowTypeUtils.getUniqueName(aggCalls.map(_.getName),
inputNameList)
-val aggTypes = aggCalls.map(_.getType)
-
-val typeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
-typeFactory.createStructType(inputTypeList ++ aggTypes, inputNameList ++
aggNames)
- }
-
private def adjustGroup(
groupBuffer: ArrayBuffer[Window.Group],
groupIdx: Int,
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalOverAggregateRule.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalOverAggregateRule.scala
index 60fdaceb93c..7004bcdf5e7 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalOverAggregateRule.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalOverAggregateRule.scala
@@ -22,6 +22,7 @@ import
org.apache.flink.table.planner.plan.`trait`.FlinkRelDistribution
import org.apache.flink.table.planner.plan.nodes.FlinkConventions
import
org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggregate
imp
(flink) branch master updated: [FLINK-26939][table] Add the built-in function TRANSLATE
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 41b68c298a6 [FLINK-26939][table] Add the built-in function TRANSLATE
41b68c298a6 is described below
commit 41b68c298a6f99b57de04e27e64567f914aab638
Author: dylanhz <53137516+dyla...@users.noreply.github.com>
AuthorDate: Sat Aug 3 22:36:28 2024 +0800
[FLINK-26939][table] Add the built-in function TRANSLATE
This closes #25122
---
docs/data/sql_functions.yml| 7 +
docs/data/sql_functions_zh.yml | 7 +
.../docs/reference/pyflink.table/expressions.rst | 1 +
flink-python/pyflink/table/expression.py | 7 +
.../flink/table/api/internal/BaseExpressions.java | 19 +++
.../functions/BuiltInFunctionDefinitions.java | 20 +++
.../planner/functions/StringFunctionsITCase.java | 146 +
.../table/data/binary/BinaryStringDataUtil.java| 10 ++
.../functions/scalar/TranslateFunction.java| 112
.../flink/table/data/BinaryStringDataTest.java | 12 ++
10 files changed, 341 insertions(+)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 69bfded1797..2cd2a3c415f 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -293,6 +293,13 @@ string:
- sql: REPLACE(string1, string2, string3)
table: STRING1.replace(STRING2, STRING3)
description: Returns a new string which replaces all the occurrences of
STRING2 with STRING3 (non-overlapping) from STRING1. E.g., 'hello
world'.replace('world', 'flink') returns 'hello flink';
'ababab'.replace('abab', 'z') returns 'zab'.
+ - sql: TRANSLATE(expr, fromStr, toStr)
+table: expr.translate(fromStr, toStr)
+description: |
+ Translate an expr where all characters in fromStr have been replaced
with those in toStr.
+ If toStr has a shorter length than fromStr, unmatched characters are
removed.
+ expr [ | ], fromStr [ | ], toStr [ |
]
+ Returns a STRING of translated expr.
- sql: REGEXP_EXTRACT(string1, string2[, integer])
table: STRING1.regexpExtract(STRING2[, INTEGER1])
description: |
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index 4f40783a291..192a06c302a 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -369,6 +369,13 @@ string:
返回一个新字符串，它用 STRING3（非重叠）替换 STRING1 中所有出现的 STRING2。
例如 `'hello world'.replace('world', 'flink')` 返回 `'hello flink'`；
`'ababab'.replace('abab', 'z')` 返回 `'zab'`。
+ - sql: TRANSLATE(expr, fromStr, toStr)
+table: expr.translate(fromStr, toStr)
+description: |
+ 将 expr 中所有出现在 fromStr 之中的字符替换为 toStr 中的相应字符。
+ 如果 toStr 的长度短于 fromStr，则未匹配的字符将被移除。
+ expr [ | ], fromStr [ | ], toStr [ |
]
+ 返回 STRING 格式的转换结果。
- sql: REGEXP_EXTRACT(string1, string2[, integer])
table: STRING1.regexpExtract(STRING2[, INTEGER1])
description: |
diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst
b/flink-python/docs/reference/pyflink.table/expressions.rst
index fbf24e15fa7..3a41b16df26 100644
--- a/flink-python/docs/reference/pyflink.table/expressions.rst
+++ b/flink-python/docs/reference/pyflink.table/expressions.rst
@@ -165,6 +165,7 @@ string functions
Expression.trim_trailing
Expression.trim
Expression.replace
+Expression.translate
Expression.char_length
Expression.upper_case
Expression.lower_case
diff --git a/flink-python/pyflink/table/expression.py
b/flink-python/pyflink/table/expression.py
index c48a488bfc9..ebaed656d23 100644
--- a/flink-python/pyflink/table/expression.py
+++ b/flink-python/pyflink/table/expression.py
@@ -1084,6 +1084,13 @@ class Expression(Generic[T]):
"""
return _ternary_op("replace")(self, search, replacement)
+def translate(self, from_str, to_str) -> 'Expression':
+"""
+Translate an expr where all characters in from_str have been replaced
with those in to_str.
+If to_str has a shorter length than from_str, unmatched characters are
removed.
+"""
+return _ternary_op("translate")(self, from_str, to_str)
+
@property
def char_length(self) -> 'Expression[int]':
"""
diff --git
a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
index ff8f2b26501..6d
(flink) branch release-1.20 updated: [FLINK-35804][table-planner] Fix incorrect calc merge during decorrelate phase
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new acc348613e5 [FLINK-35804][table-planner] Fix incorrect calc merge
during decorrelate phase
acc348613e5 is described below
commit acc348613e5c0f955e34e5fd456d3cbd5a29b5de
Author: lincoln lee
AuthorDate: Mon Jul 15 15:17:58 2024 +0800
[FLINK-35804][table-planner] Fix incorrect calc merge during decorrelate
phase
This closes #25084
Co-authored-by: zhaorongsheng
---
.../apache/calcite/sql2rel/RelDecorrelator.java| 38 --
.../logical/FlinkFilterProjectTransposeRule.java | 4 +++
.../table/planner/plan/batch/sql/CalcTest.xml | 31 ++
.../table/planner/plan/stream/sql/CalcTest.xml | 31 ++
.../table/planner/plan/batch/sql/CalcTest.scala| 16 +
.../table/planner/plan/stream/sql/CalcTest.scala | 16 +
6 files changed, 119 insertions(+), 17 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 10c2764a807..8aed985a7b0 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -16,6 +16,8 @@
*/
package org.apache.calcite.sql2rel;
+import
org.apache.flink.table.planner.plan.rules.logical.FlinkFilterProjectTransposeRule;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -125,9 +127,9 @@ import static
org.apache.calcite.linq4j.Nullness.castNonNull;
* Copied to fix calcite issues. FLINK modifications are at lines
*
*
- * Was changed within FLINK-29280, FLINK-28682: Line 216 ~ 223
- * Should be removed after fix of FLINK-29540: Line 289 ~ 295
- * Should be removed after fix of FLINK-29540: Line 307 ~ 313
+ * Was changed within FLINK-29280, FLINK-28682, FLINK-35804: Line 218 ~
225, Line 273 ~ 288
+ * Should be removed after fix of FLINK-29540: Line 293 ~ 299
+ * Should be removed after fix of FLINK-29540: Line 311 ~ 317
*
*/
public class RelDecorrelator implements ReflectiveVisitor {
@@ -268,20 +270,22 @@ public class RelDecorrelator implements ReflectiveVisitor
{
.FilterIntoJoinRuleConfig.class)
.toRule())
.addRuleInstance(
-CoreRules.FILTER_PROJECT_TRANSPOSE
-.config
-.withRelBuilderFactory(f)
-
.as(FilterProjectTransposeRule.Config.class)
-.withOperandFor(
-Filter.class,
-filter ->
-
!RexUtil.containsCorrelation(
-
filter.getCondition()),
-Project.class,
-project -> true)
-.withCopyFilter(true)
-.withCopyProject(true)
-.toRule())
+// - FLINK MODIFICATION BEGIN -
+FlinkFilterProjectTransposeRule.build(
+CoreRules.FILTER_PROJECT_TRANSPOSE
+.config
+.withRelBuilderFactory(f)
+
.as(FilterProjectTransposeRule.Config.class)
+.withOperandFor(
+Filter.class,
+filter ->
+
!RexUtil.containsCorrelation(
+
filter.getCondition()),
+Project.class,
+project -> true)
+.withCopyFilter(true)
+.withCopyProject(true)))
+// - FLINK MODIFICATION END -
.add
(flink) branch master updated: [FLINK-35804][table-planner] Fix incorrect calc merge during decorrelate phase
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new a021082fa06 [FLINK-35804][table-planner] Fix incorrect calc merge
during decorrelate phase
a021082fa06 is described below
commit a021082fa062e364e1c296a8ac33315b91c47cdd
Author: zhaorongsheng
AuthorDate: Sun Jul 14 21:23:37 2024 +0800
[FLINK-35804][table-planner] Fix incorrect calc merge during decorrelate
phase
This closes #25068
---
.../apache/calcite/sql2rel/RelDecorrelator.java| 38 --
.../logical/FlinkFilterProjectTransposeRule.java | 4 +++
.../table/planner/plan/batch/sql/CalcTest.xml | 31 ++
.../table/planner/plan/stream/sql/CalcTest.xml | 31 ++
.../table/planner/plan/batch/sql/CalcTest.scala| 16 +
.../table/planner/plan/stream/sql/CalcTest.scala | 16 +
6 files changed, 119 insertions(+), 17 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 10c2764a807..8aed985a7b0 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -16,6 +16,8 @@
*/
package org.apache.calcite.sql2rel;
+import
org.apache.flink.table.planner.plan.rules.logical.FlinkFilterProjectTransposeRule;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -125,9 +127,9 @@ import static
org.apache.calcite.linq4j.Nullness.castNonNull;
* Copied to fix calcite issues. FLINK modifications are at lines
*
*
- * Was changed within FLINK-29280, FLINK-28682: Line 216 ~ 223
- * Should be removed after fix of FLINK-29540: Line 289 ~ 295
- * Should be removed after fix of FLINK-29540: Line 307 ~ 313
+ * Was changed within FLINK-29280, FLINK-28682, FLINK-35804: Line 218 ~
225, Line 273 ~ 288
+ * Should be removed after fix of FLINK-29540: Line 293 ~ 299
+ * Should be removed after fix of FLINK-29540: Line 311 ~ 317
*
*/
public class RelDecorrelator implements ReflectiveVisitor {
@@ -268,20 +270,22 @@ public class RelDecorrelator implements ReflectiveVisitor
{
.FilterIntoJoinRuleConfig.class)
.toRule())
.addRuleInstance(
-CoreRules.FILTER_PROJECT_TRANSPOSE
-.config
-.withRelBuilderFactory(f)
-
.as(FilterProjectTransposeRule.Config.class)
-.withOperandFor(
-Filter.class,
-filter ->
-
!RexUtil.containsCorrelation(
-
filter.getCondition()),
-Project.class,
-project -> true)
-.withCopyFilter(true)
-.withCopyProject(true)
-.toRule())
+// - FLINK MODIFICATION BEGIN -
+FlinkFilterProjectTransposeRule.build(
+CoreRules.FILTER_PROJECT_TRANSPOSE
+.config
+.withRelBuilderFactory(f)
+
.as(FilterProjectTransposeRule.Config.class)
+.withOperandFor(
+Filter.class,
+filter ->
+
!RexUtil.containsCorrelation(
+
filter.getCondition()),
+Project.class,
+project -> true)
+.withCopyFilter(true)
+.withCopyProject(true)))
+// - FLINK MODIFICATION END -
.add
(flink) branch release-1.20 updated: [FLINK-35829][table-planner] The requireWatermark property of StreamPhysicalWindowTableFunction needs to check the window's time type
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new 7f96af3ea82 [FLINK-35829][table-planner] The requireWatermark property
of StreamPhysicalWindowTableFunction needs to check the window's time type
7f96af3ea82 is described below
commit 7f96af3ea82a1a1efab57f65db9674fac5bc41bf
Author: Xuyang
AuthorDate: Sat Jul 13 20:27:03 2024 +0800
[FLINK-35829][table-planner] The requireWatermark property of
StreamPhysicalWindowTableFunction needs to check the window's time type
This closes #25083
---
.../stream/StreamPhysicalWindowTableFunction.scala | 2 +-
.../plan/stream/sql/WindowTableFunctionTest.xml| 68 --
.../plan/stream/sql/WindowTableFunctionTest.scala | 37
3 files changed, 100 insertions(+), 7 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
index 633ff355418..7578db0222c 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
@@ -40,7 +40,7 @@ class StreamPhysicalWindowTableFunction(
extends CommonPhysicalWindowTableFunction(cluster, traitSet, inputRel,
outputRowType, windowing)
with StreamPhysicalRel {
- override def requireWatermark: Boolean = true
+ override def requireWatermark: Boolean = windowing.isRowtime
override def copy(traitSet: RelTraitSet, inputs: util.List[RelNode]):
RelNode = {
new StreamPhysicalWindowTableFunction(
diff --git
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
index bf45b5f6de5..d793d021e85 100644
---
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
+++
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
@@ -316,6 +316,34 @@ Calc(select=[a, b, c, d, rowtime,
PROCTIME_MATERIALIZE(proctime) AS proctime, wi
+- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL
SECOND)])
+- Calc(select=[a, b, c, d, rowtime, PROCTIME() AS proctime])
+- TableSourceScan(table=[[default_catalog, default_database,
MyTable]], fields=[a, b, c, d, rowtime])
+]]>
+
+
+
+
+
+
+
+
+
+
+
@@ -407,17 +435,17 @@ Calc(select=[a, b, c, d, rowtime,
PROCTIME_MATERIALIZE(proctime) AS proctime, wi
]]>
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
(flink) branch release-1.20 updated: [FLINK-35816][table-planner] Non-mergeable proctime tvf window aggregate needs to fallback to group aggregate
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new 41a14091740 [FLINK-35816][table-planner] Non-mergeable proctime tvf
window aggregate needs to fallback to group aggregate
41a14091740 is described below
commit 41a140917403365c30429b40fbab9a0c85806269
Author: lincoln lee
AuthorDate: Sat Jul 13 11:09:42 2024 +0800
[FLINK-35816][table-planner] Non-mergeable proctime tvf window aggregate
needs to fallback to group aggregate
This closes #25082
---
.../stream/StreamPhysicalWindowAggregate.scala | 9 +-
.../table/planner/plan/utils/WindowUtil.scala | 113 +++-
.../plan/stream/sql/agg/WindowAggregateTest.xml| 614 +
.../plan/stream/sql/agg/WindowAggregateTest.scala | 101 +++-
4 files changed, 808 insertions(+), 29 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
index 93d245139df..9bb859be10b 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
@@ -17,8 +17,9 @@
*/
package org.apache.flink.table.planner.plan.nodes.physical.stream
+import org.apache.flink.table.api.TableException
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.logical.WindowingStrategy
+import
org.apache.flink.table.planner.plan.logical.{WindowAttachedWindowingStrategy,
WindowingStrategy}
import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
import
org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecWindowAggregate
import org.apache.flink.table.planner.plan.utils._
@@ -112,6 +113,12 @@ class StreamPhysicalWindowAggregate(
override def translateToExecNode(): ExecNode[_] = {
checkEmitConfiguration(unwrapTableConfig(this))
+
+if (windowing.isInstanceOf[WindowAttachedWindowingStrategy] &&
windowing.isProctime) {
+ throw new TableException(
+"Non-mergeable processing time window tvf aggregation is invalid,
should fallback to group " +
+ "aggregation instead. This is a bug and should not happen. Please
file an issue.")
+}
new StreamExecWindowAggregate(
unwrapTableConfig(this),
grouping,
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
index 995a89a8a03..e664b41c1d2 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
@@ -24,7 +24,7 @@ import
org.apache.flink.table.planner.functions.sql.{FlinkSqlOperatorTable, SqlW
import org.apache.flink.table.planner.plan.`trait`.RelWindowProperties
import org.apache.flink.table.planner.plan.logical._
import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery
-import
org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate,
FlinkLogicalJoin, FlinkLogicalRank, FlinkLogicalTableFunctionScan}
+import
org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate,
FlinkLogicalMatch, FlinkLogicalOverAggregate, FlinkLogicalRank,
FlinkLogicalTableFunctionScan}
import
org.apache.flink.table.planner.plan.utils.AggregateUtil.inferAggAccumulatorNames
import
org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED,
TABLE_EXEC_EMIT_LATE_FIRE_ENABLED}
import org.apache.flink.table.planner.typeutils.RowTypeUtils
@@ -35,7 +35,7 @@ import
org.apache.flink.table.types.logical.utils.LogicalTypeChecks.canBeTimeAtt
import org.apache.calcite.plan.volcano.RelSubset
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.{RelNode, SingleRel}
+import org.apache.calcite.rel.{BiRel, RelNode, RelVisitor}
import org.apache.calcite.rel.core._
import org.apache.calcite.rex._
import org.apache.calcite.sql.`type`.SqlTypeFamily
@@ -45,10 +45,9 @@ import org.apache.calcite.util.{ImmutableBitSet, Util}
import java.time.Duration
import java.util.Collections
-import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.collection.mutable
-import scala.collection
(flink) branch master updated: [FLINK-35816][table-planner] Non-mergeable proctime tvf window aggregate needs to fallback to group aggregate
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 30196294eb8 [FLINK-35816][table-planner] Non-mergeable proctime tvf
window aggregate needs to fallback to group aggregate
30196294eb8 is described below
commit 30196294eb863a2ea288ea562812b9ebaffece12
Author: lincoln lee
AuthorDate: Fri Jul 12 23:18:34 2024 +0800
[FLINK-35816][table-planner] Non-mergeable proctime tvf window aggregate
needs to fallback to group aggregate
This closes #25075
---
.../stream/StreamPhysicalWindowAggregate.scala | 9 +-
.../table/planner/plan/utils/WindowUtil.scala | 113 +++-
.../plan/stream/sql/agg/WindowAggregateTest.xml| 614 +
.../plan/stream/sql/agg/WindowAggregateTest.scala | 101 +++-
4 files changed, 808 insertions(+), 29 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
index 93d245139df..9bb859be10b 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowAggregate.scala
@@ -17,8 +17,9 @@
*/
package org.apache.flink.table.planner.plan.nodes.physical.stream
+import org.apache.flink.table.api.TableException
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.logical.WindowingStrategy
+import
org.apache.flink.table.planner.plan.logical.{WindowAttachedWindowingStrategy,
WindowingStrategy}
import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
import
org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecWindowAggregate
import org.apache.flink.table.planner.plan.utils._
@@ -112,6 +113,12 @@ class StreamPhysicalWindowAggregate(
override def translateToExecNode(): ExecNode[_] = {
checkEmitConfiguration(unwrapTableConfig(this))
+
+if (windowing.isInstanceOf[WindowAttachedWindowingStrategy] &&
windowing.isProctime) {
+ throw new TableException(
+"Non-mergeable processing time window tvf aggregation is invalid,
should fallback to group " +
+ "aggregation instead. This is a bug and should not happen. Please
file an issue.")
+}
new StreamExecWindowAggregate(
unwrapTableConfig(this),
grouping,
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
index 995a89a8a03..e664b41c1d2 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
@@ -24,7 +24,7 @@ import
org.apache.flink.table.planner.functions.sql.{FlinkSqlOperatorTable, SqlW
import org.apache.flink.table.planner.plan.`trait`.RelWindowProperties
import org.apache.flink.table.planner.plan.logical._
import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery
-import
org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate,
FlinkLogicalJoin, FlinkLogicalRank, FlinkLogicalTableFunctionScan}
+import
org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate,
FlinkLogicalMatch, FlinkLogicalOverAggregate, FlinkLogicalRank,
FlinkLogicalTableFunctionScan}
import
org.apache.flink.table.planner.plan.utils.AggregateUtil.inferAggAccumulatorNames
import
org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED,
TABLE_EXEC_EMIT_LATE_FIRE_ENABLED}
import org.apache.flink.table.planner.typeutils.RowTypeUtils
@@ -35,7 +35,7 @@ import
org.apache.flink.table.types.logical.utils.LogicalTypeChecks.canBeTimeAtt
import org.apache.calcite.plan.volcano.RelSubset
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.{RelNode, SingleRel}
+import org.apache.calcite.rel.{BiRel, RelNode, RelVisitor}
import org.apache.calcite.rel.core._
import org.apache.calcite.rex._
import org.apache.calcite.sql.`type`.SqlTypeFamily
@@ -45,10 +45,9 @@ import org.apache.calcite.util.{ImmutableBitSet, Util}
import java.time.Duration
import java.util.Collections
-import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
+imp
(flink) branch master updated: [FLINK-35829][table-planner] The requireWatermark property of StreamPhysicalWindowTableFunction needs to check the window's time type
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 2958b65ae4f [FLINK-35829][table-planner] The requireWatermark property
of StreamPhysicalWindowTableFunction needs to check the window's time type
2958b65ae4f is described below
commit 2958b65ae4f9a02a66ec4635a1c6d76954099a48
Author: Xuyang
AuthorDate: Fri Jul 12 23:15:56 2024 +0800
[FLINK-35829][table-planner] The requireWatermark property of
StreamPhysicalWindowTableFunction needs to check the window's time type
This closes #25079
---
.../stream/StreamPhysicalWindowTableFunction.scala | 2 +-
.../plan/stream/sql/WindowTableFunctionTest.xml| 68 --
.../plan/stream/sql/WindowTableFunctionTest.scala | 37
3 files changed, 100 insertions(+), 7 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
index 633ff355418..7578db0222c 100644
---
a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
+++
b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWindowTableFunction.scala
@@ -40,7 +40,7 @@ class StreamPhysicalWindowTableFunction(
extends CommonPhysicalWindowTableFunction(cluster, traitSet, inputRel,
outputRowType, windowing)
with StreamPhysicalRel {
- override def requireWatermark: Boolean = true
+ override def requireWatermark: Boolean = windowing.isRowtime
override def copy(traitSet: RelTraitSet, inputs: util.List[RelNode]):
RelNode = {
new StreamPhysicalWindowTableFunction(
diff --git
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
index bf45b5f6de5..d793d021e85 100644
---
a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
+++
b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.xml
@@ -316,6 +316,34 @@ Calc(select=[a, b, c, d, rowtime,
PROCTIME_MATERIALIZE(proctime) AS proctime, wi
+- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL
SECOND)])
+- Calc(select=[a, b, c, d, rowtime, PROCTIME() AS proctime])
+- TableSourceScan(table=[[default_catalog, default_database,
MyTable]], fields=[a, b, c, d, rowtime])
+]]>
+
+
+
+
+
+
+
+
+
+
+
@@ -407,17 +435,17 @@ Calc(select=[a, b, c, d, rowtime,
PROCTIME_MATERIALIZE(proctime) AS proctime, wi
]]>
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
(flink) branch release-1.19 updated: [FLINK-35498][table] Fix unexpected argument name conflicts when extracting method parameter names from UDF
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 2581c26e4f0 [FLINK-35498][table] Fix unexpected argument name
conflicts when extracting method parameter names from UDF
2581c26e4f0 is described below
commit 2581c26e4f0e38886ba29f36268d7d7657eb8531
Author: Xuyang
AuthorDate: Thu Jul 4 21:17:02 2024 +0800
[FLINK-35498][table] Fix unexpected argument name conflicts when extracting
method parameter names from UDF
This closes #25020
---
.../table/types/extraction/ExtractionUtils.java| 49 -
.../types/extraction/ExtractionUtilsTest.java | 110 +
.../planner/runtime/stream/sql/FunctionITCase.java | 43
3 files changed, 198 insertions(+), 4 deletions(-)
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
index 9854e62eb0c..5968b41fc84 100644
---
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.types.extraction;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.table.annotation.ArgumentHint;
import org.apache.flink.table.api.DataTypes;
@@ -58,6 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.TreeMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -747,7 +749,8 @@ public final class ExtractionUtils {
return fieldNames;
}
-private static @Nullable List extractExecutableNames(Executable
executable) {
+@VisibleForTesting
+static @Nullable List extractExecutableNames(Executable
executable) {
final int offset;
if (!Modifier.isStatic(executable.getModifiers())) {
// remove "this" as first parameter
@@ -824,6 +827,40 @@ public final class ExtractionUtils {
*
* }
* }
+ *
+ * If a constructor or method has multiple identical local variables
that are not initialized
+ * like:
+ *
+ * {@code
+ * String localVariable;
+ * if (generic == null) {
+ * localVariable = "null";
+ * } else if (generic < 0) {
+ * localVariable = "negative";
+ * } else if (generic > 0) {
+ * localVariable = "positive";
+ * } else {
+ * localVariable = "zero";
+ * }
+ * }
+ *
+ * Its local variable table is as follows:
+ *
+ * {@code
+ * Start Length Slot Name Signature
+ * 7 3 2 localVariable Ljava/lang/String;
+ * 22 3 2 localVariable Ljava/lang/String;
+ * 37 3 2 localVariable Ljava/lang/String;
+ * 0 69 0 this...;
+ * 0 69 1 generic Ljava/lang/Long;
+ * 43 26 2 localVariable Ljava/lang/String;
+ * }
+ *
+ * The method parameters are always at the head in the 'slot' list.
+ *
+ * NOTE: the first parameter may be "this" if the function is not
static. See more at https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html";>3.6.
Receiving
+ * Arguments
*/
private static class ParameterExtractor extends ClassVisitor {
@@ -831,7 +868,7 @@ public final class ExtractionUtils {
private final String methodDescriptor;
-private final List parameterNames = new ArrayList<>();
+private final Map parameterNamesWithIndex = new
TreeMap<>();
ParameterExtractor(Constructor constructor) {
super(OPCODE);
@@ -844,7 +881,11 @@ public final class ExtractionUtils {
}
List getParameterNames() {
-return parameterNames;
+// method parameters are always at the head in the 'index' list
+// NOTE: the first parameter may be "this" if the function is not
static
+// See more at Chapter "3.6. Receiving Arguments" in
+// https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html
+return new ArrayList<>(parameterNamesWithIndex.values());
}
@Override
@@ -860,7 +901,7 @@ public final class ExtractionUtils {
Label start
(flink) branch release-1.20 updated: [FLINK-35498][table] Fix unexpected argument name conflicts when extracting method parameter names from UDF
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.20 by this push:
new ce29a69bcbe [FLINK-35498][table] Fix unexpected argument name
conflicts when extracting method parameter names from UDF
ce29a69bcbe is described below
commit ce29a69bcbe384cdfaec7db98ce4447b760ce3a2
Author: Xuyang
AuthorDate: Thu Jul 4 21:13:48 2024 +0800
[FLINK-35498][table] Fix unexpected argument name conflicts when extracting
method parameter names from UDF
This closes #25019
---
.../table/types/extraction/ExtractionUtils.java| 49 -
.../types/extraction/ExtractionUtilsTest.java | 110 +
.../planner/runtime/stream/sql/FunctionITCase.java | 43
3 files changed, 198 insertions(+), 4 deletions(-)
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
index 9854e62eb0c..5968b41fc84 100644
---
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.types.extraction;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.table.annotation.ArgumentHint;
import org.apache.flink.table.api.DataTypes;
@@ -58,6 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.TreeMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -747,7 +749,8 @@ public final class ExtractionUtils {
return fieldNames;
}
-private static @Nullable List extractExecutableNames(Executable
executable) {
+@VisibleForTesting
+static @Nullable List extractExecutableNames(Executable
executable) {
final int offset;
if (!Modifier.isStatic(executable.getModifiers())) {
// remove "this" as first parameter
@@ -824,6 +827,40 @@ public final class ExtractionUtils {
*
* }
* }
+ *
+ * If a constructor or method has multiple identical local variables
that are not initialized
+ * like:
+ *
+ * {@code
+ * String localVariable;
+ * if (generic == null) {
+ * localVariable = "null";
+ * } else if (generic < 0) {
+ * localVariable = "negative";
+ * } else if (generic > 0) {
+ * localVariable = "positive";
+ * } else {
+ * localVariable = "zero";
+ * }
+ * }
+ *
+ * Its local variable table is as follows:
+ *
+ * {@code
+ * Start Length Slot Name Signature
+ * 7 3 2 localVariable Ljava/lang/String;
+ * 22 3 2 localVariable Ljava/lang/String;
+ * 37 3 2 localVariable Ljava/lang/String;
+ * 0 69 0 this...;
+ * 0 69 1 generic Ljava/lang/Long;
+ * 43 26 2 localVariable Ljava/lang/String;
+ * }
+ *
+ * The method parameters are always at the head in the 'slot' list.
+ *
+ * NOTE: the first parameter may be "this" if the function is not
static. See more at https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html";>3.6.
Receiving
+ * Arguments
*/
private static class ParameterExtractor extends ClassVisitor {
@@ -831,7 +868,7 @@ public final class ExtractionUtils {
private final String methodDescriptor;
-private final List parameterNames = new ArrayList<>();
+private final Map parameterNamesWithIndex = new
TreeMap<>();
ParameterExtractor(Constructor constructor) {
super(OPCODE);
@@ -844,7 +881,11 @@ public final class ExtractionUtils {
}
List getParameterNames() {
-return parameterNames;
+// method parameters are always at the head in the 'index' list
+// NOTE: the first parameter may be "this" if the function is not
static
+// See more at Chapter "3.6. Receiving Arguments" in
+// https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html
+return new ArrayList<>(parameterNamesWithIndex.values());
}
@Override
@@ -860,7 +901,7 @@ public final class ExtractionUtils {
Label start
(flink) branch master updated: [FLINK-35498][table] Fix unexpected argument name conflicts when extracting method parameter names from UDF
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 84165b23cfc [FLINK-35498][table] Fix unexpected argument name
conflicts when extracting method parameter names from UDF
84165b23cfc is described below
commit 84165b23cfc756881d43cdab1b26c814d87e6227
Author: Xuyang
AuthorDate: Tue Jul 2 09:12:10 2024 +0800
[FLINK-35498][table] Fix unexpected argument name conflicts when extracting
method parameter names from UDF
This closes #24890
---
.../table/types/extraction/ExtractionUtils.java| 49 -
.../types/extraction/ExtractionUtilsTest.java | 110 +
.../planner/runtime/stream/sql/FunctionITCase.java | 43
3 files changed, 198 insertions(+), 4 deletions(-)
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
index 9854e62eb0c..5968b41fc84 100644
---
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.types.extraction;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.table.annotation.ArgumentHint;
import org.apache.flink.table.api.DataTypes;
@@ -58,6 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.TreeMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -747,7 +749,8 @@ public final class ExtractionUtils {
return fieldNames;
}
-private static @Nullable List extractExecutableNames(Executable
executable) {
+@VisibleForTesting
+static @Nullable List extractExecutableNames(Executable
executable) {
final int offset;
if (!Modifier.isStatic(executable.getModifiers())) {
// remove "this" as first parameter
@@ -824,6 +827,40 @@ public final class ExtractionUtils {
*
* }
* }
+ *
+ * If a constructor or method has multiple identical local variables
that are not initialized
+ * like:
+ *
+ * {@code
+ * String localVariable;
+ * if (generic == null) {
+ * localVariable = "null";
+ * } else if (generic < 0) {
+ * localVariable = "negative";
+ * } else if (generic > 0) {
+ * localVariable = "positive";
+ * } else {
+ * localVariable = "zero";
+ * }
+ * }
+ *
+ * Its local variable table is as follows:
+ *
+ * {@code
+ * Start Length Slot Name Signature
+ * 7 3 2 localVariable Ljava/lang/String;
+ * 22 3 2 localVariable Ljava/lang/String;
+ * 37 3 2 localVariable Ljava/lang/String;
+ * 0 69 0 this...;
+ * 0 69 1 generic Ljava/lang/Long;
+ * 43 26 2 localVariable Ljava/lang/String;
+ * }
+ *
+ * The method parameters are always at the head in the 'slot' list.
+ *
+ * NOTE: the first parameter may be "this" if the function is not
static. See more at https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html";>3.6.
Receiving
+ * Arguments
*/
private static class ParameterExtractor extends ClassVisitor {
@@ -831,7 +868,7 @@ public final class ExtractionUtils {
private final String methodDescriptor;
-private final List parameterNames = new ArrayList<>();
+private final Map parameterNamesWithIndex = new
TreeMap<>();
ParameterExtractor(Constructor constructor) {
super(OPCODE);
@@ -844,7 +881,11 @@ public final class ExtractionUtils {
}
List getParameterNames() {
-return parameterNames;
+// method parameters are always at the head in the 'index' list
+// NOTE: the first parameter may be "this" if the function is not
static
+// See more at Chapter "3.6. Receiving Arguments" in
+// https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-3.html
+return new ArrayList<>(parameterNamesWithIndex.values());
}
@Override
@@ -860,7 +901,7 @@ public final class ExtractionUtils {
Label start
(flink) branch release-1.19 updated: Update japicmp configuration for 1.19.0
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch release-1.19 in repository https://gitbox.apache.org/repos/asf/flink.git The following commit(s) were added to refs/heads/release-1.19 by this push: new 6eeae5fe6c9 Update japicmp configuration for 1.19.0 6eeae5fe6c9 is described below commit 6eeae5fe6c9c48ee1e7546f26decbac429f248e1 Author: lincoln lee AuthorDate: Fri Mar 22 11:20:35 2024 +0800 Update japicmp configuration for 1.19.0 This closes #24514 --- pom.xml | 11 ++- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 9e021a5d5fd..df2d047e49b 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ under the License. 3.2.4 2.3.9 1.5.6 - 1.18.0 + 1.19.0 tools/japicmp-output 2.27.1 3.4.3 @@ -2348,19 +2348,12 @@ under the License. @org.apache.flink.annotation.Public - + @org.apache.flink.annotation.PublicEvolving @org.apache.flink.annotation.Experimental - @org.apache.flink.annotation.PublicEvolving @org.apache.flink.annotation.Internal - org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#fromData(java.lang.Object[]) - org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#fromData(org.apache.flink.api.common.typeinfo.TypeInformation,java.lang.Object[]) - org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#fromData(java.lang.Class,java.lang.Object[]) - org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#fromData(java.util.Collection) - org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#fromData(java.util.Collection,org.apache.flink.api.common.typeinfo.TypeInformation) - org.apache.flink.api.common.functions.RuntimeContext public
(flink) branch master updated (cf0d75c4bb3 -> a8e0936f1c4)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from cf0d75c4bb3 [FLINK-34731][runtime] Remove SpeculativeScheduler and incorporate its features into AdaptiveBatchScheduler. add a8e0936f1c4 [FLINK-34707][tests] Update base version for japicmp check No new revisions were added by this update. Summary of changes: pom.xml | 8 +--- 1 file changed, 1 insertion(+), 7 deletions(-)
(flink) branch master updated (23c2fd0a32d -> 8ee552a326e)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 23c2fd0a32d [FLINK-34893] Bump checkstyle to 9.3 add 8ee552a326e [FLINK-34716][release] Build 1.19 docs in GitHub Action and mark 1.19 as stable in docs No new revisions were added by this update. Summary of changes: .github/workflows/docs.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
(flink) branch release-1.19 updated: [FLINK-34706][docs] Deprecates 1.17 docs.
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch release-1.19 in repository https://gitbox.apache.org/repos/asf/flink.git The following commit(s) were added to refs/heads/release-1.19 by this push: new 8fd8b3aebf4e [FLINK-34706][docs] Deprecates 1.17 docs. 8fd8b3aebf4e is described below commit 8fd8b3aebf4eabfba70e3395bd11b5db11e3eeaf Author: lincoln lee AuthorDate: Thu Mar 21 21:07:17 2024 +0800 [FLINK-34706][docs] Deprecates 1.17 docs. This closes #24547 --- docs/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.toml b/docs/config.toml index e5d14e470371..c137110bbeda 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -27,7 +27,7 @@ pygmentsUseClasses = true IsStable = true # Flag to indicate whether an outdated warning should be shown. - ShowOutDatedWarning = false + ShowOutDatedWarning = true # This are the version referenced in the docs. Please only use these variables # to reference a specific Flink version, because this is the only place where
(flink) branch master updated (a9cde49118b -> 023c9c881c7)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from a9cde49118b [Flink 32701] [cep] Fix CEP Operator Memory Leak Issue (#24084) add 023c9c881c7 [FLINK-34706][docs] Add 1.19 to PreviousDocs list. No new revisions were added by this update. Summary of changes: docs/config.toml | 1 + 1 file changed, 1 insertion(+)
(flink-docker) branch master updated: [FLINK-34701][release] Update docker-entrypoint.sh for 1.19.0
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
The following commit(s) were added to refs/heads/master by this push:
new 20017e8 [FLINK-34701][release] Update docker-entrypoint.sh for 1.19.0
20017e8 is described below
commit 20017e8f0e81d54fe74c0f9f6a3a988ea609be8f
Author: lincoln lee
AuthorDate: Tue Mar 19 11:11:48 2024 +0800
[FLINK-34701][release] Update docker-entrypoint.sh for 1.19.0
This closes #189
---
1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh | 8
1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh | 8
1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh | 8
3 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh
b/1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh
index e884375..e081109 100755
--- a/1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh
+++ b/1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh
@@ -65,19 +65,19 @@ set_config_options() {
local bin_dir="$FLINK_HOME/bin"
local lib_dir="$FLINK_HOME/lib"
-local config_params=""
+local config_params=()
while [ $# -gt 0 ]; do
local key="$1"
local value="$2"
-config_params+=" -D${key}=${value}"
+config_params+=("-D${key}=${value}")
shift 2
done
-if [ ! -z "${config_params}" ]; then
-eval "${config_parser_script} ${config_dir} ${bin_dir} ${lib_dir}
${config_params}"
+if [ "${#config_params[@]}" -gt 0 ]; then
+"${config_parser_script}" "${config_dir}" "${bin_dir}" "${lib_dir}"
"${config_params[@]}"
fi
}
diff --git a/1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh
b/1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh
index e884375..e081109 100755
--- a/1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh
+++ b/1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh
@@ -65,19 +65,19 @@ set_config_options() {
local bin_dir="$FLINK_HOME/bin"
local lib_dir="$FLINK_HOME/lib"
-local config_params=""
+local config_params=()
while [ $# -gt 0 ]; do
local key="$1"
local value="$2"
-config_params+=" -D${key}=${value}"
+config_params+=("-D${key}=${value}")
shift 2
done
-if [ ! -z "${config_params}" ]; then
-eval "${config_parser_script} ${config_dir} ${bin_dir} ${lib_dir}
${config_params}"
+if [ "${#config_params[@]}" -gt 0 ]; then
+"${config_parser_script}" "${config_dir}" "${bin_dir}" "${lib_dir}"
"${config_params[@]}"
fi
}
diff --git a/1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh
b/1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh
index e884375..e081109 100755
--- a/1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh
+++ b/1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh
@@ -65,19 +65,19 @@ set_config_options() {
local bin_dir="$FLINK_HOME/bin"
local lib_dir="$FLINK_HOME/lib"
-local config_params=""
+local config_params=()
while [ $# -gt 0 ]; do
local key="$1"
local value="$2"
-config_params+=" -D${key}=${value}"
+config_params+=("-D${key}=${value}")
shift 2
done
-if [ ! -z "${config_params}" ]; then
-eval "${config_parser_script} ${config_dir} ${bin_dir} ${lib_dir}
${config_params}"
+if [ "${#config_params[@]}" -gt 0 ]; then
+"${config_parser_script}" "${config_dir}" "${bin_dir}" "${lib_dir}"
"${config_params[@]}"
fi
}
(flink-docker) branch dev-master updated: [hotfix] Improve docker-entrypoint.sh to Use Arrays for Configuration Parameters Instead of Eval and String Concatenation
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch dev-master
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
The following commit(s) were added to refs/heads/dev-master by this push:
new dd78da0 [hotfix] Improve docker-entrypoint.sh to Use Arrays for
Configuration Parameters Instead of Eval and String Concatenation
dd78da0 is described below
commit dd78da0d9c4e5f081156e18aacf8a12b5ee0f7b7
Author: Junrui Lee
AuthorDate: Tue Mar 19 10:27:47 2024 +0800
[hotfix] Improve docker-entrypoint.sh to Use Arrays for Configuration
Parameters Instead of Eval and String Concatenation
This closes #188
---
docker-entrypoint.sh | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index e884375..e081109 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -65,19 +65,19 @@ set_config_options() {
local bin_dir="$FLINK_HOME/bin"
local lib_dir="$FLINK_HOME/lib"
-local config_params=""
+local config_params=()
while [ $# -gt 0 ]; do
local key="$1"
local value="$2"
-config_params+=" -D${key}=${value}"
+config_params+=("-D${key}=${value}")
shift 2
done
-if [ ! -z "${config_params}" ]; then
-eval "${config_parser_script} ${config_dir} ${bin_dir} ${lib_dir}
${config_params}"
+if [ "${#config_params[@]}" -gt 0 ]; then
+"${config_parser_script}" "${config_dir}" "${bin_dir}" "${lib_dir}"
"${config_params[@]}"
fi
}
(flink) branch release-1.19 updated: [FLINK-34716][release] Build 1.19 docs in GitHub Action and mark 1.19 as stable in docs
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new a6a4667202a [FLINK-34716][release] Build 1.19 docs in GitHub Action
and mark 1.19 as stable in docs
a6a4667202a is described below
commit a6a4667202a0f89fe63ff4f2e476c0200ec66e63
Author: lincoln lee
AuthorDate: Mon Mar 18 18:41:00 2024 +0800
[FLINK-34716][release] Build 1.19 docs in GitHub Action and mark 1.19 as
stable in docs
This closes #24518
---
.github/workflows/docs.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index a945fdd640e..eb966addd6d 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -28,9 +28,9 @@ jobs:
matrix:
branch:
- master
+ - release-1.19
- release-1.18
- release-1.17
- - release-1.16
steps:
- uses: actions/checkout@v3
with:
@@ -42,8 +42,8 @@ jobs:
echo "flink_branch=${currentBranch}" >> ${GITHUB_ENV}
if [ "${currentBranch}" = "master" ]; then
-echo "flink_alias=release-1.19" >> ${GITHUB_ENV}
- elif [ "${currentBranch}" = "release-1.18" ]; then
+echo "flink_alias=release-1.20" >> ${GITHUB_ENV}
+ elif [ "${currentBranch}" = "release-1.19" ]; then
echo "flink_alias=stable" >> ${GITHUB_ENV}
fi
- name: Build documentation
(flink-web) 01/02: Add announcement blog post for Flink 1.19 (#721)
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/flink-web.git
commit 46b47589e5a77906febed94a17e2a2ba879e1015
Author: lincoln lee
AuthorDate: Tue Mar 5 14:18:51 2024 +0800
Add announcement blog post for Flink 1.19 (#721)
---
docs/config.toml | 4 +-
docs/content/posts/2024-03-18-release-1.19.0.md| 501 +
docs/data/flink.yml| 10 +
docs/data/release_archive.yml | 6 +-
.../2024-03-18-release-1.19.0/minibatch_join.png | Bin 0 -> 249117 bytes
.../2024-03-18-release-1.19.0/profiling-res.png| Bin 0 -> 359154 bytes
.../blog/2024-03-18-release-1.19.0/profiling.png | Bin 0 -> 1798443 bytes
7 files changed, 518 insertions(+), 3 deletions(-)
diff --git a/docs/config.toml b/docs/config.toml
index 5b935421b..7b78b521f 100644
--- a/docs/config.toml
+++ b/docs/config.toml
@@ -43,8 +43,8 @@ posts = "/:year/:month/:day/:title/"
# 4. Copy the invitation link by clicking on "Copy invite link".
FlinkSlackInviteUrl =
"https://join.slack.com/t/apache-flink/shared_invite/zt-1t4khgllz-Fm1CnXzdBbUchBz4HzJCAg";
- FlinkStableVersion = "1.18.0"
- FlinkStableShortVersion = "1.18"
+ FlinkStableVersion = "1.19.0"
+ FlinkStableShortVersion = "1.19"
StateFunStableVersion = "3.3.0"
StateFunStableShortVersion = "3.3"
FlinkMLStableVersion = "2.3.0"
diff --git a/docs/content/posts/2024-03-18-release-1.19.0.md
b/docs/content/posts/2024-03-18-release-1.19.0.md
new file mode 100644
index 0..84bebb1d1
--- /dev/null
+++ b/docs/content/posts/2024-03-18-release-1.19.0.md
@@ -0,0 +1,501 @@
+---
+authors:
+- LincolnLee:
+ name: "Lincoln Lee"
+ twitter: lincoln_86xy
+
+date: "2024-03-18T08:00:00Z"
+subtitle: ""
+title: Announcing the Release of Apache Flink 1.19
+aliases:
+- /news/2024/03/18/release-1.19.0.html
+---
+
+The Apache Flink PMC is pleased to announce the release of Apache Flink
1.19.0. As usual, we are
+looking at a packed release with a wide variety of improvements and new
features. Overall, 162
+people contributed to this release completing 33 FLIPs and 600+ issues. Thank
you!
+
+Let's dive into the highlights.
+
+# Flink SQL Improvements
+
+## Custom Parallelism for Table/SQL Sources
+
+Now in Flink 1.19, you can set a custom parallelism for performance tuning via
the `scan.parallelism`
+option. The first available connector is DataGen (Kafka connector is on the
way). Here is an example
+using SQL Client:
+
+```sql
+-- set parallelism within the ddl
+CREATE TABLE Orders (
+order_number BIGINT,
+priceDECIMAL(32,2),
+buyerROW,
+order_time TIMESTAMP(3)
+) WITH (
+'connector' = 'datagen',
+'scan.parallelism' = '4'
+);
+
+-- or set parallelism via dynamic table option
+SELECT * FROM Orders /*+ OPTIONS('scan.parallelism'='4') */;
+```
+
+**More Information**
+*
[Documentation](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/dev/table/sourcessinks/#scan-table-source)
+* [FLIP-367: Support Setting Parallelism for Table/SQL
Sources](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=263429150)
+
+
+## Configurable SQL Gateway Java Options
+
+A new option `env.java.opts.sql-gateway` for specifying the Java options is
introduced in Flink 1.19,
+so you can fine-tune the memory settings, garbage collection behavior, and
other relevant Java
+parameters for SQL Gateway.
+
+**More Information**
+* [FLINK-33203](https://issues.apache.org/jira/browse/FLINK-33203)
+
+
+## Configure Different State TTLs Using SQL Hint
+
+Starting from Flink 1.18, Table API and SQL users can set state time-to-live
(TTL) individually for
+stateful operators via the SQL compiled plan. In Flink 1.19, users have a more
flexible way to
+specify custom TTL values for regular joins and group aggregations directly
within their queries by [utilizing the STATE_TTL
hint](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/dev/table/sql/queries/hints/#state-ttl-hints).
+This improvement means that you no longer need to alter your compiled plan to
set specific TTLs for
+these frequently used operators. With the introduction of `STATE_TTL` hints,
you can streamline your workflow and
+dynamically adjust the TTL based on your operational requirements.
+
+Here is an example:
+```sql
+-- set state ttl for join
+SELECT /*+ STATE_TTL('Orders'= '1d', 'Customers' = '20d') */ *
+FROM Orders LEFT OUTER JOIN Customers
+ON Orders.o_custkey = Customers.c_custkey;
+
+-- set state ttl for aggregation
+SELECT /*+ STATE_TTL('o' = '1d') */ o_orderkey, SUM(o_totalpr
(flink-web) branch asf-site updated (0fccd9ad3 -> f11ba16e6)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/flink-web.git from 0fccd9ad3 Rebuild website new 46b47589e Add announcement blog post for Flink 1.19 (#721) new f11ba16e6 Rebuild website The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../08/26/apache-flink-0.6-available/index.html| 10 +- .../09/26/apache-flink-0.6.1-available/index.html | 10 +- content/2014/10/03/upcoming-events/index.html | 10 +- .../11/04/apache-flink-0.7.0-available/index.html | 10 +- .../11/18/hadoop-compatibility-in-flink/index.html | 90 +- .../index.html | 10 +- .../01/21/apache-flink-0.8.0-available/index.html | 10 +- .../january-2015-in-the-flink-community/index.html | 10 +- .../02/09/introducing-flink-streaming/index.html | 584 ++--- .../index.html | 20 +- .../index.html | 10 +- .../march-2015-in-the-flink-community/index.html | 10 +- .../index.html | 10 +- .../05/11/juggling-with-bits-and-bytes/index.html | 20 +- .../april-2015-in-the-flink-community/index.html | 10 +- .../06/24/announcing-apache-flink-0.9.0/index.html | 10 +- .../index.html | 194 +- .../09/01/apache-flink-0.9.1-available/index.html | 10 +- .../09/03/announcing-flink-forward-2015/index.html | 10 +- .../index.html | 60 +- .../16/announcing-apache-flink-0.10.0/index.html | 10 +- .../2015/11/27/flink-0.10.1-released/index.html| 10 +- .../index.html | 10 +- .../index.html | 128 +- .../index.html | 10 +- .../2016/02/11/flink-0.10.2-released/index.html| 10 +- .../03/08/announcing-apache-flink-1.0.0/index.html | 10 +- content/2016/04/06/flink-1.0.1-released/index.html | 10 +- .../index.html | 156 +- .../index.html | 10 +- content/2016/04/22/flink-1.0.2-released/index.html | 10 +- content/2016/05/11/flink-1.0.3-released/index.html | 10 +- .../index.html | 10 +- .../08/04/announcing-apache-flink-1.1.0/index.html | 72 +- content/2016/08/04/flink-1.1.1-released/index.html | 10 +- .../index.html | 10 +- .../09/05/apache-flink-1.1.2-released/index.html | 10 +- .../10/12/apache-flink-1.1.3-released/index.html | 16 +- .../apache-flink-in-2016-year-in-review/index.html | 10 +- .../12/21/apache-flink-1.1.4-released/index.html | 20 +- .../02/06/announcing-apache-flink-1.2.0/index.html | 10 +- .../03/23/apache-flink-1.1.5-released/index.html | 10 +- .../index.html | 10 +- .../index.html | 10 +- .../04/26/apache-flink-1.2.1-released/index.html | 10 +- .../index.html | 10 +- .../index.html | 10 +- .../06/23/apache-flink-1.3.1-released/index.html | 10 +- .../index.html | 80 +- .../08/05/apache-flink-1.3.2-released/index.html | 10 +- .../index.html | 10 +- .../index.html | 10 +- .../apache-flink-in-2017-year-in-review/index.html | 10 +- .../index.html | 20 +- .../02/15/apache-flink-1.4.1-released/index.html | 10 +- .../index.html | 10 +- .../03/08/apache-flink-1.4.2-released/index.html | 10 +- .../03/15/apache-flink-1.3.3-released/index.html | 10 +- .../index.html | 10 +- .../07/12/apache-flink-1.5.1-released/index.html | 10 +- .../07/31/apache-flink-1.5.2-released/index.html | 10 +- .../index.html | 10 +- .../08/21/apache-flink-1.5.3-released/index.html | 10 +- .../09/20/apache-flink-1.5.4-released/index.html | 10 +- .../09/20/apache-flink-1.6.1-released/index.html | 10 +- .../10/29/apache-flink-1.5.5-released/index.html | 10 +- .../10/29/apache-flink-1.6.2-released/index.html | 10 +- .../index.html | 10 +- .../12/21/apache-flink-1.7.1-released/index.html | 10 +- .../12/22/apache-flink-1.6.3-released/index.html | 10 +- .../12/26/apache-flink-1.5.6-released/index.html | 1
(flink) branch release-1.19 updated: [FLINK-34593][release] Add release note for version 1.19
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 511814bbc44 [FLINK-34593][release] Add release note for version 1.19
511814bbc44 is described below
commit 511814bbc44ba0cc2a56147a29240445d571106b
Author: lincoln lee
AuthorDate: Mon Mar 18 14:57:02 2024 +0800
[FLINK-34593][release] Add release note for version 1.19
This closes #24505
---
docs/content.zh/_index.md | 3 +-
docs/content.zh/release-notes/flink-1.19.md | 362
docs/content/_index.md | 1 +
docs/content/release-notes/flink-1.19.md| 362
4 files changed, 727 insertions(+), 1 deletion(-)
diff --git a/docs/content.zh/_index.md b/docs/content.zh/_index.md
index f25de48eb0f..1516a960740 100644
--- a/docs/content.zh/_index.md
+++ b/docs/content.zh/_index.md
@@ -85,7 +85,8 @@ under the License.
For some reason Hugo will only allow linking to the
release notes if there is a leading '/' and file extension.
-->
-请参阅 [Flink 1.18]({{< ref "/release-notes/flink-1.18.md" >}}),
+请参阅 [Flink 1.19]({{< ref "/release-notes/flink-1.19.md" >}})，
+[Flink 1.18]({{< ref "/release-notes/flink-1.18.md" >}})，
[Flink 1.17]({{< ref "/release-notes/flink-1.17.md" >}})，
[Flink 1.16]({{< ref "/release-notes/flink-1.16.md" >}})，
[Flink 1.15]({{< ref "/release-notes/flink-1.15.md" >}})，
diff --git a/docs/content.zh/release-notes/flink-1.19.md
b/docs/content.zh/release-notes/flink-1.19.md
new file mode 100644
index 000..efb5a3228b6
--- /dev/null
+++ b/docs/content.zh/release-notes/flink-1.19.md
@@ -0,0 +1,362 @@
+---
+title: "Release Notes - Flink 1.19"
+---
+
+
+# Release notes - Flink 1.19
+
+These release notes discuss important aspects, such as configuration, behavior
or dependencies,
+that changed between Flink 1.18 and Flink 1.19. Please read these notes
carefully if you are
+planning to upgrade your Flink version to 1.19.
+
+## Dependency upgrades
+
+ Drop support for python 3.7
+
+# [FLINK-33029](https://issues.apache.org/jira/browse/FLINK-33029)
+
+ Add support for python 3.11
+
+# [FLINK-33030](https://issues.apache.org/jira/browse/FLINK-33030)
+
+## Build System
+
+ Support Java 21
+
+# [FLINK-33163](https://issues.apache.org/jira/browse/FLINK-33163)
+Apache Flink was made ready to compile and run with Java 21. This feature is
still in beta mode.
+Issues should be reported in Flink's bug tracker.
+
+## Checkpoints
+
+ Deprecate RestoreMode#LEGACY
+
+# [FLINK-34190](https://issues.apache.org/jira/browse/FLINK-34190)
+
+`RestoreMode#LEGACY` is deprecated. Please use `RestoreMode#CLAIM` or
`RestoreMode#NO_CLAIM` mode
+instead to get a clear state file ownership when restoring.
+
+ CheckpointsCleaner clean individual checkpoint states in parallel
+
+# [FLINK-33090](https://issues.apache.org/jira/browse/FLINK-33090)
+
+Now when disposing of no longer needed checkpoints, every state handle/state
file will be disposed
+in parallel by the ioExecutor, vastly improving the disposing speed of a
single checkpoint (for
+large checkpoints, the disposal time can be improved from 10 minutes to < 1
minute). The old
+behavior can be restored by setting `state.checkpoint.cleaner.parallel-mode`
to false.
+
+ Support using larger checkpointing interval when source is processing
backlog
+
+# [FLINK-32514](https://issues.apache.org/jira/browse/FLINK-32514)
+
+`ProcessingBacklog` is introduced to demonstrate whether a record should be
processed with low latency
+or high throughput. `ProcessingBacklog` can be set by source operators and can
be used to change the
+checkpoint interval of a job during runtime.
+
+ Allow triggering Checkpoints through command line client
+
+# [FLINK-6755](https://issues.apache.org/jira/browse/FLINK-6755)
+
+The command line interface supports triggering a checkpoint manually. Usage:
+```
+./bin/flink checkpoint $JOB_ID [-full]
+```
+By specifying the '-full' option, a full checkpoint is triggered. Otherwise an
incremental
+checkpoint is triggered if the job is configured to take incremental ones
periodically.
+
+
+## Runtime & Coordination
+
+ Migrate TypeSerializerSnapshot#resolveSchemaCompatibility
+
+# [FLINK-30613](https://issues.apache.org/jira/browse/FLINK-30613)
+
+In Flink 1.19, the old method of resolving schema compatibility has been
deprecated and the new one
+is introduced. See
[FLIP-263](https://cwiki.apache.org/confluence/display/FLINK/FLIP-263%3A+Improve+resolving+schema+compatibility?src=contextnavpagetreemode)
for more details.
+Please migrate to the new method following
[link](http
(flink-docker) branch dev-1.19 updated: Add GPG key for 1.19.0 release
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch dev-1.19 in repository https://gitbox.apache.org/repos/asf/flink-docker.git The following commit(s) were added to refs/heads/dev-1.19 by this push: new 7549f2d Add GPG key for 1.19.0 release 7549f2d is described below commit 7549f2d044a10d0af43c5087f7264ac379e4607e Author: lincoln lee AuthorDate: Mon Mar 18 14:56:23 2024 +0800 Add GPG key for 1.19.0 release This closes #187 --- add-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/add-version.sh b/add-version.sh index 1e511fc..c79e142 100755 --- a/add-version.sh +++ b/add-version.sh @@ -102,6 +102,8 @@ elif [ "$flink_version" = "1.17.0" ]; then gpg_key="A1BD477F79D036D2C30CA7DBCA8AEEC2F6EB040B" elif [ "$flink_version" = "1.18.0" ]; then gpg_key="96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82" +elif [ "$flink_version" = "1.19.0" ]; then +gpg_key="028B6605F51BC296B56A5042E57D30ABEE75CA06" else error "Missing GPG key ID for this release" fi
(flink-docker) branch master updated: Update Dockerfiles for 1.19.0 release
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-docker.git The following commit(s) were added to refs/heads/master by this push: new 206aa67 Update Dockerfiles for 1.19.0 release 206aa67 is described below commit 206aa670a6f70dee91ba4474768078450a263958 Author: lincoln lee AuthorDate: Sun Mar 17 00:35:36 2024 +0800 Update Dockerfiles for 1.19.0 release This closes #186 --- 1.19/scala_2.12-java11-ubuntu/Dockerfile | 105 1.19/scala_2.12-java11-ubuntu/docker-entrypoint.sh | 190 + 1.19/scala_2.12-java11-ubuntu/release.metadata | 2 + 1.19/scala_2.12-java17-ubuntu/Dockerfile | 105 1.19/scala_2.12-java17-ubuntu/docker-entrypoint.sh | 190 + 1.19/scala_2.12-java17-ubuntu/release.metadata | 2 + 1.19/scala_2.12-java8-ubuntu/Dockerfile| 105 1.19/scala_2.12-java8-ubuntu/docker-entrypoint.sh | 190 + 1.19/scala_2.12-java8-ubuntu/release.metadata | 2 + 9 files changed, 891 insertions(+) diff --git a/1.19/scala_2.12-java11-ubuntu/Dockerfile b/1.19/scala_2.12-java11-ubuntu/Dockerfile new file mode 100644 index 000..b8e946d --- /dev/null +++ b/1.19/scala_2.12-java11-ubuntu/Dockerfile @@ -0,0 +1,105 @@ +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +FROM eclipse-temurin:11-jre-jammy + +# Install dependencies +RUN set -ex; \ + apt-get update; \ + apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \ + rm -rf /var/lib/apt/lists/* + +# Grab gosu for easy step-down from root +ENV GOSU_VERSION 1.11 +RUN set -ex; \ + wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ + wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ + done && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + chmod +x /usr/local/bin/gosu; \ + gosu nobody true + +# Configure Flink version +ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.19.0/flink-1.19.0-bin-scala_2.12.tgz \ + FLINK_ASC_URL=https://downloads.apache.org/flink/flink-1.19.0/flink-1.19.0-bin-scala_2.12.tgz.asc \ +GPG_KEY=028B6605F51BC296B56A5042E57D30ABEE75CA06 \ +CHECK_GPG=true + +# Prepare environment +ENV FLINK_HOME=/opt/flink +ENV PATH=$FLINK_HOME/bin:$PATH +RUN groupadd --system --gid= flink && \ +useradd --system --home-dir $FLINK_HOME --uid= --gid=flink flink +WORKDIR $FLINK_HOME + +# Install Flink +RUN set -ex; \ + wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ + \ + if [ "$CHECK_GPG" = "true" ]; then \ +wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ +export GNUPGHOME="$(mktemp -d)"; \ +for server in ha.pool.sks-keyservers.net $(shuf -e \ +hkp://p80.pool.sks-keyservers.net:80 \ +keyserver.ubuntu.com \ +hkp://keyserver.ubuntu.com:80 \ +pgp.mit.edu) ; do \ +gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ +done && \ +gpg --batch --verify flink.
(flink-docker) branch dev-master updated: Add GPG key for 1.19.0 release
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch dev-master in repository https://gitbox.apache.org/repos/asf/flink-docker.git The following commit(s) were added to refs/heads/dev-master by this push: new 66a2fdf Add GPG key for 1.19.0 release 66a2fdf is described below commit 66a2fdf74e186844b058eebe4dd9530e97b00337 Author: lincoln lee AuthorDate: Sat Mar 16 23:26:21 2024 +0800 Add GPG key for 1.19.0 release This closes #185 --- add-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/add-version.sh b/add-version.sh index 1e511fc..c79e142 100755 --- a/add-version.sh +++ b/add-version.sh @@ -102,6 +102,8 @@ elif [ "$flink_version" = "1.17.0" ]; then gpg_key="A1BD477F79D036D2C30CA7DBCA8AEEC2F6EB040B" elif [ "$flink_version" = "1.18.0" ]; then gpg_key="96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82" +elif [ "$flink_version" = "1.19.0" ]; then +gpg_key="028B6605F51BC296B56A5042E57D30ABEE75CA06" else error "Missing GPG key ID for this release" fi
(flink) branch master updated: [FLINK-34593][release] Add release note for version 1.19
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 875a431950d [FLINK-34593][release] Add release note for version 1.19
875a431950d is described below
commit 875a431950da066aa696f2c3a335f3267b85194a
Author: lincoln lee
AuthorDate: Fri Mar 15 17:37:16 2024 +0800
[FLINK-34593][release] Add release note for version 1.19
This closes #24394
---
docs/content.zh/_index.md | 3 +-
docs/content.zh/release-notes/flink-1.19.md | 362
docs/content/_index.md | 1 +
docs/content/release-notes/flink-1.19.md| 362
4 files changed, 727 insertions(+), 1 deletion(-)
diff --git a/docs/content.zh/_index.md b/docs/content.zh/_index.md
index f25de48eb0f..1516a960740 100644
--- a/docs/content.zh/_index.md
+++ b/docs/content.zh/_index.md
@@ -85,7 +85,8 @@ under the License.
For some reason Hugo will only allow linking to the
release notes if there is a leading '/' and file extension.
-->
-请参阅 [Flink 1.18]({{< ref "/release-notes/flink-1.18.md" >}}),
+请参阅 [Flink 1.19]({{< ref "/release-notes/flink-1.19.md" >}})，
+[Flink 1.18]({{< ref "/release-notes/flink-1.18.md" >}})，
[Flink 1.17]({{< ref "/release-notes/flink-1.17.md" >}})，
[Flink 1.16]({{< ref "/release-notes/flink-1.16.md" >}})，
[Flink 1.15]({{< ref "/release-notes/flink-1.15.md" >}})，
diff --git a/docs/content.zh/release-notes/flink-1.19.md
b/docs/content.zh/release-notes/flink-1.19.md
new file mode 100644
index 000..efb5a3228b6
--- /dev/null
+++ b/docs/content.zh/release-notes/flink-1.19.md
@@ -0,0 +1,362 @@
+---
+title: "Release Notes - Flink 1.19"
+---
+
+
+# Release notes - Flink 1.19
+
+These release notes discuss important aspects, such as configuration, behavior
or dependencies,
+that changed between Flink 1.18 and Flink 1.19. Please read these notes
carefully if you are
+planning to upgrade your Flink version to 1.19.
+
+## Dependency upgrades
+
+ Drop support for python 3.7
+
+# [FLINK-33029](https://issues.apache.org/jira/browse/FLINK-33029)
+
+ Add support for python 3.11
+
+# [FLINK-33030](https://issues.apache.org/jira/browse/FLINK-33030)
+
+## Build System
+
+ Support Java 21
+
+# [FLINK-33163](https://issues.apache.org/jira/browse/FLINK-33163)
+Apache Flink was made ready to compile and run with Java 21. This feature is
still in beta mode.
+Issues should be reported in Flink's bug tracker.
+
+## Checkpoints
+
+ Deprecate RestoreMode#LEGACY
+
+# [FLINK-34190](https://issues.apache.org/jira/browse/FLINK-34190)
+
+`RestoreMode#LEGACY` is deprecated. Please use `RestoreMode#CLAIM` or
`RestoreMode#NO_CLAIM` mode
+instead to get a clear state file ownership when restoring.
+
+ CheckpointsCleaner clean individual checkpoint states in parallel
+
+# [FLINK-33090](https://issues.apache.org/jira/browse/FLINK-33090)
+
+Now when disposing of no longer needed checkpoints, every state handle/state
file will be disposed
+in parallel by the ioExecutor, vastly improving the disposing speed of a
single checkpoint (for
+large checkpoints, the disposal time can be improved from 10 minutes to < 1
minute). The old
+behavior can be restored by setting `state.checkpoint.cleaner.parallel-mode`
to false.
+
+ Support using larger checkpointing interval when source is processing
backlog
+
+# [FLINK-32514](https://issues.apache.org/jira/browse/FLINK-32514)
+
+`ProcessingBacklog` is introduced to demonstrate whether a record should be
processed with low latency
+or high throughput. `ProcessingBacklog` can be set by source operators and can
be used to change the
+checkpoint interval of a job during runtime.
+
+ Allow triggering Checkpoints through command line client
+
+# [FLINK-6755](https://issues.apache.org/jira/browse/FLINK-6755)
+
+The command line interface supports triggering a checkpoint manually. Usage:
+```
+./bin/flink checkpoint $JOB_ID [-full]
+```
+By specifying the '-full' option, a full checkpoint is triggered. Otherwise an
incremental
+checkpoint is triggered if the job is configured to take incremental ones
periodically.
+
+
+## Runtime & Coordination
+
+ Migrate TypeSerializerSnapshot#resolveSchemaCompatibility
+
+# [FLINK-30613](https://issues.apache.org/jira/browse/FLINK-30613)
+
+In Flink 1.19, the old method of resolving schema compatibility has been
deprecated and the new one
+is introduced. See
[FLIP-263](https://cwiki.apache.org/confluence/display/FLINK/FLIP-263%3A+Improve+resolving+schema+compatibility?src=contextnavpagetreemode)
for more details.
+Please migrate to the new method following
[link](https://nightli
(flink) annotated tag release-1.19.0 updated (eaffd227d85 -> 518023aa69e)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to annotated tag release-1.19.0 in repository https://gitbox.apache.org/repos/asf/flink.git *** WARNING: tag release-1.19.0 was modified! *** from eaffd227d85 (commit) to 518023aa69e (tag) tagging eaffd227d853e0cdef03f1af5016e00f950930a9 (commit) replaces pre-apache-rename by lincoln lee on Fri Mar 15 17:03:07 2024 +0800 - Log - Release Flink 1.19.0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEEAotmBfUbwpa1alBC5X0wq+51ygYFAmX0DssACgkQ5X0wq+51 ygYlKRAAiyAf6JuuLwaUrdBmXc5BawfuT3LMSD28VQc1ehOnyAHu2idMPW8HCM6M EGtFyBKAS763GsAz3G5xMjIbyRl0fFhhSIBe7hTxPMY15apTCCgQveeFEyLia9Ni WMDT1dZ81qllhdaBlLEeEclxNNftfso1hsP0JZJktuBSJPoNI0hnrqVs+H/zGt1Q TW5qFN1CIDNPetTuW9dF24GtzC4tgYaTW0YAE8uy1clSCVQR3bbK2mm0R29SgDgM kQrkBF3U/2366Wb18+g1OBiuMWVnK9teXV5QDUMXCYyaIIQebH0XfVPjHoRe0KGz LaDrurMuUrbwzwzqlgxY/fv2O3Tvg74pnYsuqNy6ZPylGvuCZUsfc1UWX/HmUZ43 rx8nuaVoKss1rGOhchUH6BOWIABCLHFcNWVAZEvMt/RaFvSZ93GaZ9nSL9kP6a/l NdPpYSUzE6f3cSYImerKfvvr+Y+MKv6R228rUPmkyPt5nbFL7Q1RkR+RJsZ/YrOL b2NvALDuzB9cHM154BXtR3en5Z+tNO5mTBOEgrAzIKG5zF74N1fSjO64a4os5Mm+ Ls+MySQjE/58Y9CyPvpFS/SFwcvoxFATD9OcEBJW1A2him4j65Cjxiz+lSDpu9pp AET8g+ZTUAXEPksLlD1KyPL1BtWH/zf9vYqEum3UTJdNAcE+VMI= =rE1A -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
(flink) annotated tag release-1.19.0-rc2 updated (eaffd227d85 -> 4bf5901d42b)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to annotated tag release-1.19.0-rc2 in repository https://gitbox.apache.org/repos/asf/flink.git *** WARNING: tag release-1.19.0-rc2 was modified! *** from eaffd227d85 (commit) to 4bf5901d42b (tag) tagging eaffd227d853e0cdef03f1af5016e00f950930a9 (commit) replaces pre-apache-rename by lincoln lee on Wed Mar 6 22:18:54 2024 +0800 - Log - release-1.19.0-rc2 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEEAotmBfUbwpa1alBC5X0wq+51ygYFAmXoe04ACgkQ5X0wq+51 ygbLXRAAhIYYjOxXVyZH+UgTIFBlCahoGBHMMIm+0J2FvsFO1clzCwtakMnpVpTM PCQ+bNZeUWQDichwqE6q7rSR3nTrCwsePehHONhqtzNVFp3DpUofAExvRvsE+PDZ +uFIXsKS8S0M350x4rn2mIGOBFJVjqLb6Si2ybyqEYbwZI7nNpm3yMAMQgfrzNXX wMjAKNcsjbyM6sEDPSZWuvLj2EGhNXGAddU55ngoqcAObo48spRVaVzm62scsenQ z9nsl3hRFX57kukVbLGz/tHfU9yYKqI+PZN3y2yjlmDmQ5ndMS6eyuFpt1VUYsSV DzvfrPz6Px2DyfsMYjVo9BEtB/uKA3cl/Bzgs7+ZmKhM7cEh0XlK0M0RgvnshsEy dTz4GoQTJ8fCXEAtXcMMVPJlKw7ElkDeZgw11V49gNVPjCMJk8KhAEDvKVIApFhs NNw6ZqLjNjcy5Dhop8QbkO2DRz8S2Of4UdjhpS4n7/OSBaBdPA3NTUD17KeACSqj fkOLDbaJ7iTHwDxGRtzjoD82u0UFrCOYNjQBpKGsKESr9EnomCbvz7McBqOkof1F Ff7k8xeDltSnHCwbpOyoobKye5az+w1gQcSa7kbbRBm718RZE8GIn9/nSVfSpQgG L/CAbrTT57BYvJDCkxypbCOEq34nKofkEq2+XpsCMlfEJQEHCJk= =ipFL -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r67774 - in /dev/flink/flink-1.19.0-rc2: ./ python/
Author: lincoln Date: Thu Mar 7 09:41:35 2024 New Revision: 67774 Log: Add flink-1.19.0-rc2 Added: dev/flink/flink-1.19.0-rc2/ dev/flink/flink-1.19.0-rc2/flink-1.19.0-bin-scala_2.12.tgz (with props) dev/flink/flink-1.19.0-rc2/flink-1.19.0-bin-scala_2.12.tgz.asc (with props) dev/flink/flink-1.19.0-rc2/flink-1.19.0-bin-scala_2.12.tgz.sha512 dev/flink/flink-1.19.0-rc2/flink-1.19.0-src.tgz (with props) dev/flink/flink-1.19.0-rc2/flink-1.19.0-src.tgz.asc (with props) dev/flink/flink-1.19.0-rc2/flink-1.19.0-src.tgz.sha512 dev/flink/flink-1.19.0-rc2/python/ dev/flink/flink-1.19.0-rc2/python/apache-flink-1.19.0.tar.gz (with props) dev/flink/flink-1.19.0-rc2/python/apache-flink-1.19.0.tar.gz.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache-flink-1.19.0.tar.gz.sha512 dev/flink/flink-1.19.0-rc2/python/apache-flink-libraries-1.19.0.tar.gz (with props) dev/flink/flink-1.19.0-rc2/python/apache-flink-libraries-1.19.0.tar.gz.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache-flink-libraries-1.19.0.tar.gz.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc2/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl.sha512 Added: dev/flink/flink-1.19.0-rc2/flink-1.19.0-bin
svn commit: r67758 - /dev/flink/flink-1.19.0-rc1/
Author: lincoln Date: Wed Mar 6 16:16:20 2024 New Revision: 67758 Log: Remove flink-1.19.0-rc1 Removed: dev/flink/flink-1.19.0-rc1/
svn commit: r67716 - in /dev/flink/flink-1.19.0-rc1: ./ python/
Author: lincoln Date: Tue Mar 5 04:10:00 2024 New Revision: 67716 Log: Add flink-1.19.0-rc1 Added: dev/flink/flink-1.19.0-rc1/ dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz (with props) dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz.asc (with props) dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz.sha512 dev/flink/flink-1.19.0-rc1/flink-1.19.0-src.tgz (with props) dev/flink/flink-1.19.0-rc1/flink-1.19.0-src.tgz.asc (with props) dev/flink/flink-1.19.0-rc1/flink-1.19.0-src.tgz.sha512 dev/flink/flink-1.19.0-rc1/python/ dev/flink/flink-1.19.0-rc1/python/apache-flink-1.19.0.tar.gz (with props) dev/flink/flink-1.19.0-rc1/python/apache-flink-1.19.0.tar.gz.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache-flink-1.19.0.tar.gz.sha512 dev/flink/flink-1.19.0-rc1/python/apache-flink-libraries-1.19.0.tar.gz (with props) dev/flink/flink-1.19.0-rc1/python/apache-flink-libraries-1.19.0.tar.gz.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache-flink-libraries-1.19.0.tar.gz.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp310-cp310-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp311-cp311-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp38-cp38-manylinux1_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-macosx_11_0_arm64.whl.sha512 dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl.asc (with props) dev/flink/flink-1.19.0-rc1/python/apache_flink-1.19.0-cp39-cp39-manylinux1_x86_64.whl.sha512 Added: dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz == Binary file - no diff available. Propchange: dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz -- svn:mime-type = application/x-gzip Added: dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz.asc == Binary file - no diff available. Propchange: dev/flink/flink-1.19.0-rc1/flink-1.19.0-bin-scala_2.12.tgz.asc
(flink) annotated tag release-1.19.0-rc1 updated (8f94b337772 -> 527d8284586)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to annotated tag release-1.19.0-rc1 in repository https://gitbox.apache.org/repos/asf/flink.git *** WARNING: tag release-1.19.0-rc1 was modified! *** from 8f94b337772 (commit) to 527d8284586 (tag) tagging 8f94b33777249ffd09f06d29059ee34d87511734 (commit) replaces pre-apache-rename by lincoln lee on Tue Mar 5 00:25:20 2024 +0800 - Log - release-1.19.0-rc1 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEEAotmBfUbwpa1alBC5X0wq+51ygYFAmXl9fAACgkQ5X0wq+51 ygaLAw/7BXtEk60eGd2/KWNv640wbyMiLyxYs5cOvYvwa0GX4Hy9hns8104nTTWV KYHudBOvYQP5Ox9zLzKvRpx5iYqXmEOLFGMolrzEXEHsazcwfJB6bzI2N2jixOCG YCvgGkCtGruB9rHXcOghawBjT5fGR4d/PaOxnnAOzR/r6HUbvj/CoQ/4XMGG1eLq SUivv01aTCzF9TpP59D6vEOFaG4oyRHMXhFPcoHWVBlqqtdTRTFfyPtvLp1Do6T+ VsB24G3A+gIHKNwI/EyyfAnjfd96y6wTtolGcNKD/NQwfccYCnJm3Fmaf2/Syp2n BOzZfVQS1JAqxL7QIFfof0K2WnQg99vsmo5odB+ye2cLKyZ+CYY0q7NggCZDckAB 0/g7VzDnJChSavHDr+Db3k0d/rHmcu2vN6rpygy64eImNxNwAz6d1R2SZT9t5A6p hQIjVYcCQEpR0sWAsX8olY5F/ata/qLuky0w0xoRzK7hzJecGjmQC16VTe80K/C3 ICLARF+NJAJmTs72gP27PIFsU4eKHsVsQklJhbLNRKiUsCOPXaR1E9Jue5E9gQnC rna50sWbvhwCmBiJqbSgKPwZjMXcqNmyTLaX0du1E5hqJDPBNbs9sSHDoBPMPePm jsHEqJ0RSem/rLq9DYNuIFic4FAXZDTPVztSURdIoDoeDNK0jt0= =s3zq -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
(flink-docker) branch dev-master updated: [FLINK-34282] Update dev-master point to 1.20 (#178)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch dev-master in repository https://gitbox.apache.org/repos/asf/flink-docker.git The following commit(s) were added to refs/heads/dev-master by this push: new bb53713 [FLINK-34282] Update dev-master point to 1.20 (#178) bb53713 is described below commit bb53713d836121839b96a2e2eeb97a7bd32d0407 Author: lincoln lee AuthorDate: Thu Feb 8 08:48:17 2024 +0800 [FLINK-34282] Update dev-master point to 1.20 (#178) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a486206..6e45a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v3 - name: "Build images" run: | - ./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.19-SNAPSHOT-bin-scala_2.12.tgz"; -j 8 -n test-java8 - ./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.19-SNAPSHOT-bin-scala_2.12.tgz"; -j 11 -n test-java11 + ./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.20-SNAPSHOT-bin-scala_2.12.tgz"; -j 8 -n test-java8 + ./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.20-SNAPSHOT-bin-scala_2.12.tgz"; -j 11 -n test-java11 - name: "Test images" run: testing/run_tests.sh
(flink-benchmarks) branch master updated: [FLINK-34282] Upgrade Flink version to 1.20-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-benchmarks.git The following commit(s) were added to refs/heads/master by this push: new 6e6e1fc [FLINK-34282] Upgrade Flink version to 1.20-SNAPSHOT 6e6e1fc is described below commit 6e6e1fcb168e2ef5dad60224f162f2977e43a7a3 Author: lincoln.lil AuthorDate: Wed Feb 7 14:05:59 2024 +0800 [FLINK-34282] Upgrade Flink version to 1.20-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 23162a6..3b8a8e1 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ under the License. UTF-8 - 1.19-SNAPSHOT + 1.20-SNAPSHOT 16.1 2.0.54.Final 1.8
(flink-benchmarks) branch dev-1.19 created (now fc87459)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch dev-1.19 in repository https://gitbox.apache.org/repos/asf/flink-benchmarks.git at fc87459 [FLINK-34164] Fix compilation issue (#85) No new revisions were added by this update.
(flink-docker) branch dev-1.19 created (now 59f0b72)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch dev-1.19 in repository https://gitbox.apache.org/repos/asf/flink-docker.git at 59f0b72 [FLINK-34282] Initialize dev-1.19 branch This branch includes the following new commits: new 59f0b72 [FLINK-34282] Initialize dev-1.19 branch The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(flink-docker) 01/01: [FLINK-34282] Initialize dev-1.19 branch
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a commit to branch dev-1.19 in repository https://gitbox.apache.org/repos/asf/flink-docker.git commit 59f0b7261f93ad6cbf58cb4ca93297b6c4c960ba Author: lincoln.lil AuthorDate: Wed Feb 7 12:22:22 2024 +0800 [FLINK-34282] Initialize dev-1.19 branch --- .github/workflows/snapshot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 1d7d53a..bcbb079 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -39,7 +39,7 @@ jobs: java_version: [8, 11] build: - flink_version: 1.19-SNAPSHOT -branch: dev-master +branch: dev-1.19 - flink_version: 1.18-SNAPSHOT branch: dev-1.18 - flink_version: 1.17-SNAPSHOT
(flink) branch master updated (03b45844228 -> b8b2596a1e4)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 03b45844228 [FLINK-34337][Core] Sink.InitContextWrapper should implement metadataConsumer method. This closes #24249 add bf5068ce920 Update version to 1.20-SNAPSHOT add b8b2596a1e4 Add new version 1.20 and update GHA nightly build versions No new revisions were added by this update. Summary of changes: .github/workflows/nightly-trigger.yml | 2 +- docs/config.toml | 4 ++-- flink-annotations/pom.xml | 2 +- flink-annotations/src/main/java/org/apache/flink/FlinkVersion.java| 3 ++- flink-architecture-tests/flink-architecture-tests-base/pom.xml| 2 +- flink-architecture-tests/flink-architecture-tests-production/pom.xml | 2 +- flink-architecture-tests/flink-architecture-tests-test/pom.xml| 2 +- flink-architecture-tests/pom.xml | 2 +- flink-clients/pom.xml | 2 +- flink-connectors/flink-connector-base/pom.xml | 2 +- flink-connectors/flink-connector-datagen-test/pom.xml | 2 +- flink-connectors/flink-connector-datagen/pom.xml | 2 +- flink-connectors/flink-connector-files/pom.xml| 2 +- flink-connectors/flink-connector-hive/pom.xml | 2 +- flink-connectors/flink-file-sink-common/pom.xml | 2 +- flink-connectors/flink-hadoop-compatibility/pom.xml | 2 +- flink-connectors/flink-sql-connector-hive-2.3.9/pom.xml | 2 +- flink-connectors/flink-sql-connector-hive-3.1.3/pom.xml | 2 +- flink-connectors/pom.xml | 2 +- flink-container/pom.xml | 2 +- flink-contrib/flink-connector-wikiedits/pom.xml | 2 +- flink-contrib/pom.xml | 2 +- flink-core/pom.xml| 2 +- flink-dist-scala/pom.xml | 2 +- flink-dist/pom.xml| 2 +- flink-docs/pom.xml| 2 +- flink-dstl/flink-dstl-dfs/pom.xml | 2 +- flink-dstl/pom.xml| 2 +- flink-end-to-end-tests/flink-batch-sql-test/pom.xml | 2 +- flink-end-to-end-tests/flink-cli-test/pom.xml | 2 +- flink-end-to-end-tests/flink-confluent-schema-registry/pom.xml| 2 +- flink-end-to-end-tests/flink-dataset-allround-test/pom.xml| 2 +- .../flink-dataset-fine-grained-recovery-test/pom.xml | 2 +- flink-end-to-end-tests/flink-datastream-allround-test/pom.xml | 2 +- flink-end-to-end-tests/flink-distributed-cache-via-blob-test/pom.xml | 2 +- flink-end-to-end-tests/flink-end-to-end-tests-common/pom.xml | 2 +- flink-end-to-end-tests/flink-end-to-end-tests-hive/pom.xml| 2 +- flink-end-to-end-tests/flink-end-to-end-tests-jdbc-driver/pom.xml | 2 +- flink-end-to-end-tests/flink-end-to-end-tests-restclient/pom.xml | 2 +- flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml | 2 +- flink-end-to-end-tests/flink-end-to-end-tests-sql/pom.xml | 2 +- flink-end-to-end-tests/flink-failure-enricher-test/pom.xml| 2 +- flink-end-to-end-tests/flink-file-sink-test/pom.xml | 2 +- flink-end-to-end-tests/flink-heavy-deployment-stress-test/pom.xml | 2 +- flink-end-to-end-tests/flink-high-parallelism-iterations-test/pom.xml | 2 +- .../flink-local-recovery-and-allocation-test/pom.xml | 2 +- flink-end-to-end-tests/flink-metrics-availability-test/pom.xml| 2 +- flink-end-to-end-tests/flink-metrics-reporter-prometheus-test/pom.xml | 2 +- .../flink-netty-shuffle-memory-control-test/pom.xml | 2 +- .../flink-parent-child-classloading-test-lib-package/pom.xml | 2 +- .../flink-parent-child-classloading-test-program/pom.xml | 2 +- flink-end-to-end-tests/flink-plugins-test/another-dummy-fs/pom.xml| 2 +- flink-end-to-end-tests/flink-plugins-test/dummy-fs/pom.xml| 2 +- flink-end-to-end-tests/flink-plugins-test/pom.xml | 2 +- flink-end-to-end-tests/flink-python-test/pom.xml | 2 +- flink-end-to-end-tests/flink-queryable-state-test/pom.xml | 2 +- flink-end-to-end-tests/flink-quickstart-test-dummy-dependency/pom.xml | 2 +- flink-end-to-end-tests/flink-quickstart-test
(flink) branch release-1.19 created (now 3d67a2b9b20)
This is an automated email from the ASF dual-hosted git repository. lincoln pushed a change to branch release-1.19 in repository https://gitbox.apache.org/repos/asf/flink.git at 3d67a2b9b20 Update for 1.19.0 This branch includes the following new commits: new 3d67a2b9b20 Update for 1.19.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(flink) 01/01: Update for 1.19.0
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3d67a2b9b208ed662f81c1ef43d76d10a82a7d88
Author: lincoln lee
AuthorDate: Wed Feb 7 10:15:20 2024 +0800
Update for 1.19.0
---
docs/config.toml | 23 +++---
.../test-scripts/common_docker.sh | 2 +-
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/docs/config.toml b/docs/config.toml
index 3776b6f895d..e5d14e47037 100644
--- a/docs/config.toml
+++ b/docs/config.toml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-baseURL = '//nightlies.apache.org/flink/flink-docs-master'
+baseURL = '//nightlies.apache.org/flink/flink-docs-release-1.19'
languageCode = "en-us"
title = "Apache Flink"
enableGitInfo = false
@@ -24,7 +24,7 @@ pygmentsUseClasses = true
[params]
# Flag whether this is a stable version or not.
# Used for the quickstart page.
- IsStable = false
+ IsStable = true
# Flag to indicate whether an outdated warning should be shown.
ShowOutDatedWarning = false
@@ -34,14 +34,14 @@ pygmentsUseClasses = true
# we change the version for the complete docs when forking of a release
branch
# etc.
# The full version string as referenced in Maven (e.g. 1.2.1)
- Version = "1.19-SNAPSHOT"
+ Version = "1.19.0"
# For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot
# release this should be the same as the regular version
- VersionTitle = "1.19-SNAPSHOT"
+ VersionTitle = "1.19"
# The branch for this version of Apache Flink
- Branch = "master"
+ Branch = "release-1.19"
# The github repository for Apache Flink
Repo = "//github.com/apache/flink"
@@ -60,22 +60,23 @@ pygmentsUseClasses = true
ZhDownloadPage = "//flink.apache.org/zh/downloads.html"
- JavaDocs = "//nightlies.apache.org/flink/flink-docs-master/api/java/"
+ JavaDocs = "//nightlies.apache.org/flink/flink-docs-release-1.19/api/java/"
- ScalaDocs =
"//nightlies.apache.org/flink/flink-docs-master/api/scala/index.html#org.apache.flink.api.scala.package"
+ ScalaDocs =
"//nightlies.apache.org/flink/flink-docs-release-1.19/api/scala/index.html#org.apache.flink.api.scala.package"
- PyDocs = "//nightlies.apache.org/flink/flink-docs-master/api/python/"
+ PyDocs = "//nightlies.apache.org/flink/flink-docs-release-1.19/api/python/"
# External links at the bottom
# of the menu
MenuLinks = [
["Project Homepage", "//flink.apache.org"],
-["JavaDocs", "//nightlies.apache.org/flink/flink-docs-master/api/java/"],
-["ScalaDocs",
"//nightlies.apache.org/flink/flink-docs-master/api/scala/index.html#org.apache.flink.api.scala.package"],
-["PyDocs", "//nightlies.apache.org/flink/flink-docs-master/api/python/"]
+["JavaDocs",
"//nightlies.apache.org/flink/flink-docs-release-1.19/api/java/"],
+["ScalaDocs",
"//nightlies.apache.org/flink/flink-docs-release-1.19/api/scala/index.html#org.apache.flink.api.scala.package/"],
+["PyDocs",
"//nightlies.apache.org/flink/flink-docs-release-1.19/api/python/"]
]
PreviousDocs = [
+["1.19", "http://nightlies.apache.org/flink/flink-docs-release-1.19";],
["1.18", "http://nightlies.apache.org/flink/flink-docs-release-1.18";],
["1.17", "http://nightlies.apache.org/flink/flink-docs-release-1.17";],
["1.16", "http://nightlies.apache.org/flink/flink-docs-release-1.16";],
diff --git a/flink-end-to-end-tests/test-scripts/common_docker.sh
b/flink-end-to-end-tests/test-scripts/common_docker.sh
index 2ce590cd424..75459e55229 100644
--- a/flink-end-to-end-tests/test-scripts/common_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/common_docker.sh
@@ -48,7 +48,7 @@ function build_image() {
local server_pid=$!
echo "Preparing Dockeriles"
-retry_times_with_exponential_backoff 5 git clone
https://github.com/apache/flink-docker.git --branch dev-master --single-branch
+retry_times_with_exponential_backoff 5 git clone
https://github.com/apache/flink-docker.git --branch dev-1.19 --single-branch
local java_version=8
if [[ ${PROFILE} == *"jdk11"* ]]; then
(flink) branch release-1.18 updated: [FLINK-34166][table] Fix KeyedLookupJoinWrapper incorrectly process delete message for inner join when previous lookup result is empty
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.18
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.18 by this push:
new 78064d93cbb [FLINK-34166][table] Fix KeyedLookupJoinWrapper
incorrectly process delete message for inner join when previous lookup result
is empty
78064d93cbb is described below
commit 78064d93cbb762372d5567a0211c0e8e55e40a7f
Author: lincoln lee
AuthorDate: Thu Jan 25 09:58:07 2024 +0800
[FLINK-34166][table] Fix KeyedLookupJoinWrapper incorrectly process delete
message for inner join when previous lookup result is empty
This closes #24166
---
.../join/lookup/KeyedLookupJoinWrapper.java| 29 ---
.../operators/join/lookup/LookupJoinRunner.java| 2 +-
.../operators/join/KeyedLookupJoinHarnessTest.java | 92 --
3 files changed, 105 insertions(+), 18 deletions(-)
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
index 17eb2c8a319..3b0e2318691 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
@@ -134,34 +134,46 @@ public class KeyedLookupJoinWrapper extends
KeyedProcessFunction rightRows = state.value();
if (null == rightRows) {
-stateStaledErrorHandle(in, out);
+stateStaledErrorHandle();
} else {
for (RowData row : rightRows) {
-collectDeleteRow(in, row, out);
+if (!emptyRow.equals(row)) {
+collectDeleteRow(in, row, out);
+collected = true;
+}
}
}
}
// clear state at last
deleteState();
+
+// pad null for left join if no delete row collected from state,
here we can't use the
+// collector's status to determine whether the row is collected or
not, because data
+// fetched from state is not collected by the collector
+if (lookupJoinRunner.isLeftOuterJoin && !collected) {
+collectDeleteRow(in, lookupJoinRunner.nullRow, out);
+}
}
}
@@ -222,12 +234,9 @@ public class KeyedLookupJoinWrapper extends
KeyedProcessFunction {
private transient FlatMapFunction fetcher;
protected transient ListenableCollector collector;
protected transient JoinedRowData outRow;
-private transient GenericRowData nullRow;
+protected transient GenericRowData nullRow;
public LookupJoinRunner(
GeneratedFunction>
generatedFetcher,
diff --git
a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
index a4ed9af4b44..daaec0c55d3 100644
---
a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
+++
b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
@@ -74,8 +74,6 @@ public class KeyedLookupJoinHarnessTest {
DataTypes.STRING().getLogicalType()
});
-StateTtlConfig ttlConfig = StateConfigUtil.createTtlConfig(10_000_000);
-
@Test
public void testTemporalInnerJoin() throws Exception {
OneInputStreamOperatorTestHarness testHarness =
@@ -200,7 +198,6 @@ public class KeyedLookupJoinHarnessTest {
List expectedOutput = new ArrayList<>();
expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
-expectedOutput.add(deleteRecord(3, "c", null, null));
expectedOutput.add(insertRecord(3, "c2", 6, "Jark-2"));
expectedOutput.add(deleteRecord(3, "c2", 6, "Jark-2"));
expectedOutput.add(insertRecord(3, "c3", 9, "Jark-3"));
@@ -288,6 +285,8 @@ public class KeyedLookupJoinHarnessTest {
testHarness.processElement(updateAfterRecord(3, "c2"));
testHarness.processElement(deleteRecord(3, "c2"));
testHarness.processElement(insertRecord(3, "c3"));
+testHarness.processEle
(flink) branch release-1.17 updated: [FLINK-34166][table] Fix KeyedLookupJoinWrapper incorrectly process delete message for inner join when previous lookup result is empty
This is an automated email from the ASF dual-hosted git repository.
lincoln pushed a commit to branch release-1.17
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.17 by this push:
new 4e795b5f01a [FLINK-34166][table] Fix KeyedLookupJoinWrapper
incorrectly process delete message for inner join when previous lookup result
is empty
4e795b5f01a is described below
commit 4e795b5f01a42659367a55d9913e60b599247a92
Author: lincoln lee
AuthorDate: Tue Jan 23 19:41:04 2024 +0800
[FLINK-34166][table] Fix KeyedLookupJoinWrapper incorrectly process delete
message for inner join when previous lookup result is empty
This closes #24167
---
.../join/lookup/KeyedLookupJoinWrapper.java| 29 ---
.../operators/join/lookup/LookupJoinRunner.java| 2 +-
.../operators/join/KeyedLookupJoinHarnessTest.java | 92 --
3 files changed, 105 insertions(+), 18 deletions(-)
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
index 17eb2c8a319..3b0e2318691 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/lookup/KeyedLookupJoinWrapper.java
@@ -134,34 +134,46 @@ public class KeyedLookupJoinWrapper extends
KeyedProcessFunction rightRows = state.value();
if (null == rightRows) {
-stateStaledErrorHandle(in, out);
+stateStaledErrorHandle();
} else {
for (RowData row : rightRows) {
-collectDeleteRow(in, row, out);
+if (!emptyRow.equals(row)) {
+collectDeleteRow(in, row, out);
+collected = true;
+}
}
}
}
// clear state at last
deleteState();
+
+// pad null for left join if no delete row collected from state,
here we can't use the
+// collector's status to determine whether the row is collected or
not, because data
+// fetched from state is not collected by the collector
+if (lookupJoinRunner.isLeftOuterJoin && !collected) {
+collectDeleteRow(in, lookupJoinRunner.nullRow, out);
+}
}
}
@@ -222,12 +234,9 @@ public class KeyedLookupJoinWrapper extends
KeyedProcessFunction {
private transient FlatMapFunction fetcher;
protected transient ListenableCollector collector;
protected transient JoinedRowData outRow;
-private transient GenericRowData nullRow;
+protected transient GenericRowData nullRow;
public LookupJoinRunner(
GeneratedFunction>
generatedFetcher,
diff --git
a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
index a4ed9af4b44..daaec0c55d3 100644
---
a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
+++
b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/KeyedLookupJoinHarnessTest.java
@@ -74,8 +74,6 @@ public class KeyedLookupJoinHarnessTest {
DataTypes.STRING().getLogicalType()
});
-StateTtlConfig ttlConfig = StateConfigUtil.createTtlConfig(10_000_000);
-
@Test
public void testTemporalInnerJoin() throws Exception {
OneInputStreamOperatorTestHarness testHarness =
@@ -200,7 +198,6 @@ public class KeyedLookupJoinHarnessTest {
List expectedOutput = new ArrayList<>();
expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
-expectedOutput.add(deleteRecord(3, "c", null, null));
expectedOutput.add(insertRecord(3, "c2", 6, "Jark-2"));
expectedOutput.add(deleteRecord(3, "c2", 6, "Jark-2"));
expectedOutput.add(insertRecord(3, "c3", 9, "Jark-3"));
@@ -288,6 +285,8 @@ public class KeyedLookupJoinHarnessTest {
testHarness.processElement(updateAfterRecord(3, "c2"));
testHarness.processElement(deleteRecord(3, "c2"));
testHarness.processElement(insertRecord(3, "c3"));
+testHarness.processEle
