Build failed in Jenkins: Kafka » Kafka Branch Builder » 3.3 #73

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 572042 lines...]
[2022-09-14T02:04:14.521Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutDownClientIfGlobalStreamThreadWantsToReplaceThread PASSED
[2022-09-14T02:04:14.521Z] 
[2022-09-14T02:04:14.521Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldReplaceSingleThread STARTED
[2022-09-14T02:04:16.257Z] 
[2022-09-14T02:04:16.257Z] Exception: java.lang.AssertionError thrown from the 
UncaughtExceptionHandler in thread 
"appId_StreamsUncaughtExceptionHandlerIntegrationTestnull-06ed4388-e80a-4229-86d4-07a57352db9c-StreamThread-1"
[2022-09-14T02:04:16.257Z] 
[2022-09-14T02:04:16.257Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldReplaceSingleThread PASSED
[2022-09-14T02:04:16.257Z] 
[2022-09-14T02:04:16.257Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownMultipleThreadApplication STARTED
[2022-09-14T02:04:30.046Z] 
[2022-09-14T02:04:30.046Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownMultipleThreadApplication PASSED
[2022-09-14T02:04:30.046Z] 
[2022-09-14T02:04:30.046Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownClient STARTED
[2022-09-14T02:04:32.652Z] 
[2022-09-14T02:04:32.652Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownClient PASSED
[2022-09-14T02:04:32.652Z] 
[2022-09-14T02:04:32.652Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownSingleThreadApplication STARTED
[2022-09-14T02:04:41.010Z] 
[2022-09-14T02:04:41.010Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldShutdownSingleThreadApplication PASSED
[2022-09-14T02:04:41.010Z] 
[2022-09-14T02:04:41.010Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldEmitSameRecordAfterFailover STARTED
[2022-09-14T02:04:43.615Z] 
[2022-09-14T02:04:43.615Z] 
org.apache.kafka.streams.integration.StreamsUncaughtExceptionHandlerIntegrationTest
 > shouldEmitSameRecordAfterFailover PASSED
[2022-09-14T02:04:45.351Z] 
[2022-09-14T02:04:45.351Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[at_least_once] STARTED
[2022-09-14T02:05:42.253Z] 
[2022-09-14T02:05:42.253Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[at_least_once] PASSED
[2022-09-14T02:05:42.253Z] 
[2022-09-14T02:05:42.253Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[exactly_once] STARTED
[2022-09-14T02:06:31.355Z] 
[2022-09-14T02:06:31.355Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[exactly_once] PASSED
[2022-09-14T02:06:31.355Z] 
[2022-09-14T02:06:31.355Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[exactly_once_v2] STARTED
[2022-09-14T02:07:20.074Z] 
[2022-09-14T02:07:20.074Z] 
org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest > 
shouldRecoverBufferAfterShutdown[exactly_once_v2] PASSED
[2022-09-14T02:07:20.998Z] 
[2022-09-14T02:07:20.998Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftInner[caching enabled = true] STARTED
[2022-09-14T02:07:27.936Z] 
[2022-09-14T02:07:27.936Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftInner[caching enabled = true] PASSED
[2022-09-14T02:07:27.936Z] 
[2022-09-14T02:07:27.936Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftOuter[caching enabled = true] STARTED
[2022-09-14T02:07:34.874Z] 
[2022-09-14T02:07:34.874Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftOuter[caching enabled = true] PASSED
[2022-09-14T02:07:34.874Z] 
[2022-09-14T02:07:34.874Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftLeft[caching enabled = true] STARTED
[2022-09-14T02:07:40.560Z] 
[2022-09-14T02:07:40.560Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftLeft[caching enabled = true] PASSED
[2022-09-14T02:07:40.560Z] 
[2022-09-14T02:07:40.560Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterLeft[caching enabled = true] STARTED
[2022-09-14T02:07:47.495Z] 
[2022-09-14T02:07:47.495Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterLeft[caching enabled = true] PASSED
[2022-09-14T02:07:47.495Z] 
[2022-09-14T02:07:47.495Z]

Build failed in Jenkins: Kafka » Kafka Branch Builder » 3.3 #72

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 573300 lines...]
[2022-09-13T18:35:20.019Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterOuter[caching enabled = true] PASSED
[2022-09-13T18:35:20.019Z] 
[2022-09-13T18:35:20.019Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftInner[caching enabled = false] STARTED
[2022-09-13T18:35:42.047Z] 
[2022-09-13T18:35:42.047Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftInner[caching enabled = false] PASSED
[2022-09-13T18:35:42.047Z] 
[2022-09-13T18:35:42.047Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftOuter[caching enabled = false] STARTED
[2022-09-13T18:36:00.964Z] 
[2022-09-13T18:36:00.964Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftOuter[caching enabled = false] PASSED
[2022-09-13T18:36:00.964Z] 
[2022-09-13T18:36:00.964Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftLeft[caching enabled = false] STARTED
[2022-09-13T18:36:19.573Z] 
[2022-09-13T18:36:19.573Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeftLeft[caching enabled = false] PASSED
[2022-09-13T18:36:19.573Z] 
[2022-09-13T18:36:19.573Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterLeft[caching enabled = false] STARTED
[2022-09-13T18:36:39.097Z] 
[2022-09-13T18:36:39.097Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterLeft[caching enabled = false] PASSED
[2022-09-13T18:36:39.097Z] 
[2022-09-13T18:36:39.097Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInner[caching enabled = false] STARTED
[2022-09-13T18:36:55.877Z] 
[2022-09-13T18:36:55.877Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInner[caching enabled = false] PASSED
[2022-09-13T18:36:55.877Z] 
[2022-09-13T18:36:55.877Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuter[caching enabled = false] STARTED
[2022-09-13T18:37:15.415Z] 
[2022-09-13T18:37:15.415Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuter[caching enabled = false] PASSED
[2022-09-13T18:37:15.415Z] 
[2022-09-13T18:37:15.415Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeft[caching enabled = false] STARTED
[2022-09-13T18:37:30.052Z] 
[2022-09-13T18:37:30.052Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testLeft[caching enabled = false] PASSED
[2022-09-13T18:37:30.052Z] 
[2022-09-13T18:37:30.052Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerInner[caching enabled = false] STARTED
[2022-09-13T18:37:43.406Z] 
[2022-09-13T18:37:43.406Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerInner[caching enabled = false] PASSED
[2022-09-13T18:37:43.406Z] 
[2022-09-13T18:37:43.406Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerOuter[caching enabled = false] STARTED
[2022-09-13T18:38:02.242Z] 
[2022-09-13T18:38:02.242Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerOuter[caching enabled = false] PASSED
[2022-09-13T18:38:02.242Z] 
[2022-09-13T18:38:02.242Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerLeft[caching enabled = false] STARTED
[2022-09-13T18:38:16.067Z] 
[2022-09-13T18:38:16.067Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testInnerLeft[caching enabled = false] PASSED
[2022-09-13T18:38:16.067Z] 
[2022-09-13T18:38:16.067Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterInner[caching enabled = false] STARTED
[2022-09-13T18:38:31.934Z] 
[2022-09-13T18:38:31.934Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterInner[caching enabled = false] PASSED
[2022-09-13T18:38:31.934Z] 
[2022-09-13T18:38:31.934Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterOuter[caching enabled = false] STARTED
[2022-09-13T18:38:49.008Z] 
[2022-09-13T18:38:49.008Z] 
org.apache.kafka.streams.integration.TableTableJoinIntegrationTest > 
testOuterOuter[caching enabled = false] PASSED
[2022-09-13T18:38:49.008Z] 
[2022-09-13T18:38:49.008Z] 
org.apache.kafka.streams.integration.TaskMetadataIntegrationTest > 
shouldReportCorrectEndOffsetInformation STARTED
[2022-09-13T18:38:52.101Z] 
[2022-09-13T18:38:52.101Z] 
org.apache.kafka.streams.integration.TaskMetadataIntegrationTest > 
shouldReportCorrectEndOffsetInformation PASSED
[2022-09-13T18:38:52.101Z] 
[2022-09-13T18:38:52.101Z] 
org.apache.kafka.streams.integration.TaskMetadataIntegrationTest > 
shouldReportCorrectCommittedOffsetInformation STARTED
[2022-09-13T18:38:55.376Z]

[jira] [Created] (KAFKA-14232) Support for nested structures: InsertField

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14232:


 Summary: Support for nested structures: InsertField
 Key: KAFKA-14232
 URL: https://issues.apache.org/jira/browse/KAFKA-14232
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14231) Support for nested structures: ReplaceField

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14231:


 Summary: Support for nested structures: ReplaceField
 Key: KAFKA-14231
 URL: https://issues.apache.org/jira/browse/KAFKA-14231
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14230) Support for nested structures: Cast

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14230:


 Summary: Support for nested structures: Cast
 Key: KAFKA-14230
 URL: https://issues.apache.org/jira/browse/KAFKA-14230
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14228) Support for nested structures: ValueToKey

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14228:


 Summary: Support for nested structures: ValueToKey
 Key: KAFKA-14228
 URL: https://issues.apache.org/jira/browse/KAFKA-14228
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14229) Support for nested structures: HoistValue

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14229:


 Summary: Support for nested structures: HoistValue
 Key: KAFKA-14229
 URL: https://issues.apache.org/jira/browse/KAFKA-14229
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14227) Support for nested structures: MaskField

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14227:


 Summary: Support for nested structures: MaskField
 Key: KAFKA-14227
 URL: https://issues.apache.org/jira/browse/KAFKA-14227
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya






--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Created] (KAFKA-14226) Introduce support for nested structures

2022-09-13 Thread Jorge Esteban Quilcate Otoya (Jira) 
Jorge Esteban Quilcate Otoya created KAFKA-14226:


 Summary: Introduce support for nested structures
 Key: KAFKA-14226
 URL: https://issues.apache.org/jira/browse/KAFKA-14226
 Project: Kafka
  Issue Type: Sub-task
Reporter: Jorge Esteban Quilcate Otoya
Assignee: Jorge Esteban Quilcate Otoya


Abstraction for FieldPath and initial SMTs:
 * ExtractField
 * HeaderFrom
 * TimestampConverter



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Build failed in Jenkins: Kafka » Kafka Branch Builder » trunk #1223

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 505219 lines...]
[2022-09-13T22:25:48.512Z] > Task :connect:api:testSrcJar
[2022-09-13T22:25:48.512Z] > Task 
:connect:api:publishMavenJavaPublicationToMavenLocal
[2022-09-13T22:25:48.512Z] > Task :connect:api:publishToMavenLocal
[2022-09-13T22:25:48.512Z] 
[2022-09-13T22:25:48.512Z] > Task :streams:javadoc
[2022-09-13T22:25:48.512Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/processor/StreamPartitioner.java:50:
 warning - Tag @link: reference not found: 
org.apache.kafka.clients.producer.internals.DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:854:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:49.453Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:50.393Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:84:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:50.393Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:136:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:50.393Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:147:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:50.393Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:101:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:50.393Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:167:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: missing '#': "org.apache.kafka.streams.StreamsBuilder()"
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: can't find org.apache.kafka.streams.StreamsBuilder() in 
org.apache.kafka.streams.TopologyConfig
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/TopologyDescription.java:38:
 warning - Tag @link: reference not found: ProcessorContext#forward(Object, 
Object) forwards
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/query/Position.java:44:
 warning - Tag @link: can't find query(Query,
[2022-09-13T22:25:51.333Z]  PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:109:
 warning - Tag @link: reference not found: this#getResult()
[2022-09-13T22:25:51.333Z] 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:116:
 warning - Tag @link: reference not found: this#getFailureReason()
[2022-09-13T22:25:51.333Z]

KafkaConsumer refactor proposal

2022-09-13 Thread Philip Nee 
Hi all,

Here is the proposal to refactor the Kafka Consumer
.
The 1-pager is on the wiki, so please take a look at it.  Also, this is a
prerequisite for KIP-848 (the next gen rebalance protocol).

Cheers,
P

Re: [VOTE] KIP-862: Self-join optimization for stream-stream joins

2022-09-13 Thread Jim Hughes 
Hi Vicky,

I'm +1 (non-binding); thanks for the KIP (and PR)!

Cheers,

Jim

On Tue, Sep 13, 2022 at 12:05 PM Guozhang Wang  wrote:

> Thank Vicky! I'm +1.
>
> Guozhang
>
> On Mon, Sep 12, 2022 at 7:02 PM John Roesler  wrote:
>
> > Thanks for the updates, Vicky!
> >
> > I've reviewed the KIP and your POC PR,
> > and I'm +1 (binding).
> >
> > Thanks!
> > -John
> >
> > On Mon, Sep 12, 2022, at 09:13, Vasiliki Papavasileiou wrote:
> > > Hey Guozhang,
> > >
> > > Great suggestion, I made the change.
> > >
> > > Best,
> > > Vicky
> > >
> > > On Fri, Sep 9, 2022 at 10:43 PM Guozhang Wang 
> > wrote:
> > >
> > >> Thanks Vicky, that reads much clearer now.
> > >>
> > >> Just regarding the value string name itself: "self.join" may be
> > confusing
> > >> compared to other values that people would think before this config is
> > >> enabled, self-join are not allowed at all. Maybe we can rename it to
> > >> "single.store.self.join"?
> > >>
> > >> Guozhang
> > >>
> > >> On Fri, Sep 9, 2022 at 2:15 AM Vasiliki Papavasileiou
> > >>  wrote:
> > >>
> > >> > Hey Guozhang,
> > >> >
> > >> > Ah it seems my text was not very clear :)
> > >> > With "TOPOLOGY_OPTIMIZATION_CONFIG will be extended to accept a list
> > of
> > >> > optimization rule configs" I meant that it will accept the new value
> > >> > strings for each optimization rule. Let me rephrase that in the KIP
> to
> > >> make
> > >> > it clearer.
> > >> > Is it better now?
> > >> >
> > >> > Best,
> > >> > Vicky
> > >> >
> > >> > On Thu, Sep 8, 2022 at 9:07 PM Guozhang Wang 
> > wrote:
> > >> >
> > >> > > Thanks Vicky,
> > >> > >
> > >> > > I read through the KIP again and it looks good to me. Just a quick
> > >> > question
> > >> > > regarding the public config changes: you mentioned "No public
> > >> interfaces
> > >> > > will be impacted. The config TOPOLOGY_OPTIMIZATION_CONFIG will be
> > >> > extended
> > >> > > to accept a list of optimization rule configs in addition to the
> > global
> > >> > > values "all" and "none" . But there are no new value strings
> > mentioned
> > >> in
> > >> > > this KIP, so that means we will apply this optimization only when
> > `all`
> > >> > is
> > >> > > specified in the config right?
> > >> > >
> > >> > >
> > >> > > Guozhang
> > >> > >
> > >> > >
> > >> > > On Thu, Sep 8, 2022 at 12:02 PM Vasiliki Papavasileiou
> > >> > >  wrote:
> > >> > >
> > >> > > > Hello everyone,
> > >> > > >
> > >> > > > I'd like to open the vote for KIP-862, which proposes to
> optimize
> > >> > > > stream-stream self-joins by using a single state store for the
> > join.
> > >> > > >
> > >> > > > The proposal is here:
> > >> > > >
> > >> > > >
> > >> > >
> > >> >
> > >>
> >
> https://cwiki.apache.org/confluence/display/KAFKA/KIP-862%3A+Self-join+optimization+for+stream-stream+joins
> > >> > > >
> > >> > > > Thanks to all who reviewed the proposal, and thanks in advance
> for
> > >> > taking
> > >> > > > the time to vote!
> > >> > > >
> > >> > > > Thank you,
> > >> > > > Vicky
> > >> > > >
> > >> > >
> > >> > >
> > >> > > --
> > >> > > -- Guozhang
> > >> > >
> > >> >
> > >>
> > >>
> > >> --
> > >> -- Guozhang
> > >>
> >
>
>
> --
> -- Guozhang
>

Build failed in Jenkins: Kafka » Kafka Branch Builder » trunk #1222

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 504968 lines...]
[Pipeline] sh
[2022-09-13T19:55:35.872Z] + grep ^version= gradle.properties
[2022-09-13T19:55:35.872Z] + cut -d= -f 2
[2022-09-13T19:55:36.403Z] 
[2022-09-13T19:55:36.403Z] KStreamAggregationIntegrationTest > 
shouldCount(TestInfo) PASSED
[2022-09-13T19:55:36.403Z] 
[2022-09-13T19:55:36.403Z] KStreamAggregationIntegrationTest > 
shouldGroupByKey(TestInfo) STARTED
[Pipeline] dir
[2022-09-13T19:55:36.937Z] Running in 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk_2@2/streams/quickstart
[Pipeline] {
[Pipeline] sh
[2022-09-13T19:55:38.887Z] 
[2022-09-13T19:55:38.887Z] KStreamAggregationIntegrationTest > 
shouldGroupByKey(TestInfo) PASSED
[2022-09-13T19:55:38.887Z] 
[2022-09-13T19:55:38.887Z] KStreamAggregationIntegrationTest > 
shouldCountWithInternalStore(TestInfo) STARTED
[2022-09-13T19:55:39.773Z] + mvn clean install -Dgpg.skip
[2022-09-13T19:55:40.735Z] [INFO] Scanning for projects...
[2022-09-13T19:55:40.735Z] [INFO] 

[2022-09-13T19:55:40.735Z] [INFO] Reactor Build Order:
[2022-09-13T19:55:40.735Z] [INFO] 
[2022-09-13T19:55:40.735Z] [INFO] Kafka Streams :: Quickstart   
 [pom]
[2022-09-13T19:55:40.735Z] [INFO] streams-quickstart-java   
 [maven-archetype]
[2022-09-13T19:55:40.735Z] [INFO] 
[2022-09-13T19:55:40.735Z] [INFO] < 
org.apache.kafka:streams-quickstart >-
[2022-09-13T19:55:40.735Z] [INFO] Building Kafka Streams :: Quickstart 
3.4.0-SNAPSHOT[1/2]
[2022-09-13T19:55:40.735Z] [INFO] [ pom 
]-
[2022-09-13T19:55:40.735Z] [INFO] 
[2022-09-13T19:55:40.735Z] [INFO] --- maven-clean-plugin:3.0.0:clean 
(default-clean) @ streams-quickstart ---
[2022-09-13T19:55:40.735Z] [INFO] 
[2022-09-13T19:55:40.735Z] [INFO] --- maven-remote-resources-plugin:1.5:process 
(process-resource-bundles) @ streams-quickstart ---
[2022-09-13T19:55:41.800Z] [INFO] 
[2022-09-13T19:55:41.800Z] [INFO] --- maven-site-plugin:3.5.1:attach-descriptor 
(attach-descriptor) @ streams-quickstart ---
[2022-09-13T19:55:41.800Z] [INFO] 
[2022-09-13T19:55:41.800Z] [INFO] --- maven-gpg-plugin:1.6:sign 
(sign-artifacts) @ streams-quickstart ---
[2022-09-13T19:55:41.800Z] [INFO] 
[2022-09-13T19:55:41.800Z] [INFO] --- maven-install-plugin:2.5.2:install 
(default-install) @ streams-quickstart ---
[2022-09-13T19:55:42.863Z] [INFO] Installing 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk_2@2/streams/quickstart/pom.xml
 to 
/home/jenkins/.m2/repository/org/apache/kafka/streams-quickstart/3.4.0-SNAPSHOT/streams-quickstart-3.4.0-SNAPSHOT.pom
[2022-09-13T19:55:42.863Z] [INFO] 
[2022-09-13T19:55:42.863Z] [INFO] --< 
org.apache.kafka:streams-quickstart-java >--
[2022-09-13T19:55:42.863Z] [INFO] Building streams-quickstart-java 
3.4.0-SNAPSHOT[2/2]
[2022-09-13T19:55:42.863Z] [INFO] --[ maven-archetype 
]---
[2022-09-13T19:55:42.863Z] [INFO] 
[2022-09-13T19:55:42.863Z] [INFO] --- maven-clean-plugin:3.0.0:clean 
(default-clean) @ streams-quickstart-java ---
[2022-09-13T19:55:42.863Z] [INFO] 
[2022-09-13T19:55:42.863Z] [INFO] --- maven-remote-resources-plugin:1.5:process 
(process-resource-bundles) @ streams-quickstart-java ---
[2022-09-13T19:55:42.863Z] [INFO] 
[2022-09-13T19:55:42.863Z] [INFO] --- maven-resources-plugin:2.7:resources 
(default-resources) @ streams-quickstart-java ---
[2022-09-13T19:55:42.863Z] [INFO] Using 'UTF-8' encoding to copy filtered 
resources.
[2022-09-13T19:55:42.863Z] [INFO] Copying 6 resources
[2022-09-13T19:55:42.863Z] [INFO] Copying 3 resources
[2022-09-13T19:55:42.863Z] [INFO] 
[2022-09-13T19:55:42.864Z] [INFO] --- maven-resources-plugin:2.7:testResources 
(default-testResources) @ streams-quickstart-java ---
[2022-09-13T19:55:42.864Z] [INFO] Using 'UTF-8' encoding to copy filtered 
resources.
[2022-09-13T19:55:42.864Z] [INFO] Copying 2 resources
[2022-09-13T19:55:42.864Z] [INFO] Copying 3 resources
[2022-09-13T19:55:42.864Z] [INFO] 
[2022-09-13T19:55:42.864Z] [INFO] --- maven-archetype-plugin:2.2:jar 
(default-jar) @ streams-quickstart-java ---
[2022-09-13T19:55:42.864Z] [INFO] Building archetype jar: 
/home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk_2@2/streams/quickstart/java/target/streams-quickstart-java-3.4.0-SNAPSHOT
[2022-09-13T19:55:42.864Z] [INFO] 
[2022-09-13T19:55:42.864Z] [INFO] --- maven-site-plugin:3.5.1:attach-descriptor 
(attach-descriptor) @ streams-quickstart-java ---
[2022-09-13T19:55:42.864Z] [INFO] 
[2022-09-13T19:55:42.864Z] [INFO] --- 
maven-archetype-plugin:2.2:integration-test (default-integration-test) @ 
streams-quickstart-java ---
[2022-09-13T19:55:42.864Z] [INFO]

[GitHub] [kafka-site] bbejeck commented on pull request #431: Brain Station 23 adopted Kafka

2022-09-13 Thread GitBox 


bbejeck commented on PR #431:
URL: https://github.com/apache/kafka-site/pull/431#issuecomment-1245799227

   Hi, @sadatrafsan I'm thinking you closed this unintentionally - I'm going to 
reopen the PR and wait a day then merge this.
   If you still want to add this to the `Powered By` you don't need to take any 
further action.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscr...@kafka.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[jira] [Created] (KAFKA-14225) lazy val exemptSensor Could Cause Deadlock

2022-09-13 Thread Huilin Shi (Jira) 
Huilin Shi created KAFKA-14225:
--

 Summary: lazy val exemptSensor Could Cause Deadlock
 Key: KAFKA-14225
 URL: https://issues.apache.org/jira/browse/KAFKA-14225
 Project: Kafka
  Issue Type: Bug
  Components: core
Affects Versions: 2.5.1
Reporter: Huilin Shi


There is a chance to cause deadlock when multiple threads access 
ClientRequestQuotaManager. In the version of Scala 2.12, the lazy val 
initialization is under the object lock. The deadlock could happen in the 
following condition: In thread a, when ClientRequestQuotaManager.exemptSensor 
is being initialized, it has acquired the object lock and enters the the actual 
initialization block. The initialization of 'exemptSensor' requires another 
lock private val lock = new ReentrantReadWriteLock() and it is waiting for this 
lock. In thread b, at the same time, ClientQuotaManager.updateQuota() is called 
and it has already acquired ReentrantReadWriteLock lock by calling 
lock.writeLock().lock(). And then it executes info(). If this is the first time 
accessing Logging.logger, which is also a lazy val, it need to wait for the 
object lock. Since the lazy val initialization is under the object lock, we 
should avoid using lazy val if the initialization function holds another lock 
to prevent holding two locks at the same time which is prone for deadlock. 



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Resolved] (KAFKA-14073) Logging the reason for creating a snapshot

2022-09-13 Thread Jose Armando Garcia Sancio (Jira) 


 [ 
https://issues.apache.org/jira/browse/KAFKA-14073?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Jose Armando Garcia Sancio resolved KAFKA-14073.

Resolution: Fixed

> Logging the reason for creating a snapshot
> --
>
> Key: KAFKA-14073
> URL: https://issues.apache.org/jira/browse/KAFKA-14073
> Project: Kafka
>  Issue Type: Improvement
>Reporter: dengziming
>Priority: Minor
>  Labels: kraft, newbie
>
> So far we have two reasons for creating a snapshot. 1. X bytes were applied. 
> 2. the metadata version changed. we should log the reason when creating 
> snapshot both in the broker side and controller side. see 
> https://github.com/apache/kafka/pull/12265#discussion_r915972383



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Re: Transactions, delivery timeout and changing transactional producer behavior

2022-09-13 Thread Chris Egerton 
Hi Colt,

You can certainly review PRs, but you cannot merge them. Reviews are still
valuable as they help lighten the workload on committers and, to some
degree, provide a signal of how high-priority a change is to the community.

Cheers,

Chris

On Sun, Sep 11, 2022 at 12:19 PM Colt McNealy  wrote:

> Hi all—
>
> I'm not a committer so I can't review this PR (or is that not true?).
> However, I'd like to bump this as well. I believe that I've encountered
> this bug during chaos testing with the transactional producer. I can
> sometimes produce this error when killing a broker during a long-running
> transaction, which causes a batch to encounter delivery timeout as
> described in the Jira. I have observed some inconsistencies with
> the consumer offset being advanced prematurely (i.e. perhaps after the
> delivery of the EndTxnRequest).
>
> Daniel, thank you for the PR.
>
> Cheers,
> Colt McNealy
> *Founder, LittleHorse.io*
>
> On Fri, Sep 9, 2022 at 9:54 AM Dániel Urbán  wrote:
>
> > Hi all,
> >
> > I would like to bump this and bring some attention to the issue.
> > This is a nasty bug in the transactional producer, would be nice if I
> could
> > get some feedback on the PR: https://github.com/apache/kafka/pull/12392
> >
> > Thanks in advance,
> > Daniel
> >
> > Viktor Somogyi-Vass  ezt írta
> > (időpont: 2022. júl. 25., H, 15:28):
> >
> > > Hi Luke & Artem,
> > >
> > > We prepared the fix, would you please help in getting a
> > committer-reviewer
> > > to get this issue resolved?
> > >
> > > Thanks,
> > > Viktor
> > >
> > > On Fri, Jul 8, 2022 at 12:57 PM Dániel Urbán 
> > > wrote:
> > >
> > > > Submitted a PR with the fix:
> > https://github.com/apache/kafka/pull/12392
> > > > In the PR I tried keeping the producer in a usable state after the
> > forced
> > > > bump. I understand that it might be the cleanest solution, but the
> only
> > > > other option I know of is to transition into a fatal state, meaning
> > that
> > > > the producer has to be recreated after a delivery timeout. I think
> that
> > > is
> > > > still fine compared to the out-of-order messages.
> > > >
> > > > Looking forward to your reviews,
> > > > Daniel
> > > >
> > > > Dániel Urbán  ezt írta (időpont: 2022. júl.
> 7.,
> > > Cs,
> > > > 12:04):
> > > >
> > > > > Thanks for the feedback, I created
> > > > > https://issues.apache.org/jira/browse/KAFKA-14053 and started
> > working
> > > on
> > > > > a PR.
> > > > >
> > > > > Luke, for the workaround, we used the transaction admin tool
> released
> > > in
> > > > > 3.0 to "abort" these hanging batches manually.
> > > > > Naturally, the cluster health should be stabilized. This issue
> popped
> > > up
> > > > > most frequently around times when some partitions went into a few
> > > minute
> > > > > window of unavailability. The infinite retries on the producer side
> > > > caused
> > > > > a situation where the last retry was still in-flight, but the
> > delivery
> > > > > timeout was triggered on the client side. We reduced the retries
> and
> > > > > increased the delivery timeout to avoid such situations.
> > > > > Still, the issue can occur in other scenarios, for example a client
> > > > > queueing up many batches in the producer buffer, and causing those
> > > > batches
> > > > > to spend most of the delivery timeout window in the client memory.
> > > > >
> > > > > Thanks,
> > > > > Daniel
> > > > >
> > > > > Luke Chen  ezt írta (időpont: 2022. júl. 7.,
> Cs,
> > > > 5:13):
> > > > >
> > > > >> Hi Daniel,
> > > > >>
> > > > >> Thanks for reporting the issue, and the investigation.
> > > > >> I'm curious, so, what's your workaround for this issue?
> > > > >>
> > > > >> I agree with Artem, it makes sense. Please file a bug in JIRA.
> > > > >> And looking forward to your PR! :)
> > > > >>
> > > > >> Thank you.
> > > > >> Luke
> > > > >>
> > > > >> On Thu, Jul 7, 2022 at 3:07 AM Artem Livshits
> > > > >>  wrote:
> > > > >>
> > > > >> > Hi Daniel,
> > > > >> >
> > > > >> > What you say makes sense.  Could you file a bug and put this
> info
> > > > there
> > > > >> so
> > > > >> > that it's easier to track?
> > > > >> >
> > > > >> > -Artem
> > > > >> >
> > > > >> > On Wed, Jul 6, 2022 at 8:34 AM Dániel Urbán <
> > urb.dani...@gmail.com>
> > > > >> wrote:
> > > > >> >
> > > > >> > > Hello everyone,
> > > > >> > >
> > > > >> > > I've been investigating some transaction related issues in a
> > very
> > > > >> > > problematic cluster. Besides finding some interesting issues,
> I
> > > had
> > > > >> some
> > > > >> > > ideas about how transactional producer behavior could be
> > improved.
> > > > >> > >
> > > > >> > > My suggestion in short is: when the transactional producer
> > > > encounters
> > > > >> an
> > > > >> > > error which doesn't necessarily mean that the in-flight
> request
> > > was
> > > > >> > > processed (for example a client side timeout), the producer
> > should
> > > > not
> > > > >> > send
> > > > >> > > an EndTxnRequest on abort, but instead it should bump the
> >

Jenkins build is unstable: Kafka » Kafka Branch Builder » trunk #1221

2022-09-13 Thread Apache Jenkins Server 
See

Re: [VOTE] KIP-865: Support --bootstrap-server in kafka-streams-application-reset

2022-09-13 Thread Nikolay Izhikov 
Thanks all!

This KIP has passed with one +1 (non-binding) votes from myself
and
three +1 (binding) votes from Chris Egerton, Bill Bejeck, Guozhang Wang.

вт, 13 сент. 2022 г. в 19:30, Nikolay Izhikov :

> +1 (non-binding)
>
> пн, 12 сент. 2022 г. в 21:16, Chris Egerton :
>
>> +1 (binding). Thanks!
>>
>> On Mon, Sep 12, 2022 at 1:43 PM Bill Bejeck  wrote:
>>
>> > Thanks for the KIP!
>> >
>> > +1(binding)
>> >
>> > -Bill
>> >
>> > On Mon, Sep 12, 2022 at 1:39 PM Николай Ижиков 
>> > wrote:
>> >
>> > > Community, please, share your vote for this KIP.
>> > >
>> > > > 9 сент. 2022 г., в 19:55, Guozhang Wang 
>> > написал(а):
>> > > >
>> > > > +1. Thanks.
>> > > >
>> > > > Guozhang
>> > > >
>> > > > On Fri, Sep 9, 2022 at 9:52 AM Николай Ижиков 
>> > > wrote:
>> > > >
>> > > >> Hello.
>> > > >>
>> > > >> I'd like to start a vote on KIP-865 which adds support of
>> > > >> —bootstrap-server parameter in kafka-streams-application-reset tool
>> > > >>
>> > > >> Discuss Thread:
>> > > >> https://lists.apache.org/thread/5c1plw7mgmzd4zzqh1w59cqopn8kv21c
>> > > >> KIP:
>> > > >>
>> > >
>> >
>> https://cwiki.apache.org/confluence/display/KAFKA/KIP-865%3A+Support+--bootstrap-server+in+kafka-streams-application-reset
>> > > >> JIRA: https://issues.apache.org/jira/browse/KAFKA-12878
>> > > >
>> > > >
>> > > >
>> > > > --
>> > > > -- Guozhang
>> > >
>> > >
>> >
>>
>

Re: [VOTE] KIP-865: Support --bootstrap-server in kafka-streams-application-reset

2022-09-13 Thread Nikolay Izhikov 
+1 (non-binding)

пн, 12 сент. 2022 г. в 21:16, Chris Egerton :

> +1 (binding). Thanks!
>
> On Mon, Sep 12, 2022 at 1:43 PM Bill Bejeck  wrote:
>
> > Thanks for the KIP!
> >
> > +1(binding)
> >
> > -Bill
> >
> > On Mon, Sep 12, 2022 at 1:39 PM Николай Ижиков 
> > wrote:
> >
> > > Community, please, share your vote for this KIP.
> > >
> > > > 9 сент. 2022 г., в 19:55, Guozhang Wang 
> > написал(а):
> > > >
> > > > +1. Thanks.
> > > >
> > > > Guozhang
> > > >
> > > > On Fri, Sep 9, 2022 at 9:52 AM Николай Ижиков 
> > > wrote:
> > > >
> > > >> Hello.
> > > >>
> > > >> I'd like to start a vote on KIP-865 which adds support of
> > > >> —bootstrap-server parameter in kafka-streams-application-reset tool
> > > >>
> > > >> Discuss Thread:
> > > >> https://lists.apache.org/thread/5c1plw7mgmzd4zzqh1w59cqopn8kv21c
> > > >> KIP:
> > > >>
> > >
> >
> https://cwiki.apache.org/confluence/display/KAFKA/KIP-865%3A+Support+--bootstrap-server+in+kafka-streams-application-reset
> > > >> JIRA: https://issues.apache.org/jira/browse/KAFKA-12878
> > > >
> > > >
> > > >
> > > > --
> > > > -- Guozhang
> > >
> > >
> >
>

Re: [VOTE] KIP-862: Self-join optimization for stream-stream joins

2022-09-13 Thread Guozhang Wang 
Thank Vicky! I'm +1.

Guozhang

On Mon, Sep 12, 2022 at 7:02 PM John Roesler  wrote:

> Thanks for the updates, Vicky!
>
> I've reviewed the KIP and your POC PR,
> and I'm +1 (binding).
>
> Thanks!
> -John
>
> On Mon, Sep 12, 2022, at 09:13, Vasiliki Papavasileiou wrote:
> > Hey Guozhang,
> >
> > Great suggestion, I made the change.
> >
> > Best,
> > Vicky
> >
> > On Fri, Sep 9, 2022 at 10:43 PM Guozhang Wang 
> wrote:
> >
> >> Thanks Vicky, that reads much clearer now.
> >>
> >> Just regarding the value string name itself: "self.join" may be
> confusing
> >> compared to other values that people would think before this config is
> >> enabled, self-join are not allowed at all. Maybe we can rename it to
> >> "single.store.self.join"?
> >>
> >> Guozhang
> >>
> >> On Fri, Sep 9, 2022 at 2:15 AM Vasiliki Papavasileiou
> >>  wrote:
> >>
> >> > Hey Guozhang,
> >> >
> >> > Ah it seems my text was not very clear :)
> >> > With "TOPOLOGY_OPTIMIZATION_CONFIG will be extended to accept a list
> of
> >> > optimization rule configs" I meant that it will accept the new value
> >> > strings for each optimization rule. Let me rephrase that in the KIP to
> >> make
> >> > it clearer.
> >> > Is it better now?
> >> >
> >> > Best,
> >> > Vicky
> >> >
> >> > On Thu, Sep 8, 2022 at 9:07 PM Guozhang Wang 
> wrote:
> >> >
> >> > > Thanks Vicky,
> >> > >
> >> > > I read through the KIP again and it looks good to me. Just a quick
> >> > question
> >> > > regarding the public config changes: you mentioned "No public
> >> interfaces
> >> > > will be impacted. The config TOPOLOGY_OPTIMIZATION_CONFIG will be
> >> > extended
> >> > > to accept a list of optimization rule configs in addition to the
> global
> >> > > values "all" and "none" . But there are no new value strings
> mentioned
> >> in
> >> > > this KIP, so that means we will apply this optimization only when
> `all`
> >> > is
> >> > > specified in the config right?
> >> > >
> >> > >
> >> > > Guozhang
> >> > >
> >> > >
> >> > > On Thu, Sep 8, 2022 at 12:02 PM Vasiliki Papavasileiou
> >> > >  wrote:
> >> > >
> >> > > > Hello everyone,
> >> > > >
> >> > > > I'd like to open the vote for KIP-862, which proposes to optimize
> >> > > > stream-stream self-joins by using a single state store for the
> join.
> >> > > >
> >> > > > The proposal is here:
> >> > > >
> >> > > >
> >> > >
> >> >
> >>
> https://cwiki.apache.org/confluence/display/KAFKA/KIP-862%3A+Self-join+optimization+for+stream-stream+joins
> >> > > >
> >> > > > Thanks to all who reviewed the proposal, and thanks in advance for
> >> > taking
> >> > > > the time to vote!
> >> > > >
> >> > > > Thank you,
> >> > > > Vicky
> >> > > >
> >> > >
> >> > >
> >> > > --
> >> > > -- Guozhang
> >> > >
> >> >
> >>
> >>
> >> --
> >> -- Guozhang
> >>
>


-- 
-- Guozhang

Re: [DISCUSS] KIP-848: The Next Generation of the Consumer Rebalance Protocol

2022-09-13 Thread David Jacot 
Hi Jun,

Thanks for your feedback.

40. The reason is set in the object returned by
PartitionAssignor#metadata, which is consulted every time the HB
request is constructed. Note that PartitionAssignor is a new interface
here that replaces ConsumerPartitionAssignor.

41. partition.assignment.strategy is not used when group.protocol is
set to consumer so I am inclined to leave it as it is. We should
perhaps deprecate it. Note that it is replaced by two new
configurations: group.remote.assignor and group.local.assignors as
described in the KIP.

42. We transition a member to its target assignment when it has
revoked the partitions that he had to revoke. However, we may not
provide all partitions in the target assignment to the member just yet
because some partitions may be still owned by other members. In this
case, we provide only the partitions which are free. The metadata
(bytes) attached to the assignment are delivered immediately though.
In the Streams case, this metadata contains references to
tasks/partitions which may not be assigned yet for the reason that I
just explained. I hope that it makes it clearer.

43.
43.1 We definitely do. If you look in the response schema, the
possible error codes are listed. GROUP_AUTHORIZATION_FAILED,
NOT_COORDINATOR, COORDINATOR_NOT_AVAILABLE and
COORDINATOR_LOAD_IN_PROGRESS are reused.
43.2 Yeah, I agree with you. We could use a boolean in the response to
signal that. Another approach would be to directly provide the group
state in the response. This would elimitage the need for the
ConsumerGroupPrepareAssignemnt API and therefore save a roundtrip
between the member and the group coordinator. What do you think about
this?

44.
44.1 The client provides a list of client-side assignors. This is
required in order to support switching from one to another for
instance. So the AssignorName is there to let the member know which
one it should use.
44.2 The issue with the current owned partitions is that they could
change while the new assignment computation is inflight - a member
could converge during this time window. If this happens, the assignor
would not know it and likely undo this last change. I think that it is
preferable to use the current target and continue from there if we
want the partitions to be sticky.

45. You're right. Thanks.

46. The list is in the "Group Configurations" section. At the moment,
there are group.consumer.session.timeout.ms and
group.consumer.heartbeat.interval.ms.

47. The client side session timeout is no longer used in this
protocol. The group coordinator uses group.consumer.session.timeout.ms
defined at the broker level or at the group level. I should deprecate
the client side configuration as well.

48. Let me check them.
B - epoch=2, partitions=[foo-2] -> That seems correct to me. I suppose
that the confusion comes from foo-2, isn't it? The model used there
represents the persisted metadata. When B transitions to its target
assignment, including foo-2, it is stored directly as-is even if foo-2
is not free yet. The dependency is managed in memory.
C - epoch=3, partitions=[foo-1] -> It is similar as above.
C - epoch=22, partitions=[foo-2, foo-5] -> Same as well.
Does it make sense? I could switch the examples to use the in-memory
representation if it helps.

Best,
David

On Tue, Sep 13, 2022 at 1:53 AM Jun Rao  wrote:
>
> Hi, David,
>
> Thanks for the reply. A few more comments.
>
> 40. A member updates its assignors' reason to trigger a rebalance: What API
> in the consumer does the power user use to change the assignor reason?
>
> 41. partition.assignment.strategy defaults class
> org.apache.kafka.clients.consumer.RangeAssignor,class
> org.apache.kafka.clients.consumer.CooperativeStickyAssignor. Does that
> still make sense since the assignor name on the broker is not the full
> class name? Should it default to a strategy determined by the broker?
>
> 42. "This means that the assignment metadata may already reference
> partitions which are not assigned to the member yet." Not sure I
> follow this one.
>
> 43. ConsumerGroupHeartbeat:
> 43.1 Do we need an ErrorCode to indicate that the group state is being
> loaded (during coordinator failover)? If so, do we need the same ErrorCode
> for other requests?
> 43.2 COMPUTE_ASSIGNMENT : It's a bit weird to represent this as an error
> code since there is no error. Could we represent that in the payload of
> ConsumerGroupHeartbeat response?
>
> 44. ConsumerGroupPrepareAssignmentResponse:
> 44.1 Why does it have AssignorName since the client decides the assignor?
> 44.2 TopicPartitions: The target topic-partitions of the member. Should
> that be the topic partition currently owned by the member?
>
> 45. List group: Should the ACL be Describe (Cluster) instead of Describe
> Group?
>
> 46. AlterIncrementalConfigs: what group configs could be changed?
>
> 47. group.consumer.session.timeout.ms: how does the client communicate
> its session timeout config to the broker?
>
> 48. There still seems to be

Build failed in Jenkins: Kafka » Kafka Branch Builder » 3.3 #71

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 578671 lines...]
[2022-09-13T14:09:23.352Z] > Task :streams:javadoc
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:854:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:84:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:136:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:147:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:101:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:167:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: missing '#': "org.apache.kafka.streams.StreamsBuilder()"
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: can't find org.apache.kafka.streams.StreamsBuilder() in 
org.apache.kafka.streams.TopologyConfig
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyDescription.java:38:
 warning - Tag @link: reference not found: ProcessorContext#forward(Object, 
Object) forwards
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/Position.java:44:
 warning - Tag @link: can't find query(Query,
[2022-09-13T14:09:23.352Z]  PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:44:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:36:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:57:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:74:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T14:09:23.352Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:110:
 warning - Tag @link: reference not found: this#getResult()
[2022-09-13T14:09:23.352Z]

Build failed in Jenkins: Kafka » Kafka Branch Builder » trunk #1220

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 437765 lines...]
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
[2022-09-13T14:00:31.949Z]  at com.sun.proxy.$Proxy90.onOutput(Unknown 
Source)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.api.internal.tasks.testing.results.TestListenerAdapter.output(TestListenerAdapter.java:56)
[2022-09-13T14:00:31.949Z]  at 
sun.reflect.GeneratedMethodAccessor492.invoke(Unknown Source)
[2022-09-13T14:00:31.949Z]  at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[2022-09-13T14:00:31.949Z]  at 
java.lang.reflect.Method.invoke(Method.java:498)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.AbstractBroadcastDispatch.dispatch(AbstractBroadcastDispatch.java:43)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.BroadcastDispatch$SingletonDispatch.dispatch(BroadcastDispatch.java:245)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.BroadcastDispatch$SingletonDispatch.dispatch(BroadcastDispatch.java:157)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.AbstractBroadcastDispatch.dispatch(AbstractBroadcastDispatch.java:61)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.BroadcastDispatch$CompositeDispatch.dispatch(BroadcastDispatch.java:346)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.BroadcastDispatch$CompositeDispatch.dispatch(BroadcastDispatch.java:249)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.ListenerBroadcast.dispatch(ListenerBroadcast.java:141)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.event.ListenerBroadcast.dispatch(ListenerBroadcast.java:37)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
[2022-09-13T14:00:31.949Z]  at com.sun.proxy.$Proxy91.output(Unknown Source)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.api.internal.tasks.testing.results.StateTrackingTestResultProcessor.output(StateTrackingTestResultProcessor.java:112)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.testretry.internal.executer.RetryTestResultProcessor.output(RetryTestResultProcessor.java:127)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.api.internal.tasks.testing.results.AttachParentTestResultProcessor.output(AttachParentTestResultProcessor.java:48)
[2022-09-13T14:00:31.949Z]  at 
sun.reflect.GeneratedMethodAccessor494.invoke(Unknown Source)
[2022-09-13T14:00:31.949Z]  at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[2022-09-13T14:00:31.949Z]  at 
java.lang.reflect.Method.invoke(Method.java:498)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.FailureHandlingDispatch.dispatch(FailureHandlingDispatch.java:30)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.AsyncDispatch.dispatchMessages(AsyncDispatch.java:87)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.AsyncDispatch.access$000(AsyncDispatch.java:36)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.dispatch.AsyncDispatch$1.run(AsyncDispatch.java:71)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.concurrent.InterruptibleRunnable.run(InterruptibleRunnable.java:42)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.operations.CurrentBuildOperationPreservingRunnable.run(CurrentBuildOperationPreservingRunnable.java:42)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
[2022-09-13T14:00:31.949Z]  at 
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
[2022-09-13T14:00:31.949Z]  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[2022-09-13T14:00:31.949Z]  at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[2022-09-13T14:00:31.949Z]  at java.lang.Thread.run(Thread.java:748)
[2022-09-13T14:00:31.949Z] java.io.IOException: No space left on device
[2022-09-13T14:00:31.949Z] com.esotericsoftware.kryo.KryoException: 
java.io.IOException: No space left on device
[2022-09-13T14:00:31.949Z]  at 
com.esotericsoftware.kryo.io.Output.flush(Output.java:165)

Build failed in Jenkins: Kafka » Kafka Branch Builder » 3.3 #70

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 578785 lines...]
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:854:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:890:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:919:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java:939:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:84:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:136:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Produced.java:147:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:101:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:38.195Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java:167:
 warning - Tag @link: reference not found: DefaultPartitioner
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: missing '#': "org.apache.kafka.streams.StreamsBuilder()"
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyConfig.java:58:
 warning - Tag @link: can't find org.apache.kafka.streams.StreamsBuilder() in 
org.apache.kafka.streams.TopologyConfig
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/TopologyDescription.java:38:
 warning - Tag @link: reference not found: ProcessorContext#forward(Object, 
Object) forwards
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/Position.java:44:
 warning - Tag @link: can't find query(Query,
[2022-09-13T09:40:39.146Z]  PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:44:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:36:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:57:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:74:
 warning - Tag @link: can't find query(Query, PositionBound, boolean) in 
org.apache.kafka.streams.processor.StateStore
[2022-09-13T09:40:39.146Z] 
/home/jenkins/workspace/Kafka_kafka_3.3/streams/src/main/java/org/apache/kafka/streams/query/QueryResult.java:110:
 warning - Tag @link: reference not found: this#getResult()
[2022-09-13T09:40:39.146Z]

[jira] [Resolved] (KAFKA-14196) Duplicated consumption during rebalance, causing OffsetValidationTest to act flaky

2022-09-13 Thread Luke Chen (Jira) 


 [ 
https://issues.apache.org/jira/browse/KAFKA-14196?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Luke Chen resolved KAFKA-14196.
---
Resolution: Fixed

> Duplicated consumption during rebalance, causing OffsetValidationTest to act 
> flaky
> --
>
> Key: KAFKA-14196
> URL: https://issues.apache.org/jira/browse/KAFKA-14196
> Project: Kafka
>  Issue Type: Bug
>  Components: clients, consumer
>Affects Versions: 3.2.1
>Reporter: Philip Nee
>Assignee: Philip Nee
>Priority: Blocker
>  Labels: new-consumer-threading-should-fix
> Fix For: 3.3.0, 3.2.3
>
>
> Several flaky tests under OffsetValidationTest are indicating potential 
> consumer duplication issue, when autocommit is enabled.  I believe this is 
> affecting *3.2* and onward.  Below shows the failure message:
>  
> {code:java}
> Total consumed records 3366 did not match consumed position 3331 {code}
>  
> After investigating the log, I discovered that the data consumed between the 
> start of a rebalance event and the async commit was lost for those failing 
> tests.  In the example below, the rebalance event kicks in at around 
> 1662054846995 (first record), and the async commit of the offset 3739 is 
> completed at around 1662054847015 (right before partitions_revoked).
>  
> {code:java}
> {"timestamp":1662054846995,"name":"records_consumed","count":3,"partitions":[{"topic":"test_topic","partition":0,"count":3,"minOffset":3739,"maxOffset":3741}]}
> {"timestamp":1662054846998,"name":"records_consumed","count":2,"partitions":[{"topic":"test_topic","partition":0,"count":2,"minOffset":3742,"maxOffset":3743}]}
> {"timestamp":1662054847008,"name":"records_consumed","count":2,"partitions":[{"topic":"test_topic","partition":0,"count":2,"minOffset":3744,"maxOffset":3745}]}
> {"timestamp":1662054847016,"name":"partitions_revoked","partitions":[{"topic":"test_topic","partition":0}]}
> {"timestamp":1662054847031,"name":"partitions_assigned","partitions":[{"topic":"test_topic","partition":0}]}
> {"timestamp":1662054847038,"name":"records_consumed","count":23,"partitions":[{"topic":"test_topic","partition":0,"count":23,"minOffset":3739,"maxOffset":3761}]}
>  {code}
> A few things to note here:
>  # Manually calling commitSync in the onPartitionsRevoke cb seems to 
> alleviate the issue
>  # Setting includeMetadataInTimeout to false also seems to alleviate the 
> issue.
> The above tries seems to suggest that contract between poll() and 
> asyncCommit() is broken.  AFAIK, we implicitly uses poll() to ack the 
> previously fetched data, and the consumer would (try to) commit these offsets 
> in the current poll() loop.  However, it seems like as the poll continues to 
> loop, the "acked" data isn't being committed.
>  
> I believe this could be introduced in  KAFKA-14024, which originated from 
> KAFKA-13310.
> More specifically, (see the comments below), the ConsumerCoordinator will 
> alway return before async commit, due to the previous incomplete commit.  
> However, this is a bit contradictory here because:
>  # I think we want to commit asynchronously while the poll continues, and if 
> we do that, we are back to KAFKA-14024, that the consumer will get rebalance 
> timeout and get kicked out of the group.
>  # But we also need to commit all the "acked" offsets before revoking the 
> partition, and this has to be blocked.
> *Steps to Reproduce the Issue:*
>  # Check out AK 3.2
>  # Run this several times: (Recommend to only run runs with autocommit 
> enabled in consumer_test.py to save time)
> {code:java}
> _DUCKTAPE_OPTIONS="--debug" 
> TC_PATHS="tests/kafkatest/tests/client/consumer_test.py::OffsetValidationTest.test_consumer_failure"
>  bash tests/docker/run_tests.sh {code}
>  
> *Steps to Diagnose the Issue:*
>  # Open the test results in *results/*
>  # Go to the consumer log.  It might look like this
>  
> {code:java}
> results/2022-09-03--005/OffsetValidationTest/test_consumer_failure/clean_shutdown=True.enable_autocommit=True.metadata_quorum=ZK/2/VerifiableConsumer-0-xx/dockerYY
>  {code}
> 3. Find the docker instance that has partition getting revoked and rejoined.  
> Observed the offset before and after.
> *Propose Fixes:*
>  TBD
>  
> https://github.com/apache/kafka/pull/12603



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

[jira] [Resolved] (KAFKA-14208) KafkaConsumer#commitAsync throws unexpected WakeupException

2022-09-13 Thread Luke Chen (Jira) 


 [ 
https://issues.apache.org/jira/browse/KAFKA-14208?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Luke Chen resolved KAFKA-14208.
---
Resolution: Fixed

> KafkaConsumer#commitAsync throws unexpected WakeupException
> ---
>
> Key: KAFKA-14208
> URL: https://issues.apache.org/jira/browse/KAFKA-14208
> Project: Kafka
>  Issue Type: Bug
>  Components: clients
>Affects Versions: 3.2.1
>Reporter: Qingsheng Ren
>Assignee: Guozhang Wang
>Priority: Blocker
> Fix For: 3.3.0, 3.2.2
>
>
> We recently encountered a bug after upgrading Kafka client to 3.2.1 in Flink 
> Kafka connector (FLINK-29153). Here's the exception:
> {code:java}
> org.apache.kafka.common.errors.WakeupException
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.maybeTriggerWakeup(ConsumerNetworkClient.java:514)
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:278)
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236)
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:215)
>   at 
> org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:252)
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnready(ConsumerCoordinator.java:493)
>   at 
> org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.commitOffsetsAsync(ConsumerCoordinator.java:1055)
>   at 
> org.apache.kafka.clients.consumer.KafkaConsumer.commitAsync(KafkaConsumer.java:1573)
>   at 
> org.apache.flink.streaming.connectors.kafka.internals.KafkaConsumerThread.run(KafkaConsumerThread.java:226)
>  {code}
> As {{WakeupException}} is not listed in the JavaDoc of 
> {{{}KafkaConsumer#commitAsync{}}}, Flink Kafka connector doesn't catch the 
> exception thrown directly from KafkaConsumer#commitAsync but handles all 
> exceptions in the callback.
> I checked the source code and suspect this is caused by KAFKA-13563. Also we 
> never had this exception in commitAsync when we used Kafka client 2.4.1 & 
> 2.8.1. 
> I'm wondering if this is kind of breaking the public API as the 
> WakeupException is not listed in JavaDoc, and maybe it's better to invoke the 
> callback to handle the {{WakeupException}} instead of throwing it directly 
> from the method itself. 



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Re: PR Review

2022-09-13 Thread Nandini Anagondi 
Hi,

Can anyone help in reviewing the below pr please..?

Thanks,
Nandini A.


On Mon, Sep 5, 2022 at 11:32 AM Nandini Anagondi 
wrote:

> Hi,
>
> Can you help me in reviewing this pr please.?
> https://github.com/apache/kafka/pull/11516/files.
>
> Thanks,
> Nandini A.
>
>

Build failed in Jenkins: Kafka » Kafka Branch Builder » trunk #1219

2022-09-13 Thread Apache Jenkins Server 
See 


Changes:


--
[...truncated 504165 lines...]
[2022-09-13T05:54:45.832Z] > Run with --stacktrace option to get the stack 
trace.
[2022-09-13T05:54:45.832Z] > Run with --info or --debug option to get more log 
output.
[2022-09-13T05:54:45.832Z] > Run with --scan to get full insights.
[2022-09-13T05:54:45.832Z] 
[2022-09-13T05:54:45.832Z] * Get more help at https://help.gradle.org
[2022-09-13T05:54:45.832Z] 
[2022-09-13T05:54:45.832Z] BUILD FAILED in 2h 27m 3s
[2022-09-13T05:54:45.832Z] 212 actionable tasks: 115 executed, 97 up-to-date
[2022-09-13T05:54:45.832Z] 
[2022-09-13T05:54:45.832Z] See the profiling report at: 
file:///home/jenkins/jenkins-agent/workspace/Kafka_kafka_trunk/build/reports/profile/profile-2022-09-13-03-27-49.html
[2022-09-13T05:54:45.832Z] A fine-grained performance profile is available: use 
the --scan option.
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // timestamps
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch JDK 11 and Scala 2.12
[2022-09-13T05:54:46.469Z] 
[2022-09-13T05:54:46.469Z] AdjustStreamThreadCountTest > 
shouldAddAndRemoveThreadsMultipleTimes() PASSED
[2022-09-13T05:54:46.469Z] 
[2022-09-13T05:54:46.469Z] AdjustStreamThreadCountTest > 
shouldnNotRemoveStreamThreadWithinTimeout() STARTED
[2022-09-13T05:54:49.879Z] 
[2022-09-13T05:54:49.879Z] AdjustStreamThreadCountTest > 
shouldnNotRemoveStreamThreadWithinTimeout() PASSED
[2022-09-13T05:54:49.879Z] 
[2022-09-13T05:54:49.879Z] AdjustStreamThreadCountTest > 
shouldAddAndRemoveStreamThreadsWhileKeepingNamesCorrect() STARTED
[2022-09-13T05:55:12.904Z] 
[2022-09-13T05:55:12.904Z] AdjustStreamThreadCountTest > 
shouldAddAndRemoveStreamThreadsWhileKeepingNamesCorrect() PASSED
[2022-09-13T05:55:12.904Z] 
[2022-09-13T05:55:12.904Z] AdjustStreamThreadCountTest > 
shouldAddStreamThread() STARTED
[2022-09-13T05:55:17.482Z] 
[2022-09-13T05:55:17.482Z] AdjustStreamThreadCountTest > 
shouldAddStreamThread() PASSED
[2022-09-13T05:55:17.482Z] 
[2022-09-13T05:55:17.482Z] AdjustStreamThreadCountTest > 
shouldRemoveStreamThreadWithStaticMembership() STARTED
[2022-09-13T05:55:23.507Z] 
[2022-09-13T05:55:23.507Z] AdjustStreamThreadCountTest > 
shouldRemoveStreamThreadWithStaticMembership() PASSED
[2022-09-13T05:55:23.507Z] 
[2022-09-13T05:55:23.507Z] AdjustStreamThreadCountTest > 
shouldRemoveStreamThread() STARTED
[2022-09-13T05:55:32.881Z] 
[2022-09-13T05:55:32.881Z] AdjustStreamThreadCountTest > 
shouldRemoveStreamThread() PASSED
[2022-09-13T05:55:32.881Z] 
[2022-09-13T05:55:32.881Z] AdjustStreamThreadCountTest > 
shouldResizeCacheAfterThreadRemovalTimesOut() STARTED
[2022-09-13T05:55:32.881Z] 
[2022-09-13T05:55:32.881Z] AdjustStreamThreadCountTest > 
shouldResizeCacheAfterThreadRemovalTimesOut() PASSED
[2022-09-13T05:55:36.129Z] 
[2022-09-13T05:55:36.129Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithNoCommittedOffsetsWithGlobalAutoOffsetResetLatest()
 STARTED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithNoCommittedOffsetsWithGlobalAutoOffsetResetLatest()
 PASSED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldThrowExceptionOverlappingPattern() STARTED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldThrowExceptionOverlappingPattern() PASSED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldThrowExceptionOverlappingTopic() STARTED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldThrowExceptionOverlappingTopic() PASSED
[2022-09-13T05:55:37.192Z] 
[2022-09-13T05:55:37.192Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithInvalidCommittedOffsets() STARTED
[2022-09-13T05:56:23.204Z] 
[2022-09-13T05:56:23.204Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithInvalidCommittedOffsets() PASSED
[2022-09-13T05:56:23.204Z] 
[2022-09-13T05:56:23.204Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithNoCommittedOffsetsWithDefaultGlobalAutoOffsetResetEarliest()
 STARTED
[2022-09-13T05:57:10.027Z] 
[2022-09-13T05:57:10.027Z] FineGrainedAutoResetIntegrationTest > 
shouldOnlyReadRecordsWhereEarliestSpecifiedWithNoCommittedOffsetsWithDefaultGlobalAutoOffsetResetEarliest()
 PASSED
[2022-09-13T05:57:10.027Z] 
[2022-09-13T05:57:10.027Z] FineGrainedAutoResetIntegrationTest > 
shouldThrowStreamsExceptionNoResetSpecified() STARTED
[2022-09-13T05:57:10.027Z] 
[2022-09-13T05:57:10.027Z] FineGrainedAutoResetIntegrationTest >