Build failed in Jenkins: kafka-trunk-jdk7 #1643

[Apache Jenkins Server]

See 

Changes:

[wangguoz] HOTFIX: follow up on KAFKA-4275

[wangguoz] KAFKA-4269: Update topic subscription when regex pattern specified 
out

--
[...truncated 14242 lines...]
org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testSubscription PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldSetClusterMetadataOnAssignment STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldSetClusterMetadataOnAssignment PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopics STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopics PASSED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
testSpecificPartition STARTED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
testSpecificPartition PASSED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
shouldThrowStreamsExceptionAfterMaxAttempts STARTED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
shouldThrowStreamsExceptionAfterMaxAttempts PASSED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
shouldRetryWhenTimeoutExceptionOccursOnSend STARTED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
shouldRetryWhenTimeoutExceptionOccursOnSend PASSED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
testStreamPartitioner STARTED

org.apache.kafka.streams.processor.internals.RecordCollectorTest > 
testStreamPartitioner PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldHaveCompactionPropSetIfSupplied STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldHaveCompactionPropSetIfSupplied PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldThrowIfNameIsNull STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldThrowIfNameIsNull PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldConfigureRetentionMsWithAdditionalRetentionWhenCompactAndDelete STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldConfigureRetentionMsWithAdditionalRetentionWhenCompactAndDelete PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldBeCompactedIfCleanupPolicyCompactOrCompactAndDelete STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldBeCompactedIfCleanupPolicyCompactOrCompactAndDelete PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotBeCompactedWhenCleanupPolicyIsDelete STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotBeCompactedWhenCleanupPolicyIsDelete PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldHavePropertiesSuppliedByUser STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldHavePropertiesSuppliedByUser PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldUseCleanupPolicyFromConfigIfSupplied STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldUseCleanupPolicyFromConfigIfSupplied PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotConfigureRetentionMsWhenCompact STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotConfigureRetentionMsWhenCompact PASSED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotConfigureRetentionMsWhenDelete STARTED

org.apache.kafka.streams.processor.internals.InternalTopicConfigTest > 
shouldNotConfigureRetentionMsWhenDelete PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullProcessorSupplier STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullProcessorSupplier PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotSetApplicationIdToNull STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotSetApplicationIdToNull PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > testSourceTopics 
STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > testSourceTopics PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullNameWhenAddingSink STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullNameWhenAddingSink PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
testNamedTopicMatchesAlreadyProvidedPattern STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
testNamedTopicMatchesAlreadyProvidedPattern PASSED

org.apache.kafka.streams.processor.Topo

Build failed in Jenkins: kafka-trunk-jdk8 #992

[Apache Jenkins Server]

See 

Changes:

[wangguoz] HOTFIX: follow up on KAFKA-4275

[wangguoz] KAFKA-4269: Update topic subscription when regex pattern specified 
out

--
[...truncated 7893 lines...]

kafka.integration.SaslPlaintextTopicMetadataTest > 
testIsrAfterBrokerShutDownAndJoinsBack STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testIsrAfterBrokerShutDownAndJoinsBack PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAutoCreateTopicWithCollision STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAutoCreateTopicWithCollision PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokerListWithNoTopics STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokerListWithNoTopics PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > testAutoCreateTopic STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > testAutoCreateTopic PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > testGetAllTopicMetadata 
STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > testGetAllTopicMetadata 
PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterNewBrokerStartup STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterNewBrokerStartup PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > testBasicTopicMetadata 
STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > testBasicTopicMetadata PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAutoCreateTopicWithInvalidReplication STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAutoCreateTopicWithInvalidReplication PASSED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterABrokerShutdown STARTED

kafka.integration.SaslPlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterABrokerShutdown PASSED

kafka.integration.PrimitiveApiTest > testMultiProduce STARTED

kafka.integration.PrimitiveApiTest > testMultiProduce PASSED

kafka.integration.PrimitiveApiTest > testDefaultEncoderProducerAndFetch STARTED

kafka.integration.PrimitiveApiTest > testDefaultEncoderProducerAndFetch PASSED

kafka.integration.PrimitiveApiTest > testFetchRequestCanProperlySerialize 
STARTED

kafka.integration.PrimitiveApiTest > testFetchRequestCanProperlySerialize PASSED

kafka.integration.PrimitiveApiTest > testPipelinedProduceRequests STARTED

kafka.integration.PrimitiveApiTest > testPipelinedProduceRequests PASSED

kafka.integration.PrimitiveApiTest > testProduceAndMultiFetch STARTED

kafka.integration.PrimitiveApiTest > testProduceAndMultiFetch PASSED

kafka.integration.PrimitiveApiTest > 
testDefaultEncoderProducerAndFetchWithCompression STARTED

kafka.integration.PrimitiveApiTest > 
testDefaultEncoderProducerAndFetchWithCompression PASSED

kafka.integration.PrimitiveApiTest > testConsumerEmptyTopic STARTED

kafka.integration.PrimitiveApiTest > testConsumerEmptyTopic PASSED

kafka.integration.PrimitiveApiTest > testEmptyFetchRequest STARTED

kafka.integration.PrimitiveApiTest > testEmptyFetchRequest PASSED

kafka.integration.PlaintextTopicMetadataTest > 
testIsrAfterBrokerShutDownAndJoinsBack STARTED

kafka.integration.PlaintextTopicMetadataTest > 
testIsrAfterBrokerShutDownAndJoinsBack PASSED

kafka.integration.PlaintextTopicMetadataTest > testAutoCreateTopicWithCollision 
STARTED

kafka.integration.PlaintextTopicMetadataTest > testAutoCreateTopicWithCollision 
PASSED

kafka.integration.PlaintextTopicMetadataTest > testAliveBrokerListWithNoTopics 
STARTED

kafka.integration.PlaintextTopicMetadataTest > testAliveBrokerListWithNoTopics 
PASSED

kafka.integration.PlaintextTopicMetadataTest > testAutoCreateTopic STARTED

kafka.integration.PlaintextTopicMetadataTest > testAutoCreateTopic PASSED

kafka.integration.PlaintextTopicMetadataTest > testGetAllTopicMetadata STARTED

kafka.integration.PlaintextTopicMetadataTest > testGetAllTopicMetadata PASSED

kafka.integration.PlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterNewBrokerStartup STARTED

kafka.integration.PlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterNewBrokerStartup PASSED

kafka.integration.PlaintextTopicMetadataTest > testBasicTopicMetadata STARTED

kafka.integration.PlaintextTopicMetadataTest > testBasicTopicMetadata PASSED

kafka.integration.PlaintextTopicMetadataTest > 
testAutoCreateTopicWithInvalidReplication STARTED

kafka.integration.PlaintextTopicMetadataTest > 
testAutoCreateTopicWithInvalidReplication PASSED

kafka.integration.PlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterABrokerShutdown STARTED

kafka.integration.PlaintextTopicMetadataTest > 
testAliveBrokersListWithNoTopicsAfterABrokerShutdown PASSED

kafka.integration.AutoOffsetResetTest > testResetToLatestWhenOffsetTooLow 
STARTED

kafka.integration.AutoOffsetResetTest > testResetToLat

[jira] [Commented] (KAFKA-4269) Multiple KStream instances with at least one Regex source causes NPE when using multiple consumers

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4269?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15590710#comment-15590710
 ] 

ASF GitHub Bot commented on KAFKA-4269:
---

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2005


> Multiple KStream instances with at least one Regex source causes NPE when 
> using multiple consumers
> --
>
> Key: KAFKA-4269
> URL: https://issues.apache.org/jira/browse/KAFKA-4269
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.1.0
>Reporter: Bill Bejeck
>Assignee: Bill Bejeck
> Fix For: 0.10.2.0
>
>
> I discovered this issue while doing testing for for KAFKA-4114. 
> KAFKA-4131 fixed the issue of a _single_ KStream with a regex source on 
> partitioned topics across multiple consumers.
> //KAFKA-4131 fixed this case assuming an "foo*" topics are partitioned
> KStream kstream = builder.source(Pattern.compile("foo.*"));
> KafkaStream stream = new KafkaStreams(builder, props);
> stream.start();  
> This is a new issue where there are _multiple_
> KStream instances (and one has a regex source) within a single KafkaStreams 
> object. When running the second or "following"
> consumer there are NPE errors generated in the RecordQueue.addRawRecords 
> method when attempting to consume records. 
> For example:
> KStream kstream = builder.source(Pattern.compile("foo.*"));
> KStream kstream2 = builder.source(.): //can be regex or named topic 
> sources
> KafkaStream stream = new KafkaStreams(builder, props);
> stream.start();
> By adding an additional KStream instance like above (whether Regex or Named 
> topic) causes a NPE when run as "follower"
> From my initial debugging I can see the TopicPartition assignments being set 
> on the "follower" KafkaStreams instance, but need to track down why and where 
> all assignments aren't being set.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka pull request #2005: KAFKA-4269 extracted code updating topics when reg...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2005


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Updated] (KAFKA-4269) Multiple KStream instances with at least one Regex source causes NPE when using multiple consumers

[Guozhang Wang (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4269?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Guozhang Wang updated KAFKA-4269:
-
   Resolution: Fixed
Fix Version/s: (was: 0.10.1.1)
   0.10.2.0
   Status: Resolved  (was: Patch Available)

Issue resolved by pull request 2005
[https://github.com/apache/kafka/pull/2005]

> Multiple KStream instances with at least one Regex source causes NPE when 
> using multiple consumers
> --
>
> Key: KAFKA-4269
> URL: https://issues.apache.org/jira/browse/KAFKA-4269
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.1.0
>Reporter: Bill Bejeck
>Assignee: Bill Bejeck
> Fix For: 0.10.2.0
>
>
> I discovered this issue while doing testing for for KAFKA-4114. 
> KAFKA-4131 fixed the issue of a _single_ KStream with a regex source on 
> partitioned topics across multiple consumers.
> //KAFKA-4131 fixed this case assuming an "foo*" topics are partitioned
> KStream kstream = builder.source(Pattern.compile("foo.*"));
> KafkaStream stream = new KafkaStreams(builder, props);
> stream.start();  
> This is a new issue where there are _multiple_
> KStream instances (and one has a regex source) within a single KafkaStreams 
> object. When running the second or "following"
> consumer there are NPE errors generated in the RecordQueue.addRawRecords 
> method when attempting to consume records. 
> For example:
> KStream kstream = builder.source(Pattern.compile("foo.*"));
> KStream kstream2 = builder.source(.): //can be regex or named topic 
> sources
> KafkaStream stream = new KafkaStreams(builder, props);
> stream.start();
> By adding an additional KStream instance like above (whether Regex or Named 
> topic) causes a NPE when run as "follower"
> From my initial debugging I can see the TopicPartition assignments being set 
> on the "follower" KafkaStreams instance, but need to track down why and where 
> all assignments aren't being set.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4275) Check of State-Store-assignment to Processor-Nodes is not enabled

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4275?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15590693#comment-15590693
 ] 

ASF GitHub Bot commented on KAFKA-4275:
---

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2039


> Check of State-Store-assignment to Processor-Nodes is not enabled
> -
>
> Key: KAFKA-4275
> URL: https://issues.apache.org/jira/browse/KAFKA-4275
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Reporter: Matthias J. Sax
>Assignee: Matthias J. Sax
> Fix For: 0.10.1.0, 0.10.2.0
>
>
> In {{ProcessorContextImpl#getStateStores()}} we should check if a store was 
> connected to the processor and thus, if the processor is allowed to access 
> the store. This check is currently disabled.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka pull request #2039: HOTFIX: follow up on KAFKA-4275

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2039


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (KAFKA-4295) kafka-console-consumer.sh does not delete the temporary group in zookeeper

[huxi (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15590599#comment-15590599
 ] 

huxi commented on KAFKA-4295:
-

The design is not changed. 0.9.x still removes the zk nodes for inactive 
console consumer groups which used a randomly-generated group id. But if the 
consumer is stopped before the zk node 'offsets' got created, the program 
failed to remove the path, which I prefer is a bug.  

> kafka-console-consumer.sh does not delete the temporary group in zookeeper
> --
>
> Key: KAFKA-4295
> URL: https://issues.apache.org/jira/browse/KAFKA-4295
> Project: Kafka
>  Issue Type: Bug
>  Components: admin
>Affects Versions: 0.9.0.1, 0.10.0.0, 0.10.0.1
>Reporter: Sswater Shi
>Assignee: huxi
>Priority: Minor
>
> I'm not sure it is a bug or you guys designed it.
> Since 0.9.x.x, the kafka-console-consumer.sh will not delete the group 
> information in zookeeper/consumers on exit when without "--new-consumer". 
> There will be a lot of abandoned zookeeper/consumers/console-consumer-xxx if 
> kafka-console-consumer.sh runs a lot of times.
> When 0.8.x.x,  the kafka-console-consumer.sh can be followed by an argument 
> "group". If not specified, the kafka-console-consumer.sh will create a 
> temporary group name like 'console-consumer-'. If the group name is 
> specified by "group", the information in the zookeeper/consumers will be kept 
> on exit. If the group name is a temporary one, the information in the 
> zookeeper will be deleted when kafka-console-consumer.sh is quitted by 
> Ctrl+C. Why this is changed from 0.9.x.x.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Re: [VOTE] KIP-73 - Replication Quotas

[Jun Rao]

Hi,

While testing KIP-73, we found an issue described in
https://issues.apache.org/jira/browse/KAFKA-4313. Basically, when there are
mixed high-volume and low-volume partitions, when replication throttling is
specified, ISRs for those low volume partitions could thrash. KAFKA-4313
fixes this issue by avoiding throttling those replicas in the throttled
replica list that are already in sync. Those in-sync replicas traffic will
still be accounted for the throttled traffic though. Just want to bring
this up since it slightly changes the behavior described in the KIP. If
anyone has concerns on this, please comment on the jira.

Thanks,

Jun

On Tue, Aug 23, 2016 at 3:25 PM, Ismael Juma  wrote:

> For the record, there were 4 binding +1s.
>
> Ismael
>
> On Tue, Aug 23, 2016 at 11:16 PM, Ben Stopford  wrote:
>
> > Thanks everyone. It looks like this KIP has now been accepted.
> >
> > There is a corresponding PR 
> > for the implementation also.
> >
> > All the best
> >
> > B
> >
> >
> > > On 23 Aug 2016, at 22:39, Joel Koshy  wrote:
> > >
> > > +1
> > > (sent some very minor edits to you off-thread)
> > >
> > > On Fri, Aug 19, 2016 at 1:21 AM, Ben Stopford 
> wrote:
> > >
> > >> I’d like to initiate the voting process for KIP-73:
> > >> https://cwiki.apache.org/confluence/display/KAFKA/KIP-
> > >> 73+Replication+Quotas  > >> confluence/display/KAFKA/KIP-73+Replication+Quotas>
> > >>
> > >> Ben
> >
> >
>

Build failed in Jenkins: kafka-0.10.1-jdk7 #79

[Apache Jenkins Server]

See 

Changes:

[wangguoz] MINOR: Added more basic concepts to the documentation

--
[...truncated 7393 lines...]
kafka.log.BrokerCompressionTest > testBrokerSideCompression[1] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[1] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[2] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[2] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[3] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[3] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[4] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[4] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[5] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[5] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[6] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[6] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[7] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[7] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[8] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[8] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[9] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[9] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[10] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[10] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[11] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[11] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[12] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[12] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[13] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[13] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[14] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[14] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[15] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[15] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[16] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[16] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[17] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[17] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[18] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[18] PASSED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[19] STARTED

kafka.log.BrokerCompressionTest > testBrokerSideCompression[19] PASSED

kafka.log.LogSegmentTest > testRecoveryWithCorruptMessage STARTED

kafka.log.LogSegmentTest > testRecoveryWithCorruptMessage PASSED

kafka.log.LogSegmentTest > testRecoveryFixesCorruptIndex STARTED

kafka.log.LogSegmentTest > testRecoveryFixesCorruptIndex PASSED

kafka.log.LogSegmentTest > testReadFromGap STARTED

kafka.log.LogSegmentTest > testReadFromGap PASSED

kafka.log.LogSegmentTest > testTruncate STARTED

kafka.log.LogSegmentTest > testTruncate PASSED

kafka.log.LogSegmentTest > testReadBeforeFirstOffset STARTED

kafka.log.LogSegmentTest > testReadBeforeFirstOffset PASSED

kafka.log.LogSegmentTest > testCreateWithInitFileSizeAppendMessage STARTED

kafka.log.LogSegmentTest > testCreateWithInitFileSizeAppendMessage PASSED

kafka.log.LogSegmentTest > testChangeFileSuffixes STARTED

kafka.log.LogSegmentTest > testChangeFileSuffixes PASSED

kafka.log.LogSegmentTest > testRecoveryFixesCorruptTimeIndex STARTED

kafka.log.LogSegmentTest > testRecoveryFixesCorruptTimeIndex PASSED

kafka.log.LogSegmentTest > testReloadLargestTimestampAfterTruncation STARTED

kafka.log.LogSegmentTest > testReloadLargestTimestampAfterTruncation PASSED

kafka.log.LogSegmentTest > testMaxOffset STARTED

kafka.log.LogSegmentTest > testMaxOffset PASSED

kafka.log.LogSegmentTest > testNextOffsetCalculation STARTED

kafka.log.LogSegmentTest > testNextOffsetCalculation PASSED

kafka.log.LogSegmentTest > testFindOffsetByTimestamp STARTED

kafka.log.LogSegmentTest > testFindOffsetByTimestamp PASSED

kafka.log.LogSegmentTest > testReadOnEmptySegment STARTED

kafka.log.LogSegmentTest > testReadOnEmptySegment PASSED

kafka.log.LogSegmentTest > testReadAfterLast STARTED

kafka.log.LogSegmentTest > testReadAfterLast PASSED

kafka.log.LogSegmentTest > testCreateWithInitFileSizeClearShutdown STARTED

kafka.log.LogSegmentTest > testCreateWithInitFileSizeClearShutdown PASSED

kafka.log.LogSegmentTest > testTruncateFull STARTED

kafka.log.LogSegmentTest > testTruncateFull PASSED

kafka.log.LogConfigTest > shouldValidateThrottledReplicasConfig STARTED

k

Jenkins build is back to normal : kafka-trunk-jdk8 #990

[Apache Jenkins Server]

See 

[jira] [Resolved] (KAFKA-4250) make ProducerRecord and ConsumerRecord extensible

[Joel Koshy (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4250?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Joel Koshy resolved KAFKA-4250.
---
   Resolution: Fixed
 Assignee: radai rosenblatt
 Reviewer: Joel Koshy
Fix Version/s: 0.10.2.0

> make ProducerRecord and ConsumerRecord extensible
> -
>
> Key: KAFKA-4250
> URL: https://issues.apache.org/jira/browse/KAFKA-4250
> Project: Kafka
>  Issue Type: Improvement
>  Components: core
>Affects Versions: 0.10.0.1
>Reporter: radai rosenblatt
>Assignee: radai rosenblatt
> Fix For: 0.10.2.0
>
>
> KafkaProducer and KafkaConsumer implement interfaces are are designed to be 
> extensible (or at least allow it).
> ProducerRecord and ConsumerRecord, however, are final, making extending 
> producer/consumer very difficult.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4250) make ProducerRecord and ConsumerRecord extensible

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4250?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15590239#comment-15590239
 ] 

ASF GitHub Bot commented on KAFKA-4250:
---

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/1961


> make ProducerRecord and ConsumerRecord extensible
> -
>
> Key: KAFKA-4250
> URL: https://issues.apache.org/jira/browse/KAFKA-4250
> Project: Kafka
>  Issue Type: Improvement
>  Components: core
>Affects Versions: 0.10.0.1
>Reporter: radai rosenblatt
>
> KafkaProducer and KafkaConsumer implement interfaces are are designed to be 
> extensible (or at least allow it).
> ProducerRecord and ConsumerRecord, however, are final, making extending 
> producer/consumer very difficult.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4312) void KTableImpl.writeAsText(String filePath) throws NullPointerException when filePath is empty String

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4312?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15590240#comment-15590240
 ] 

ASF GitHub Bot commented on KAFKA-4312:
---

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2042


> void KTableImpl.writeAsText(String filePath) throws NullPointerException when 
> filePath is empty String
> --
>
> Key: KAFKA-4312
> URL: https://issues.apache.org/jira/browse/KAFKA-4312
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Reporter: Seweryn Habdank-Wojewodzki
>Assignee: Bill Bejeck
>Priority: Minor
> Fix For: 0.10.2.0
>
>
> The KTableImpl method 
> void writeAsText(String filePath) 
> throws NullPointerException when filePath is empty String = "".
> It is pretty uninformative.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka pull request #2042: KAFKA-4312: If filePath is empty string writeAsTex...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2042


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] kafka pull request #1961: KAFKA-4250: make ProducerRecord and ConsumerRecord...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/1961


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Re: KStreams / add support for sink processor with dynamic topics

[Guozhang Wang]

We can consider adding this feature to with a StreamsAdminClient that we
are adding as part of KAFKA-4060. However, I'm still not sure if it should
be added on the DSL layer or on the Processor API layer.

Florian, what do you mean that the Processor is not "completely safe"? Do
you mean not strong typed? I'm wondering why that would be an issue if you
just want to dynamically create topics on-the-fly based on the message
content?


Guozhang


On Tue, Oct 18, 2016 at 7:36 AM, Florian Hussonnois 
wrote:

> Thank you Matthias for your answers.
>
> The mailing list that you linked shows a solution using the Processor API.
>
> Actually, the set of subtypes is not known in advance this is why I need to
> compute output topics from messages. So the branch method is of any help in
> my context.
>
> I think, this feature should be supported by the DSL as the Processor API
> solution is not completely safe.
>
>
> 2016-10-18 10:01 GMT+02:00 Damian Guy :
>
> > Hi Florian,
> >
> > Do you know the set of subtypes in advance? I.e, could you use:
> >
> > KStream[] branches = stream.branch(predicates);
> >
> > to split the stream based on the subtypes?
> >
> > Thanks,
> > Damian
> >
> >
> > On Tue, 18 Oct 2016 at 00:43 Matthias J. Sax 
> > wrote:
> >
> > > -BEGIN PGP SIGNED MESSAGE-
> > > Hash: SHA512
> > >
> > > Hi,
> > >
> > > using DSL you cannot do this. However, if you use Processor API you
> can.
> > >
> > > There are similar question on the mailing list already. For example:
> > > http://search-hadoop.com/m/uyzND1lghNN1tzbf41&subj=kafka+
> stream+to+new+t
> > > opic+based+on+message+key
> > >
> > > As we got this request multiple times already, it might be worth
> > > adding it IMHO. Not sure what the opinion of other is? We should make
> > > sure that the feature gets accepted before you put a lot of effort in
> > > it. :)
> > >
> > >
> > > - -Matthias
> > >
> > > On 10/17/16 2:10 PM, Florian Hussonnois wrote:
> > > > Hi All,
> > > >
> > > > Currently, it seems not possible with KStream to produce messages
> > > > to topics which are not known until runtime.
> > > >
> > > > For a new project I am evaluating the Kafka Connect / Kafka
> > > > Streams architecture but without that feature I cannot retain the
> > > > KStreams API.
> > > >
> > > > Our use case is pretty basic. We have xml messages in input of our
> > > > topology. Each message is splitted into subtypes and formatted in
> > > > Avro before being sent to a dedicated topic.
> > > >
> > > > So the output topics depend of the subtype of each message.
> > > >
> > > > I think it would be nice to add methods into the KStream interface
> > > > to provide such feature.
> > > >
> > > > If you think that feature would be usefull I can create a jira and
> > > > contribute to it. Also, do I need to create a new KIP as this
> > > > requires changes on a public API ?
> > > >
> > > > Thanks,
> > > >
> > > -BEGIN PGP SIGNATURE-
> > > Comment: GPGTools - https://gpgtools.org
> > >
> > > iQIcBAEBCgAGBQJYBWIhAAoJECnhiMLycopPfTQQAI69Uii5xd8KvaEo/Aeqs0Xw
> > > AzdPHekdVoHANzo1h45W1x3/lnyeMU/n2v09Agsz46cxb+Xbz9NOKGqT3v9Ye0Ic
> > > Eyl5yib1B6sWr41rGuYmwDH8zBoC8dPfGZiWhfXL4Sypey3RWzQlVAUWg8Ob4tqF
> > > rFeubMjWp7yopKRe/7n//JHF029hVK/ePk1vdEsI+2lBI4N7q9ONT/1wKkeCAtdd
> > > CCkI2WP/WbHzCcUVmOL41KoqgQFnmrH7BtLH67jumzEIR16H+ZenGZmS1uzde56E
> > > 9mEsl4wmAvfB5GJu6y7JnS7FnQotw7pV7ZneQrA2q8eCZHZqs2fkXf+6ZJNYRir+
> > > rysqt8wJG69ZN9bSNO1Q6/fNbRiSjYi0I7JnzkErP6scfDKlf3bWzlw6Ejc0+iUr
> > > Cd0x2m/RlCepVleMT0UZNDlJd0Ml9Q77npP1lyntHVYHjVvtZLdlB5BQYdTMAx3N
> > > KCLZ+WkY2CBKcwh/KuMr9kW2eWSxH89JSwEule+1bN4vSKyBA6vtrwDoshf6N23g
> > > dEhTiY5NsgkvAe1JEK6d7PLN2Tq1Tq4OJNoP8PZlqW+YSFl41klQUblo8yT1jSlF
> > > iCyQS4rgNRabjBs1iZnZNoZ5eodoJMcUyWPhHUYHne+MXuSr1cNNGeNcbS5W0UyE
> > > dPCe2IiY4zErzxW/Mjmw
> > > =4DpY
> > > -END PGP SIGNATURE-
> > >
> >
>
>
>
> --
> Florian HUSSONNOIS
>



-- 
-- Guozhang

[jira] [Updated] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Guozhang Wang (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Guozhang Wang updated KAFKA-4316:
-
Affects Version/s: 0.10.0.1

> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.0.1
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Guozhang Wang (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Guozhang Wang updated KAFKA-4316:
-
Component/s: streams

> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.0.1
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka-site issue #25: Enable 0.10.1 documentation

[gwenshap]

Github user gwenshap commented on the issue:

https://github.com/apache/kafka-site/pull/25
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] kafka-site pull request #25: Enable 0.10.1 documentation

[hachikuji]

GitHub user hachikuji opened a pull request:

https://github.com/apache/kafka-site/pull/25

Enable 0.10.1 documentation



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/hachikuji/kafka-site activate-0.10.1-docs

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/kafka-site/pull/25.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #25


commit ac2567c7e234fc6dbe496fc1b92670a6e4689812
Author: Jason Gustafson 
Date:   2016-10-19T22:40:04Z

Enable 0.10.1 documentation




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Build failed in Jenkins: kafka-trunk-jdk8 #989

[Apache Jenkins Server]

See 

Changes:

[wangguoz] HOTFIX: Fix putAll and putIfAbsent logic for correct eviction 
behavior

--
[...truncated 14202 lines...]
org.apache.kafka.streams.processor.internals.AbstractTaskTest > 
shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenAuthorizationException 
PASSED

org.apache.kafka.streams.processor.internals.AbstractTaskTest > 
shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaException STARTED

org.apache.kafka.streams.processor.internals.AbstractTaskTest > 
shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaException PASSED

org.apache.kafka.streams.processor.internals.AbstractTaskTest > 
shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException STARTED

org.apache.kafka.streams.processor.internals.AbstractTaskTest > 
shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldThrowExceptionIfApplicationServerConfigIsNotHostPortPair STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldThrowExceptionIfApplicationServerConfigIsNotHostPortPair PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldMapUserEndPointToTopicPartitions STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldMapUserEndPointToTopicPartitions PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldAddUserDefinedEndPointToSubscription STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldAddUserDefinedEndPointToSubscription PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithStandbyReplicas STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithStandbyReplicas PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithNewTasks STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithNewTasks PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldExposeHostStateToTopicPartitionsOnAssignment STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldExposeHostStateToTopicPartitionsOnAssignment PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithStates STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithStates PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldThrowExceptionIfApplicationServerConfigPortIsNotAnInteger STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldThrowExceptionIfApplicationServerConfigPortIsNotAnInteger PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldReturnEmptyClusterMetadataIfItHasntBeenBuilt STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldReturnEmptyClusterMetadataIfItHasntBeenBuilt PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignBasic STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignBasic PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testOnAssignment STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testOnAssignment PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testSubscription STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testSubscription PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldSetClusterMetadataOnAssignment STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
shouldSetClusterMetadataOnAssignment PASSED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopics STARTED

org.apache.kafka.streams.processor.internals.StreamPartitionAssignorTest > 
testAssignWithInternalTopics PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullProcessorSupplier STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotAllowNullProcessorSupplier PASSED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotSetApplicationIdToNull STARTED

org.apache.kafka.streams.processor.TopologyBuilderTest > 
shouldNotSetApplication

[jira] [Commented] (KAFKA-3995) Add a new configuration "enable.comrpession.ratio.estimation" to the producer config

[Mayuresh Gharat (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-3995?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589938#comment-15589938
 ] 

Mayuresh Gharat commented on KAFKA-3995:


If we disable compression, we would not have this issue right? Of course that's 
not recommended.
The other way would be to reduce the linger.ms to be very very low.

On thinking more about this I plan to reopen and work on this. Since this is a 
new config, we would probably require a KIP for this right? If Yes, I can write 
up a KIP and submit for review.

> Add a new configuration "enable.comrpession.ratio.estimation" to the producer 
> config
> 
>
> Key: KAFKA-3995
> URL: https://issues.apache.org/jira/browse/KAFKA-3995
> Project: Kafka
>  Issue Type: Improvement
>  Components: clients
>Affects Versions: 0.10.0.0
>Reporter: Jiangjie Qin
>Assignee: Mayuresh Gharat
> Fix For: 0.10.2.0
>
>
> We recently see a few cases where RecordTooLargeException is thrown because 
> the compressed message sent by KafkaProducer exceeded the max message size.
> The root cause of this issue is because the compressor is estimating the 
> batch size using an estimated compression ratio based on heuristic 
> compression ratio statistics. This does not quite work for the traffic with 
> highly variable compression ratios. 
> For example, if the batch size is set to 1MB and the max message size is 1MB. 
> Initially a the producer is sending messages (each message is 1MB) to topic_1 
> whose data can be compressed to 1/10 of the original size. After a while the 
> estimated compression ratio in the compressor will be trained to 1/10 and the 
> producer would put 10 messages into one batch. Now the producer starts to 
> send messages (each message is also 1MB) to topic_2 whose message can only be 
> compress to 1/5 of the original size. The producer would still use 1/10 as 
> the estimated compression ratio and put 10 messages into a batch. That batch 
> would be 2 MB after compression which exceeds the maximum message size. In 
> this case the user do not have many options other than resend everything or 
> close the producer if they care about ordering.
> This is especially an issue for services like MirrorMaker whose producer is 
> shared by many different topics.
> To solve this issue, we can probably add a configuration 
> "enable.compression.ratio.estimation" to the producer. So when this 
> configuration is set to false, we stop estimating the compressed size but 
> will close the batch once the uncompressed bytes in the batch reaches the 
> batch size.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Reopened] (KAFKA-3995) Add a new configuration "enable.comrpession.ratio.estimation" to the producer config

[Mayuresh Gharat (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-3995?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Mayuresh Gharat reopened KAFKA-3995:


> Add a new configuration "enable.comrpession.ratio.estimation" to the producer 
> config
> 
>
> Key: KAFKA-3995
> URL: https://issues.apache.org/jira/browse/KAFKA-3995
> Project: Kafka
>  Issue Type: Improvement
>  Components: clients
>Affects Versions: 0.10.0.0
>Reporter: Jiangjie Qin
>Assignee: Mayuresh Gharat
> Fix For: 0.10.2.0
>
>
> We recently see a few cases where RecordTooLargeException is thrown because 
> the compressed message sent by KafkaProducer exceeded the max message size.
> The root cause of this issue is because the compressor is estimating the 
> batch size using an estimated compression ratio based on heuristic 
> compression ratio statistics. This does not quite work for the traffic with 
> highly variable compression ratios. 
> For example, if the batch size is set to 1MB and the max message size is 1MB. 
> Initially a the producer is sending messages (each message is 1MB) to topic_1 
> whose data can be compressed to 1/10 of the original size. After a while the 
> estimated compression ratio in the compressor will be trained to 1/10 and the 
> producer would put 10 messages into one batch. Now the producer starts to 
> send messages (each message is also 1MB) to topic_2 whose message can only be 
> compress to 1/5 of the original size. The producer would still use 1/10 as 
> the estimated compression ratio and put 10 messages into a batch. That batch 
> would be 2 MB after compression which exceeds the maximum message size. In 
> this case the user do not have many options other than resend everything or 
> close the producer if they care about ordering.
> This is especially an issue for services like MirrorMaker whose producer is 
> shared by many different topics.
> To solve this issue, we can probably add a configuration 
> "enable.compression.ratio.estimation" to the producer. So when this 
> configuration is set to false, we stop estimating the compressed size but 
> will close the batch once the uncompressed bytes in the batch reaches the 
> batch size.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4312) void KTableImpl.writeAsText(String filePath) throws NullPointerException when filePath is empty String

[Guozhang Wang (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4312?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Guozhang Wang updated KAFKA-4312:
-
   Resolution: Fixed
Fix Version/s: (was: 0.10.1.1)
   0.10.2.0
   Status: Resolved  (was: Patch Available)

Issue resolved by pull request 2042
[https://github.com/apache/kafka/pull/2042]

> void KTableImpl.writeAsText(String filePath) throws NullPointerException when 
> filePath is empty String
> --
>
> Key: KAFKA-4312
> URL: https://issues.apache.org/jira/browse/KAFKA-4312
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Reporter: Seweryn Habdank-Wojewodzki
>Assignee: Bill Bejeck
>Priority: Minor
> Fix For: 0.10.2.0
>
>
> The KTableImpl method 
> void writeAsText(String filePath) 
> throws NullPointerException when filePath is empty String = "".
> It is pretty uninformative.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka pull request #2030: MINOR: Added more basic concepts to the documentat...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2030


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] kafka pull request #2036: MINOR: Update rocksDB dependency to 4.11.2

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2036


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] kafka pull request #2038: HOTFIX: Fix put logic

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2038


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Michael Noll (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589806#comment-15589806
 ] 

Michael Noll commented on KAFKA-4316:
-

[~habdank]:  Sounds good.  FYI: The vote on releasing Kafka 0.10.1 just passed, 
which means an official 0.10.1 release should happen until end of this month.

> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Seweryn Habdank-Wojewodzki (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589756#comment-15589756
 ] 

Seweryn Habdank-Wojewodzki commented on KAFKA-4316:
---

Thanks a lot for very quick response!

We are using bleeding edge features of the Kafka (especially Streams), so we 
are waiting for every new release and all bug fixes :-).
I will talk with our team and we will schedule this update, when Kafka 0.10.1 
comes.

> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Re: Kafka KIP meeting Oct 19 at 11:00am PST

[Jun Rao]

The following are the notes from today's KIP discussion.


   - KIP-82 - add record header: We agreed that there are use cases for
   third-party vendors building tools around Kafka. We haven't reached the
   conclusion whether the added complexity justifies the use cases. We will
   follow up on the mailing list with use cases, container format people have
   been using, and details on the proposal.


The video will be uploaded soon in https://cwiki.apache.org/
confluence/display/KAFKA/Kafka+Improvement+Proposals .

Thanks,

Jun

On Mon, Oct 17, 2016 at 10:49 AM, Jun Rao  wrote:

> Hi, Everyone.,
>
> We plan to have a Kafka KIP meeting this coming Wednesday at 11:00am PST.
> If you plan to attend but haven't received an invite, please let me know.
> The following is the tentative agenda.
>
> Agenda:
> KIP-82: add record header
>
> Thanks,
>
> Jun
>

Jenkins build is back to normal : kafka-trunk-jdk7 #1640

[Apache Jenkins Server]

See 

Build failed in Jenkins: kafka-trunk-jdk8 #988

[Apache Jenkins Server]

See 

Changes:

[jason] KAFKA-4319; Parallelize shutdown of fetchers in AbstractFetcherManager

--
[...truncated 14188 lines...]
org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedProducerConfigs PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfRestoreConsumerConfig STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfRestoreConsumerConfig PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfProducerConfig STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfProducerConfig PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldBeSupportNonPrefixedConsumerConfigs STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldBeSupportNonPrefixedConsumerConfigs PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfConsumerConfig STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedPropertiesThatAreNotPartOfConsumerConfig PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportMultipleBootstrapServers STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportMultipleBootstrapServers PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldThrowStreamsExceptionIfKeySerdeConfigFails STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldThrowStreamsExceptionIfKeySerdeConfigFails PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportNonPrefixedProducerConfigs STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportNonPrefixedProducerConfigs PASSED

org.apache.kafka.streams.StreamsConfigTest > testGetRestoreConsumerConfigs 
STARTED

org.apache.kafka.streams.StreamsConfigTest > testGetRestoreConsumerConfigs 
PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldBeSupportNonPrefixedRestoreConsumerConfigs STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldBeSupportNonPrefixedRestoreConsumerConfigs PASSED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedRestoreConsumerConfigs STARTED

org.apache.kafka.streams.StreamsConfigTest > 
shouldSupportPrefixedRestoreConsumerConfigs PASSED

org.apache.kafka.streams.KafkaStreamsTest > shouldNotGetAllTasksWhenNotRunning 
STARTED

org.apache.kafka.streams.KafkaStreamsTest > shouldNotGetAllTasksWhenNotRunning 
PASSED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetTaskWithKeyAndPartitionerWhenNotRunning STARTED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetTaskWithKeyAndPartitionerWhenNotRunning PASSED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetTaskWithKeyAndSerializerWhenNotRunning STARTED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetTaskWithKeyAndSerializerWhenNotRunning PASSED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetAllTasksWithStoreWhenNotRunning STARTED

org.apache.kafka.streams.KafkaStreamsTest > 
shouldNotGetAllTasksWithStoreWhenNotRunning PASSED

org.apache.kafka.streams.KafkaStreamsTest > testCannotStartOnceClosed STARTED

org.apache.kafka.streams.KafkaStreamsTest > testCannotStartOnceClosed PASSED

org.apache.kafka.streams.KafkaStreamsTest > testCleanup STARTED

org.apache.kafka.streams.KafkaStreamsTest > testCleanup PASSED

org.apache.kafka.streams.KafkaStreamsTest > testStartAndClose STARTED

org.apache.kafka.streams.KafkaStreamsTest > testStartAndClose PASSED

org.apache.kafka.streams.KafkaStreamsTest > testCloseIsIdempotent STARTED

org.apache.kafka.streams.KafkaStreamsTest > testCloseIsIdempotent PASSED

org.apache.kafka.streams.KafkaStreamsTest > testCannotCleanupWhileRunning 
STARTED

org.apache.kafka.streams.KafkaStreamsTest > testCannotCleanupWhileRunning PASSED

org.apache.kafka.streams.KafkaStreamsTest > testCannotStartTwice STARTED

org.apache.kafka.streams.KafkaStreamsTest > testCannotStartTwice PASSED

org.apache.kafka.streams.integration.KStreamKTableJoinIntegrationTest > 
shouldCountClicksPerRegion[0] STARTED

org.apache.kafka.streams.integration.KStreamKTableJoinIntegrationTest > 
shouldCountClicksPerRegion[0] PASSED

org.apache.kafka.streams.integration.KStreamKTableJoinIntegrationTest > 
shouldCountClicksPerRegion[1] STARTED

org.apache.kafka.streams.integration.KStreamKTableJoinIntegrationTest > 
shouldCountClicksPerRegion[1] PASSED

org.apache.kafka.streams.integration.QueryableStateIntegrationTest > 
shouldBeAbleToQueryState[0] STARTED

org.apache.kafka.streams.integration.QueryableStateIntegrationTest > 
shouldBeAbleToQueryState[0] PASSED

org.apache.kafka.streams.integration.QueryableStateIntegrationTest > 
shouldNotMakeStoreAvailableUntilAllStoresAvailable[0] STARTED

org.apache.kafka.streams.integration.QueryableStateIntegrationTest > 
shouldNotMakeStoreAvailableUntilAllStoresAvailable[0] PASSED

org.apache.kaf

[GitHub] kafka pull request #2044: Changes for supporting unsgined tinyint values

[vamossagar12]

GitHub user vamossagar12 opened a pull request:

https://github.com/apache/kafka/pull/2044

Changes for supporting unsgined tinyint values

Based upon the theory provided in the official jdbc documentation, I have 
added a condition to include both Short and Byte as INT8 types in ConnectSchema.

(https://docs.oracle.com/javase/6/docs/technotes/guides/jdbc/getstart/mapping.html)
Here's the doc:

> 8.3.5 SMALLINT
> 
> The JDBC type SMALLINT represents a 16-bit signed integer value between 
-32768 and 32767.
> 
> The corresponding SQL type, SMALLINT, is defined in SQL-92 and is 
supported by all the major databases. The SQL-92 standard leaves the precision 
of SMALLINT up to the implementation, but in practice, all the major databases 
support at least 16 bits.
> 
> The recommended Java mapping for the JDBC SMALLINT type is as a Java 
short.

This would allow kafka-connect-jdbc to allow both signed and unsigned 
values. Currently it fails for unsigned values(i.e values > 127). This is 
specifically needed when fetching unsigned tinyint values from dbs using 
kafka-connect-jdbc. I have sent the PR to them and schema-registry as well to 
address the issue.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/vamossagar12/kafka trunk

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/kafka/pull/2044.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #2044


commit be3213d98c94b6569743254fc11bd619584cfce2
Author: Rao 
Date:   2016-10-19T18:05:05Z

Changes for supporting unsgined tinyint values




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Updated] (KAFKA-4321) Make client.id available to MetricReporter and (De)Serializers in clients

[Sumit Arrawatia (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4321?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Sumit Arrawatia updated KAFKA-4321:
---
Description: Currently, only the interceptors get the default client.id 
(which is generated if client.id is not set by the users) in configure(...) 
method. It is useful to pass the default client.id for the MetricReporters and 
(De)Serializers too.  (was: Currently, only the interceptors get the client.id 
in configure(...) method. It is useful to pass the client.id for the 
MetricReporters and (De)Serializers too.)

> Make client.id available to MetricReporter and (De)Serializers in clients
> -
>
> Key: KAFKA-4321
> URL: https://issues.apache.org/jira/browse/KAFKA-4321
> Project: Kafka
>  Issue Type: Improvement
>  Components: clients
>Reporter: Sumit Arrawatia
>Assignee: Sumit Arrawatia
>Priority: Minor
>
> Currently, only the interceptors get the default client.id (which is 
> generated if client.id is not set by the users) in configure(...) method. It 
> is useful to pass the default client.id for the MetricReporters and 
> (De)Serializers too.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Resolved] (KAFKA-4215) Consumers miss messages during partition reassignment

[Apurva Mehta (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4215?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Apurva Mehta resolved KAFKA-4215.
-
Resolution: Won't Fix

This is the expected behavior. When the replication factor is 1, and when we 
have unclean leader election, and when we lose brokers, we expect some data 
loss. 

> Consumers miss messages during partition reassignment
> -
>
> Key: KAFKA-4215
> URL: https://issues.apache.org/jira/browse/KAFKA-4215
> Project: Kafka
>  Issue Type: Bug
>Reporter: Apurva Mehta
>Assignee: Apurva Mehta
>
> In the specific case where the replication-factor of a topic is 1, when 
> partition reassignment is ongoing, and when a broker is bounced, consumers 
> reliably lose some messages in the stream. 
> This can be reproduced in system tests where the following error message sis 
> observed:
> {noformat}
> AssertionError: 737 acked message did not make it to the Consumer. They are: 
> 22530, 45059, 22534, 45063, 22538, 45067, 22542, 45071, 22546, 45075, 22550, 
> 45079, 22554, 45083, 22558, 45087, 22562, 45091, 22566, 45095, ...plus 717 
> more. Total Acked: 51809, Total Consumed: 51073. We validated that the first 
> 737 of these missing messages correctly made it into Kafka's data files. This 
> suggests they were lost on their way to the consumer.
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Re: [VOTE] 0.10.1.0 RC3

[Jason Gustafson]

+1 from myself too.

The vote passes with 9 +1 votes and no 0 or -1 votes.

+1 votes
PMC Members:
* Gwen Shapira
* Jun Rao
* Neha Narkhede

Committers:
* Ismael Juma
* Jason Gustafson

Community:
* Eno Thereska
* Manikumar Reddy
* Dana Powers
* Magnus Edenhill

0 votes
* No votes

-1 votes
* No votes

I'll continue with the release process and the release announcement will
follow shortly.

Thanks,
Jason



On Wed, Oct 19, 2016 at 9:08 AM, Magnus Edenhill  wrote:

> +1 (non-binding) passes librdkafka test suites
>
> 2016-10-19 15:55 GMT+02:00 Ismael Juma :
>
> > +1 (non-binding).
> >
> > Verified source and Scala 2.11 binary artifacts, ran ./gradlew test with
> > JDK 7u80, quick start on source artifact and Scala 2.11 binary artifacts.
> >
> > Thanks for managing the release!
> >
> > Ismael
> >
> > On Sat, Oct 15, 2016 at 12:29 AM, Jason Gustafson 
> > wrote:
> >
> > > Hello Kafka users, developers and client-developers,
> > >
> > > One more RC for 0.10.1.0. We're hoping this is the final one so that we
> > can
> > > meet the release target date of Oct. 17 (Monday). Please let me know as
> > > soon as possible if you find any major problems.
> > >
> > > Release plan: https://cwiki.apache.org/confluence/display/KAFKA/Rele
> > > ase+Plan+0.10.1.
> > >
> > > Release notes for the 0.10.1.0 release:
> > > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/RELEASE_NOTES.html
> > >
> > > *** Please download, test and vote by Monday, Oct 17, 5pm PT
> > >
> > > Kafka's KEYS file containing PGP keys we use to sign the release:
> > > http://kafka.apache.org/KEYS
> > >
> > > * Release artifacts to be voted upon (source and binary):
> > > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/
> > >
> > > * Maven artifacts to be voted upon:
> > > https://repository.apache.org/content/groups/staging/
> > >
> > > * Javadoc:
> > > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/javadoc/
> > >
> > > * Tag to be voted upon (off 0.10.1 branch) is the 0.10.1.0-rc3 tag:
> > > https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=tag;h=
> > > 50f30a44f31fca1bd9189d2814388d51bd56b06b
> > >
> > > * Documentation:
> > > http://kafka.apache.org/0101/documentation.html
> > >
> > > * Protocol:
> > > http://kafka.apache.org/0101/protocol.html
> > >
> > > * Tests:
> > > Unit tests: https://builds.apache.org/job/kafka-0.10.1-jdk7/71/
> > > System tests:
> > > http://testing.confluent.io/confluent-kafka-0-10-1-system-
> > > test-results/?prefix=2016-10-13--001.1476369986--apache--0.
> > 10.1--ee212d1/
> > >
> > > (Note that these tests do not include a couple patches merged today. I
> > will
> > > send links to updated test builds as soon as they are available)
> > >
> > > Thanks,
> > >
> > > Jason
> > >
> >
>

[jira] [Created] (KAFKA-4321) Make client.id available to MetricReporter and (De)Serializers in clients

[Sumit Arrawatia (JIRA)]

Sumit Arrawatia created KAFKA-4321:
--

 Summary: Make client.id available to MetricReporter and 
(De)Serializers in clients
 Key: KAFKA-4321
 URL: https://issues.apache.org/jira/browse/KAFKA-4321
 Project: Kafka
  Issue Type: Improvement
  Components: clients
Reporter: Sumit Arrawatia
Assignee: Sumit Arrawatia
Priority: Minor


Currently, only the interceptors get the client.id in configure(...) method. It 
is useful to pass the client.id for the MetricReporters and (De)Serializers too.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] kafka pull request #2023: KAFKA-4319; AbstractFetcherManager: shutdown speed...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2023


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Updated] (KAFKA-4319) AbstractFetcherManager shutdown speedup

[Jason Gustafson (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4319?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Jason Gustafson updated KAFKA-4319:
---
   Resolution: Fixed
Fix Version/s: 0.10.2.0
   Status: Resolved  (was: Patch Available)

Issue resolved by pull request 2023
[https://github.com/apache/kafka/pull/2023]

> AbstractFetcherManager shutdown speedup
> ---
>
> Key: KAFKA-4319
> URL: https://issues.apache.org/jira/browse/KAFKA-4319
> Project: Kafka
>  Issue Type: Improvement
>Reporter: Alexey Ozeritskiy
> Fix For: 0.10.2.0
>
>
> While a shutdown proccess, AbstractFetcherManager closed all worker-threads 
> sequentially which slows the final time of shutdown dramatically on huge 
> clusters (approximately 15 minutes for 100 nodes, for example).
> This can be improved by parallel mode. On the first stage 
> AbstractFetcherManager can send the stop signal and then join all the workers 
> to the thread



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4319) AbstractFetcherManager shutdown speedup

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4319?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589236#comment-15589236
 ] 

ASF GitHub Bot commented on KAFKA-4319:
---

Github user asfgit closed the pull request at:

https://github.com/apache/kafka/pull/2023


> AbstractFetcherManager shutdown speedup
> ---
>
> Key: KAFKA-4319
> URL: https://issues.apache.org/jira/browse/KAFKA-4319
> Project: Kafka
>  Issue Type: Improvement
>Reporter: Alexey Ozeritskiy
> Fix For: 0.10.2.0
>
>
> While a shutdown proccess, AbstractFetcherManager closed all worker-threads 
> sequentially which slows the final time of shutdown dramatically on huge 
> clusters (approximately 15 minutes for 100 nodes, for example).
> This can be improved by parallel mode. On the first stage 
> AbstractFetcherManager can send the stop signal and then join all the workers 
> to the thread



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Re: [VOTE] 0.10.1.0 RC3

[Magnus Edenhill]

+1 (non-binding) passes librdkafka test suites

2016-10-19 15:55 GMT+02:00 Ismael Juma :

> +1 (non-binding).
>
> Verified source and Scala 2.11 binary artifacts, ran ./gradlew test with
> JDK 7u80, quick start on source artifact and Scala 2.11 binary artifacts.
>
> Thanks for managing the release!
>
> Ismael
>
> On Sat, Oct 15, 2016 at 12:29 AM, Jason Gustafson 
> wrote:
>
> > Hello Kafka users, developers and client-developers,
> >
> > One more RC for 0.10.1.0. We're hoping this is the final one so that we
> can
> > meet the release target date of Oct. 17 (Monday). Please let me know as
> > soon as possible if you find any major problems.
> >
> > Release plan: https://cwiki.apache.org/confluence/display/KAFKA/Rele
> > ase+Plan+0.10.1.
> >
> > Release notes for the 0.10.1.0 release:
> > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/RELEASE_NOTES.html
> >
> > *** Please download, test and vote by Monday, Oct 17, 5pm PT
> >
> > Kafka's KEYS file containing PGP keys we use to sign the release:
> > http://kafka.apache.org/KEYS
> >
> > * Release artifacts to be voted upon (source and binary):
> > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/
> >
> > * Maven artifacts to be voted upon:
> > https://repository.apache.org/content/groups/staging/
> >
> > * Javadoc:
> > http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/javadoc/
> >
> > * Tag to be voted upon (off 0.10.1 branch) is the 0.10.1.0-rc3 tag:
> > https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=tag;h=
> > 50f30a44f31fca1bd9189d2814388d51bd56b06b
> >
> > * Documentation:
> > http://kafka.apache.org/0101/documentation.html
> >
> > * Protocol:
> > http://kafka.apache.org/0101/protocol.html
> >
> > * Tests:
> > Unit tests: https://builds.apache.org/job/kafka-0.10.1-jdk7/71/
> > System tests:
> > http://testing.confluent.io/confluent-kafka-0-10-1-system-
> > test-results/?prefix=2016-10-13--001.1476369986--apache--0.
> 10.1--ee212d1/
> >
> > (Note that these tests do not include a couple patches merged today. I
> will
> > send links to updated test builds as soon as they are available)
> >
> > Thanks,
> >
> > Jason
> >
>

[jira] [Commented] (KAFKA-4311) Exception in NamedCache.flush - Key found in dirty key set, but entry is null

[Damian Guy (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4311?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589089#comment-15589089
 ] 

Damian Guy commented on KAFKA-4311:
---

Thanks - i'll take a look at it what your repository and get back to you. 
Appreciate your help

> Exception in NamedCache.flush -  Key found in dirty key set, but entry is 
> null 
> ---
>
> Key: KAFKA-4311
> URL: https://issues.apache.org/jira/browse/KAFKA-4311
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.1.0
>Reporter: Damian Guy
>Assignee: Damian Guy
> Fix For: 0.10.1.1
>
>
> Reported on the mailing list. Needs looking into how it could get in this 
> state.
> [StreamThread-1] ERROR
> org.apache.kafka.streams.processor.internals.StreamThread - stream-thread
> [StreamThread-1] Failed to close state manager for StreamTask 0_0:
> org.apache.kafka.streams.errors.ProcessorStateException: task [0_0] Failed
> to close state store addr-organization
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:342)
> at
> org.apache.kafka.streams.processor.internals.AbstractTask.closeStateManager(AbstractTask.java:121)
> at
> org.apache.kafka.streams.processor.internals.StreamThread$2.apply(StreamThread.java:341)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.performOnAllTasks(StreamThread.java:322)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.closeAllStateManagers(StreamThread.java:338)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdownTasksAndState(StreamThread.java:299)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdown(StreamThread.java:262)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:245)
> Caused by: java.lang.IllegalStateException: Key found in dirty key set, but
> entry is null
> at
> org.apache.kafka.streams.state.internals.NamedCache.flush(NamedCache.java:112)
> at
> org.apache.kafka.streams.state.internals.ThreadCache.flush(ThreadCache.java:100)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.flush(CachingKeyValueStore.java:111)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.close(CachingKeyValueStore.java:117)
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:340)
> ... 7 more



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4311) Exception in NamedCache.flush - Key found in dirty key set, but entry is null

[Frank Lyaruu (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4311?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15589074#comment-15589074
 ] 

Frank Lyaruu commented on KAFKA-4311:
-

I'm working on it, and frankly I'm at a loss.

I've made a repository here: https://github.com/flyaruu/kafka4311.git

I've isolated the code that does the joining. If I run this against my customer 
data (which is person data I'm afraid I cannot share) I run into this bug 
pretty quickly, especially when I increase the number of threads. If I set the 
CACHE_MAX_BYTES_BUFFERING_CONFIG to 0, I don't see it.

I've tried to create a synthetic dataset (The code is in the GenerateTestData 
class), but I've been unable to reproduce the bug with that dataset.

It seems to be data related, but at the same time the cache and threading 
settings seem to have influence.

So either there is a bug that only manifests with very specific data, or there 
is a bug in my code that somehow gets sidestepped when the cache is off.

Any ideas? Is there something obvious I'm doing wrong?

regards, Frank

> Exception in NamedCache.flush -  Key found in dirty key set, but entry is 
> null 
> ---
>
> Key: KAFKA-4311
> URL: https://issues.apache.org/jira/browse/KAFKA-4311
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.1.0
>Reporter: Damian Guy
>Assignee: Damian Guy
> Fix For: 0.10.1.1
>
>
> Reported on the mailing list. Needs looking into how it could get in this 
> state.
> [StreamThread-1] ERROR
> org.apache.kafka.streams.processor.internals.StreamThread - stream-thread
> [StreamThread-1] Failed to close state manager for StreamTask 0_0:
> org.apache.kafka.streams.errors.ProcessorStateException: task [0_0] Failed
> to close state store addr-organization
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:342)
> at
> org.apache.kafka.streams.processor.internals.AbstractTask.closeStateManager(AbstractTask.java:121)
> at
> org.apache.kafka.streams.processor.internals.StreamThread$2.apply(StreamThread.java:341)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.performOnAllTasks(StreamThread.java:322)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.closeAllStateManagers(StreamThread.java:338)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdownTasksAndState(StreamThread.java:299)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdown(StreamThread.java:262)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:245)
> Caused by: java.lang.IllegalStateException: Key found in dirty key set, but
> entry is null
> at
> org.apache.kafka.streams.state.internals.NamedCache.flush(NamedCache.java:112)
> at
> org.apache.kafka.streams.state.internals.ThreadCache.flush(ThreadCache.java:100)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.flush(CachingKeyValueStore.java:111)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.close(CachingKeyValueStore.java:117)
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:340)
> ... 7 more



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Frequent UNKNOWN_MEMBER_ID errors in kafka consumer

[dhiraj prajapati]

Hi,
I have a consumer which implements new consumer api (0.9.0.1). I see below
errors quite frequently in the consumer application logs:

ERROR [pool-4-thread-5] - o.a.k.c.c.i.ConsumerCoordinator - Error
UNKNOWN_MEMBER_ID occurred while committing offsets for group
audit.consumer.group

Can you please enlighten me about the reason of its occurrence?

Re: [VOTE] 0.10.1.0 RC3

[Ismael Juma]

+1 (non-binding).

Verified source and Scala 2.11 binary artifacts, ran ./gradlew test with
JDK 7u80, quick start on source artifact and Scala 2.11 binary artifacts.

Thanks for managing the release!

Ismael

On Sat, Oct 15, 2016 at 12:29 AM, Jason Gustafson 
wrote:

> Hello Kafka users, developers and client-developers,
>
> One more RC for 0.10.1.0. We're hoping this is the final one so that we can
> meet the release target date of Oct. 17 (Monday). Please let me know as
> soon as possible if you find any major problems.
>
> Release plan: https://cwiki.apache.org/confluence/display/KAFKA/Rele
> ase+Plan+0.10.1.
>
> Release notes for the 0.10.1.0 release:
> http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/RELEASE_NOTES.html
>
> *** Please download, test and vote by Monday, Oct 17, 5pm PT
>
> Kafka's KEYS file containing PGP keys we use to sign the release:
> http://kafka.apache.org/KEYS
>
> * Release artifacts to be voted upon (source and binary):
> http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/
>
> * Maven artifacts to be voted upon:
> https://repository.apache.org/content/groups/staging/
>
> * Javadoc:
> http://home.apache.org/~jgus/kafka-0.10.1.0-rc3/javadoc/
>
> * Tag to be voted upon (off 0.10.1 branch) is the 0.10.1.0-rc3 tag:
> https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=tag;h=
> 50f30a44f31fca1bd9189d2814388d51bd56b06b
>
> * Documentation:
> http://kafka.apache.org/0101/documentation.html
>
> * Protocol:
> http://kafka.apache.org/0101/protocol.html
>
> * Tests:
> Unit tests: https://builds.apache.org/job/kafka-0.10.1-jdk7/71/
> System tests:
> http://testing.confluent.io/confluent-kafka-0-10-1-system-
> test-results/?prefix=2016-10-13--001.1476369986--apache--0.10.1--ee212d1/
>
> (Note that these tests do not include a couple patches merged today. I will
> send links to updated test builds as soon as they are available)
>
> Thanks,
>
> Jason
>

[jira] [Commented] (KAFKA-3995) Add a new configuration "enable.comrpession.ratio.estimation" to the producer config

[Jiangjie Qin (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-3995?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15588765#comment-15588765
 ] 

Jiangjie Qin commented on KAFKA-3995:
-

[~mgharat] Could you elaborate on what is the workaround?

> Add a new configuration "enable.comrpession.ratio.estimation" to the producer 
> config
> 
>
> Key: KAFKA-3995
> URL: https://issues.apache.org/jira/browse/KAFKA-3995
> Project: Kafka
>  Issue Type: Improvement
>  Components: clients
>Affects Versions: 0.10.0.0
>Reporter: Jiangjie Qin
>Assignee: Mayuresh Gharat
> Fix For: 0.10.2.0
>
>
> We recently see a few cases where RecordTooLargeException is thrown because 
> the compressed message sent by KafkaProducer exceeded the max message size.
> The root cause of this issue is because the compressor is estimating the 
> batch size using an estimated compression ratio based on heuristic 
> compression ratio statistics. This does not quite work for the traffic with 
> highly variable compression ratios. 
> For example, if the batch size is set to 1MB and the max message size is 1MB. 
> Initially a the producer is sending messages (each message is 1MB) to topic_1 
> whose data can be compressed to 1/10 of the original size. After a while the 
> estimated compression ratio in the compressor will be trained to 1/10 and the 
> producer would put 10 messages into one batch. Now the producer starts to 
> send messages (each message is also 1MB) to topic_2 whose message can only be 
> compress to 1/5 of the original size. The producer would still use 1/10 as 
> the estimated compression ratio and put 10 messages into a batch. That batch 
> would be 2 MB after compression which exceeds the maximum message size. In 
> this case the user do not have many options other than resend everything or 
> close the producer if they care about ordering.
> This is especially an issue for services like MirrorMaker whose producer is 
> shared by many different topics.
> To solve this issue, we can probably add a configuration 
> "enable.compression.ratio.estimation" to the producer. So when this 
> configuration is set to false, we stop estimating the compressed size but 
> will close the batch once the uncompressed bytes in the batch reaches the 
> batch size.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Issue Comment Deleted] (KAFKA-4295) kafka-console-consumer.sh does not delete the temporary group in zookeeper

[huxi (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

huxi updated KAFKA-4295:

Comment: was deleted

(was: I think we'd better follow what 0.8.x does for us.  Removed consumer 
group's persisitent nodes after it became inactive in case of polluting 
zookeeper, if we did not specify "group.id" for the console consumer.)

> kafka-console-consumer.sh does not delete the temporary group in zookeeper
> --
>
> Key: KAFKA-4295
> URL: https://issues.apache.org/jira/browse/KAFKA-4295
> Project: Kafka
>  Issue Type: Bug
>  Components: admin
>Affects Versions: 0.9.0.1, 0.10.0.0, 0.10.0.1
>Reporter: Sswater Shi
>Assignee: huxi
>Priority: Minor
>
> I'm not sure it is a bug or you guys designed it.
> Since 0.9.x.x, the kafka-console-consumer.sh will not delete the group 
> information in zookeeper/consumers on exit when without "--new-consumer". 
> There will be a lot of abandoned zookeeper/consumers/console-consumer-xxx if 
> kafka-console-consumer.sh runs a lot of times.
> When 0.8.x.x,  the kafka-console-consumer.sh can be followed by an argument 
> "group". If not specified, the kafka-console-consumer.sh will create a 
> temporary group name like 'console-consumer-'. If the group name is 
> specified by "group", the information in the zookeeper/consumers will be kept 
> on exit. If the group name is a temporary one, the information in the 
> zookeeper will be deleted when kafka-console-consumer.sh is quitted by 
> Ctrl+C. Why this is changed from 0.9.x.x.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4295) kafka-console-consumer.sh does not delete the temporary group in zookeeper

[huxi (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15588733#comment-15588733
 ] 

huxi commented on KAFKA-4295:
-

I think we'd better follow what 0.8.x does for us.  Removed consumer group's 
persisitent nodes after it became inactive in case of polluting zookeeper, if 
we did not specify "group.id" for the console consumer.

> kafka-console-consumer.sh does not delete the temporary group in zookeeper
> --
>
> Key: KAFKA-4295
> URL: https://issues.apache.org/jira/browse/KAFKA-4295
> Project: Kafka
>  Issue Type: Bug
>  Components: admin
>Affects Versions: 0.9.0.1, 0.10.0.0, 0.10.0.1
>Reporter: Sswater Shi
>Assignee: huxi
>Priority: Minor
>
> I'm not sure it is a bug or you guys designed it.
> Since 0.9.x.x, the kafka-console-consumer.sh will not delete the group 
> information in zookeeper/consumers on exit when without "--new-consumer". 
> There will be a lot of abandoned zookeeper/consumers/console-consumer-xxx if 
> kafka-console-consumer.sh runs a lot of times.
> When 0.8.x.x,  the kafka-console-consumer.sh can be followed by an argument 
> "group". If not specified, the kafka-console-consumer.sh will create a 
> temporary group name like 'console-consumer-'. If the group name is 
> specified by "group", the information in the zookeeper/consumers will be kept 
> on exit. If the group name is a temporary one, the information in the 
> zookeeper will be deleted when kafka-console-consumer.sh is quitted by 
> Ctrl+C. Why this is changed from 0.9.x.x.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Assigned] (KAFKA-4295) kafka-console-consumer.sh does not delete the temporary group in zookeeper

[huxi (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

huxi reassigned KAFKA-4295:
---

Assignee: huxi

> kafka-console-consumer.sh does not delete the temporary group in zookeeper
> --
>
> Key: KAFKA-4295
> URL: https://issues.apache.org/jira/browse/KAFKA-4295
> Project: Kafka
>  Issue Type: Bug
>  Components: admin
>Affects Versions: 0.9.0.1, 0.10.0.0, 0.10.0.1
>Reporter: Sswater Shi
>Assignee: huxi
>Priority: Minor
>
> I'm not sure it is a bug or you guys designed it.
> Since 0.9.x.x, the kafka-console-consumer.sh will not delete the group 
> information in zookeeper/consumers on exit when without "--new-consumer". 
> There will be a lot of abandoned zookeeper/consumers/console-consumer-xxx if 
> kafka-console-consumer.sh runs a lot of times.
> When 0.8.x.x,  the kafka-console-consumer.sh can be followed by an argument 
> "group". If not specified, the kafka-console-consumer.sh will create a 
> temporary group name like 'console-consumer-'. If the group name is 
> specified by "group", the information in the zookeeper/consumers will be kept 
> on exit. If the group name is a temporary one, the information in the 
> zookeeper will be deleted when kafka-console-consumer.sh is quitted by 
> Ctrl+C. Why this is changed from 0.9.x.x.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4295) kafka-console-consumer.sh does not delete the temporary group in zookeeper

[huxi (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15588725#comment-15588725
 ] 

huxi commented on KAFKA-4295:
-

I think we'd better follow what 0.8.x does for us.  Removed consumer group's 
persisitent nodes after it became inactive in case of polluting zookeeper, if 
we did not specify "group.id" for the console consumer.

> kafka-console-consumer.sh does not delete the temporary group in zookeeper
> --
>
> Key: KAFKA-4295
> URL: https://issues.apache.org/jira/browse/KAFKA-4295
> Project: Kafka
>  Issue Type: Bug
>  Components: admin
>Affects Versions: 0.9.0.1, 0.10.0.0, 0.10.0.1
>Reporter: Sswater Shi
>Priority: Minor
>
> I'm not sure it is a bug or you guys designed it.
> Since 0.9.x.x, the kafka-console-consumer.sh will not delete the group 
> information in zookeeper/consumers on exit when without "--new-consumer". 
> There will be a lot of abandoned zookeeper/consumers/console-consumer-xxx if 
> kafka-console-consumer.sh runs a lot of times.
> When 0.8.x.x,  the kafka-console-consumer.sh can be followed by an argument 
> "group". If not specified, the kafka-console-consumer.sh will create a 
> temporary group name like 'console-consumer-'. If the group name is 
> specified by "group", the information in the zookeeper/consumers will be kept 
> on exit. If the group name is a temporary one, the information in the 
> zookeeper will be deleted when kafka-console-consumer.sh is quitted by 
> Ctrl+C. Why this is changed from 0.9.x.x.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4320) Log compaction docs update

[Dustin Cote (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4320?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Dustin Cote updated KAFKA-4320:
---
Labels: newbie  (was: )

> Log compaction docs update
> --
>
> Key: KAFKA-4320
> URL: https://issues.apache.org/jira/browse/KAFKA-4320
> Project: Kafka
>  Issue Type: Bug
>  Components: documentation
>Reporter: Dustin Cote
>Priority: Minor
>  Labels: newbie
>
> The log compaction docs are out of date.  At least the default is said to be 
> that log compaction is disabled which is not true as of 0.9.0.1.  Probably 
> the whole section needs a once over to make sure it's in line with what is 
> currently there.  This is the section:
> [http://kafka.apache.org/documentation#design_compactionconfig]



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (KAFKA-4320) Log compaction docs update

[Dustin Cote (JIRA)]

Dustin Cote created KAFKA-4320:
--

 Summary: Log compaction docs update
 Key: KAFKA-4320
 URL: https://issues.apache.org/jira/browse/KAFKA-4320
 Project: Kafka
  Issue Type: Bug
  Components: documentation
Reporter: Dustin Cote
Priority: Minor


The log compaction docs are out of date.  At least the default is said to be 
that log compaction is disabled which is not true as of 0.9.0.1.  Probably the 
whole section needs a once over to make sure it's in line with what is 
currently there.  This is the section:
[http://kafka.apache.org/documentation#design_compactionconfig]



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4319) AbstractFetcherManager shutdown speedup

[Alexey Ozeritskiy (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4319?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Alexey Ozeritskiy updated KAFKA-4319:
-
Status: Patch Available  (was: Open)

https://github.com/apache/kafka/pull/2023

> AbstractFetcherManager shutdown speedup
> ---
>
> Key: KAFKA-4319
> URL: https://issues.apache.org/jira/browse/KAFKA-4319
> Project: Kafka
>  Issue Type: Improvement
>Reporter: Alexey Ozeritskiy
>
> While a shutdown proccess, AbstractFetcherManager closed all worker-threads 
> sequentially which slows the final time of shutdown dramatically on huge 
> clusters (approximately 15 minutes for 100 nodes, for example).
> This can be improved by parallel mode. On the first stage 
> AbstractFetcherManager can send the stop signal and then join all the workers 
> to the thread



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (KAFKA-4319) AbstractFetcherManager shutdown speedup

[Alexey Ozeritskiy (JIRA)]

Alexey Ozeritskiy created KAFKA-4319:


 Summary: AbstractFetcherManager shutdown speedup
 Key: KAFKA-4319
 URL: https://issues.apache.org/jira/browse/KAFKA-4319
 Project: Kafka
  Issue Type: Improvement
Reporter: Alexey Ozeritskiy


While a shutdown proccess, AbstractFetcherManager closed all worker-threads 
sequentially which slows the final time of shutdown dramatically on huge 
clusters (approximately 15 minutes for 100 nodes, for example).

This can be improved by parallel mode. On the first stage 
AbstractFetcherManager can send the stop signal and then join all the workers 
to the thread



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Kafka running on Ceph

[jupiter]

Hi,

I found following post from achieve, but no any response. Any comments,
future development and roadmap for that feature?

---

From: Connie Yang 
To: us...@kafka.apache.org
Content-Type: multipart/alternative; boundary=001a11c38a9c2aab6905338628c7

--001a11c38a9c2aab6905338628c7
Content-Type: text/plain; charset=UTF-8

Hi All,

Does anyone have any performance metrics running Kafka on Ceph?

I briefly gathered at the 2016 Kafka Summit that there's an ongoing work
between the Kafka community and RedHat in getting Kafka running
successfully on Ceph.  Is this correct?  If so, what's timeline for that?

Thanks
Connie

[jira] [Created] (KAFKA-4318) Migrate ProducerSendTest to the new consumer

[Balint Molnar (JIRA)]

Balint Molnar created KAFKA-4318:


 Summary: Migrate ProducerSendTest to the new consumer
 Key: KAFKA-4318
 URL: https://issues.apache.org/jira/browse/KAFKA-4318
 Project: Kafka
  Issue Type: Bug
  Components: unit tests
Reporter: Balint Molnar
Assignee: Balint Molnar
Priority: Minor


BaseProducerSendTest contains a 
TODO: "we need to migrate to new consumers when 0.9 is final"




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Work started] (KAFKA-4318) Migrate ProducerSendTest to the new consumer

[Balint Molnar (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4318?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Work on KAFKA-4318 started by Balint Molnar.

> Migrate ProducerSendTest to the new consumer
> 
>
> Key: KAFKA-4318
> URL: https://issues.apache.org/jira/browse/KAFKA-4318
> Project: Kafka
>  Issue Type: Bug
>  Components: unit tests
>Reporter: Balint Molnar
>Assignee: Balint Molnar
>Priority: Minor
>
> BaseProducerSendTest contains a 
> TODO: "we need to migrate to new consumers when 0.9 is final"



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Michael Noll (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Michael Noll updated KAFKA-4316:

Description: 
We had encountered the problem that starting application with Kafka Streams 
0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the Windows 
x64 machine. 

Part of the stacktrace:

{code}
Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
inside JAR.
at 
org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
{code}

It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
broken Java API. 
See: 
https://github.com/facebook/rocksdb/issues/1177
https://github.com/facebook/rocksdb/issues/1302

This critical (for Windows) bug was fixed in RocksDB 4.9.0.

Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
So the line shall be rocksDB: "4.9.0".

I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
and it was promissing and definitely the bug was away.

  was:
We had encountered the problem that starting application with Kafka Streams 
0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the Windows 
x64 machine. 

Part of the stacktrace:

{code}
Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
inside JAR.
at 
org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
{code}

It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
broken Java API. 
See: 
https://github.com/facebook/rocksdb/issues/1177
https://github.com/facebook/rocksdb/issues/1302

This critical (for Windows) bug was fixed in RocksDB 4.9.0.

Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
So the line shall be rocksDB: "4.9.0".

I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
and it was promissing and definitely the bug was away.


> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RocksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Michael Noll (JIRA)]

 [ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Michael Noll updated KAFKA-4316:

Description: 
We had encountered the problem that starting application with Kafka Streams 
0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the Windows 
x64 machine. 

Part of the stacktrace:

{code}
Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
inside JAR.
at 
org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
{code}

It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
broken Java API. 
See: 
https://github.com/facebook/rocksdb/issues/1177
https://github.com/facebook/rocksdb/issues/1302

This critical (for Windows) bug was fixed in RocksDB 4.9.0.

Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
So the line shall be rocksDB: "4.9.0".

I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
and it was promissing and definitely the bug was away.

  was:
We had encountered the problem that starting application with Kafka Streams 
0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the Windows 
x64 machine. 

Part of the stacktrace:
Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
inside JAR.
at 
org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)

It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
broken Java API. 
See: 
https://github.com/facebook/rocksdb/issues/1177
https://github.com/facebook/rocksdb/issues/1302

This critical (for Windows) bug was fixed in RocksDB 4.9.0.

Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
So the line shall be rocksDB: "4.9.0".

I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
and it was promissing and definitely the bug was away.


> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> {code}
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> {code}
> It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Michael Noll (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4316?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15588149#comment-15588149
 ] 

Michael Noll commented on KAFKA-4316:
-

[~habdank]: Thanks for reporting this.

FYI:  The upcoming 0.10.1 version of Kafka already uses RocksDB 4.9.0, hence 
upgrading to 0.10.1 would be an option for you? FWIW, there are several 
critical bug fixes relating to Kafka Streams in 0.10.1 anyway, so we definitely 
recommend upgrading from 0.10.0 to 0.10.1.

I don't know whether there are plans to change the RocksDB version of the 
0.10.0.x release line (perhaps others can chime in here).

> Kafka Streams 0.10.0.1 does not run on Windows x64
> --
>
> Key: KAFKA-4316
> URL: https://issues.apache.org/jira/browse/KAFKA-4316
> Project: Kafka
>  Issue Type: Bug
>Reporter: Seweryn Habdank-Wojewodzki
>
> We had encountered the problem that starting application with Kafka Streams 
> 0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the 
> Windows x64 machine. 
> Part of the stacktrace:
> Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
> inside JAR.
> at 
> org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)
> It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
> broken Java API. 
> See: 
> https://github.com/facebook/rocksdb/issues/1177
> https://github.com/facebook/rocksdb/issues/1302
> This critical (for Windows) bug was fixed in RocksDB 4.9.0.
> Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
> So the line shall be rocksDB: "4.9.0".
> I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
> and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4311) Exception in NamedCache.flush - Key found in dirty key set, but entry is null

[Damian Guy (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4311?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15588096#comment-15588096
 ] 

Damian Guy commented on KAFKA-4311:
---

[~flyaruu]
We are unable to reproduce or find the problem you are seeing. If you could 
provide us with some sample code that can reproduce the problem it would help 
us to investigate.

Thanks,
Damian

> Exception in NamedCache.flush -  Key found in dirty key set, but entry is 
> null 
> ---
>
> Key: KAFKA-4311
> URL: https://issues.apache.org/jira/browse/KAFKA-4311
> Project: Kafka
>  Issue Type: Bug
>  Components: streams
>Affects Versions: 0.10.1.0
>Reporter: Damian Guy
>Assignee: Damian Guy
> Fix For: 0.10.1.1
>
>
> Reported on the mailing list. Needs looking into how it could get in this 
> state.
> [StreamThread-1] ERROR
> org.apache.kafka.streams.processor.internals.StreamThread - stream-thread
> [StreamThread-1] Failed to close state manager for StreamTask 0_0:
> org.apache.kafka.streams.errors.ProcessorStateException: task [0_0] Failed
> to close state store addr-organization
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:342)
> at
> org.apache.kafka.streams.processor.internals.AbstractTask.closeStateManager(AbstractTask.java:121)
> at
> org.apache.kafka.streams.processor.internals.StreamThread$2.apply(StreamThread.java:341)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.performOnAllTasks(StreamThread.java:322)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.closeAllStateManagers(StreamThread.java:338)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdownTasksAndState(StreamThread.java:299)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.shutdown(StreamThread.java:262)
> at
> org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:245)
> Caused by: java.lang.IllegalStateException: Key found in dirty key set, but
> entry is null
> at
> org.apache.kafka.streams.state.internals.NamedCache.flush(NamedCache.java:112)
> at
> org.apache.kafka.streams.state.internals.ThreadCache.flush(ThreadCache.java:100)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.flush(CachingKeyValueStore.java:111)
> at
> org.apache.kafka.streams.state.internals.CachingKeyValueStore.close(CachingKeyValueStore.java:117)
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.close(ProcessorStateManager.java:340)
> ... 7 more



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (KAFKA-4317) RocksDB checkpoint files lost on kill -9

[Greg Fodor (JIRA)]

Greg Fodor created KAFKA-4317:
-

 Summary: RocksDB checkpoint files lost on kill -9
 Key: KAFKA-4317
 URL: https://issues.apache.org/jira/browse/KAFKA-4317
 Project: Kafka
  Issue Type: Improvement
  Components: streams
Affects Versions: 0.10.0.1
Reporter: Greg Fodor
Assignee: Guozhang Wang


Right now, the checkpoint files for logged RocksDB stores are written during a 
graceful shutdown, and removed upon restoration. Unfortunately this means that 
in a scenario where the process is forcibly killed, the checkpoint files are 
not there, so all RocksDB stores are rematerialized from scratch on the next 
launch.

In a way, this is good, because it simulates bootstrapping a new node (for 
example, its a good way to see how much I/O is used to rematerialize the 
stores) however it leads to longer recovery times when a non-graceful shutdown 
occurs and we want to get the job up and running again.

It seems that two possible things to consider:

- Simply do not remove checkpoint files on restoring. This way a kill -9 will 
result in only repeating the restoration of all the data generated in the 
source topics since the last graceful shutdown.

- Continually update the checkpoint files (perhaps on commit) -- this would 
result in the least amount of overhead/latency in restarting, but the 
additional complexity may not be worth it.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4113) Allow KTable bootstrap

[Greg Fodor (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4113?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15587991#comment-15587991
 ] 

Greg Fodor commented on KAFKA-4113:
---

I guess what I would argue is that KStreamBuilder#table should have identical 
semantics to a logged state store backed KTable, except you are specifying the 
topic and (obv) it's not mutable from the job's POV. It should first check if 
it has a local, checkpointed rocksdb, and if so, it should just read from the 
checkpoint forward. If not, it should rematerialize from offset 0 and block the 
start of the job until it does. On shutdown, it should write the checkpoint 
file. It seems to me that this might boil down to just having it be "use this 
topic for the logged state store backing this KTableImpl."

I'm sure there are cases I'm missing, but having that be the behavior for 
KStreamBuilder#table would effectively solve all of our problems as far as I 
can tell. The semantics + I/O impact of this approach back out to the same 
exact ones you have when you use a normal user-created persistent state store, 
but just are managing the topic writes yourself.

> Allow KTable bootstrap
> --
>
> Key: KAFKA-4113
> URL: https://issues.apache.org/jira/browse/KAFKA-4113
> Project: Kafka
>  Issue Type: Sub-task
>  Components: streams
>Reporter: Matthias J. Sax
>Assignee: Guozhang Wang
>
> On the mailing list, there are multiple request about the possibility to 
> "fully populate" a KTable before actual stream processing start.
> Even if it is somewhat difficult to define, when the initial populating phase 
> should end, there are multiple possibilities:
> The main idea is, that there is a rarely updated topic that contains the 
> data. Only after this topic got read completely and the KTable is ready, the 
> application should start processing. This would indicate, that on startup, 
> the current partition sizes must be fetched and stored, and after KTable got 
> populated up to those offsets, stream processing can start.
> Other discussed ideas are:
> 1) an initial fixed time period for populating
> (it might be hard for a user to estimate the correct value)
> 2) an "idle" period, ie, if no update to a KTable for a certain time is
> done, we consider it as populated
> 3) a timestamp cut off point, ie, all records with an older timestamp
> belong to the initial populating phase
> The API change is not decided yet, and the API desing is part of this JIRA.
> One suggestion (for option (4)) was:
> {noformat}
> KTable table = builder.table("topic", 1000); // populate the table without 
> reading any other topics until see one record with timestamp 1000.
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (KAFKA-4316) Kafka Streams 0.10.0.1 does not run on Windows x64

[Seweryn Habdank-Wojewodzki (JIRA)]

Seweryn Habdank-Wojewodzki created KAFKA-4316:
-

 Summary: Kafka Streams 0.10.0.1 does not run on Windows x64
 Key: KAFKA-4316
 URL: https://issues.apache.org/jira/browse/KAFKA-4316
 Project: Kafka
  Issue Type: Bug
Reporter: Seweryn Habdank-Wojewodzki


We had encountered the problem that starting application with Kafka Streams 
0.10.0.1 leads to runtime exception, that rocksdb DLL is missing on the Windows 
x64 machine. 

Part of the stacktrace:
Caused by: java.lang.RuntimeException: librocksdbjni-win64.dll was not found 
inside JAR.
at 
org.rocksdb.NativeLibraryLoader.loadLibraryFromJarToTemp(NativeLibraryLoader.java:106)

It is true, as Kafka 0.10.0.1 uses RoscksDB 4.8.0. This RocksDB release has 
broken Java API. 
See: 
https://github.com/facebook/rocksdb/issues/1177
https://github.com/facebook/rocksdb/issues/1302

This critical (for Windows) bug was fixed in RocksDB 4.9.0.

Please update Kafka gradle\dependencies.gradle to use at least 4.9.0:
So the line shall be rocksDB: "4.9.0".

I had tested some basic functionality with Kafka Streams with RocksDB 4.11.2 
and it was promissing and definitely the bug was away.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (KAFKA-4113) Allow KTable bootstrap

[Greg Fodor (JIRA)]

[ 
https://issues.apache.org/jira/browse/KAFKA-4113?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15587966#comment-15587966
 ] 

Greg Fodor commented on KAFKA-4113:
---

Having played around with Kafka Streams for a while now, I am still confused 
(and we still get burned) by this. Let me walk through a case, and see if you 
guys can find out where I am misunderstanding.

Say we have a topic that's a user table changelog that has user id keys and 
user records. And we have a clickstream topic that is just a user id to url. 
For the sake of this example, lets assume our kafka streams job has been 
running from t = 0 where both topics were empty, so there's no bootstrapping 
problem.

In the Kafka Streams DSL, I would tap the user table topic via 
`KStreamBuilder#table`. As best I can tell, this creates a KTable with:
- An unlogged rocksdb state store (which is going to land on disk)
- A new source that is the table topic

After this, I'm going to tap + inner join the click stream as a KStream on user 
id, and just for this example lets assume I'll sink it all out too to a new 
topic.

As my node is humming along, it is writing the user id -> user record k/vs to 
the local rocksdb but is *not* storing the changes to the rocksdb in a topic, 
because it is not marked as logged. When it reads a record from the KStream, 
the join is performed by looking for the key in the state store. As mentioned, 
my understanding is that the join against the stream will wait until the 
records for the KTable which have earlier timestamps have been consumed. This 
makes sense.

If I terminate and restart the java process, the kafka consumer for the KTable 
will pick off at the last committed offset for the user table topic. It may 
re-consume a few seconds worth of records, and re-write a few keys in the rocks 
db store, but after that it's still got the full historical state of the topic. 
So joins against any user id will continue to work.

Where things completely stop making sense for me is if I lose the node. If I 
lose the node, i lose my rocksdb, which is not logged so is not backed by a 
changelog topic. When I bring up a new node, my understanding is that the 
consumer will *not* start at the beginning of the topic used for the KTable, it 
will just pick up at the last commit. So what I end up with is a rocksdb that 
only contains the last couple of records from the user table topic. This is 
obviously really broken, because now my joins will start failling. (And it 
seems I was lulled into complaency here since I was robust across JVM restarts, 
but not across node failures.) I believe this problem also happens in a more 
nefarious way upon rebalances, since if a partition of the KTable gets 
reassigned, it will also have a partially complete rocksdb store for that 
partition since it will just consume from the last committed offset. Similarly, 
and even scarier, if it gets assigned back to the original node, that node now 
has a rocksdb store with a very small gap, for the key changes that happened 
during the period where it was assigned to another node.

I am not sure if I am missing something here but this has been the behavior we 
have seen. The workarounds we have done for this problem are:
- write a routine to let us reset the KTable topics consumer offsets to zero 
(still doesn't help with a rebalance)
- perform a "touch" to the database records we are flushing to kafka, so new 
copies of all of the records are appended to the topic via kafka connect, and 
are forced into the rocksdb stores (this works well, but obviously is terrible)
- put a dummy aggregation/reduce after the tap of the KTable topic, which 
forces things into a logged state store that will be fully materialized on 
startup if it is missing

Thoughts?

> Allow KTable bootstrap
> --
>
> Key: KAFKA-4113
> URL: https://issues.apache.org/jira/browse/KAFKA-4113
> Project: Kafka
>  Issue Type: Sub-task
>  Components: streams
>Reporter: Matthias J. Sax
>Assignee: Guozhang Wang
>
> On the mailing list, there are multiple request about the possibility to 
> "fully populate" a KTable before actual stream processing start.
> Even if it is somewhat difficult to define, when the initial populating phase 
> should end, there are multiple possibilities:
> The main idea is, that there is a rarely updated topic that contains the 
> data. Only after this topic got read completely and the KTable is ready, the 
> application should start processing. This would indicate, that on startup, 
> the current partition sizes must be fetched and stored, and after KTable got 
> populated up to those offsets, stream processing can start.
> Other discussed ideas are:
> 1) an initial fixed time period for populating
> (it might be hard for a user to estimate the correct value)
> 2) an "idle" perio

[jira] [Created] (KAFKA-4315) Kafka Connect documentation problems

[Seweryn Habdank-Wojewodzki (JIRA)]

Seweryn Habdank-Wojewodzki created KAFKA-4315:
-

 Summary: Kafka Connect documentation problems
 Key: KAFKA-4315
 URL: https://issues.apache.org/jira/browse/KAFKA-4315
 Project: Kafka
  Issue Type: Bug
Reporter: Seweryn Habdank-Wojewodzki


On the base of documentation of the Kafka Connect - 
http://kafka.apache.org/documentation#connect, I had tried to build example in 
Java. It was not possible. 

The code pieces available on the webpage are taken out of any context and they 
are not compiling. 

Also it seems they are taken completely from other code software parts, so even 
putting them together shows, that they are not building any reasonable example. 
And they tend to be very complex. where I would expect that the API examples 
are driving "Hello World" like code.

Also there are weak connections between examples from the Kafka documentation 
and Kafka Connect tools code parts available in the Kafka source.

Finally I would be nice to have a kind of statement in the Kafka documentation 
which parts of API are stable and which are unstable or experimental.
I saw much (~20) of such a remarks in the Kafka code - I mean that API is 
unstable. This note is very important, as we will plan additional effort to 
prepare some facades for unstable code.

In my opinion it is nothing wrong in experimental API, but all those matters 
when documented shall be well documented. The current status of the main Kafka 
documentation makes impression that Kafka Connect is well tested and consistent 
and stable feature set, but it is not. What leads to confusion on the effort 
management.





--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (KAFKA-4314) Kafka Streams documentation needs definitive rework and improvement

[Seweryn Habdank-Wojewodzki (JIRA)]

Seweryn Habdank-Wojewodzki created KAFKA-4314:
-

 Summary: Kafka Streams documentation needs definitive rework and 
improvement
 Key: KAFKA-4314
 URL: https://issues.apache.org/jira/browse/KAFKA-4314
 Project: Kafka
  Issue Type: Bug
Reporter: Seweryn Habdank-Wojewodzki


On the base of documentation of the Kafka Stream, I had tried to build example 
in Java. It was not possible. The code pieces available on the webpage: 
http://kafka.apache.org/documentation#streams are taken out of any context and 
they are not compiling. 
Also it seems they are taken completely from other code software parts, so even 
putting them together shows, that they are not building any reasonable example.

I took the code of the Kafka itself and there there are some examples of the 
Kafka streams, which are at least consistent. It is very good basis to repair 
the main documentation.




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Build failed in Jenkins: kafka-trunk-jdk7 #1639

[Apache Jenkins Server]

See 

Changes:

[jason] MINOR: Replaced unnecessary map and getOrElse with exists

--
[...truncated 12372 lines...]
org.apache.kafka.common.record.RecordTest > testFields[185] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[186] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[186] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[186] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[186] PASSED

org.apache.kafka.common.record.RecordTest > testFields[186] STARTED

org.apache.kafka.common.record.RecordTest > testFields[186] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[187] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[187] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[187] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[187] PASSED

org.apache.kafka.common.record.RecordTest > testFields[187] STARTED

org.apache.kafka.common.record.RecordTest > testFields[187] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[188] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[188] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[188] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[188] PASSED

org.apache.kafka.common.record.RecordTest > testFields[188] STARTED

org.apache.kafka.common.record.RecordTest > testFields[188] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[189] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[189] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[189] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[189] PASSED

org.apache.kafka.common.record.RecordTest > testFields[189] STARTED

org.apache.kafka.common.record.RecordTest > testFields[189] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[190] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[190] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[190] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[190] PASSED

org.apache.kafka.common.record.RecordTest > testFields[190] STARTED

org.apache.kafka.common.record.RecordTest > testFields[190] PASSED

org.apache.kafka.common.record.RecordTest > testChecksum[191] STARTED

org.apache.kafka.common.record.RecordTest > testChecksum[191] PASSED

org.apache.kafka.common.record.RecordTest > testEquality[191] STARTED

org.apache.kafka.common.record.RecordTest > testEquality[191] PASSED

org.apache.kafka.common.record.RecordTest > testFields[191] STARTED

org.apache.kafka.common.record.RecordTest > testFields[191] PASSED

org.apache.kafka.common.record.SimpleRecordTest > 
testIsValidWithFourBytesBuffer STARTED

org.apache.kafka.common.record.SimpleRecordTest > 
testIsValidWithFourBytesBuffer PASSED

org.apache.kafka.common.record.SimpleRecordTest > 
testIsValidWithChecksumMismatch STARTED

org.apache.kafka.common.record.SimpleRecordTest > 
testIsValidWithChecksumMismatch PASSED

org.apache.kafka.common.record.SimpleRecordTest > testIsValidWithTooSmallBuffer 
STARTED

org.apache.kafka.common.record.SimpleRecordTest > testIsValidWithTooSmallBuffer 
PASSED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[0] STARTED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[0] PASSED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[1] STARTED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[1] PASSED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[2] STARTED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[2] PASSED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[3] STARTED

org.apache.kafka.common.record.KafkaLZ4Test > testKafkaLZ4[3] PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[0] 
STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[0] 
PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[0] STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[0] PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[1] 
STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[1] 
PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[1] STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[1] PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[2] 
STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[2] 
PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[2] STARTED

org.apache.kafka.common.record.MemoryRecordsTest > testIterator[2] PASSED

org.apache.kafka.common.record.MemoryRecordsTest > testHasRoomForMethod[3] 
STARTED

org.apache.kafka.common.record.MemoryRecordsTest