rabbah commented on a change in pull request #3072: Enhance kafka message provider URL: https://github.com/apache/incubator-openwhisk/pull/3072#discussion_r164444571
########## File path: common/scala/src/main/scala/whisk/connector/kafka/KafkaConsumerConnector.scala ########## @@ -21,46 +21,91 @@ import java.util.Properties import scala.collection.JavaConversions.iterableAsScalaIterable import scala.collection.JavaConversions.seqAsJavaList -import scala.concurrent.duration.Duration -import scala.concurrent.duration.DurationInt -import scala.concurrent.duration.FiniteDuration - +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.common.serialization.ByteArrayDeserializer - +import org.apache.kafka.common.errors.{RetriableException, WakeupException} +import akka.actor.ActorSystem import whisk.common.Logging import whisk.core.connector.MessageConsumer -class KafkaConsumerConnector(kafkahost: String, - groupid: String, - topic: String, - override val maxPeek: Int = Int.MaxValue, - readeos: Boolean = true, - sessionTimeout: FiniteDuration = 30.seconds, - autoCommitInterval: FiniteDuration = 10.seconds, - maxPollInterval: FiniteDuration = 5.minutes)(implicit logging: Logging) +class KafkaConsumerConnector( + kafkahost: String, + groupid: String, + topic: String, + override val maxPeek: Int = Int.MaxValue, + readeos: Boolean = true, + sessionTimeout: FiniteDuration = 30.seconds, + autoCommitInterval: FiniteDuration = 10.seconds, + maxPollInterval: FiniteDuration = 5.minutes)(implicit logging: Logging, actorSystem: ActorSystem) extends MessageConsumer { + implicit val ec: ExecutionContext = actorSystem.dispatcher + private val gracefulWaitTime = 100.milliseconds.toMillis + /** * Long poll for messages. Method returns once message are available but no later than given * duration. * * @param duration the maximum duration for the long poll */ - override def peek(duration: Duration = 500.milliseconds) = { - val records = consumer.poll(duration.toMillis) - records map { r => - (r.topic, r.partition, r.offset, r.value) - } + override def peek(duration: Duration = 500.milliseconds, + retry: Int = 3): Iterable[(String, Int, Long, Array[Byte])] = { + // Since kafka client can be infinitely blocked in poll(timeout), we should interrupt it using the wakeup() method + val wakeUpTask = + actorSystem.scheduler.scheduleOnce(sessionTimeout + gracefulWaitTime.milliseconds)(consumer.wakeup()) + + try { + consumer.poll(duration.toMillis).map(r => (r.topic, r.partition, r.offset, r.value)) + } catch { + // Happens if the peek hangs. + case e: WakeupException => + if (retry > 0) { + logging.error(this, s"poll timeout occurred. Retrying $retry more times.") + Thread.sleep(gracefulWaitTime) // using Thread.sleep is okay, since `poll` is blocking anyway + peek(duration, retry - 1) + } else { + recreateConsumer() + throw e + } + case e: RetriableException => + if (retry > 0) { + logging.error(this, s"$e: Retrying $retry more times") + wakeUpTask.cancel() + Thread.sleep(gracefulWaitTime) // using Thread.sleep is okay, since `poll` is blocking anyway + peek(duration, retry - 1) + } else { + recreateConsumer() + throw e + } + // Every other error results in a restart of the consumer + case e: Throwable => + recreateConsumer() + throw e + } finally wakeUpTask.cancel() } /** * Commits offsets from last poll. */ - def commit() = consumer.commitSync() + def commit(retry: Int = 3): Unit = + try { + consumer.commitSync() + } catch { + case e: RetriableException => + if (retry > 0) { + logging.error(this, s"$e: retrying $retry more times") Review comment: (nit warning ahead): how about `retries left: $retry` to avoid this reading `retrying 1 more times` here and elsewhere above and below. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services