package com.lightbend.jmx

// This class collects JMX data from Kafka Brokers. The set of data is loosly based on https://blog.serverdensity.com/how-to-monitor-kafka/
//
//    Metric	                                                      Comments	                                        Suggested Alert
// Memory usage	                  | Kafka should run entirely on RAM. JVM heap size shouldn’t be bigger than your 	| None
//                                | available RAM. That is to avoid swapping.                                       |
// Swap usage	                    | Watch for swap usage, as it will degrade performance on Kafka and lead to       | When used swap is > 128MB.
//                                | operations timing out (set vm.swappiness = 0).                                  |
// Network bandwidth	            | Kafka servers can incur a high network usage. Keep an eye on this, especially if| None
//                                | you notice any performance degradation. Also look out for dropped packet errors.|
// Disk usage	                    | Make sure you always have free space for new data, temporary files, snapshot or | When disk is > 85% usage.
//                                | backups.	                                                                      |
// Disk IO	                      | Kafka partitions are stored asynchronously as a sequential write ahead log.     | None
//                                | Thus, disk reads and writes in Kafka are sequential, with very few random seeks.|
// UnderReplicatedPartitions	    | kafka.server: type=ReplicaManager, name=UnderReplicatedPartitions – Number of   | When UnderReplicatedPartitions > 0.
//                                | under-replicated partitions.	                                                  |
// OfflinePartitionsCount	        | kafka.controller: type=KafkaController, name=OfflinePartitionsCount – Number of | When OfflinePartitionsCount > 0.
//                                | partitions without an active leader, therefore not readable nor writeable.	    |
// ActiveControllerCount	        | kafka.controller: type=KafkaController, name=ActiveControllerCount – Number of  | When ActiveControllerCount != 1.
//                                | active controller brokers.	                                                    |
// MessagesInPerSec	              | kafka.server: type=BrokerTopicMetrics, name=MessagesInPerSec – Incoming messages| None
//                                | per second.                                                                     |
// BytesInPerSec / BytesOutPerSec	| kafka.server: type=BrokerTopicMetrics, name=BytesInPerSec – kafka.server:       | None
//                                | type=BrokerTopicMetrics, name=BytesOutPerSec – Incoming/outgoing bytes per second.
// RequestsPerSec	                | kafka.network: type=RequestMetrics, name=RequestsPerSec, request={Produce|      | None
//                                | FetchConsumer|FetchFollower} – Number of requests per second.	                  |
// TotalTimeMs	                  | kafka.network: type=RequestMetrics, name=TotalTimeMs, request=                  | None
//                                | {Produce|FetchConsumer|FetchFollower} – Total time it takes to process a        |
//                                | request. You can also monitor split times for QueueTimeMs, LocalTimeMs,         |
//                                | RemoteTimeMs and RemoteTimeMs.                                                  |
// UncleanLeaderElectionsPerSec	  | kafka.controller: type=ControllerStats, name=LeaderElectionRateAndTimeMs –      | When UncleanLeaderElectionsPerSec != 0.
//                                | Number of disputed leader elections rate.	                                      |
// LogFlushRateAndTimeMs	        | kafka.log: type=LogFlushStats, name=LogFlushRateAndTimeMs – Asynchronous disk   | None
//                                | log flush and time in ms.	                                                      |
// UncleanLeaderElectionsPerSec	  | kafka.controller: type=ControllerStats, name=UncleanLeaderElectionsPerSec –     | When UncleanLeaderElectionsPerSec != 0.
//                                | Unclean leader election rate.	                                                  |
// PartitionCount	                | kafka.server: type=ReplicaManager, name=PartitionCount – Number of partitions on| When PartitionCount != your_num_partitions.
//                                | your system.	                                                                  |
// ISR shrink/expansion           | rate	kafka.server: type=ReplicaManager, name=IsrShrinksPerSec – kafka.server:  | IsrShrinksPerSec | IsrExpandsPerSec != 0.
//                                | type=ReplicaManager, name=IsrExpandsPerSec – When a broker goes down, ISR will  |
//                                | shrink for some of the partitions. When that broker is up again, ISR will be    |
//                                | expanded once the replicas have fully caught up.	                              |
// NetworkProcessorAvgIdlePercent	| kafka.network: type=SocketServer, name=NetworkProcessorAvgIdlePercent – The     | When NetworkProcessorAvgIdlePercent < 0.3
//                                | average fraction of time the network processors are idle.	                      |.
// RequestHandlerAvgIdlePercent	  | kafka.server: type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent – | When RequestHandlerAvgIdlePercent < 0.3.
//                                | The average fraction of time the request handler threads are idle.	            |
// Heap Memory Usage	            | Memory allocated dynamically by the Java process, Zookeeper in this case.	      | None

import java.util.concurrent.TimeUnit
import javax.management.{ MBeanServerConnection, ObjectName }
import javax.management.openmbean.CompositeData
import javax.management.remote.{ JMXConnector, JMXConnectorFactory, JMXServiceURL }

import com.lightbend.broker.BrokerWithJMX
import kafka.utils.ZkUtils
import org.influxdb.dto.Point

import scala.concurrent.duration._
import scala.concurrent.duration.Duration
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.concurrent.Future
import scala.collection.mutable.{ Map => mutableMap }
import scala.concurrent.ExecutionContext.Implicits.global

case class JMXCollector(cluster: String, broker: BrokerWithJMX) {

  val url = new JMXServiceURL(s"service:jmx:rmi:///jndi/rmi://${broker.host}:${broker.jmxPort}/jmxrmi")
  var connector: JMXConnector = null
  var server: MBeanServerConnection = null

  private def connect(): Unit = {
    var attempt = 0
    while ((server == null) && (attempt < 3)) {
      try {
        attempt = attempt + 1
        connector = JMXConnectorFactory.connect(url)
        server = connector.getMBeanServerConnection()
      } catch {
        case e: Throwable => {
          e.printStackTrace()
          if (null != connector) {
            connector.close()
          }
          connector = null
          server = null
        }
      }
    }
    server match {
      case null => println(s"Failed to (re)connected to ${broker.host} after $attempt attempts")
      case _ => println(s"Successfully (re)connected to ${broker.host} with $attempt attempts")
    }
  }

  def getStats(timestamp: Long): Future[Seq[Point]] = Future {

    val start = System.currentTimeMillis()
    //println(s"JMX collector started at $start, server $server")

    import JMXCollector._

    if (server == null)
      connect()

    val measurements = ListBuffer[Point]()
    try {
      // Memory metrics
      val memoryData = server.getAttribute(memoryObject, "HeapMemoryUsage").asInstanceOf[CompositeData]
      val memoryPoint = Point.measurement("broker_memory").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- MemoryUsage) memoryPoint.addField(field, memoryData.get(key).toString.toLong)
      measurements += memoryPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      val nonHeapData = server.getAttribute(memoryObject, "NonHeapMemoryUsage").asInstanceOf[CompositeData]
      val nonHeapPoint = Point.measurement("broker_nonheapmemory").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- MemoryUsage) nonHeapPoint.addField(field, nonHeapData.get(key).toString.toLong)
      measurements += nonHeapPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done memory")
      // OS Metrics
      val osPoint = Point.measurement("broker_OSUsage").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- OperatingSystemLong) osPoint.addField(field, server.getAttribute(osObject, key).asInstanceOf[Long])
      for ((key, field) <- OperatingSystemDouble) osPoint.addField(field, convertDouble(server.getAttribute(osObject, key).asInstanceOf[Double]))
      measurements += osPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done OS")
      // Broker topics metrics
      val bytesInPerSecPoint = Point.measurement("broker_bytesInPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) bytesInPerSecPoint.addField(field, server.getAttribute(bytesInPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) bytesInPerSecPoint.addField(field, convertDouble(server.getAttribute(bytesInPerSec, key).asInstanceOf[Double]))
      measurements += bytesInPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done bytes-in")

      val bytesOutPerSecPoint = Point.measurement("broker_bytesOutPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) bytesOutPerSecPoint.addField(field, server.getAttribute(bytesOutPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) bytesOutPerSecPoint.addField(field, convertDouble(server.getAttribute(bytesOutPerSec, key).asInstanceOf[Double]))
      measurements += bytesOutPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done bytes-out")

      val bytesRejectedPerSecPoint = Point.measurement("broker_bytesRejectedPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) bytesRejectedPerSecPoint.addField(field, server.getAttribute(bytesRejectedPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) bytesRejectedPerSecPoint.addField(field, convertDouble(server.getAttribute(bytesRejectedPerSec, key).asInstanceOf[Double]))
      measurements += bytesRejectedPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done rejected")

      val messagesInPerSecPoint = Point.measurement("broker_messagesInPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) messagesInPerSecPoint.addField(field, server.getAttribute(messagesInPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) messagesInPerSecPoint.addField(field, convertDouble(server.getAttribute(messagesInPerSec, key).asInstanceOf[Double]))
      measurements += messagesInPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done message-in")

      val totalFetchRequestsPerSecPoint = Point.measurement("broker_totalFetchRequestsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) totalFetchRequestsPerSecPoint.addField(field, server.getAttribute(totalFetchRequestsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) totalFetchRequestsPerSecPoint.addField(field, convertDouble(server.getAttribute(totalFetchRequestsPerSec, key).asInstanceOf[Double]))
      measurements += totalFetchRequestsPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done Fetch request")

      val totalProduceRequestsPerSecPoint = Point.measurement("broker_totalProduceRequestsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) totalProduceRequestsPerSecPoint.addField(field, server.getAttribute(totalProduceRequestsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) totalProduceRequestsPerSecPoint.addField(field, convertDouble(server.getAttribute(totalProduceRequestsPerSec, key).asInstanceOf[Double]))
      measurements += totalProduceRequestsPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done producer requests")

      val requestHandlerPoolPoint = Point.measurement("broker_requestHandlerAvgIdlePercent").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) requestHandlerPoolPoint.addField(field, server.getAttribute(requestHandlerAvgIdlePercent, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) requestHandlerPoolPoint.addField(field, convertDouble(server.getAttribute(requestHandlerAvgIdlePercent, key).asInstanceOf[Double]))
      measurements += requestHandlerPoolPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done request handler pool")

      val serverCountsPoint = Point.measurement("broker_serverCounts").time(timestamp, TimeUnit.MILLISECONDS)
      serverCountsPoint.addField("leaderCount", server.getAttribute(leaderCount, "Value").asInstanceOf[Int])
      serverCountsPoint.addField("partitionsCount", server.getAttribute(partitionCount, "Value").asInstanceOf[Int])
      serverCountsPoint.addField("underreplicatedPartitions", server.getAttribute(underReplicatedPartitions, "Value").asInstanceOf[Int])
      serverCountsPoint.addField("fetchQueueSize", convertDouble(server.getAttribute(fetchCount, "queue-size").asInstanceOf[Double]))
      measurements += serverCountsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done server counts")

      val replicaMangementPoint = Point.measurement("broker_replicaManagement").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) replicaMangementPoint.addField("isr_expand_" + field, server.getAttribute(isrExpandsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) replicaMangementPoint.addField("isr_expand_" + field, convertDouble(server.getAttribute(isrExpandsPerSec, key).asInstanceOf[Double]))
      for ((key, field) <- BrokerTopicsMetricsLong) replicaMangementPoint.addField("isr_shrink_" + field, server.getAttribute(isrExpandsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) replicaMangementPoint.addField("isr_shrink_" + field, convertDouble(server.getAttribute(isrExpandsPerSec, key).asInstanceOf[Double]))
      measurements += replicaMangementPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done replication")

      for (networkProcessor <- server.queryMBeans(brokerSocketMetrics, null).asScala) {
        val objectName = networkProcessor.getObjectName
        val processor = objectName.getCanonicalName.split(",")(0).split(":")(1).split("=")(1)
        val networkProcessorPoint = Point.measurement("broker_networkProcessor").time(timestamp, TimeUnit.MILLISECONDS)
        for ((key, field) <- BrokerSocketMetrics) networkProcessorPoint.addField(field, convertDouble(server.getAttribute(objectName, key).asInstanceOf[Double]))
        measurements += networkProcessorPoint.tag("broker", broker.id.toString).tag("cluster", cluster).tag("networkProcessor", processor).build()
      }
      /*
      // Does not exist in version .10 and up
      val logFlushMetricsPoint = Point.measurement("broker_logFlushMetrics").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong)logFlushMetricsPoint.addField(field, server.getAttribute(logFlushMetrics, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) logFlushMetricsPoint.addField(field, convertDouble(server.getAttribute(logFlushMetrics, key).asInstanceOf[Double]))
      measurements += logFlushMetricsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
*/
      val leaderElectionMetricsPoint = Point.measurement("broker_leaderElectionRateAndTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) leaderElectionMetricsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) leaderElectionMetricsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += leaderElectionMetricsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()
      //    println(s"JMX collector started at $start done leader election")

      val uncleanLeaderElectionMetricsPoint = Point.measurement("broker_uncleanLeaderElectionsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) uncleanLeaderElectionMetricsPoint.addField(field, server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) uncleanLeaderElectionMetricsPoint.addField(field, convertDouble(server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Double]))
      measurements += uncleanLeaderElectionMetricsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      measurements += Point.measurement("broker_offlinePartitionsCount").time(timestamp, TimeUnit.MILLISECONDS).
        addField("value", server.getAttribute(offlinePartitionsCount, "Value").asInstanceOf[Int]).tag("broker", broker.id.toString).tag("cluster", cluster).build()

      measurements += Point.measurement("broker_activeControllerCount").time(timestamp, TimeUnit.MILLISECONDS).
        addField("value", server.getAttribute(activeContollerCount, "Value").asInstanceOf[Int]).tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val produceRequestsPerSecPoint = Point.measurement("broker_produceRequestsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) produceRequestsPerSecPoint.addField(field, server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) produceRequestsPerSecPoint.addField(field, convertDouble(server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Double]))
      measurements += produceRequestsPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchConsumerRequestsPerSecPoint = Point.measurement("broker_fetchConsumerRequestsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) fetchConsumerRequestsPerSecPoint.addField(field, server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) fetchConsumerRequestsPerSecPoint.addField(field, convertDouble(server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Double]))
      measurements += fetchConsumerRequestsPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchFollowerRequestsPerSecPoint = Point.measurement("broker_fetchFollowerRequestsPerSec").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerTopicsMetricsLong) fetchFollowerRequestsPerSecPoint.addField(field, server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Long])
      for ((key, field) <- BrokerTopicsMetricsDouble) fetchFollowerRequestsPerSecPoint.addField(field, convertDouble(server.getAttribute(uncleanLeaderElectionsPerSec, key).asInstanceOf[Double]))
      measurements += fetchFollowerRequestsPerSecPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val produceTotalTimeMsPoint = Point.measurement("broker_produceTotalTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) produceTotalTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) produceTotalTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += produceTotalTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchConsumerTotalTimeMsPoint = Point.measurement("broker_fetchConsumerTotalTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) fetchConsumerTotalTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) fetchConsumerTotalTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += fetchConsumerTotalTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchFollowerTotalTimeMsPoint = Point.measurement("broker_fetchFollowerTotalTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) fetchFollowerTotalTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) fetchFollowerTotalTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += fetchFollowerTotalTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchFollowerRequestQueueTimeMsPoint = Point.measurement("broker_fetchFollowerRequestQueueTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) fetchFollowerRequestQueueTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) fetchFollowerRequestQueueTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += fetchFollowerRequestQueueTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val fetchConsumerRequestQueueTimeMsPoint = Point.measurement("broker_fetchConsumerRequestQueueTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) fetchConsumerRequestQueueTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) fetchConsumerRequestQueueTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += fetchConsumerRequestQueueTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      val produceRequestQueueTimeMsPoint = Point.measurement("broker_produceRequestQueueTimeMs").time(timestamp, TimeUnit.MILLISECONDS)
      for ((key, field) <- BrokerLogMetricsLong) produceRequestQueueTimeMsPoint.addField(field, server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Long])
      for ((key, field) <- BrokerLogMetricsDouble) produceRequestQueueTimeMsPoint.addField(field, convertDouble(server.getAttribute(leaderElectionRateAndTimeMs, key).asInstanceOf[Double]))
      measurements += produceRequestQueueTimeMsPoint.tag("broker", broker.id.toString).tag("cluster", cluster).build()

      measurements += Point.measurement("broker_networkProcessorAvgIdlePercent").time(timestamp, TimeUnit.MILLISECONDS).
        addField("value", convertDouble(server.getAttribute(networkProcessorAvgIdlePercent, "Value").asInstanceOf[Double])).tag("broker", broker.id.toString).tag("cluster", cluster).build()

    } catch {
      case e: Throwable =>
        e.printStackTrace()
        if (null != connector) {
          connector.close()
        }
        connector = null
        server = null
    }

    //println(s"JMX collector started at $start got ${measurements.size} points in ${System.currentTimeMillis() - start} ms")

    measurements
  }

  def cleanup(): Unit = {
    if (connector != null)
      connector.close()
  }
}

object JMXCollector {

  // JMX objects
  val memoryObject = new ObjectName("java.lang:type=Memory")
  val osObject = new ObjectName("java.lang:type=OperatingSystem")

  val bytesRejectedPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec")
  val totalFetchRequestsPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=TotalFetchRequestsPerSec")
  val totalProduceRequestsPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=TotalProduceRequestsPerSec")
  val bytesOutPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec")
  val bytesInPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec")
  val messagesInPerSec = new ObjectName("kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec")

  val produceRequestsPerSec = new ObjectName("kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce")
  val fetchConsumerRequestsPerSec = new ObjectName("kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer")
  val fetchFollowerRequestsPerSec = new ObjectName("kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower")
  val produceTotalTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce")
  val fetchConsumerTotalTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer")
  val fetchFollowerTotalTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower")
  val fetchFollowerRequestQueueTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchFollowe")
  val fetchConsumerRequestQueueTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchConsumer")
  val produceRequestQueueTimeMs = new ObjectName("kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=Produce")
  val networkProcessorAvgIdlePercent = new ObjectName("kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent")

  val leaderElectionRateAndTimeMs = new ObjectName("kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs")
  val uncleanLeaderElectionsPerSec = new ObjectName("kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec")

  val requestHandlerAvgIdlePercent = new ObjectName("kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent")

  val leaderCount = new ObjectName("kafka.server:type=ReplicaManager,name=LeaderCount")
  val partitionCount = new ObjectName("kafka.server:type=ReplicaManager,name=PartitionCount")
  val underReplicatedPartitions = new ObjectName("kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions")
  val fetchCount = new ObjectName("kafka.server:type=Fetch")

  val isrExpandsPerSec = new ObjectName("kafka.server:type=ReplicaManager,name=IsrExpandsPerSec")
  val isrShrinksPerSec = new ObjectName("kafka.server:type=ReplicaManager,name=IsrShrinksPerSec")

  val brokerSocketMetrics = new ObjectName("kafka.server:type=socket-server-metrics,*")

  val logFlushMetrics = new ObjectName("kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs")

  val offlinePartitionsCount = new ObjectName("kafka.controller:type=KafkaController,name=OfflinePartitionsCount")
  val activeContollerCount = new ObjectName("kafka.controller:type=KafkaController,name=ActiveControllerCount")

  // Mappings
  val MemoryUsage = Map("committed" -> "committed", "used" -> "used", "init" -> "init", "max" -> "max")

  val OperatingSystemLong = Map(
    "CommittedVirtualMemorySize" -> "CommittedVirtualMemory",
    "TotalSwapSpaceSize" -> "TotalSwapSpace",
    "FreeSwapSpaceSize" -> "FreeSwapSpace",
    "ProcessCpuTime" -> "ProcessCpuTime",
    "FreePhysicalMemorySize" -> "FreePhysicalMemory",
    "TotalPhysicalMemorySize" -> "TotalPhysicalMemory"
  )
  val OperatingSystemDouble = Map(
    "SystemCpuLoad" -> "SystemCpuLoad",
    "ProcessCpuLoad" -> "ProcessCpuLoad",
    "SystemLoadAverage" -> "SystemLoadAverage"
  )

  val BrokerTopicsMetricsDouble = Map(
    "FifteenMinuteRate" -> "FifteenMinuteRate",
    "FiveMinuteRate" -> "FiveMinuteRate",
    "OneMinuteRate" -> "OneMinuteRate",
    "MeanRate" -> "MeanRate"
  )
  val BrokerTopicsMetricsLong = Map("Count" -> "Count")

  val BrokerSocketMetrics = Map(
    "connection-close-rate" -> "ConnectionCloseRate",
    "connection-count" -> "ConnectionCount",
    "connection-creation-rate" -> "ConnectionCreationRate",
    "incoming-byte-rate" -> "IncomingBytesRate",
    "io-ratio" -> "IORatio",
    "io-time-ns-avg" -> "IOAverageTime",
    "io-wait-ratio" -> "IOWaitRatio",
    "io-wait-time-ns-avg" -> "IOWaitAverageTime",
    "network-io-rate" -> "NetworkIORate",
    "outgoing-byte-rate" -> "OutgoingByteRate",
    "request-rate" -> "RequestRate",
    "request-size-avg" -> "AverageRequestSize",
    "request-size-max" -> "MaxRequestSize",
    "response-rate" -> "ResponseRate",
    "select-rate" -> "SelectRate"
  )

  val BrokerLogMetricsDouble = Map(
    "50thPercentile" -> "50thPercentile",
    "75thPercentile" -> "75thPercentile",
    "95thPercentile" -> "95thPercentile",
    "98thPercentile" -> "98thPercentile",
    "99thPercentile" -> "99thPercentile",
    "999thPercentile" -> "999thPercentile",
    "FifteenMinuteRate" -> "FifteenMinuteRate",
    "FiveMinuteRate" -> "FiveMinuteRate",
    "OneMinuteRate" -> "OneMinuteRate",
    "MeanRate" -> "MeanRate",
    "Mean" -> "Mean",
    "Min" -> "Min",
    "Max" -> "Max",
    "StdDev" -> "StdDev"
  )
  val BrokerLogMetricsLong = Map("Count" -> "count")

  val collectorMap = mutableMap[String, (ZkUtils, ListBuffer[JMXCollector])]() // Map of collectors per cluster

  private def convertDouble(value: Double): Double = value match {
    case v if (v < 1E-5) => .0
    case _ => value
  }

  def addCluster(name: String, zkUtils: ZkUtils): Unit = collectorMap.put(name, (zkUtils, ListBuffer[JMXCollector]()))
  def removeCluster(name: String): Unit = {
    collectorMap.get(name) match {
      case Some(collectors) => {
        for (collector <- collectors._2) collector.cleanup()
        collectors._1.close()
        collectorMap.remove(name)
      }
      case _ =>
    }
  }

  private def updateBrokerList(cluster: String, brokers: Seq[BrokerWithJMX]): Unit = {

    collectorMap.get(cluster) match {
      case Some(currentCollectors) => {
        val currentBrokers = currentCollectors._2.map(_.broker)
        // Calculate differences
        val newBrokers = brokers.diff(currentBrokers) // List of brokers that are in brokers and not in the brokerList
        val removedBrokers = currentBrokers.diff(brokers) // List of brokers that are in brokerList and not in the broker
        for (collector <- currentCollectors._2.filter(c => removedBrokers.contains(c.broker))) {
          collector.cleanup()
          currentCollectors._2 -= collector
        }
        for (broker <- newBrokers) currentCollectors._2 += new JMXCollector(cluster, broker)
        collectorMap.put(cluster, currentCollectors)
      }
      case _ =>
    }
  }

  def getJMXPoints(ts: Long): Future[Seq[Point]] = {
    val points = ListBuffer[Future[Seq[Point]]]()
    for (cluster <- collectorMap.keys) {
      updateBrokerList(cluster, BrokerWithJMX.getBrokers(collectorMap.get(cluster).get._1, Some(cluster)))
      for (collector <- collectorMap.get(cluster).get._2) points += collector.getStats(ts)
    }
    Future.sequence(points).map(_.flatten)
  }

  def main(args: Array[String]): Unit = {

    val zkSessionTimeout: Duration = 30 seconds
    val zkConnectionTimeout: Duration = 30 seconds
    val zkUtils = ZkUtils("localhost:2181", zkSessionTimeout.toMillis.toInt, zkConnectionTimeout.toMillis.toInt, false)

    addCluster("kafka", zkUtils)
    while (true) {
      for {
        points <- getJMXPoints(System.currentTimeMillis())
      } yield points.foreach(println(_))
      Thread.sleep(2000)
    }
  }
}
