http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_alerts.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_alerts.png b/docs/docs/include/images/alert_alerts.png new file mode 100644 index 0000000..56991ed Binary files /dev/null and b/docs/docs/include/images/alert_alerts.png differ
http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_define_policy.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_define_policy.png b/docs/docs/include/images/alert_define_policy.png new file mode 100644 index 0000000..16312bc Binary files /dev/null and b/docs/docs/include/images/alert_define_policy.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_details.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_details.png b/docs/docs/include/images/alert_details.png new file mode 100644 index 0000000..dcf0476 Binary files /dev/null and b/docs/docs/include/images/alert_details.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_engine.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_engine.png b/docs/docs/include/images/alert_engine.png new file mode 100644 index 0000000..3276afc Binary files /dev/null and b/docs/docs/include/images/alert_engine.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_engine_coordination.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_engine_coordination.png b/docs/docs/include/images/alert_engine_coordination.png new file mode 100644 index 0000000..3b134f7 Binary files /dev/null and b/docs/docs/include/images/alert_engine_coordination.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_engine_policy_spec.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_engine_policy_spec.png b/docs/docs/include/images/alert_engine_policy_spec.png new file mode 100644 index 0000000..dc723af Binary files /dev/null and b/docs/docs/include/images/alert_engine_policy_spec.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/alert_policies.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/alert_policies.png b/docs/docs/include/images/alert_policies.png new file mode 100644 index 0000000..9c9936f Binary files /dev/null and b/docs/docs/include/images/alert_policies.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/configure_site.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/configure_site.png b/docs/docs/include/images/configure_site.png new file mode 100644 index 0000000..7aa62e9 Binary files /dev/null and b/docs/docs/include/images/configure_site.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/dashboard.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/dashboard.png b/docs/docs/include/images/dashboard.png new file mode 100644 index 0000000..f94d6f2 Binary files /dev/null and b/docs/docs/include/images/dashboard.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/define_jmx_alert_policy.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/define_jmx_alert_policy.png b/docs/docs/include/images/define_jmx_alert_policy.png new file mode 100644 index 0000000..c9e4469 Binary files /dev/null and b/docs/docs/include/images/define_jmx_alert_policy.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/delete_icon.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/delete_icon.png b/docs/docs/include/images/delete_icon.png new file mode 100644 index 0000000..0f58236 Binary files /dev/null and b/docs/docs/include/images/delete_icon.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/eagle_arch_v0.5.0.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/eagle_arch_v0.5.0.png b/docs/docs/include/images/eagle_arch_v0.5.0.png new file mode 100644 index 0000000..abe1de7 Binary files /dev/null and b/docs/docs/include/images/eagle_arch_v0.5.0.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/eagle_ecosystem.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/eagle_ecosystem.png b/docs/docs/include/images/eagle_ecosystem.png new file mode 100644 index 0000000..77e946e Binary files /dev/null and b/docs/docs/include/images/eagle_ecosystem.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/eagle_web_interface.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/eagle_web_interface.png b/docs/docs/include/images/eagle_web_interface.png new file mode 100644 index 0000000..49b9cdd Binary files /dev/null and b/docs/docs/include/images/eagle_web_interface.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/edit_icon.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/edit_icon.png b/docs/docs/include/images/edit_icon.png new file mode 100644 index 0000000..b086c1c Binary files /dev/null and b/docs/docs/include/images/edit_icon.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/favicon.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/favicon.png b/docs/docs/include/images/favicon.png new file mode 100644 index 0000000..3bede2a Binary files /dev/null and b/docs/docs/include/images/favicon.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_1.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_1.png b/docs/docs/include/images/hadoop_queue_monitor_1.png new file mode 100644 index 0000000..15040ee Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_1.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_2.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_2.png b/docs/docs/include/images/hadoop_queue_monitor_2.png new file mode 100644 index 0000000..3597a5d Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_2.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_3.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_3.png b/docs/docs/include/images/hadoop_queue_monitor_3.png new file mode 100644 index 0000000..287401c Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_3.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_4.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_4.png b/docs/docs/include/images/hadoop_queue_monitor_4.png new file mode 100644 index 0000000..2bbd4f1 Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_4.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_5.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_5.png b/docs/docs/include/images/hadoop_queue_monitor_5.png new file mode 100644 index 0000000..5a65f1e Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_5.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_6.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_6.png b/docs/docs/include/images/hadoop_queue_monitor_6.png new file mode 100644 index 0000000..ddc92be Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_6.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hadoop_queue_monitor_7.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hadoop_queue_monitor_7.png b/docs/docs/include/images/hadoop_queue_monitor_7.png new file mode 100644 index 0000000..812795d Binary files /dev/null and b/docs/docs/include/images/hadoop_queue_monitor_7.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hdfs_audit_log.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hdfs_audit_log.png b/docs/docs/include/images/hdfs_audit_log.png new file mode 100644 index 0000000..f260ac3 Binary files /dev/null and b/docs/docs/include/images/hdfs_audit_log.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hdfs_install_1.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hdfs_install_1.png b/docs/docs/include/images/hdfs_install_1.png new file mode 100644 index 0000000..ecb2607 Binary files /dev/null and b/docs/docs/include/images/hdfs_install_1.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hdfs_install_2.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hdfs_install_2.png b/docs/docs/include/images/hdfs_install_2.png new file mode 100644 index 0000000..c865e03 Binary files /dev/null and b/docs/docs/include/images/hdfs_install_2.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hdfs_install_3.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hdfs_install_3.png b/docs/docs/include/images/hdfs_install_3.png new file mode 100644 index 0000000..b325739 Binary files /dev/null and b/docs/docs/include/images/hdfs_install_3.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/hdfs_policy_1.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/hdfs_policy_1.png b/docs/docs/include/images/hdfs_policy_1.png new file mode 100644 index 0000000..1aebad5 Binary files /dev/null and b/docs/docs/include/images/hdfs_policy_1.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/health_check_installation.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/health_check_installation.png b/docs/docs/include/images/health_check_installation.png new file mode 100644 index 0000000..a2763ac Binary files /dev/null and b/docs/docs/include/images/health_check_installation.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/health_check_policy.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/health_check_policy.png b/docs/docs/include/images/health_check_policy.png new file mode 100644 index 0000000..05b2df5 Binary files /dev/null and b/docs/docs/include/images/health_check_policy.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/health_check_settings.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/health_check_settings.png b/docs/docs/include/images/health_check_settings.png new file mode 100644 index 0000000..c93e71e Binary files /dev/null and b/docs/docs/include/images/health_check_settings.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/health_check_stream.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/health_check_stream.png b/docs/docs/include/images/health_check_stream.png new file mode 100644 index 0000000..c6de29e Binary files /dev/null and b/docs/docs/include/images/health_check_stream.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/install_jmx_2.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/install_jmx_2.png b/docs/docs/include/images/install_jmx_2.png new file mode 100644 index 0000000..9c73d34 Binary files /dev/null and b/docs/docs/include/images/install_jmx_2.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/install_jmx_3.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/install_jmx_3.png b/docs/docs/include/images/install_jmx_3.png new file mode 100644 index 0000000..26c812d Binary files /dev/null and b/docs/docs/include/images/install_jmx_3.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/install_jmx_6.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/install_jmx_6.png b/docs/docs/include/images/install_jmx_6.png new file mode 100644 index 0000000..f325df1 Binary files /dev/null and b/docs/docs/include/images/install_jmx_6.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/integration_applications.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/integration_applications.png b/docs/docs/include/images/integration_applications.png new file mode 100644 index 0000000..8a71e7c Binary files /dev/null and b/docs/docs/include/images/integration_applications.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/integration_sites.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/integration_sites.png b/docs/docs/include/images/integration_sites.png new file mode 100644 index 0000000..8b27ad2 Binary files /dev/null and b/docs/docs/include/images/integration_sites.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/jpm.jpg ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/jpm.jpg b/docs/docs/include/images/jpm.jpg new file mode 100644 index 0000000..f93adb5 Binary files /dev/null and b/docs/docs/include/images/jpm.jpg differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/jpm_configure.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/jpm_configure.png b/docs/docs/include/images/jpm_configure.png new file mode 100644 index 0000000..638b6ea Binary files /dev/null and b/docs/docs/include/images/jpm_configure.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/jpm_define_policy.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/jpm_define_policy.png b/docs/docs/include/images/jpm_define_policy.png new file mode 100644 index 0000000..29a58bb Binary files /dev/null and b/docs/docs/include/images/jpm_define_policy.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/jpm_streams.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/jpm_streams.png b/docs/docs/include/images/jpm_streams.png new file mode 100644 index 0000000..28c5eda Binary files /dev/null and b/docs/docs/include/images/jpm_streams.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/new_site.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/new_site.png b/docs/docs/include/images/new_site.png new file mode 100644 index 0000000..998e478 Binary files /dev/null and b/docs/docs/include/images/new_site.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/overview.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/overview.png b/docs/docs/include/images/overview.png new file mode 100644 index 0000000..5803167 Binary files /dev/null and b/docs/docs/include/images/overview.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/site_list.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/site_list.png b/docs/docs/include/images/site_list.png new file mode 100644 index 0000000..fc1734b Binary files /dev/null and b/docs/docs/include/images/site_list.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/start_icon.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/start_icon.png b/docs/docs/include/images/start_icon.png new file mode 100644 index 0000000..6ee9efe Binary files /dev/null and b/docs/docs/include/images/start_icon.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/stop_icon.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/stop_icon.png b/docs/docs/include/images/stop_icon.png new file mode 100644 index 0000000..a4ee629 Binary files /dev/null and b/docs/docs/include/images/stop_icon.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/include/images/storage_engine.png ---------------------------------------------------------------------- diff --git a/docs/docs/include/images/storage_engine.png b/docs/docs/include/images/storage_engine.png new file mode 100644 index 0000000..8c9bbd3 Binary files /dev/null and b/docs/docs/include/images/storage_engine.png differ http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/index.html ---------------------------------------------------------------------- diff --git a/docs/docs/index.html b/docs/docs/index.html new file mode 100644 index 0000000..9b64d62 --- /dev/null +++ b/docs/docs/index.html @@ -0,0 +1,6 @@ +<script> + var pathname = window.location.pathname; + if (pathname == "/" || pathname.indexOf('/index.html') == 0) { + window.location.href = "home"; + } +</script> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/index.md ---------------------------------------------------------------------- diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..b24a57e --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,93 @@ +# What is Eagle + +** Apache Eagle ** (incubating) is a highly extensible, scalable monitoring and alerting platform, designed with its flexible application framework and proven big data technologies, such as Kafka, Spark and Storm. It ships a rich set of applications for big data platform monitoring, e.g. HDFS/HBase/YARN service health check, JMX metrics, daemon logs, audit logs and yarn applications. External Eagle developers can define applications to monitoring their NoSQLs or Web Servers, and publish to Eagle application repository at your own discretion. It also provides the state-of-art alert engine to report security breaches, service failures, and application anomalies, highly customizable by the alert policy definition. + + + +## Terminology + +### Site + +* A virtual concept in Apache Eagle. You can use it to manage a group of application instances, and distinguish the applications if you have a certain application installed for multiple times. + +### Application + +* Application(or Monitoring Application) is the first-class citizen in Apache Eagle, it stands for an end-to-end monitoring/alerting solution, which usually contains the monitoring source onboarding, source schema specification, alerting policy and dashboard definition. + +### Stream + +* Stream is the input for Alert Engine, each Application should have its own stream to be defined by the developer. Usually, it will have a POJO-like structure included in the stream definition. Once it's defined, Application should have the logic to write data into Kafka. + +### Data Activity Monitoring + +* A built-in monitoring application to monitor HDFS/HBase/Hive operations, and allow users to define certain policies to detect sensitive data access and malicious data operations in real-time. + +### Alert Engine + +* A specific built-in application shared for all other monitoring applications, it reads data from Kafka, and processes the data by applying the policy in real-time manner, and generates alert notification. So we call this application as the Alert Engine. + +### Policy + +* A rule used by Alert Engine to match the data input from Kafka. Policy is defined in [SiddhiQL](https://docs.wso2.com/display/CEP300/Siddhi+Language+Specification) format. + +### Alert + +* If any data input to Alert Engine meets the policy, the Alert Engine will generate a message and publish it through alert publisher. We call such messages as the alerts. + +### Alert Publisher + +* It will publish the alert to external channels which can be the SMTP channel, the Kafka channel, Slack channel or other storage systems. + +## Key Qualities + +### Extensible + +* Apache Eagle built its core framework around the application concept, application itself includes the logic for monitoring source data collection, pre-processing and normalization. Developer can easily develop his own out-of-box monitoring applications using Eagle's application framework, and deploy into Eagle. + +### Scalable + +* The Eagle core team has chosen the proven big data technologies to build its fundamental runtime, and apply a scalable core to make it adaptive according to the throughput of data stream as well as the number of monitored applications. + +### Real-time + +* Storm or Spark Streaming based computing engine allow us to apply the policy to data stream and generate alerts in real-time manner. + +### Dynamic + +* The user can freely enable or disable a monitoring application without restarting the service. Eagle user can dynamically add/delet/change their alert policies without any impact to the underlying runtime. + +### Easy-of-Use + +* User can enable the monitoring for a service within minutes effort by just choosing the corresponding monitoring application and configuring few parameters for the service. + +### Non-Invasive + +* Apache Eagle uses the out-of-box applications to monitor services, you don't need any change to your existing services. + +--- + +# Use Case Examples + +## Data Activity Monitoring + +* Data activity represents how user explores data provided by big data platforms. Analyzing data activity and alerting for insecure access are fundamental requirements for securing enterprise data. As data volume is increasing exponentially with Hadoop, Hive, Spark technology, understanding data activities for every user becomes extremely hard, let alone to alert for a single malicious event in real time among petabytes streaming data per day. + +* Securing enterprise data starts from understanding data activities for every user. Apache Eagle (incubating, called Eagle in the following) has integrated with many popular big data platforms e.g. Hadoop, Hive, Spark, Cassandra etc. With Eagle user can browse data hierarchy, mark sensitive data and then create comprehensive policy to alert for insecure data access. + +## Job Performance Analysis + +* Running map/reduce job is the most popular way people use to analyze data in Hadoop system. Analyzing job performance and providing tuning suggestions are critical for Hadoop system stability, job SLA and resource usage etc. + +* Eagle analyzes job performance with two complementing approaches. First Eagle periodically takes snapshots for all running jobs with YARN API, secondly Eagle continuously reads job lifecycle events immediately after the job is completed. With the two approaches, Eagle can analyze single job's trend, data skew problem, failure reasons etc. More interestingly, Eagle can analyze whole Hadoop cluster's performance by taking into account all jobs. + +## Cluster Performance Analytics + +* It is critical to understand why a cluster performs bad. Is that because of some crazy jobs recently on-boarded, or huge amount of tiny files, or namenode performance degrading? + +* Eagle in realtime calculates resource usage per minute out of individual jobs, e.g. CPU, memory, HDFS IO bytes, HDFS IO numOps etc. and also collects namenode JMX metrics. Correlating them together will easily help system administrator find root cause for cluster slowness. + +--- + +# Disclaimer + +* Apache Eagle now is being incubated, and therefore, across the whole documentation site, all appearances of case-insensitive word `eagle` and `apache eagle` represent `Apache Eagle (incubating)`. This could be seen as a part of disclaimer. http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/reference.md ---------------------------------------------------------------------- diff --git a/docs/docs/reference.md b/docs/docs/reference.md new file mode 100644 index 0000000..aef299f --- /dev/null +++ b/docs/docs/reference.md @@ -0,0 +1,325 @@ +# Configuration +### System Level Configuration +Eagle system level configuration is typically configured in `conf/eagle.conf`. + + # --------------------------------------------- + # Eagle REST Web Service Configuration + # --------------------------------------------- + service { + env = "testing" + host = "localhost" + port = 9090 + username = "admin" + password = "secret" + readTimeOutSeconds = 60 + context = "/rest" + timezone = "UTC" + } + + zookeeper { + zkQuorum = "localhost:2181" + zkSessionTimeoutMs : 15000 + zkRetryTimes : 3 + zkRetryInterval : 20000 + } + + # --------------------------------------------- + # Eagle Deep Storage Configuration + # --------------------------------------------- + + storage { + # storage type: ["hbase","jdbc"] + # default is "hbase" + type = "hbase" + + hbase { + # hbase configuration: hbase.zookeeper.quorum + # default is "localhost" + zookeeperQuorum = "localhost" + + # hbase configuration: hbase.zookeeper.property.clientPort + # default is 2181 + zookeeperPropertyClientPort = 2181 + + # hbase configuration: zookeeper.znode.parent + # default is "/hbase" + zookeeperZnodeParent = "/hbase-unsecure" + + # eagle web login profile: [sandbox, default] + # default is sandbox + tableNamePrefixedWithEnvironment = false + + # eagle coprocessor enabled or not: [true, false] + # default is false + coprocessorEnabled = false + } + } + + # --------------------------------------------- + # Eagle Metadata Store Configuration + # --------------------------------------------- + metadata { + store = org.apache.eagle.metadata.service.memory.MemoryMetadataStore + jdbc { + username = "root" + password = "" + driverClassName = com.mysql.jdbc.Driver + url = "jdbc:mysql://server.eagle.apache.org:3306/eagle" + } + } + + # --------------------------------------------- + # Eagle Application Configuration + # --------------------------------------------- + application { + sink { + type = org.apache.eagle.app.sink.KafkaStreamSink + } + storm { + nimbusHost = "server.eagle.apache.org" + nimbusThriftPort = 6627 + } + updateStatus: { + initialDelay: 10 + period: 10 + } + } + + # --------------------------------------------- + # Eagle Alert Engine Configuration + # --------------------------------------------- + + # Coordinator Configuration + coordinator { + policiesPerBolt = 5 + boltParallelism = 5 + policyDefaultParallelism = 5 + boltLoadUpbound = 0.8 + topologyLoadUpbound = 0.8 + numOfAlertBoltsPerTopology = 5 + zkConfig { + zkQuorum = "server.eagle.apache.org:2181" + zkRoot = "/alert" + zkSessionTimeoutMs = 10000 + connectionTimeoutMs = 10000 + zkRetryTimes = 3 + zkRetryInterval = 3000 + } + metadataService { + host = "localhost", + port = 9090, + context = "/rest" + } + metadataDynamicCheck { + initDelayMillis = 1000 + delayMillis = 30000 + } + } + + + +### Web Server Level Configuration +Web Server Level Configuration is typically configured in `conf/server.yaml`: + + server: + applicationConnectors: + - type: http + port: 9090 + adminConnectors: + - type: http + port: 9091 + + + # --------------------------------------------- + # Eagle Authentication Configuration + # --------------------------------------------- + auth: + # indicating if authentication is enabled, true for enabled, false for disabled + enabled: false + + # indicating authentication mode, "simple" or "ldap" + mode: simple + + # indicating whether to use cache: cache is usually used for authentications that may + # not handle high throughput (an RDBMS or LDAP server, for example) + caching: false + + # indicating the cache policy, containing maximumSize and expireAfterWrite, e.g. maximumSize=10000, expireAfterWrite=10m + cachePolicy: maximumSize=10000, expireAfterWrite=1m + + # indicating whether authorization is needed + authorization: false + + # indicating whether @Auth annotation on parameters is needed + annotated: true + + # for basic authentication, effective only when auth.mode=simple + simple: + # username for basic authentication, effective only when auth.mode=simple + username: admin + # password for basic authentication, effective only when auth.mode=simple + password: secret + + # for ldap authentication, effective only when auth.mode=ldap + ldap: + uri: ldaps://ldap.server.address:636 + userFilter: ou=x,dc=y,dc=z + groupFilter: ou=x,dc=y,dc=z + userNameAttribute: cn + groupNameAttribute: cn + groupMembershipAttribute: memberUid + groupClassName: posixGroup + restrictToGroups: + - user + - admin + connectTimeout: 500ms + readTimeout: 500ms + + + +As eagle server by default is based on DropWizard, so for more confgiruation details, please refer to [Dropwizard Configuration Reference](http://www.dropwizard.io/0.7.1/docs/manual/configuration.html) + +### Application Level Configuration + +Application level configurations could be set with `Settings` form during installation or modification. For more details of each application level configuration, please refer to ["Application"]("applications") guide pages. + +--- + +# REST APIs +| Method | Path | Resource | +| ------------- |--------------| ----------| +|DELETE | /rest/metadata/clusters | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/clusters/{clusterId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/datasources | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/datasources/{datasourceId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/policies | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/policies/{policyId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/publishmentTypes | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/publishmentTypes/{pubType} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/publishments | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/publishments/{name} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/streams | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/streams/{streamId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/topologies | org.apache.eagle.service.metadata.resource.MetadataResource| +|DELETE | /rest/metadata/topologies/{topologyName} | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/alerts | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/alerts/{alertId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/assignments | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/clusters | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/datasources | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/policies | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/policies/{policyId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/policies/{policyId}/publishments | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/publishmentTypes | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/publishments | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/schedulestates | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/schedulestates/{versionId} | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/streams | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/metadata/topologies | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/alerts | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/alerts/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/assignments | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/clear | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/clusters | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/clusters/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/datasources | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/datasources/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/export | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/import | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies/parse | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies/validate | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies/{policyId}/publishments | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/policies/{policyId}/status/{status} | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/publishmentTypes | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/publishmentTypes/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/publishments | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/publishments/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/schedulestates | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/streams | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/streams/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/topologies | org.apache.eagle.service.metadata.resource.MetadataResource| +|POST | /rest/metadata/topologies/batch | org.apache.eagle.service.metadata.resource.MetadataResource| +|GET | /rest/alert/topologies | org.apache.eagle.service.topology.resource.TopologyMgmtResource| +|POST | /rest/alert/topologies/{topologyName}/start | org.apache.eagle.service.topology.resource.TopologyMgmtResource| +|POST | /rest/alert/topologies/{topologyName}/stop | org.apache.eagle.service.topology.resource.TopologyMgmtResource| +|GET | /rest/coordinator/assignments | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|GET | /rest/coordinator/periodicForceBuildState | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|POST | /rest/coordinator/build | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|POST | /rest/coordinator/disablePeriodicForceBuild | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|POST | /rest/coordinator/enablePeriodicForceBuild | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|POST | /rest/coordinator/refreshUsages | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|POST | /rest/coordinator/validate | org.apache.eagle.alert.coordinator.resource.CoordinatorResource| +|DELETE | /rest/sites | org.apache.eagle.metadata.resource.SiteResource| +|DELETE | /rest/sites/{siteId} | org.apache.eagle.metadata.resource.SiteResource| +|GET | /rest/sites | org.apache.eagle.metadata.resource.SiteResource| +|GET | /rest/sites/{siteId} | org.apache.eagle.metadata.resource.SiteResource| +|POST | /rest/sites | org.apache.eagle.metadata.resource.SiteResource| +|PUT | /rest/sites | org.apache.eagle.metadata.resource.SiteResource| +|PUT | /rest/sites/{siteId} | org.apache.eagle.metadata.resource.SiteResource| +|DELETE | /rest/apps/uninstall | org.apache.eagle.app.resource.ApplicationResource| +|GET | /rest/apps | org.apache.eagle.app.resource.ApplicationResource| +|GET | /rest/apps/providers | org.apache.eagle.app.resource.ApplicationResource| +|GET | /rest/apps/providers/{type} | org.apache.eagle.app.resource.ApplicationResource| +|GET | /rest/apps/{appUuid} | org.apache.eagle.app.resource.ApplicationResource| +|POST | /rest/apps/install | org.apache.eagle.app.resource.ApplicationResource| +|POST | /rest/apps/start | org.apache.eagle.app.resource.ApplicationResource| +|POST | /rest/apps/status | org.apache.eagle.app.resource.ApplicationResource| +|POST | /rest/apps/stop | org.apache.eagle.app.resource.ApplicationResource| +|POST | /rest/apps/{appUuid} | org.apache.eagle.app.resource.ApplicationResource| +|PUT | /rest/apps/providers/reload | org.apache.eagle.app.resource.ApplicationResource| +|GET | /rest/example | org.apache.eagle.app.example.extensions.ExampleResource| +|GET | /rest/example/common | org.apache.eagle.app.example.extensions.ExampleResource| +|GET | /rest/example/config | org.apache.eagle.app.example.extensions.ExampleResource| +|GET | /rest/metadata/security/hbaseSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|GET | /rest/metadata/security/hdfsSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|GET | /rest/metadata/security/hiveSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|GET | /rest/metadata/security/ipzone | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|GET | /rest/metadata/security/oozieSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|POST | /rest/metadata/security/hbaseSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|POST | /rest/metadata/security/hdfsSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|POST | /rest/metadata/security/hiveSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|POST | /rest/metadata/security/ipzone | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|POST | /rest/metadata/security/oozieSensitivity | org.apache.eagle.security.service.SecurityExternalMetadataResource| +|GET | /rest/stream/attributeresolve | org.apache.eagle.service.alert.resolver.AttributeResolveResource| +|POST | /rest/stream/attributeresolve | org.apache.eagle.service.alert.resolver.AttributeResolveResource| +|GET | /rest/hbaseResource/columns | org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource| +|GET | /rest/hbaseResource/namespaces | org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource| +|GET | /rest/hbaseResource/tables | org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource| +|GET | /rest/oozieResource/coordinators | org.apache.eagle.service.security.oozie.res.OozieMetadataBrowseWebResource| +|DELETE | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|GET | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|GET | /rest/entities/jsonp | org.apache.eagle.service.generic.GenericEntityServiceResource| +|GET | /rest/entities/rowkey | org.apache.eagle.service.generic.GenericEntityServiceResource| +|POST | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|POST | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|POST | /rest/entities/delete | org.apache.eagle.service.generic.GenericEntityServiceResource| +|POST | /rest/entities/rowkey | org.apache.eagle.service.generic.GenericEntityServiceResource| +|PUT | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|PUT | /rest/entities | org.apache.eagle.service.generic.GenericEntityServiceResource| +|GET | /rest/list | org.apache.eagle.service.generic.ListQueryResource| +|GET | /rest/list/jsonp | org.apache.eagle.service.generic.ListQueryResource| +|GET | /rest/list/legacy | org.apache.eagle.service.generic.ListQueryResource| +|GET | /rest/meta | org.apache.eagle.service.generic.MetadataResource| +|GET | /rest/meta/resource | org.apache.eagle.service.generic.MetadataResource| +|GET | /rest/meta/service | org.apache.eagle.service.generic.MetadataResource| +|POST | /rest/metric | org.apache.eagle.service.metric.EagleMetricResource| +|GET | /rest/rowkeyquery | org.apache.eagle.service.rowkey.RowKeyQueryResource| +|DELETE | /rest/rowkey | org.apache.eagle.service.rowkey.RowkeyResource| +|GET | /rest/rowkey | org.apache.eagle.service.rowkey.RowkeyResource| +|GET | /rest/ValidateInternals | org.apache.eagle.service.selfcheck.EagleServiceSelfCheckResource| +|GET | /rest/services | org.apache.eagle.service.selfcheck.ServiceResource| +|GET | /rest/services/jsonp | org.apache.eagle.service.selfcheck.ServiceResource| +|GET | /rest/hdfsResource | org.apache.eagle.service.security.hdfs.rest.HDFSResourceWebResource| +|GET | /rest/mrJobs | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrJobs/jobCountsByDuration | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrJobs/jobMetrics/entities | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrJobs/jobMetrics/list | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrJobs/runningJobCounts | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrJobs/search | org.apache.eagle.service.jpm.MRJobExecutionResource| +|GET | /rest/mrTasks/historyTaskCount | org.apache.eagle.service.jpm.MRTaskExecutionResource| +|GET | /rest/mrTasks/taskCountsByDuration | org.apache.eagle.service.jpm.MRTaskExecutionResource| +|GET | /rest/mrTasks/taskDistribution/{counterName} | org.apache.eagle.service.jpm.MRTaskExecutionResource| +|GET | /rest/mrTasks/taskSuggestion | org.apache.eagle.service.jpm.MRTaskExecutionResource| +|GET | /rest/swagger.{type:json or yaml} | io.swagger.jaxrs.listing.ApiListingResource| http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/underlying-design.md ---------------------------------------------------------------------- diff --git a/docs/docs/underlying-design.md b/docs/docs/underlying-design.md new file mode 100644 index 0000000..9a3b448 --- /dev/null +++ b/docs/docs/underlying-design.md @@ -0,0 +1,231 @@ +# Alert Engine + + +### Distributed Real-time Streaming + +Eagle Alert Engine is buit on open source realtime streaming infrastrcuture like [Apache Storm](http://storm.apache.org/) as default execution engine and [Apache Kafka](http://kafka.apache.org/) as default messagig Bus. + + + +### Declarative Policy Evaluator +Eagle supports declarative policy with SQL (CEP) on distributed streaming environment. + + + +Here are some typical examples: + +* Example 1: Alert if hadoop namenode capacity usage exceed 90 percentages + + from hadoopJmxMetricEventStream + [metric == "hadoop.namenode.fsnamesystemstate.capacityused" and value > 0.9] + select metric, host, value, timestamp, component, site + insert into alertStream; + +* Example 2: Alert if hadoop namenode HA switches + + from every + a = hadoopJmxMetricEventStream[metric=="hadoop.namenode.fsnamesystem.hastate"] + -> + b = hadoopJmxMetricEventStream[metric==a.metric and b.host == a.host and a.value != value)] + within 10 min + select a.host, a.value as oldHaState, b.value as newHaState, b.timestamp as timestamp, b.metric as metric, b.component as component, b.site as site insert into alertStream; + +### Dynamic Coordinator + + +# Application Framework + +Eagle has multiple distributed real-time frameworks for efficiently developing highly scalable monitoring applications. + +## Application + +An "Application" or "App" is composed of data integration, policies and insights for one data source. + +## Application Descriptor + +An "Application Descriptor" is a static packaged metadata information consist of + +* __Basic__: type, name, version, description. +* __Application__: the application process to run. +* __Configuration Descriptor____: describe the configurations required by the application when starting like name, displayName, defaultValue Description, required, etc. which will automatically be visualized in configuration editor. +* __Streams__: the streams schema the application will export. +* __Docs__: application specific documentations which will be embedded in necessary area during the whole lifecyle of application management. + +Here is a ApplicationDesc example of `JPM_WEB_APP` + + { + type: "JPM_WEB_APP", + name: "Job Performance Monitoring Web ", + version: "0.5.0-incubating", + description: null, + appClass: "org.apache.eagle.app.StaticApplication", + jarPath: "/opt/eagle/0.5.0-incubating-SNAPSHOT-build-20161103T0332/eagle-0.5.0-incubating-SNAPSHOT/lib/eagle-topology-0.5.0-incubating-SNAPSHOT-hadoop-2.4.1-11-assembly.jar", + viewPath: "/apps/jpm", + providerClass: "org.apache.eagle.app.jpm.JPMWebApplicationProvider", + configuration: { + properties: [{ + name: "service.host", + displayName: "Eagle Service Host", + value: "localhost", + description: "Eagle Service Host, default: localhost", + required: false + }, { + name: "service.port", + displayName: "Eagle Service Port", + value: "8080", + description: "Eagle Service Port, default: 8080", + required: false + }] + }, + streams: null, + docs: null, + executable: false, + dependencies: [{ + type: "MR_RUNNING_JOB_APP", + version: "0.5.0-incubating", + required: true + }, { + type: "MR_HISTORY_JOB_APP", + version: "0.5.0-incubating", + required: true + }] + } + +## Application Provider + +An "application provider" in fact is a package management and loading mechanism leveraging [Java SPI](https://docs.oracle.com/javase/tutorial/ext/basics/spi.html). + +For example, in file + + META-INF/services/org.apache.eagle.app.spi.ApplicationProvider + +place the full class name of an application provider: + + org.apache.eagle.app.jpm.JPMWebApplicationProvider + +# Metric Storage + + + +## Persistence ORM Framework + +Eagle nativelly provides a light-weight ORM Framework for HBase/RDMBS, supporting easily defining persistence data model with java annotation. + +For example as below: + + + @Table("HbaseTableName") + @ColumnFamily("ColumnFamily") + @Prefix("RowkeyPrefix") + @Service("UniqueEntitytServiceName") + @JsonIgnoreProperties(ignoreUnknown = true) + @TimeSeries(false) + @Indexes({ + @Index(name="Index_1_alertExecutorId", columns = { "alertExecutorID" }, unique = true)}) + public class AlertDefinitionAPIEntity extends TaggedLogAPIEntity{ + @Column("a") + private String desc; + +## SQL-Like REST Query Language + +Eagle provide SQL-Like REST query language on NoSQL Model. For example like + + /rest/entities?query=UniqueEntitytServiceName[@site="sandbox"]{*} + +### Eagle Query API + +#### List Query + + ListQuery ::= "/" <EntityName> "?" <KeyValue> { "&" <KeyValue> } + KeyValue ::= [ <Filter> | <Output> ] + Filter ::= [ <TagFilter> | <FieldFilter> ] + Output ::= [ <OuputTag> | <OutputField> ] + TagFilter ::= "tagNameValue" "=" <Condition> <!= while used in URL, the equal "=" character has to be URL encoded with http://www.w3schools.com/tags/ref_urlencode.asp --> + FieldFilter ::= "fieldNameValue" "=" <Condition> <!-- not implemented yet, field filter has to be pushed down to hbase column filters --> + Condition ::= <Identifier> <OP> [ <Identifier> | <RegExpression> ] + OutputTag ::= <Identifier> + OutputField ::= <Identifier> + Identifier ::= <Character> { <Identifier> } + OP ::= [ "=" | "=~" ] <!-- =~ means regular expression matching, not supported yet as of now --> + RegExpression ::= <!-- java supported regular expressions http://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html , not yet supported in version 0.9 as of now --> + EntityName ::= [ "HdfsService" | "HBaseService" | "MRService" | "Host" | "Disk" | "CPU" | "Config" | "Alert" | "Action" | "Event" | "MRJob" | "MRTask" | "HBaseRPC" | "HdfsRPC" ] + +#### Aggregation Query + + AggregationQuery ::= "/" <EntityName> "?" <Filter> { "&" <Filter> } "&" <AggregationField> { "&" <AggregationField> } <AggregationFunction> { "&" <AggregationFunction> } [ "&" <AggregationFilters> ] + AggregationField ::= "gb" "=" <Identifier> + AggregationFunction ::= "function" "=" <Identifier> + AggregationFilters ::= "having" "=" <BooleanExpression> + BooleanExpression ::= <!-- as of now, boolean expression has not been yet implemented --> + +#### Bucket Query + + BucketQuery ::= "/" <EntityName> "?" <Filter> { "&" <Filter> } "&" <BucketField> { "&" <BucketField> } <BucketLimit> { "&" <BucketLimit> } + BucketField ::= "BucketField" "=" <Identifier> + BucketLimit ::= <Number> + +#### Rowkey Query + + RowkeyQuery ::= "/" <RowkeyPath> "?" <KeyFilter> "&' <TableFilter> "&" <ColumnFamilyFilter> "&" [ "all=true" | <FieldSelection> ] + RowkeyPath ::= "rowkey" + KeyFilter ::= "key" "=" <Identifier> + TableFilter ::= "table" "=" <Identifier> + ColumnFamilyFilter ::= "cf" "=" <Identifier> + FieldSelection ::= <Field> { <FieldSelection> } + Field ::= "Field" "=" <Identifier> + +#### Pagination + + Pagination ::= <CollectionQuery> "&" <PageSize> "&" <StartRowkey> + PageSize ::= "pageSize" "=" <Number> + StartRowkey ::= "startRowkey" "=" <Identifier> <!-- client application should set the rowkey where the query will start with. rowkey should be null if the query starts from the beginning --> + +#### Sorting + + Sort ::= [ <AggregationQuery> | <CollectionQuery> ] "&" <SortSpecs> + SortSpecs ::= <SortSpec> "&" { <SortSpecs> } + SortSpec ::= "st" "=" <SortField> = <SortOrder> + SortField ::= <Identifier> + SortOrder ::= [ asc | desc ] + +#### Union + + UnionQuery ::= <Query> { "+" <Query> } <!-- as of now this syntax is not yet implemented, but union is used by letting service knowing it's a union --> + Query ::= [ <CollectionQuery> | <AggregationQuery> | <BucketQuery> ] + +#### Join + + JoinQuery ::= <Query> { "^" <Query> } <!-- as of now this syntax is not yet implemented, but union is used by letting service knowing it's a join --> + +## HBase Rowkey Design +Optimized Rowkey design for time-series data, optimized for metric/entity/log, etc. different storage types + + Rowkey ::= Prefix | Partition Keys | timestamp | tagName | tagValue | ⦠+ +#### Metric Rowkey Design + + Rowkey ::= Metric Name | Partition Keys | timestamp | tagName | tagValue | ⦠+ +#### Entity Rowkey Design + + Rowkey ::= Default Prefix | Partition Keys | timestamp | tagName | tagValue | ⦠+ +##### Log Rowkey Design + + Rowkey ::= Log Type | Partition Keys | timestamp | tagName | tagValue | ⦠+ Rowvalue ::= Log Content +### Secondary Index Support + +Eagle supports to define secondary index with annotation with `@Index`. For example: + @Indexes({@Index(name="INDEX_NAME", columns = { "SECONDARY_INDEX_COLUMN_NAME" }, unique = true/false)}) + +### Native HBase Coprocessor + org.apache.eagle.storage.hbase.query.coprocessor.AggregateProtocolEndPoint + +# UI Framework + +Eagle UI is mainly consist of following parts: + +* Eagle Main UI +* Eagle App Portal/Dashboard/Widgets +* Eagle Customized Dashboard http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/docs/using-eagle.md ---------------------------------------------------------------------- diff --git a/docs/docs/using-eagle.md b/docs/docs/using-eagle.md new file mode 100644 index 0000000..885346d --- /dev/null +++ b/docs/docs/using-eagle.md @@ -0,0 +1,347 @@ +# Manage Eagle and Services + +* After Apache Eagle has been deployed (please reference [**deployment**](getting-started/#deployment)), you can enter deployment directory and use commands below to control Apache Eagle Server. + + ./bin/eagle-server.sh start|stop|status + +* After starting the Eagle server, please type http://<EAGLE_SERVER_HOST>:<PORT>/ to open the web ui of Eagle. + +--- + +# Use Eagle Web Interface + +* This is the typical Web Interface (short for WI) after setting up your Eagle monitoring environment. WI majorly contain the right main panel and left function menu. + +  + +## Home + +* This is the aggregated UI for configured sites, and the applications. It will show those created sites created, how many application installed for each sites, and alerts generated from that cluster. You can click âMore infoâ link to view the details for particular site. + +* The â**Widgets**â section is customizable; if the application developer have its application registered to Home page, you can find that in â**Widgets**â section. Please check the application developer guide about how to register applications to home widgets. It give you a shortcut to go directly to the application home. + +## Alert + +* In Alert menu, you can define the policies, list the policies and check your alerts there. + +## Integration + +* The integration page provides the management functionality for Eagle. You can list the built-in applications there, create sites, and manage the applications in your site. + +## Sites + +* It also gives you a shortcut to particular site. + +--- + +# Setup The Monitoring Application + +## Monitoring Applications + +* Eagle has an extensible framework to dynamically add new monitoring applications in Eagle environment. It also ships some built-in big data monitoring applications. + +* Go to â**Integration**â -> â**Applications**â, it will list a set of available monitoring applications which you can choose to monitor your services. + +  + +* The â**Application**â column is the display name for an application, â**Streams**â is a logical name for the data stream from the monitored source after pre-processing, which will consumed by Alert Engine. + +* At the moment, we have the below built-in applications shipped with Apache Eagle. You can refer to the application documentation to understand how to do the configuration for each monitoring application. + + | Application | Description | + | :---------: | :---------: | + | Topology Health Check | This application can be used to monitor the service healthiness for HDFS, HBase and YARN. You can get alerted once the master role or the slave role got crashed. | + | Hadoop JMX Metrics Monitoring | This application can be used to monitor the JMX metrics data from the master nodes of HDFS, HBase and YARN, e.g. NameNode, HBase Master and YARN Resource Manager. | + | HDFS Audit Log Monitor | This application can be used to monitor the data operations in HDFS, to detect sensitive data access and malicious operations; to protect from data leak or data loss. | + | HBase Audit Log Monitor | Same as HDFS Audit Log Monitor, this application is used to monitor the data operations in HBase. | + | Map Reduce History Job | This application is used to get the MapReduce history job counters from YARN history server and job running history from HDFS log directory. | + | Map Reduce Running Job | This application is used to get the MapReduce running job counter information using YARN Rest API. | + | Hadoop Queue Monitor | This application is used to get the resource scheduling and utilization info from YARN. | + | MR Metrics Aggregation | This application is used to aggregate the job counters and some resource utilization in a certain period of time (daily, weekly or monthly). | + | Job Performance Monitor Web | This application only contains the frontend, and depends on Map Reduce History Job and Map Reduce Running Job. | + | Alert Engine | Alert Engine is a special application and used to process the output data from other applications. | + + +## Managing Sites + +To enable a real monitoring use case, you have to create a site first, and install a certain application for this site, and finally start the application. We use site concept to group the running applications and avoid the application conflict. + +### Sites + +* Go to â**Integration**â -> â**Sites**â, there will be a table listing the managed sites. + +  + +### Create Site + +* Click â**New Site**â on the bottom right of the Sites page. You can fill the information in site creation dialog. + +  + +* The â**Site Id**â should not be duplicated. After the creation, you can find it in sites page. + +  + +### Configuring a Site + +* By clicking â**Edit**â button or the Site column in Sites table, you can have the Site configuration page, there you can install monitoring applications. + +  + +## Install and Run Applications in Site + +* Choose the particular application which you want to install, you probably have something to fill, e.g. the HDFS NameNode address, Zookeeper address and port. Please check each application documentation for how to configure each application. + +* After doing the installation, you can start the application by clicking  or stop the application by . You can check the â**Status**â column about the running status. Usually, it should have â**INITIALIZED**â or â**RUNNING**â for a healthy application. + +--- + +# Define Policies + +After setting up the monitoring applications, you probably want to setup some alert policies against the monitored data, so you can get notified once any violation on the data. Eagle has a centralized place for policy definition. + +## Policies + +* Go to â**Alert**â -> â**Policies**â, you can check the policies defined and take control on whether to enable the policy: + +  + +* You can apply the below actions for a certain policy: + + * : enable a policy + + * : disable a policy + + * : edit a policy + + * : purge a policy + +## Define or Edit Policies + +* If you want to create a new policy, click â**Alert**â -> â**Define Policy**â, or you can enter into the policy definition page by editing an existing policy. After that, you can go to the policy list to enable the policy dynamically. + +  + +### Source Stream + +* The source stream gives user a full view about what data stream is available for application defined for particular site, as well as the data structures in each data stream. Data stream name is suffixed by the site name. + +### Policy Name + +* The policy name should be globally unique. + +### Publish Alerts + +* In this section, you can define the alert publishment method by clicking the â**+Add Publisher**â. + +  + +* You can choose the publishment method from an existing policy or by creating new publisher. + +* There are four built-in publisher types: + + * **EmailPublisher**: org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher + + * **KafkaPublisher**: org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher + + * **SlackPublisher**: org.apache.eagle.alert.engine.publisher.impl.AlertSlackPublisher + + * **EagleStoragePlugin**: org.apache.eagle.alert.engine.publisher.impl.AlertEagleStoragePlugin + +### Policy Syntax + +* Currently, we support SiddhiQL(please view Siddhi Query Language Specification [here](https://docs.wso2.com/display/CEP300/Siddhi+Language+Specification)) + +* In order to explain how stream data is processed, let us take policy below as an example: + + from map_reduce_failed_job_stream[site=="sandbox" and currentState=="FAILED"] + select * group by jobId insert into map_reduce_failed_job_stream_out + +* This policy contains below parts: + + * **Source**: from map_reduce_failed_job_stream + + * **Filter**: [site=="sandbox" and currentState=="FAILED"] + + * **Projection**: select * + + * **GroupBy**: group by jobId + + * **Destination**: insert into map_reduce_failed_job_stream_out + +* Source Streams(schema) are defined by applications, and applications will write stream data to data sink(currently, we support kafka as data sink). + + <streams> + <stream> + <streamId>map_reduce_failed_job_stream</streamId> + <description>Map Reduce Failed Job Stream</description> + <validate>true</validate> + <columns> + <column> + <name>site</name> + <type>string</type> + </column> + â¦... + <column> + <name>jobId</name> + <type>string</type> + <column> + <name>currentState</name> + <type>string</type> + </column> + </columns> + </stream> + </streams> + +* After policy is defined, Alert engine will create siddhi execution runtime for the policy(also load stream data schema from metadata store). Since siddhi execution runtime knows the stream data schema, then it will process stream data and do the calculation. + +--- + +# Monitoring Dashboard + +* After setting the sites and applications, you can find the site item from the home page or âSitesâ menu. + +* Here is a site home example. After entering the site home, the left menu will be replaced by application dashboard links only related to that site, so you can switch between the application dashboard quickly. In the right panel, it contains the application icons installed in this site, but depends on if the application has its dashboard defined. You can click the application icon or the application links to go to the application dashboard home. Please check the application documentation about how to use the application monitoring dashboard. + +  + +--- + +# Check The Alerts + +* Eagle has all the alerts generated by all the applications stored in its database, so you can check your application alerts from Eagle WI. + +* Go to â**Alert**â -> â**Alerts**â, you can find the alerts table. + +  + +* Also you can check more detailed information by clicking â**Detail**â link for each alert item. + +  + +--- + +# How to stream audit log into Kafka + +## Logstash + +The sample configuration is tested with logstash-2.3.4. Logstash is required to be installed on the namenode host. + +* **Step 1**: Create a Kafka topic as the streaming input. + + Here is an sample Kafka command to create topic 'sandbox_hdfs_audit_log' + + cd <kafka-home> + bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log + +* **Step 2**: Create a Logstash configuration file under ${LOGSTASH_HOME}/conf. Here is a sample. + + input { + file { + type => "hdp-nn-audit" + path => "/tmp/test/hdfs-audit.log" + start_position => end + sincedb_path => "/dev/null" + } + } + output { + if [type] == "hdp-nn-audit" { + kafka { + codec => plain { + format => "%{message}" + } + bootstrap_servers => "host:9092" + topic_id => "hdfs_audit_log" + acks => "0" + timeout_ms => 10000 + + send_buffer_bytes => 102400 + client_id => "hdp-nn-audit" + + workers => 10 + compression_type => "gzip" + } + # stdout { codec => rubydebug } + } + } + +* **Step 4**: Start Logstash + + bin/logstash -f conf/sample.conf + +* **Step 5**: Check whether logs are flowing into the kafka topic specified by `topic_id` + +## Filebeat + +The sample filebeat.yml is tested with filebeat-5.0.0-beta1-linux-x86_64. The throughput can be up to 20K messages per second. Filebeat is required to be installed on the namenode host. + + filebeat.publish_async: false + filebeat.spool_size: 8192 + filebeat.idle_timeout: 5s + max_procs: 1 + queue_size: 1000 + + filebeat.prospectors: + - input_type: log + paths: + - /tmp/test/hdfs-audit.log + #tail_files: true + harvester_buffer_size: 8192 + + output.kafka: + enabled: true + hosts: ["host:9092"] + topic: "phx_hdfs_audit_log" + client_id: "client-host" + worker: 10 + max_retries: 3 + bulk_max_size: 8192 + channel_buffer_size: 512 + timeout: 10 + broker_timeout: 3s + keep_alive: 0 + compression: none + max_message_bytes: 1000000 + required_acks: 0 + flush_interval: 1 + + logging.metrics.period: 10s + + processors: + - include_fields: + fields: ["message", "beat.hostname"] + +## Log4j Kafka Appender + +This sample configuration is tested in HDP sandbox. `Restarting namenode is required` after updating the log4j configuration. + +* **Step 1**: Create a Kafka topic. Here is an example Kafka command for creating topic "sandbox_hdfs_audit_log" + + cd <kafka-home> + bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log + +* **Step 2**: Configure $HADOOP_CONF_DIR/log4j.properties, and add a log4j appender "KAFKA_HDFS_AUDIT" to hdfs audit logging + + log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender + log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log + log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667 + log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer + log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout + log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n + log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async + #log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1 + #log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1 + +* **Step 3**: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and add the reference to KAFKA_HDFS_AUDIT to HADOOP_NAMENODE_OPTS. + + -Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT + +* **Step 4**: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and append the following command to it. + + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/path/to/eagle/lib/log4jkafka/lib/* + +* **Step 5**: save the changes and restart the namenode. + +* **Step 6**: Check whether logs are flowing into Topic sandbox_hdfs_audit_log + + $ /usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic sandbox_hdfs_audit_log http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/eagle-theme/__init__.py ---------------------------------------------------------------------- diff --git a/docs/eagle-theme/__init__.py b/docs/eagle-theme/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/eagle-theme/base.html ---------------------------------------------------------------------- diff --git a/docs/eagle-theme/base.html b/docs/eagle-theme/base.html new file mode 100644 index 0000000..b19a1c8 --- /dev/null +++ b/docs/eagle-theme/base.html @@ -0,0 +1,118 @@ +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + {% if page_description %}<meta name="description" content="{{ page_description }}">{% endif %} + {% if site_author %}<meta name="author" content="{{ site_author }}">{% endif %} + {% block htmltitle %} + <title>{% if page_title %}{{ page_title }} - {% endif %}{{ site_name }}</title> + {% endblock %} + + {% if favicon %}<link rel="shortcut icon" href="{{ favicon }}"> + {% else %}<link rel="shortcut icon" href="{{ base_url }}/img/favicon.ico">{% endif %} + + {# CSS #} + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + <link rel="stylesheet" href="{{ base_url }}/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="{{ base_url }}/css/theme_extra.css" type="text/css" /> + <link rel="stylesheet" href="{{ base_url }}/css/highlight.css"> + {%- for path in extra_css %} + <link href="{{ path }}" rel="stylesheet"> + {%- endfor %} + + {% if current_page %} + <script> + // Current page data + var mkdocs_page_name = "{{ page_title }}"; + var mkdocs_page_input_path = "{{ current_page.input_path }}"; + var mkdocs_page_url = "{{ current_page.abs_url }}"; + </script> + {% endif %} + <script src="{{ base_url }}/js/jquery-2.1.1.min.js"></script> + <script src="{{ base_url }}/js/modernizr-2.8.3.min.js"></script> + <script type="text/javascript" src="{{ base_url }}/js/highlight.pack.js"></script> + <script src="{{ base_url }}/js/theme.js"></script> + + {%- block extrahead %} {% endblock %} + + {%- for path in extra_javascript %} + <script src="{{ path }}"></script> + {%- endfor %} + + {% if google_analytics %} + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', '{{ google_analytics[0] }}', '{{ google_analytics[1] }}'); + ga('send', 'pageview'); + </script> + {% endif %} +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + {# SIDE NAV, TOGGLES ON MOBILE #} + <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> + <div class="wy-side-nav-search"> + <a href="{{ homepage_url }}" class="icon icon-home"> {{ site_name }}</a> + {% include "searchbox.html" %} + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + <ul class="current"> + {% for nav_item in nav %} + <li>{% include "toc.html" %}<li> + {% endfor %} + </ul> + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + {# MOBILE NAV, TRIGGLES SIDE NAV ON TOGGLE #} + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="{{ homepage_url }}">{{ site_name }}</a> + </nav> + + {# PAGE CONTENT #} + <div class="wy-nav-content"> + <div class="rst-content"> + {% include "breadcrumbs.html" %} + <div role="main"> + <div class="section"> + {% block content %} + {{ content }} + {% endblock %} + </div> + </div> + {%- block footer %} + {% include "footer.html" %} + {% endblock %} + </div> + </div> + + </section> + + </div> + +{% include "versions.html" %} + +</body> +</html> +{% if current_page and current_page.is_homepage %} +<!-- +MkDocs version : {{ mkdocs_version }} +Build Date UTC : {{ build_date_utc }} +--> +{% endif %} http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/eagle-theme/breadcrumbs.html ---------------------------------------------------------------------- diff --git a/docs/eagle-theme/breadcrumbs.html b/docs/eagle-theme/breadcrumbs.html new file mode 100644 index 0000000..01960e6 --- /dev/null +++ b/docs/eagle-theme/breadcrumbs.html @@ -0,0 +1,25 @@ +<div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="{{ homepage_url }}">Docs</a> »</li> + {% if current_page %} + {% for doc in current_page.ancestors %} + {% if doc.link %} + <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> »</li> + {% else %} + <li>{{ doc.title }} »</li> + {% endif %} + {% endfor %} + {% endif %} + {% if current_page %}<li>{{ current_page.title }}</li>{% endif %} + <li class="wy-breadcrumbs-aside"> + {% if repo_url %} + {% if repo_name == 'GitHub' %} + <a href="{{ repo_url }}" class="icon icon-github"> Edit on GitHub</a> + {% elif repo_name == 'Bitbucket' %} + <a href="{{ repo_url }}" class="icon icon-bitbucket"> Edit on BitBucket</a> + {% endif %} + {% endif %} + </li> + </ul> + <hr/> +</div> http://git-wip-us.apache.org/repos/asf/eagle/blob/ee55054a/docs/eagle-theme/css/highlight.css ---------------------------------------------------------------------- diff --git a/docs/eagle-theme/css/highlight.css b/docs/eagle-theme/css/highlight.css new file mode 100644 index 0000000..0375453 --- /dev/null +++ b/docs/eagle-theme/css/highlight.css @@ -0,0 +1,125 @@ +/* +This is the GitHub theme for highlight.js + +github.com style (c) Vasily Polovnyov <[email protected]> + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + color: #333; + -webkit-text-size-adjust: none; +} + +.hljs-comment, +.diff .hljs-header, +.hljs-javadoc { + color: #998; + font-style: italic; +} + +.hljs-keyword, +.css .rule .hljs-keyword, +.hljs-winutils, +.nginx .hljs-title, +.hljs-subst, +.hljs-request, +.hljs-status { + color: #333; + font-weight: bold; +} + +.hljs-number, +.hljs-hexcolor, +.ruby .hljs-constant { + color: #008080; +} + +.hljs-string, +.hljs-tag .hljs-value, +.hljs-phpdoc, +.hljs-dartdoc, +.tex .hljs-formula { + color: #d14; +} + +.hljs-title, +.hljs-id, +.scss .hljs-preprocessor { + color: #900; + font-weight: bold; +} + +.hljs-list .hljs-keyword, +.hljs-subst { + font-weight: normal; +} + +.hljs-class .hljs-title, +.hljs-type, +.vhdl .hljs-literal, +.tex .hljs-command { + color: #458; + font-weight: bold; +} + +.hljs-tag, +.hljs-tag .hljs-title, +.hljs-rule .hljs-property, +.django .hljs-tag .hljs-keyword { + color: #000080; + font-weight: normal; +} + +.hljs-attribute, +.hljs-variable, +.lisp .hljs-body, +.hljs-name { + color: #008080; +} + +.hljs-regexp { + color: #009926; +} + +.hljs-symbol, +.ruby .hljs-symbol .hljs-string, +.lisp .hljs-keyword, +.clojure .hljs-keyword, +.scheme .hljs-keyword, +.tex .hljs-special, +.hljs-prompt { + color: #990073; +} + +.hljs-built_in { + color: #0086b3; +} + +.hljs-preprocessor, +.hljs-pragma, +.hljs-pi, +.hljs-doctype, +.hljs-shebang, +.hljs-cdata { + color: #999; + font-weight: bold; +} + +.hljs-deletion { + background: #fdd; +} + +.hljs-addition { + background: #dfd; +} + +.diff .hljs-change { + background: #0086b3; +} + +.hljs-chunk { + color: #aaa; +}
