Added support for Kubernetes
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/a575d479 Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/a575d479 Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/a575d479 Branch: refs/heads/master Commit: a575d479351a159a034c2b927963a78cc1f3802b Parents: 9f80900 Author: msbeard <msbe...@apache.org> Authored: Wed Apr 18 11:00:35 2018 -0400 Committer: msbeard <msbe...@apache.org> Committed: Wed Apr 18 11:00:35 2018 -0400 ---------------------------------------------------------------------- .gitignore | 1 + README.md | 2 +- docker/CLUSTER.md | 68 +++++ docker/README.md | 50 ++-- docker/docker-compose.5.6.3.yml | 133 +++++++++ docker/docker-compose.cluster.yml | 153 ++++++++++ docker/docker-compose.yml | 115 ++++---- docker/docs/images/configure_index.png | Bin 0 -> 116678 bytes docker/docs/images/confirmation.png | Bin 0 -> 158876 bytes docker/docs/images/dashboard.png | Bin 0 -> 172635 bytes docker/docs/images/management.png | Bin 0 -> 60954 bytes docker/elasticsearch/.env | 8 + docker/elasticsearch/5.6.3/Dockerfile | 29 ++ .../5.6.3/config/elasticsearch.yml | 48 +++ .../5.6.3/config/log4j2.properties | 24 ++ docker/elasticsearch/6.2.2/Dockerfile | 30 ++ .../6.2.2/config/elasticsearch.yml | 51 ++++ .../6.2.2/config/log4j2.properties | 24 ++ docker/elasticsearch/Dockerfile | 29 -- docker/elasticsearch/config/elasticsearch.yml | 49 ---- docker/elasticsearch/config/log4j2.properties | 24 -- docker/images/configure_index.png | Bin 116678 -> 0 bytes docker/images/confirmation.png | Bin 158876 -> 0 bytes docker/images/dashboard.png | Bin 172635 -> 0 bytes docker/images/management.png | Bin 60954 -> 0 bytes docker/kibana/5.6.3/Dockerfile | 25 ++ docker/kibana/5.6.3/config/kibana.yml | 18 ++ docker/kibana/5.6.3/dashboard.json | 19 ++ docker/kibana/5.6.3/visualizations.json | 58 ++++ docker/kibana/6.2.2/Dockerfile | 20 ++ docker/kibana/6.2.2/config/kibana.yml | 18 ++ docker/kibana/Dockerfile | 25 -- docker/kibana/config/kibana.yml | 18 -- docker/kibana/dashboard.json | 19 -- docker/kibana/visualizations.json | 58 ---- docker/logstash/5.6.3/Dockerfile | 30 ++ .../logstash/5.6.3/config/logstash-userale.conf | 53 ++++ docker/logstash/5.6.3/templates/userale.json | 150 ++++++++++ docker/logstash/6.2.2/Dockerfile | 31 ++ .../6.2.2/pipeline/logstash-userale.conf | 52 ++++ docker/logstash/6.2.2/templates/userale.json | 143 +++++++++ docker/logstash/Dockerfile | 30 -- docker/logstash/config/logstash-userale.conf | 53 ---- docker/logstash/templates/userale.json | 150 ---------- .../6.2.2/elasticsearch/es-client-svc.yaml | 43 +++ kubernetes/6.2.2/elasticsearch/es-client.yaml | 78 +++++ kubernetes/6.2.2/elasticsearch/es-data.yaml | 65 ++++ .../6.2.2/elasticsearch/es-master-svc.yaml | 38 +++ kubernetes/6.2.2/elasticsearch/es-master.yaml | 74 +++++ kubernetes/6.2.2/kibana/kibana-svc.yaml | 15 + kubernetes/6.2.2/kibana/kibana.yaml | 39 +++ kubernetes/6.2.2/logstash/logstash-svc.yaml | 19 ++ kubernetes/6.2.2/logstash/logstash.yaml | 52 ++++ kubernetes/README.md | 293 +++++++++++++++++++ kubernetes/configure.sh | 225 ++++++++++++++ 55 files changed, 2201 insertions(+), 548 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 4709039..ba433d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea release/incubator-* release/apache-* .sass-cache http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index c1558ec..29d471e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ SensSoft Docker Containers and Website -------------------------------------- -Specific build instructions located in docker and site folders. +Specific build instructions located in docker, kubernetes, release and site folders. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/CLUSTER.md ---------------------------------------------------------------------- diff --git a/docker/CLUSTER.md b/docker/CLUSTER.md new file mode 100644 index 0000000..93df592 --- /dev/null +++ b/docker/CLUSTER.md @@ -0,0 +1,68 @@ +How to Build SensSoft Docker Containers +--------------------------------------- + +1. Install [``Docker``](http://docker.com) on your machine. Require Docker 1.7 and above. + +1. Install ``docker-compose`` in an virtual environment. + Full instructions can be found [``here``](https://docs.docker.com/compose/install/). + + ```bash + $ python3 -m venv env + $ source env/bin/activate + $ pip install -r requirements.txt + ``` + +1. Before launching the Docker containers, ensure your ``vm_max_map_count`` + kernel setting is set to at least 262144. + Visit [``Running Elasticsearch in Production mode``](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/docker.html#docker-cli-run-prod-mode) for OS specific instructions. + + ```bash + # Example for Linux systems + $ sysctl -w vm.max_map_count=262144 + ``` + +1. Start Elasticsearch cluster: + + ```bash + $ docker-compose -f docker-compose.cluster.yaml up -d --scale elasticsearch=3 elasticsearch + $ docker-compose -f docker-compose.cluster.yaml up -d loadbalancer + ``` + + The loadbalancer node exposes port 9200 on localhost and is the only node + that has HTTP enabled. Services such as Kibana and Logstash connect to the + loadbalancer node directly. Loadbalancer accepts requests from Kibana and Logstash + and balances them across the elasticsearch worker nodes. The elasticsearch + worker nodes communicate to each other and the loadbalancer via TCP on port 9300. + + <aside class="warning"> + Starting an elasticsearch cluster is not recommended on a single server. This + is just for demonstration purposes only. Please refer to our [Kubernetes] guide to + deploy an Elasticsearch cluster. + </aside> + +1. Confirm cluster state: + ```bash + $ curl -XGET http://localhost:9200/_cluster/health?pretty + { + "cluster_name" : "SensSoft", + "status" : "green", + "timed_out" : false, + "number_of_nodes" : 4, + "number_of_data_nodes" : 3, + "active_primary_shards" : 0, + "active_shards" : 0, + "relocating_shards" : 0, + "initializing_shards" : 0, + "unassigned_shards" : 0, + "delayed_unassigned_shards" : 0, + "number_of_pending_tasks" : 0, + "number_of_in_flight_fetch" : 0, + "task_max_waiting_in_queue_millis" : 0, + "active_shards_percent_as_number" : 100.0 + } + ``` + Confirm that the `number_of_nodes` is 4 and `number_of_data_nodes` is 3. + +1. Follow remaining instructions in README.md, starting at #6. + +© Copyright 2016 The Charles Stark Draper Laboratory, Inc. All rights reserved. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/README.md ---------------------------------------------------------------------- diff --git a/docker/README.md b/docker/README.md index 036a995..78edcee 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,7 +1,7 @@ How to Build SensSoft Docker Containers --------------------------------------- -1. Install [``Docker``](http://docker.com) on your machine. +1. Install [``Docker``](http://docker.com) on your machine. Requires Docker 1.7 and above. 1. Install ``docker-compose`` in an virtual environment. Full instructions can be found [``here``](https://docs.docker.com/compose/install/). @@ -21,28 +21,22 @@ How to Build SensSoft Docker Containers $ sysctl -w vm.max_map_count=262144 ``` -1. Start Elasticsearch cluster: - - ```bash - $ docker-compose up -d --scale elasticsearch=3 elasticsearch loadbalancer - ``` - - The loadbalancer node exposes port 9200 on localhost and is the only node - that has HTTP enabled. Services such as Kibana and Logstash connect to the - loadbalancer node directly. Loadbalancer accepts requests from Kibana and Logstash - and balances them across the elasticsearch worker nodes. The elasticsearch - worker nodes communicate to each other and the loadbalancer via TCP on port 9300. - - -1. Confirm cluster state: +1. Start Elasticsearch. Give Elasticsearch about 2 minutes to start before confirming + its state. + + ```bash + $ docker-compose up -d loadbalancer + ``` + +1. Confirm state: ```bash $ curl -XGET http://localhost:9200/_cluster/health?pretty { "cluster_name" : "SensSoft", "status" : "green", "timed_out" : false, - "number_of_nodes" : 4, - "number_of_data_nodes" : 3, + "number_of_nodes" : 1, + "number_of_data_nodes" : 1, "active_primary_shards" : 0, "active_shards" : 0, "relocating_shards" : 0, @@ -55,7 +49,6 @@ How to Build SensSoft Docker Containers "active_shards_percent_as_number" : 100.0 } ``` - Confirm that the `number_of_nodes` is 4 and `number_of_data_nodes` is 3. 1. Launch logging server. Give Logstash about 2 minutes to start before confirming its state. @@ -105,8 +98,13 @@ How to Build SensSoft Docker Containers ![alt text][dashboard] +1. To Launch Tap and Distill + ```bash + $ docker-compose up -d distill tap + ``` + 1. To stop all containers. - ```sh + ```bash $ docker-compose stop ``` @@ -123,15 +121,9 @@ Having Issues? $ docker-compose logs > err.dump ``` -Todo ----- -- [ ] TAP docker deployment instructions. -- [ ] Distill docker deployment instructions. -- [ ] Apache SensSoft Docker + Kubernetes instructions. - -[configure_index]: ./images/configure_index.png "Configure Kibana index" -[confirmation]: ./images/confirmation.png "Confirm index pattern conflicts" -[dashboard]: ./images/dashboard.png "Apache Senssoft Dashboard" -[management]: ./images/management.png "Kibana management console" +[configure_index]: ./docs/images/configure_index.png "Configure Kibana index" +[confirmation]: ./docs/images/confirmation.png "Confirm index pattern conflicts" +[dashboard]: ./docs/images/dashboard.png "Apache Senssoft Dashboard" +[management]: ./docs/images/management.png "Kibana management console" © Copyright 2016 The Charles Stark Draper Laboratory, Inc. All rights reserved. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docker-compose.5.6.3.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.5.6.3.yml b/docker/docker-compose.5.6.3.yml new file mode 100644 index 0000000..00b6157 --- /dev/null +++ b/docker/docker-compose.5.6.3.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache SensSoft Single-Node Build + +version: "3" +services: + + # Startup + startup: + build: ./startup + image: senssoft/startup:latest + container_name: senssoft-start + depends_on: + - "loadbalancer" + command: bash /opt/entrypoint.sh + + # # Site + site: + build: ./site + image: senssoft/website:latest + container_name: senssoft-app + ports: + - 8080:8080 + depends_on: + - "logstash" + command: python -m SimpleHTTPServer 8080 + + # Elasticsearch Loadbalancer/Entry + loadbalancer: + build: ./elasticsearch/5.6.3 + image: senssoft/elasticsearch:5.6.3 + container_name: senssoft-loadbalancer + environment: + - NODE_MASTER=true + - NUMBER_OF_MASTERS=1 + - NODE_COORDINATE=true + - "DISCOVERY_SERVICE=elasticsearch" + - NODE_DATA=true + - NODE_INGEST=true + - MAX_LOCAL_STORAGE_NODES=1 + - NETWORK_HOST=0.0.0.0 + - HTTP_ENABLE=true + - HTTP_CORS_ENABLE=true + - HTTP_CORS_ALLOW_ORIGIN=* + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - "9200:9200" + - "9300" + networks: + - esnet + + # Kibana + kibana: + build: ./kibana/5.6.3 + image: senssoft/kibana:5.6.3 + container_name: senssoft-kibana + environment: + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + - "ELASTICSEARCH_URL=http://loadbalancer:9200"; + ports: + - "5601:5601" + depends_on: + - "loadbalancer" + networks: + - esnet + + # Logstash + logstash: + build: ./logstash/5.6.3 + image: senssoft/logstash:5.6.3 + container_name: senssoft-logstash + environment: + - "ELASTICSEARCH_URL=loadbalancer:9200" + ports: + - 8100:8100 + volumes: + - ./logstash/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf + - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json + depends_on: + - "loadbalancer" + networks: + - esnet + + # Distill + distill: + build: ./distill + image: senssoft/distill:0.1.0 + container_name: senssoft-distill + ports: + - 8090:8090 + depends_on: + - "loadbalancer" + networks: + - esnet + + # TAP's Database + db: + build: ./db + image: senssoft/postgres:latest + container_name: senssoft-postgres + ports: + - "5432:5432" + + # TAP + tap: + build: ./tap + image: senssoft/tap:0.1.0 + container_name: senssoft-tap + # command: ["bash", "./wait-for-postgres.sh"] + command: bash -c "/usr/src/app/wait-for-postgres.sh" + ports: + - "8010:8010" + depends_on: + - db + links: + - db:db + +networks: + esnet: + driver: bridge http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docker-compose.cluster.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.cluster.yml b/docker/docker-compose.cluster.yml new file mode 100644 index 0000000..5835716 --- /dev/null +++ b/docker/docker-compose.cluster.yml @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache SensSoft Cluster Build + +version: "3" +services: + + # Startup + startup: + build: ./startup + image: senssoft/startup:latest + container_name: senssoft-start + depends_on: + - "loadbalancer" + command: bash /opt/entrypoint.sh + + # # Site + site: + build: ./site + image: senssoft/website:latest + container_name: senssoft-app + ports: + - 8080:8080 + depends_on: + - "logstash" + command: python -m SimpleHTTPServer 8080 + + # Elasticsearch Swarm Workers + elasticsearch: + build: ./elasticsearch + image: senssoft/elasticsearch:6.2.2 + environment: + - NODE_MASTER=true + - NUMBER_OF_MASTERS=3 + - NODE_COORDINATE=true + - "DISCOVERY_SERVICE=elasticsearch" + - NODE_DATA=true + - NODE_INGEST=true + - MAX_LOCAL_STORAGE_NODES=1 + - NETWORK_HOST=0.0.0.0 + - HTTP_ENABLE=false + - HTTP_CORS_ENABLE=false + - HTTP_CORS_ALLOW_ORIGIN=* + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + networks: + - esnet + + # Elasticsearch Loadbalancer/Entry + loadbalancer: + build: ./elasticsearch + image: senssoft/elasticsearch:6.2.2 + container_name: senssoft-loadbalancer + environment: + - NODE_MASTER=false + - NUMBER_OF_MASTERS=1 + - NODE_COORDINATE=true + - "DISCOVERY_SERVICE=elasticsearch" + - NODE_DATA=false + - NODE_INGEST=false + - MAX_LOCAL_STORAGE_NODES=1 + - NETWORK_HOST=0.0.0.0 + - HTTP_ENABLE=true + - HTTP_CORS_ENABLE=true + - HTTP_CORS_ALLOW_ORIGIN=* + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - "9200:9200" + - "9300" + networks: + - esnet + + # Kibana + kibana: + build: ./kibana + image: senssoft/kibana:6.2.2 + container_name: senssoft-kibana + environment: + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + - "ELASTICSEARCH_URL=http://loadbalancer:9200"; + ports: + - "5601:5601" + depends_on: + - "loadbalancer" + networks: + - esnet + + # Logstash + logstash: + build: ./logstash + image: senssoft/logstash:6.2.2 + container_name: senssoft-logstash + environment: + - "ELASTICSEARCH_URL=loadbalancer:9200" + ports: + - 8100:8100 + volumes: + - ./logstash/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf + - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json + depends_on: + - "loadbalancer" + networks: + - esnet + + # Distill + distill: + build: ./distill + image: senssoft/distill:0.1.0 + container_name: senssoft-distill + ports: + - 8090:8090 + depends_on: + - "loadbalancer" + networks: + - esnet + + # TAP's Database + db: + build: ./db + image: senssoft/postgres:latest + container_name: senssoft-postgres + ports: + - "5432:5432" + + # TAP + tap: + build: ./tap + image: senssoft/tap:0.1.0 + container_name: senssoft-tap + # command: ["bash", "./wait-for-postgres.sh"] + command: bash -c "/usr/src/app/wait-for-postgres.sh" + ports: + - "8010:8010" + depends_on: + - db + links: + - db:db + +networks: + esnet: + driver: bridge http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d2b5ca6..f10b5f8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Apache SensSoft Docker Build +# Apache SensSoft Single-Node Build version: "3" services: @@ -21,16 +21,16 @@ services: # Startup startup: build: ./startup - image: senssoft.incubator.apache.org/startup:latest + image: senssoft/startup:latest container_name: senssoft-start depends_on: - "loadbalancer" command: bash /opt/entrypoint.sh - # Site + # Site site: build: ./site - image: senssoft.incubator.apache.org/website:latest + image: senssoft/website:latest container_name: senssoft-app ports: - 8080:8080 @@ -38,45 +38,34 @@ services: - "logstash" command: python -m SimpleHTTPServer 8080 - # Elasticsearch Workers elasticsearch: - build: ./elasticsearch - image: senssoft.incubator.apache.org/elasticsearch:5.6.3 - command: - - /bin/sh - - -c - - | - elasticsearch-plugin remove --silent x-pack - elasticsearch + build: ./elasticsearch/6.2.2 + image: senssoft/elasticsearch:6.2.2 environment: - NODE_MASTER=true - - NUMBER_OF_MASTERS=2 - - NODE_COORDINATE=true + - NUMBER_OF_MASTERS=1 + - NODE_COORDINATE=false - "DISCOVERY_SERVICE=elasticsearch" - - NODE_DATA=true - - NODE_INGEST=true + - NODE_DATA=false + - NODE_INGEST=false - MAX_LOCAL_STORAGE_NODES=1 - NETWORK_HOST=0.0.0.0 - HTTP_ENABLE=false - HTTP_CORS_ENABLE=false - HTTP_CORS_ALLOW_ORIGIN=* - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - "9300" networks: - esnet - # Elasticsearch Loadbalancer + # Elasticsearch Loadbalancer/Entry loadbalancer: - build: ./elasticsearch - image: senssoft.incubator.apache.org/elasticsearch:5.6.3 + build: ./elasticsearch/6.2.2 + image: senssoft/elasticsearch:6.2.2 container_name: senssoft-loadbalancer - command: - - /bin/sh - - -c - - | - elasticsearch-plugin remove --silent x-pack - elasticsearch environment: - - NODE_MASTER=true + - NODE_MASTER=false - NUMBER_OF_MASTERS=1 - NODE_COORDINATE=true - "DISCOVERY_SERVICE=elasticsearch" @@ -90,17 +79,19 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" ports: - "9200:9200" - - "9300" networks: - esnet + depends_on: + - elasticsearch # Kibana kibana: - build: ./kibana - image: senssoft.incubator.apache.org/kibana:5.6.3 + build: ./kibana/6.2.2 + image: senssoft/kibana:6.2.2 container_name: senssoft-kibana environment: - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + - "ELASTICSEARCH_URL=http://loadbalancer:9200"; ports: - "5601:5601" depends_on: @@ -110,57 +101,55 @@ services: # Logstash logstash: - build: ./logstash - image: senssoft.incubator.apache.org/logstash:5.6.3 + build: ./logstash/6.2.2 + image: senssoft/logstash:6.2.2 container_name: senssoft-logstash environment: - "ELASTICSEARCH_URL=loadbalancer:9200" - command: -f /etc/logstash/conf.d ports: - 8100:8100 volumes: - - ./logstash/config/logstash-userale.conf:/etc/logstash/conf.d/logstash-userale.conf - - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json + - ./logstash/6.2.2/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf + - ./logstash/6.2.2/templates/userale.json:/usr/share/logstash/templates/userale.json depends_on: - "loadbalancer" networks: - esnet - # Distill + # Distill distill: - build: ./distill - image: senssoft.incubator.apache.org/distill:0.1.0 - container_name: senssoft-distill - ports: - - 8090:8090 - depends_on: - - "loadbalancer" - networks: - - esnet + build: ./distill + image: senssoft/distill:0.1.0 + container_name: senssoft-distill + ports: + - 8090:8090 + depends_on: + - "loadbalancer" + networks: + - esnet - # TAP's Database + # TAP's Database db: - build: ./db - image: senssoft.incubator.apache.org/postgres:latest - container_name: senssoft-postgres - ports: - - "5432:5432" + build: ./db + image: senssoft/postgres:latest + container_name: senssoft-postgres + ports: + - "5432:5432" # TAP tap: - build: ./tap - image: senssoft.incubator.apache.org/tap:0.1.0 - container_name: senssoft-tap - # command: ["bash", "./wait-for-postgres.sh"] - command: bash -c "/usr/src/app/wait-for-postgres.sh" - ports: - - "8010:8010" - depends_on: - - db - links: - - db:db + build: ./tap + image: senssoft/tap:0.1.0 + container_name: senssoft-tap + # command: ["bash", "./wait-for-postgres.sh"] + command: bash -c "/usr/src/app/wait-for-postgres.sh" + ports: + - "8010:8010" + depends_on: + - db + links: + - db:db networks: - esnet: driver: bridge http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docs/images/configure_index.png ---------------------------------------------------------------------- diff --git a/docker/docs/images/configure_index.png b/docker/docs/images/configure_index.png new file mode 100644 index 0000000..31ebe02 Binary files /dev/null and b/docker/docs/images/configure_index.png differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docs/images/confirmation.png ---------------------------------------------------------------------- diff --git a/docker/docs/images/confirmation.png b/docker/docs/images/confirmation.png new file mode 100644 index 0000000..467b4d4 Binary files /dev/null and b/docker/docs/images/confirmation.png differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docs/images/dashboard.png ---------------------------------------------------------------------- diff --git a/docker/docs/images/dashboard.png b/docker/docs/images/dashboard.png new file mode 100644 index 0000000..7123c9e Binary files /dev/null and b/docker/docs/images/dashboard.png differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/docs/images/management.png ---------------------------------------------------------------------- diff --git a/docker/docs/images/management.png b/docker/docs/images/management.png new file mode 100644 index 0000000..90a26ae Binary files /dev/null and b/docker/docs/images/management.png differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/.env ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/.env b/docker/elasticsearch/.env new file mode 100644 index 0000000..f322e1a --- /dev/null +++ b/docker/elasticsearch/.env @@ -0,0 +1,8 @@ +NODE_MASTER=true +NODE_DATA=true +NETWORK_HOST=0.0.0.0 +NODE_COORDINATE=false +HTTP_ENABLE=true +HTTP_CORS_ENABLE=true +HTTP_CORS_ALLOW_ORIGIN="127.0.0.1" +DISCOVERY_TIMETOUT=10s \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/5.6.3/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/5.6.3/Dockerfile b/docker/elasticsearch/5.6.3/Dockerfile new file mode 100644 index 0000000..4f1e5c9 --- /dev/null +++ b/docker/elasticsearch/5.6.3/Dockerfile @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.3 + +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Override config +COPY config /usr/share/elasticsearch/config + +# Set environment +ENV CLUSTER_NAME SensSoft + +# Kubernetes requires swap is turned off +ENV MEMORY_LOCK false + +EXPOSE 9200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/5.6.3/config/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/5.6.3/config/elasticsearch.yml b/docker/elasticsearch/5.6.3/config/elasticsearch.yml new file mode 100644 index 0000000..93421af --- /dev/null +++ b/docker/elasticsearch/5.6.3/config/elasticsearch.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + name: ${CLUSTER_NAME} + +node: + master: ${NODE_MASTER} + data: ${NODE_DATA} + ingest: ${NODE_INGEST} + max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES} + +network.host: ${NETWORK_HOST} + +search.remote.connect: ${NODE_COORDINATE} + +path: + data: /usr/share/elasticsearch/data + logs: /usr/share/elasticsearch/log + +bootstrap: + memory_lock: ${MEMORY_LOCK} + +http: + enabled: ${HTTP_ENABLE} + compression: true + cors: + enabled: ${HTTP_CORS_ENABLE} + allow-origin: ${HTTP_CORS_ALLOW_ORIGIN} + +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +discovery: + zen: + ping.unicast.hosts: ${DISCOVERY_SERVICE} + minimum_master_nodes: ${NUMBER_OF_MASTERS} + http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/5.6.3/config/log4j2.properties ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/5.6.3/config/log4j2.properties b/docker/elasticsearch/5.6.3/config/log4j2.properties new file mode 100644 index 0000000..7f7817d --- /dev/null +++ b/docker/elasticsearch/5.6.3/config/log4j2.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/Dockerfile b/docker/elasticsearch/6.2.2/Dockerfile new file mode 100644 index 0000000..f1fe259 --- /dev/null +++ b/docker/elasticsearch/6.2.2/Dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2 + +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Override config +COPY config /usr/share/elasticsearch/config +RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config + +# Set environment +ENV CLUSTER_NAME SensSoft + +# Kubernetes requires swap is turned off +ENV MEMORY_LOCK false + +EXPOSE 9200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/6.2.2/config/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/config/elasticsearch.yml b/docker/elasticsearch/6.2.2/config/elasticsearch.yml new file mode 100644 index 0000000..5525441 --- /dev/null +++ b/docker/elasticsearch/6.2.2/config/elasticsearch.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom config borrowed from Pires: https://github.com/pires/docker-elasticsearch-kubernetes + +cluster: + name: ${CLUSTER_NAME:SensSoft} + +node: + master: ${NODE_MASTER:true} + data: ${NODE_DATA:true} + name: ${NODE_NAME:${HOSTNAME}} + ingest: ${NODE_INGEST:true} + max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES:1} + +network.host: ${NETWORK_HOST:0.0.0.0} + +search.remote.connect: ${NODE_COORDINATE:true} + +path: + data: /usr/share/elasticsearch/data + logs: /usr/share/elasticsearch/log + +bootstrap: + memory_lock: ${MEMORY_LOCK} + +http: + enabled: ${HTTP_ENABLE:true} + compression: true + cors: + enabled: ${HTTP_CORS_ENABLE:true} + allow-origin: ${HTTP_CORS_ALLOW_ORIGIN:"*"} + +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +discovery: + zen: + ping.unicast.hosts: ${DISCOVERY_SERVICE:127.0.0.1, [::1]} + ping.unicast.hosts.resolve_timeout: ${DISCOVERY_TIMEOUT:5s} + minimum_master_nodes: ${NUMBER_OF_MASTERS:1} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/6.2.2/config/log4j2.properties ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/config/log4j2.properties b/docker/elasticsearch/6.2.2/config/log4j2.properties new file mode 100644 index 0000000..7f7817d --- /dev/null +++ b/docker/elasticsearch/6.2.2/config/log4j2.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile deleted file mode 100644 index 4f1e5c9..0000000 --- a/docker/elasticsearch/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.3 - -MAINTAINER Michelle Beard <msbe...@apache.org> - -# Override config -COPY config /usr/share/elasticsearch/config - -# Set environment -ENV CLUSTER_NAME SensSoft - -# Kubernetes requires swap is turned off -ENV MEMORY_LOCK false - -EXPOSE 9200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/config/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/config/elasticsearch.yml b/docker/elasticsearch/config/elasticsearch.yml deleted file mode 100644 index a245d97..0000000 --- a/docker/elasticsearch/config/elasticsearch.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cluster: - name: ${CLUSTER_NAME} - -node: - master: ${NODE_MASTER} - data: ${NODE_DATA} - #name: ${NODE_NAME} - ingest: ${NODE_INGEST} - max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES} - -network.host: ${NETWORK_HOST} - -search.remote.connect: ${NODE_COORDINATE} - -path: - data: /usr/share/elasticsearch/data - logs: /usr/share/elasticsearch/log - -bootstrap: - memory_lock: ${MEMORY_LOCK} - -http: - enabled: ${HTTP_ENABLE} - compression: true - cors: - enabled: ${HTTP_CORS_ENABLE} - allow-origin: ${HTTP_CORS_ALLOW_ORIGIN} - -# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): -discovery: - zen: - ping.unicast.hosts: ${DISCOVERY_SERVICE} - minimum_master_nodes: ${NUMBER_OF_MASTERS} - http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/elasticsearch/config/log4j2.properties ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/config/log4j2.properties b/docker/elasticsearch/config/log4j2.properties deleted file mode 100644 index 7f7817d..0000000 --- a/docker/elasticsearch/config/log4j2.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/images/configure_index.png ---------------------------------------------------------------------- diff --git a/docker/images/configure_index.png b/docker/images/configure_index.png deleted file mode 100644 index 31ebe02..0000000 Binary files a/docker/images/configure_index.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/images/confirmation.png ---------------------------------------------------------------------- diff --git a/docker/images/confirmation.png b/docker/images/confirmation.png deleted file mode 100644 index 467b4d4..0000000 Binary files a/docker/images/confirmation.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/images/dashboard.png ---------------------------------------------------------------------- diff --git a/docker/images/dashboard.png b/docker/images/dashboard.png deleted file mode 100644 index 7123c9e..0000000 Binary files a/docker/images/dashboard.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/images/management.png ---------------------------------------------------------------------- diff --git a/docker/images/management.png b/docker/images/management.png deleted file mode 100644 index 90a26ae..0000000 Binary files a/docker/images/management.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/5.6.3/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/5.6.3/Dockerfile b/docker/kibana/5.6.3/Dockerfile new file mode 100644 index 0000000..91027a5 --- /dev/null +++ b/docker/kibana/5.6.3/Dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/kibana/kibana:5.6.3 + +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Override config +COPY config /usr/share/kibana/config + +RUN kibana-plugin remove x-pack + +EXPOSE 5601 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/5.6.3/config/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/5.6.3/config/kibana.yml b/docker/kibana/5.6.3/config/kibana.yml new file mode 100644 index 0000000..cc93fed --- /dev/null +++ b/docker/kibana/5.6.3/config/kibana.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.name: kibana +server.host: "0" +elasticsearch.url: http://localhost:9200 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/5.6.3/dashboard.json ---------------------------------------------------------------------- diff --git a/docker/kibana/5.6.3/dashboard.json b/docker/kibana/5.6.3/dashboard.json new file mode 100644 index 0000000..8a179a6 --- /dev/null +++ b/docker/kibana/5.6.3/dashboard.json @@ -0,0 +1,19 @@ +[ + { + "_id": "513c8680-be7d-11e7-8adf-fd0a12531eca", + "_type": "dashboard", + "_source": { + "title": "Apache SensSoft Dashboard", + "hits": 0, + "description": "", + "panelsJSON": "[{\"size_x\":6,\"size_y\":5,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"2bf40010-be7d-11e7-8adf-fd0a12531eca\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"f7ca4560-be7c-11e7-8adf-fd0a12531eca\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"bcd7c950-be7c-11e7-8adf-fd0a12531eca\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"e36b1f40-be7c-11e7-8adf-fd0a12531eca\",\"col\":1,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":true}", + "uiStateJSON": "{}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}],\"highlightAll\":true,\"version\":true}" + } + } + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/5.6.3/visualizations.json ---------------------------------------------------------------------- diff --git a/docker/kibana/5.6.3/visualizations.json b/docker/kibana/5.6.3/visualizations.json new file mode 100644 index 0000000..7052384 --- /dev/null +++ b/docker/kibana/5.6.3/visualizations.json @@ -0,0 +1,58 @@ +[ + { + "_id": "bcd7c950-be7c-11e7-8adf-fd0a12531eca", + "_type": "visualization", + "_source": { + "title": "Top Event Types", + "visState": "{\"title\":\"Top Event Types\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Top Event Types\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"type\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Event types\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "e36b1f40-be7c-11e7-8adf-fd0a12531eca", + "_type": "visualization", + "_source": { + "title": "Total Number of Logs", + "visState": "{\"title\":\"Total Number of Logs\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total number of logs\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "f7ca4560-be7c-11e7-8adf-fd0a12531eca", + "_type": "visualization", + "_source": { + "title": "Time Series", + "visState": "{\"title\":\"Time Series\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(*)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } + } + }, + { + "_id": "2bf40010-be7d-11e7-8adf-fd0a12531eca", + "_type": "visualization", + "_source": { + "title": "Events by Targets", + "visState": "{\"title\":\"Events by Targets\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"HTML Tags\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxi s-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"target\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"HTML Tags\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"JS Events\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/6.2.2/Dockerfile b/docker/kibana/6.2.2/Dockerfile new file mode 100644 index 0000000..630fe98 --- /dev/null +++ b/docker/kibana/6.2.2/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/kibana/kibana-oss:6.2.2 + +MAINTAINER Michelle Beard <msbe...@apache.org> + +EXPOSE 5601 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/6.2.2/config/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/6.2.2/config/kibana.yml b/docker/kibana/6.2.2/config/kibana.yml new file mode 100644 index 0000000..cc93fed --- /dev/null +++ b/docker/kibana/6.2.2/config/kibana.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.name: kibana +server.host: "0" +elasticsearch.url: http://localhost:9200 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile deleted file mode 100644 index 91027a5..0000000 --- a/docker/kibana/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/kibana/kibana:5.6.3 - -MAINTAINER Michelle Beard <msbe...@apache.org> - -# Override config -COPY config /usr/share/kibana/config - -RUN kibana-plugin remove x-pack - -EXPOSE 5601 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/config/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/config/kibana.yml b/docker/kibana/config/kibana.yml deleted file mode 100644 index cc93fed..0000000 --- a/docker/kibana/config/kibana.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -server.name: kibana -server.host: "0" -elasticsearch.url: http://localhost:9200 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/dashboard.json ---------------------------------------------------------------------- diff --git a/docker/kibana/dashboard.json b/docker/kibana/dashboard.json deleted file mode 100644 index 8a179a6..0000000 --- a/docker/kibana/dashboard.json +++ /dev/null @@ -1,19 +0,0 @@ -[ - { - "_id": "513c8680-be7d-11e7-8adf-fd0a12531eca", - "_type": "dashboard", - "_source": { - "title": "Apache SensSoft Dashboard", - "hits": 0, - "description": "", - "panelsJSON": "[{\"size_x\":6,\"size_y\":5,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"2bf40010-be7d-11e7-8adf-fd0a12531eca\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"f7ca4560-be7c-11e7-8adf-fd0a12531eca\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"bcd7c950-be7c-11e7-8adf-fd0a12531eca\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"e36b1f40-be7c-11e7-8adf-fd0a12531eca\",\"col\":1,\"row\":1}]", - "optionsJSON": "{\"darkTheme\":true}", - "uiStateJSON": "{}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}],\"highlightAll\":true,\"version\":true}" - } - } - } -] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/kibana/visualizations.json ---------------------------------------------------------------------- diff --git a/docker/kibana/visualizations.json b/docker/kibana/visualizations.json deleted file mode 100644 index 7052384..0000000 --- a/docker/kibana/visualizations.json +++ /dev/null @@ -1,58 +0,0 @@ -[ - { - "_id": "bcd7c950-be7c-11e7-8adf-fd0a12531eca", - "_type": "visualization", - "_source": { - "title": "Top Event Types", - "visState": "{\"title\":\"Top Event Types\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Top Event Types\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"type\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Event types\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - } - }, - { - "_id": "e36b1f40-be7c-11e7-8adf-fd0a12531eca", - "_type": "visualization", - "_source": { - "title": "Total Number of Logs", - "visState": "{\"title\":\"Total Number of Logs\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total number of logs\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - } - }, - { - "_id": "f7ca4560-be7c-11e7-8adf-fd0a12531eca", - "_type": "visualization", - "_source": { - "title": "Time Series", - "visState": "{\"title\":\"Time Series\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(*)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" - } - } - }, - { - "_id": "2bf40010-be7d-11e7-8adf-fd0a12531eca", - "_type": "visualization", - "_source": { - "title": "Events by Targets", - "visState": "{\"title\":\"Events by Targets\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"HTML Tags\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxi s-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"target\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"HTML Tags\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"JS Events\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"userale\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - } - } -] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/5.6.3/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/5.6.3/Dockerfile b/docker/logstash/5.6.3/Dockerfile new file mode 100644 index 0000000..fb46e5c --- /dev/null +++ b/docker/logstash/5.6.3/Dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/logstash/logstash:5.6.3 +MAINTAINER Michelle Beard <msbe...@apache.org> + +RUN mkdir /usr/share/logstash/templates + +RUN logstash-plugin remove x-pack + +# Known issue: https://github.com/elastic/logstash-docker/issues/45 +RUN sed -i '/xpack/d' /usr/share/logstash/config/logstash.yml + +# Copy Configuration files over +COPY config/logstash-userale.conf /etc/logstash/conf.d/logstash-userale.conf +COPY templates/userale.json /usr/share/logstash/templates/userale.json + +EXPOSE 8100 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/5.6.3/config/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/5.6.3/config/logstash-userale.conf b/docker/logstash/5.6.3/config/logstash-userale.conf new file mode 100644 index 0000000..4af6ee7 --- /dev/null +++ b/docker/logstash/5.6.3/config/logstash-userale.conf @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +input { + http { + codec => "json" + port => 8100 + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" + } + } +} + +## Add your filters / logstash plugins configuration here +filter { + mutate { + remove_field => [ "message", "@version", "headers"] + } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "${ELASTICSEARCH_URL}" + index => "userale" + document_type => "logs" + manage_template => true + template_overwrite => true + template => "/usr/share/logstash/templates/userale.json" + template_name => "userale" + } + + # Debug + # stdout { codec => rubydebug } +} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/5.6.3/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/5.6.3/templates/userale.json b/docker/logstash/5.6.3/templates/userale.json new file mode 100644 index 0000000..c1bdbb1 --- /dev/null +++ b/docker/logstash/5.6.3/templates/userale.json @@ -0,0 +1,150 @@ +{ + "template": "userale", + "order": 1, + "settings": + { + "number_of_shards": 1, + "number_of_replicas": 0, + "index.refresh_interval": "5s", + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "my_tokenizer" + } + }, + "tokenizer": { + "my_tokenizer": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": + { + "_default_": + { + "_all": + { + "enabled": false + } + }, + "logs": + { + "dynamic_templates": [ + { + "strings_as_keywords": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], + "dynamic": "true", + "properties": + { + "userAction": + { + "type": "boolean" + }, + "clientTime": + { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "count": + { + "type": "integer" + }, + "targetChange": + { + "type": "boolean" + }, + "typeChange": + { + "type": "boolean" + }, + "details": + { + "properties": + { + "alt": + { + "type": "boolean" + }, + "clicks": + { + "type": "integer" + }, + "ctrl": + { + "type": "boolean" + }, + "meta": + { + "type": "boolean" + }, + "shift": + { + "type": "boolean" + }, + "window": + { + "type": "boolean" + }, + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + }, + "z": + { + "type": "integer" + } + } + }, + "location": + { + "properties": + { + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + } + } + }, + "major_ver": + { + "type": "integer" + }, + "minor_ver": + { + "type": "integer" + }, + "patch_ver": + { + "type": "integer" + }, + "tool_major_ver": + { + "type": "integer" + }, + "tool_minor_ver": + { + "type": "integer" + }, + "tool_patch_ver": + { + "type": "integer" + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/Dockerfile b/docker/logstash/6.2.2/Dockerfile new file mode 100644 index 0000000..d041e48 --- /dev/null +++ b/docker/logstash/6.2.2/Dockerfile @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/logstash/logstash-oss:6.2.2 +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Ensure default logstash configuration is removed +RUN rm -f /usr/share/logstash/pipeline/logstash.conf + +# Copy Pipeline files over +ADD pipeline/ /usr/share/logstash/pipeline/ + +# Copy templates over +RUN mkdir /usr/share/logstash/templates +COPY templates/userale.json /usr/share/logstash/templates/userale.json + +EXPOSE 8100 + +CMD ["logstash"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/6.2.2/pipeline/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/pipeline/logstash-userale.conf b/docker/logstash/6.2.2/pipeline/logstash-userale.conf new file mode 100644 index 0000000..8b4313f --- /dev/null +++ b/docker/logstash/6.2.2/pipeline/logstash-userale.conf @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +input { + http { + codec => "json" + port => 8100 + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" + } + } +} + +## Add your filters / logstash plugins configuration here +filter { + mutate { + remove_field => [ "message", "@version", "headers"] + } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "${ELASTICSEARCH_URL}" + index => "userale" + manage_template => true + template_overwrite => true + template => "/usr/share/logstash/templates/userale.json" + template_name => "userale" + } + + # Debug + stdout { codec => rubydebug } +} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/6.2.2/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/templates/userale.json b/docker/logstash/6.2.2/templates/userale.json new file mode 100644 index 0000000..5aa92d0 --- /dev/null +++ b/docker/logstash/6.2.2/templates/userale.json @@ -0,0 +1,143 @@ +{ + "index_patterns": "userale", + "order": 1, + "settings": + { + "number_of_shards": 1, + "number_of_replicas": 0, + "index.refresh_interval": "5s", + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "my_tokenizer" + } + }, + "tokenizer": { + "my_tokenizer": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": + { + "doc": + { + "dynamic_templates": [ + { + "strings_as_keywords": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], + "dynamic": "true", + "properties": + { + "userAction": + { + "type": "boolean" + }, + "clientTime": + { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "count": + { + "type": "integer" + }, + "targetChange": + { + "type": "boolean" + }, + "typeChange": + { + "type": "boolean" + }, + "details": + { + "properties": + { + "alt": + { + "type": "boolean" + }, + "clicks": + { + "type": "integer" + }, + "ctrl": + { + "type": "boolean" + }, + "meta": + { + "type": "boolean" + }, + "shift": + { + "type": "boolean" + }, + "window": + { + "type": "boolean" + }, + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + }, + "z": + { + "type": "integer" + } + } + }, + "location": + { + "properties": + { + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + } + } + }, + "major_ver": + { + "type": "integer" + }, + "minor_ver": + { + "type": "integer" + }, + "patch_ver": + { + "type": "integer" + }, + "tool_major_ver": + { + "type": "integer" + }, + "tool_minor_ver": + { + "type": "integer" + }, + "tool_patch_ver": + { + "type": "integer" + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile deleted file mode 100644 index fb46e5c..0000000 --- a/docker/logstash/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/logstash/logstash:5.6.3 -MAINTAINER Michelle Beard <msbe...@apache.org> - -RUN mkdir /usr/share/logstash/templates - -RUN logstash-plugin remove x-pack - -# Known issue: https://github.com/elastic/logstash-docker/issues/45 -RUN sed -i '/xpack/d' /usr/share/logstash/config/logstash.yml - -# Copy Configuration files over -COPY config/logstash-userale.conf /etc/logstash/conf.d/logstash-userale.conf -COPY templates/userale.json /usr/share/logstash/templates/userale.json - -EXPOSE 8100 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/config/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/config/logstash-userale.conf b/docker/logstash/config/logstash-userale.conf deleted file mode 100644 index 4af6ee7..0000000 --- a/docker/logstash/config/logstash-userale.conf +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -input { - http { - codec => "json" - port => 8100 - response_headers => { - "Access-Control-Allow-Origin" => "*" - "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" - } - } -} - -## Add your filters / logstash plugins configuration here -filter { - mutate { - remove_field => [ "message", "@version", "headers"] - } - - grok { - match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] - match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] - } -} - -output { - # Output data to Elasticsearch instance - elasticsearch { - hosts => "${ELASTICSEARCH_URL}" - index => "userale" - document_type => "logs" - manage_template => true - template_overwrite => true - template => "/usr/share/logstash/templates/userale.json" - template_name => "userale" - } - - # Debug - # stdout { codec => rubydebug } -} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/a575d479/docker/logstash/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/templates/userale.json b/docker/logstash/templates/userale.json deleted file mode 100644 index c1bdbb1..0000000 --- a/docker/logstash/templates/userale.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "template": "userale", - "order": 1, - "settings": - { - "number_of_shards": 1, - "number_of_replicas": 0, - "index.refresh_interval": "5s", - "analysis": { - "analyzer": { - "my_analyzer": { - "tokenizer": "my_tokenizer" - } - }, - "tokenizer": { - "my_tokenizer": { - "type": "path_hierarchy" - } - } - } - }, - "mappings": - { - "_default_": - { - "_all": - { - "enabled": false - } - }, - "logs": - { - "dynamic_templates": [ - { - "strings_as_keywords": { - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ], - "dynamic": "true", - "properties": - { - "userAction": - { - "type": "boolean" - }, - "clientTime": - { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "count": - { - "type": "integer" - }, - "targetChange": - { - "type": "boolean" - }, - "typeChange": - { - "type": "boolean" - }, - "details": - { - "properties": - { - "alt": - { - "type": "boolean" - }, - "clicks": - { - "type": "integer" - }, - "ctrl": - { - "type": "boolean" - }, - "meta": - { - "type": "boolean" - }, - "shift": - { - "type": "boolean" - }, - "window": - { - "type": "boolean" - }, - "x": - { - "type": "integer" - }, - "y": - { - "type": "integer" - }, - "z": - { - "type": "integer" - } - } - }, - "location": - { - "properties": - { - "x": - { - "type": "integer" - }, - "y": - { - "type": "integer" - } - } - }, - "major_ver": - { - "type": "integer" - }, - "minor_ver": - { - "type": "integer" - }, - "patch_ver": - { - "type": "integer" - }, - "tool_major_ver": - { - "type": "integer" - }, - "tool_minor_ver": - { - "type": "integer" - }, - "tool_patch_ver": - { - "type": "integer" - } - } - } - } -} \ No newline at end of file
