Repository: incubator-senssoft Updated Branches: refs/heads/kubernetes bf43699ff -> 6c79f91a9
Split Docker and kubernetes build b/w ELK 5.6.3 and 6.2.2 Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/6c79f91a Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/6c79f91a Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/6c79f91a Branch: refs/heads/kubernetes Commit: 6c79f91a931a648f5f6314a74c8a094b5fd1d019 Parents: bf43699 Author: msbeard <[email protected]> Authored: Fri Apr 13 16:12:28 2018 -0400 Committer: msbeard <[email protected]> Committed: Fri Apr 13 16:12:28 2018 -0400 ---------------------------------------------------------------------- docker/docker-compose.yml | 38 +++-- docker/elasticsearch/6.2.2/Dockerfile | 30 ++++ .../6.2.2/config/elasticsearch.yml | 51 +++++++ .../6.2.2/config/log4j2.properties | 24 ++++ docker/elasticsearch/Dockerfile | 30 ---- docker/elasticsearch/config/elasticsearch.yml | 51 ------- docker/elasticsearch/config/log4j2.properties | 24 ---- docker/kibana/6.2.2/Dockerfile | 20 +++ docker/kibana/6.2.2/config/kibana.yml | 18 +++ docker/kibana/Dockerfile | 20 --- docker/kibana/config/kibana.yml | 18 --- docker/logstash/6.2.2/Dockerfile | 31 ++++ .../6.2.2/pipeline/logstash-userale.conf | 52 +++++++ docker/logstash/6.2.2/templates/userale.json | 143 +++++++++++++++++++ docker/logstash/Dockerfile | 31 ---- docker/logstash/pipeline/logstash-userale.conf | 52 ------- docker/logstash/templates/userale.json | 143 ------------------- .../6.2.2/elasticsearch/es-client-svc.yaml | 43 ++++++ kubernetes/6.2.2/elasticsearch/es-client.yaml | 78 ++++++++++ kubernetes/6.2.2/elasticsearch/es-data.yaml | 65 +++++++++ .../6.2.2/elasticsearch/es-master-svc.yaml | 38 +++++ kubernetes/6.2.2/elasticsearch/es-master.yaml | 74 ++++++++++ kubernetes/6.2.2/kibana/kibana-svc.yaml | 15 ++ kubernetes/6.2.2/kibana/kibana.yaml | 39 +++++ kubernetes/6.2.2/logstash/logstash-svc.yaml | 19 +++ kubernetes/6.2.2/logstash/logstash.yaml | 52 +++++++ kubernetes/elasticsearch/es-client-svc.yaml | 43 ------ kubernetes/elasticsearch/es-client.yaml | 78 ---------- kubernetes/elasticsearch/es-data.yaml | 65 --------- kubernetes/elasticsearch/es-master-svc.yaml | 38 ----- kubernetes/elasticsearch/es-master.yaml | 74 ---------- kubernetes/kibana/kibana-svc.yaml | 15 -- kubernetes/kibana/kibana.yaml | 39 ----- kubernetes/logstash/logstash-svc.yaml | 19 --- kubernetes/logstash/logstash.yaml | 52 ------- 35 files changed, 822 insertions(+), 800 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 25cc8e1..f10b5f8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -27,7 +27,7 @@ services: - "loadbalancer" command: bash /opt/entrypoint.sh - # # Site + # Site site: build: ./site image: senssoft/website:latest @@ -38,13 +38,34 @@ services: - "logstash" command: python -m SimpleHTTPServer 8080 + elasticsearch: + build: ./elasticsearch/6.2.2 + image: senssoft/elasticsearch:6.2.2 + environment: + - NODE_MASTER=true + - NUMBER_OF_MASTERS=1 + - NODE_COORDINATE=false + - "DISCOVERY_SERVICE=elasticsearch" + - NODE_DATA=false + - NODE_INGEST=false + - MAX_LOCAL_STORAGE_NODES=1 + - NETWORK_HOST=0.0.0.0 + - HTTP_ENABLE=false + - HTTP_CORS_ENABLE=false + - HTTP_CORS_ALLOW_ORIGIN=* + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - "9300" + networks: + - esnet + # Elasticsearch Loadbalancer/Entry loadbalancer: - build: ./elasticsearch + build: ./elasticsearch/6.2.2 image: senssoft/elasticsearch:6.2.2 container_name: senssoft-loadbalancer environment: - - NODE_MASTER=true + - NODE_MASTER=false - NUMBER_OF_MASTERS=1 - NODE_COORDINATE=true - "DISCOVERY_SERVICE=elasticsearch" @@ -58,13 +79,14 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" ports: - "9200:9200" - - "9300" networks: - esnet + depends_on: + - elasticsearch # Kibana kibana: - build: ./kibana + build: ./kibana/6.2.2 image: senssoft/kibana:6.2.2 container_name: senssoft-kibana environment: @@ -79,7 +101,7 @@ services: # Logstash logstash: - build: ./logstash + build: ./logstash/6.2.2 image: senssoft/logstash:6.2.2 container_name: senssoft-logstash environment: @@ -87,8 +109,8 @@ services: ports: - 8100:8100 volumes: - - ./logstash/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf - - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json + - ./logstash/6.2.2/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf + - ./logstash/6.2.2/templates/userale.json:/usr/share/logstash/templates/userale.json depends_on: - "loadbalancer" networks: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/Dockerfile b/docker/elasticsearch/6.2.2/Dockerfile new file mode 100644 index 0000000..f1fe259 --- /dev/null +++ b/docker/elasticsearch/6.2.2/Dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2 + +MAINTAINER Michelle Beard <[email protected]> + +# Override config +COPY config /usr/share/elasticsearch/config +RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config + +# Set environment +ENV CLUSTER_NAME SensSoft + +# Kubernetes requires swap is turned off +ENV MEMORY_LOCK false + +EXPOSE 9200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/6.2.2/config/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/config/elasticsearch.yml b/docker/elasticsearch/6.2.2/config/elasticsearch.yml new file mode 100644 index 0000000..5525441 --- /dev/null +++ b/docker/elasticsearch/6.2.2/config/elasticsearch.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom config borrowed from Pires: https://github.com/pires/docker-elasticsearch-kubernetes + +cluster: + name: ${CLUSTER_NAME:SensSoft} + +node: + master: ${NODE_MASTER:true} + data: ${NODE_DATA:true} + name: ${NODE_NAME:${HOSTNAME}} + ingest: ${NODE_INGEST:true} + max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES:1} + +network.host: ${NETWORK_HOST:0.0.0.0} + +search.remote.connect: ${NODE_COORDINATE:true} + +path: + data: /usr/share/elasticsearch/data + logs: /usr/share/elasticsearch/log + +bootstrap: + memory_lock: ${MEMORY_LOCK} + +http: + enabled: ${HTTP_ENABLE:true} + compression: true + cors: + enabled: ${HTTP_CORS_ENABLE:true} + allow-origin: ${HTTP_CORS_ALLOW_ORIGIN:"*"} + +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +discovery: + zen: + ping.unicast.hosts: ${DISCOVERY_SERVICE:127.0.0.1, [::1]} + ping.unicast.hosts.resolve_timeout: ${DISCOVERY_TIMEOUT:5s} + minimum_master_nodes: ${NUMBER_OF_MASTERS:1} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/6.2.2/config/log4j2.properties ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/6.2.2/config/log4j2.properties b/docker/elasticsearch/6.2.2/config/log4j2.properties new file mode 100644 index 0000000..7f7817d --- /dev/null +++ b/docker/elasticsearch/6.2.2/config/log4j2.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile deleted file mode 100644 index f1fe259..0000000 --- a/docker/elasticsearch/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2 - -MAINTAINER Michelle Beard <[email protected]> - -# Override config -COPY config /usr/share/elasticsearch/config -RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config - -# Set environment -ENV CLUSTER_NAME SensSoft - -# Kubernetes requires swap is turned off -ENV MEMORY_LOCK false - -EXPOSE 9200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/config/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/config/elasticsearch.yml b/docker/elasticsearch/config/elasticsearch.yml deleted file mode 100644 index 5525441..0000000 --- a/docker/elasticsearch/config/elasticsearch.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Custom config borrowed from Pires: https://github.com/pires/docker-elasticsearch-kubernetes - -cluster: - name: ${CLUSTER_NAME:SensSoft} - -node: - master: ${NODE_MASTER:true} - data: ${NODE_DATA:true} - name: ${NODE_NAME:${HOSTNAME}} - ingest: ${NODE_INGEST:true} - max_local_storage_nodes: ${MAX_LOCAL_STORAGE_NODES:1} - -network.host: ${NETWORK_HOST:0.0.0.0} - -search.remote.connect: ${NODE_COORDINATE:true} - -path: - data: /usr/share/elasticsearch/data - logs: /usr/share/elasticsearch/log - -bootstrap: - memory_lock: ${MEMORY_LOCK} - -http: - enabled: ${HTTP_ENABLE:true} - compression: true - cors: - enabled: ${HTTP_CORS_ENABLE:true} - allow-origin: ${HTTP_CORS_ALLOW_ORIGIN:"*"} - -# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): -discovery: - zen: - ping.unicast.hosts: ${DISCOVERY_SERVICE:127.0.0.1, [::1]} - ping.unicast.hosts.resolve_timeout: ${DISCOVERY_TIMEOUT:5s} - minimum_master_nodes: ${NUMBER_OF_MASTERS:1} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/elasticsearch/config/log4j2.properties ---------------------------------------------------------------------- diff --git a/docker/elasticsearch/config/log4j2.properties b/docker/elasticsearch/config/log4j2.properties deleted file mode 100644 index 7f7817d..0000000 --- a/docker/elasticsearch/config/log4j2.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/kibana/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/6.2.2/Dockerfile b/docker/kibana/6.2.2/Dockerfile new file mode 100644 index 0000000..630fe98 --- /dev/null +++ b/docker/kibana/6.2.2/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/kibana/kibana-oss:6.2.2 + +MAINTAINER Michelle Beard <[email protected]> + +EXPOSE 5601 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/kibana/6.2.2/config/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/6.2.2/config/kibana.yml b/docker/kibana/6.2.2/config/kibana.yml new file mode 100644 index 0000000..cc93fed --- /dev/null +++ b/docker/kibana/6.2.2/config/kibana.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.name: kibana +server.host: "0" +elasticsearch.url: http://localhost:9200 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/kibana/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile deleted file mode 100644 index 630fe98..0000000 --- a/docker/kibana/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/kibana/kibana-oss:6.2.2 - -MAINTAINER Michelle Beard <[email protected]> - -EXPOSE 5601 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/kibana/config/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/config/kibana.yml b/docker/kibana/config/kibana.yml deleted file mode 100644 index cc93fed..0000000 --- a/docker/kibana/config/kibana.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -server.name: kibana -server.host: "0" -elasticsearch.url: http://localhost:9200 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/6.2.2/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/Dockerfile b/docker/logstash/6.2.2/Dockerfile new file mode 100644 index 0000000..d041e48 --- /dev/null +++ b/docker/logstash/6.2.2/Dockerfile @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.elastic.co/logstash/logstash-oss:6.2.2 +MAINTAINER Michelle Beard <[email protected]> + +# Ensure default logstash configuration is removed +RUN rm -f /usr/share/logstash/pipeline/logstash.conf + +# Copy Pipeline files over +ADD pipeline/ /usr/share/logstash/pipeline/ + +# Copy templates over +RUN mkdir /usr/share/logstash/templates +COPY templates/userale.json /usr/share/logstash/templates/userale.json + +EXPOSE 8100 + +CMD ["logstash"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/6.2.2/pipeline/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/pipeline/logstash-userale.conf b/docker/logstash/6.2.2/pipeline/logstash-userale.conf new file mode 100644 index 0000000..8b4313f --- /dev/null +++ b/docker/logstash/6.2.2/pipeline/logstash-userale.conf @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +input { + http { + codec => "json" + port => 8100 + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" + } + } +} + +## Add your filters / logstash plugins configuration here +filter { + mutate { + remove_field => [ "message", "@version", "headers"] + } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "${ELASTICSEARCH_URL}" + index => "userale" + manage_template => true + template_overwrite => true + template => "/usr/share/logstash/templates/userale.json" + template_name => "userale" + } + + # Debug + stdout { codec => rubydebug } +} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/6.2.2/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/6.2.2/templates/userale.json b/docker/logstash/6.2.2/templates/userale.json new file mode 100644 index 0000000..5aa92d0 --- /dev/null +++ b/docker/logstash/6.2.2/templates/userale.json @@ -0,0 +1,143 @@ +{ + "index_patterns": "userale", + "order": 1, + "settings": + { + "number_of_shards": 1, + "number_of_replicas": 0, + "index.refresh_interval": "5s", + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "my_tokenizer" + } + }, + "tokenizer": { + "my_tokenizer": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": + { + "doc": + { + "dynamic_templates": [ + { + "strings_as_keywords": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], + "dynamic": "true", + "properties": + { + "userAction": + { + "type": "boolean" + }, + "clientTime": + { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "count": + { + "type": "integer" + }, + "targetChange": + { + "type": "boolean" + }, + "typeChange": + { + "type": "boolean" + }, + "details": + { + "properties": + { + "alt": + { + "type": "boolean" + }, + "clicks": + { + "type": "integer" + }, + "ctrl": + { + "type": "boolean" + }, + "meta": + { + "type": "boolean" + }, + "shift": + { + "type": "boolean" + }, + "window": + { + "type": "boolean" + }, + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + }, + "z": + { + "type": "integer" + } + } + }, + "location": + { + "properties": + { + "x": + { + "type": "integer" + }, + "y": + { + "type": "integer" + } + } + }, + "major_ver": + { + "type": "integer" + }, + "minor_ver": + { + "type": "integer" + }, + "patch_ver": + { + "type": "integer" + }, + "tool_major_ver": + { + "type": "integer" + }, + "tool_minor_ver": + { + "type": "integer" + }, + "tool_patch_ver": + { + "type": "integer" + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile deleted file mode 100644 index d041e48..0000000 --- a/docker/logstash/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM docker.elastic.co/logstash/logstash-oss:6.2.2 -MAINTAINER Michelle Beard <[email protected]> - -# Ensure default logstash configuration is removed -RUN rm -f /usr/share/logstash/pipeline/logstash.conf - -# Copy Pipeline files over -ADD pipeline/ /usr/share/logstash/pipeline/ - -# Copy templates over -RUN mkdir /usr/share/logstash/templates -COPY templates/userale.json /usr/share/logstash/templates/userale.json - -EXPOSE 8100 - -CMD ["logstash"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/pipeline/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/pipeline/logstash-userale.conf b/docker/logstash/pipeline/logstash-userale.conf deleted file mode 100644 index 8b4313f..0000000 --- a/docker/logstash/pipeline/logstash-userale.conf +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -input { - http { - codec => "json" - port => 8100 - response_headers => { - "Access-Control-Allow-Origin" => "*" - "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" - } - } -} - -## Add your filters / logstash plugins configuration here -filter { - mutate { - remove_field => [ "message", "@version", "headers"] - } - - grok { - match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] - match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] - } -} - -output { - # Output data to Elasticsearch instance - elasticsearch { - hosts => "${ELASTICSEARCH_URL}" - index => "userale" - manage_template => true - template_overwrite => true - template => "/usr/share/logstash/templates/userale.json" - template_name => "userale" - } - - # Debug - stdout { codec => rubydebug } -} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/docker/logstash/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/templates/userale.json b/docker/logstash/templates/userale.json deleted file mode 100644 index 5aa92d0..0000000 --- a/docker/logstash/templates/userale.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "index_patterns": "userale", - "order": 1, - "settings": - { - "number_of_shards": 1, - "number_of_replicas": 0, - "index.refresh_interval": "5s", - "analysis": { - "analyzer": { - "my_analyzer": { - "tokenizer": "my_tokenizer" - } - }, - "tokenizer": { - "my_tokenizer": { - "type": "path_hierarchy" - } - } - } - }, - "mappings": - { - "doc": - { - "dynamic_templates": [ - { - "strings_as_keywords": { - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ], - "dynamic": "true", - "properties": - { - "userAction": - { - "type": "boolean" - }, - "clientTime": - { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "count": - { - "type": "integer" - }, - "targetChange": - { - "type": "boolean" - }, - "typeChange": - { - "type": "boolean" - }, - "details": - { - "properties": - { - "alt": - { - "type": "boolean" - }, - "clicks": - { - "type": "integer" - }, - "ctrl": - { - "type": "boolean" - }, - "meta": - { - "type": "boolean" - }, - "shift": - { - "type": "boolean" - }, - "window": - { - "type": "boolean" - }, - "x": - { - "type": "integer" - }, - "y": - { - "type": "integer" - }, - "z": - { - "type": "integer" - } - } - }, - "location": - { - "properties": - { - "x": - { - "type": "integer" - }, - "y": - { - "type": "integer" - } - } - }, - "major_ver": - { - "type": "integer" - }, - "minor_ver": - { - "type": "integer" - }, - "patch_ver": - { - "type": "integer" - }, - "tool_major_ver": - { - "type": "integer" - }, - "tool_minor_ver": - { - "type": "integer" - }, - "tool_patch_ver": - { - "type": "integer" - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml b/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml new file mode 100644 index 0000000..0d0622e --- /dev/null +++ b/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Discovery Service for Elasticsearch client nodes over port 9200. +# +# A Kubernetes Service is an abstraction which defines a logical set of +# Pods and a policy by which to access them. +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-loadbalancer + labels: + component: elk + role: client +spec: + selector: + component: elk + role: client + ports: + - name: http + port: 9200 +# targetPort: 9200 + protocol: TCP + type: NodePort +# type: Loadbalancer only works with certain providers: +# AWS, Azure, OpenStack, CloudStack, and Google Compute Engine +# type: LoadBalancer http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/elasticsearch/es-client.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/elasticsearch/es-client.yaml b/kubernetes/6.2.2/elasticsearch/es-client.yaml new file mode 100644 index 0000000..032ba18 --- /dev/null +++ b/kubernetes/6.2.2/elasticsearch/es-client.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: es-client + labels: + component: elk + role: client +spec: + replicas: 2 + template: + metadata: + labels: + component: elk + role: client + spec: + initContainers: + - name: init-sysctl + image: busybox:1.27.2 + imagePullPolicy: IfNotPresent + command: + - sysctl + - -w + - vm.max_map_count=262144 + securityContext: + privileged: true + containers: + - name: es-client + image: senssoft/elasticsearch:6.2.2 + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # Client node + - name: NODE_MASTER + value: "false" + - name: NODE_DATA + value: "false" + - name: NODE_INGEST + value: "false" + - name: HTTP_ENABLE + value: "true" + - name: DISCOVERY_SERVICE + value: "elasticsearch-discovery" + - name: "NETWORK_HOST" + value: _site_,_lo_ + - name: ES_JAVA_OPTS + value: -Xms256m -Xmx256m + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + livenessProbe: + tcpSocket: + port: transport + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /_cluster/health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 10 + volumeMounts: + - name: storage + mountPath: /data + volumes: + - emptyDir: + medium: "" + name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/elasticsearch/es-data.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/elasticsearch/es-data.yaml b/kubernetes/6.2.2/elasticsearch/es-data.yaml new file mode 100644 index 0000000..afe5617 --- /dev/null +++ b/kubernetes/6.2.2/elasticsearch/es-data.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: es-data + labels: + component: elk + role: data +spec: + replicas: 2 + template: + metadata: + labels: + component: elk + role: data + spec: + initContainers: + - name: init-sysctl + image: busybox:1.27.2 + imagePullPolicy: IfNotPresent + command: + - sysctl + - -w + - vm.max_map_count=262144 + securityContext: + privileged: true + containers: + - name: es-data + image: senssoft/elasticsearch:6.2.2 + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # Data node + - name: NODE_MASTER + value: "false" + - name: NODE_DATA + value: "true" + - name: HTTP_ENABLE + value: "false" + - name: DISCOVERY_SERVICE + value: "elasticsearch-discovery" + - name: ES_JAVA_OPTS + value: -Xms256m -Xmx256m + ports: + - containerPort: 9300 + name: transport + livenessProbe: + tcpSocket: + port: transport + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 5 + volumeMounts: + - name: storage + mountPath: /data + volumes: + - emptyDir: + medium: "" + name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml b/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml new file mode 100644 index 0000000..0cb1de8 --- /dev/null +++ b/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Discovery Service for Elasticsearch master nodes over port 9300. +# +# A Kubernetes Service is an abstraction which defines a logical set of +# Pods and a policy by which to access them. +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-discovery + labels: + component: elk + role: master +spec: + selector: + component: elk + role: master + ports: + - name: transport + port: 9300 + protocol: TCP http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/elasticsearch/es-master.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/elasticsearch/es-master.yaml b/kubernetes/6.2.2/elasticsearch/es-master.yaml new file mode 100644 index 0000000..592a00c --- /dev/null +++ b/kubernetes/6.2.2/elasticsearch/es-master.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: es-master + labels: + component: elk + role: master +spec: + replicas: 3 + template: + metadata: + labels: + component: elk + role: master + spec: + initContainers: + - name: init-sysctl + image: busybox:1.27.2 + imagePullPolicy: IfNotPresent + command: + - sysctl + - -w + - vm.max_map_count=262144 + securityContext: + privileged: true + containers: + - name: es-master + image: senssoft/elasticsearch:6.2.2 + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # Dedicated master-eligible node + - name: NODE_MASTER + value: "true" + - name: NODE_DATA + value: "false" + - name: NODE_INGEST + value: "false" + - name: NODE_COORDINATE + value: "false" + - name: NUMBER_OF_MASTERS + value: "2" + - name: HTTP_ENABLE + value: "false" + - name: DISCOVERY_SERVICE + value: "elasticsearch-discovery" + - name: DISCOVERY_TIMEOUT + value: "30s" + - name: ES_JAVA_OPTS + value: -Xms256m -Xmx256m + ports: + - containerPort: 9300 + name: transport +# Need better readiness check policy than checking if port 9300 is active + livenessProbe: + tcpSocket: + port: transport + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 5 + volumeMounts: + - name: storage + mountPath: /data + volumes: + - emptyDir: + medium: "" + name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/kibana/kibana-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/kibana/kibana-svc.yaml b/kubernetes/6.2.2/kibana/kibana-svc.yaml new file mode 100644 index 0000000..fbefde0 --- /dev/null +++ b/kubernetes/6.2.2/kibana/kibana-svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana + labels: + component: elk + role: kibana +spec: + selector: + component: elk + role: kibana + ports: + - name: http + port: 5601 + type: NodePort \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/kibana/kibana.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/kibana/kibana.yaml b/kubernetes/6.2.2/kibana/kibana.yaml new file mode 100644 index 0000000..f0f1ed0 --- /dev/null +++ b/kubernetes/6.2.2/kibana/kibana.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: kibana + labels: + component: elk + role: kibana +spec: + replicas: 1 + selector: + matchLabels: + component: elk + role: kibana + template: + metadata: + labels: + component: elk + role: kibana + spec: + containers: + - name: kibana + image: senssoft/kibana:6.2.2 + imagePullPolicy: IfNotPresent + resources: + # need more cpu upon initialization, therefore burstable class + limits: + cpu: 1000m + requests: + cpu: 100m + env: + - name: CLUSTER_NAME + value: SensSoft + - name: ELASTICSEARCH_URL + value: "http://elasticsearch-loadbalancer:9200"; + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + ports: + - containerPort: 5601 + name: http \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/logstash/logstash-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/logstash/logstash-svc.yaml b/kubernetes/6.2.2/logstash/logstash-svc.yaml new file mode 100644 index 0000000..b77a032 --- /dev/null +++ b/kubernetes/6.2.2/logstash/logstash-svc.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: logstash-discovery + labels: + component: elk + role: logstash +spec: + selector: + component: elk + role: logstash + ports: +# - name: transport +# port: 9600 +# protocol: TCP + - name: http + port: 8100 + protocol: TCP + type: NodePort http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/6.2.2/logstash/logstash.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/6.2.2/logstash/logstash.yaml b/kubernetes/6.2.2/logstash/logstash.yaml new file mode 100644 index 0000000..8756731 --- /dev/null +++ b/kubernetes/6.2.2/logstash/logstash.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: logstash + labels: + component: elk + role: logstash +spec: + replicas: 1 + template: + metadata: + labels: + component: elk + role: logstash + spec: + containers: + - name: logstash + image: senssoft/logstash:6.2.2 + imagePullPolicy: IfNotPresent + command: + - logstash + - -f + - /etc/logstash/conf.d + env: + - name: ELASTICSEARCH_URL + value: "elasticsearch-loadbalancer:9200" + - name: ES_JAVA_OPTS + value: "-Xms1g -Xmx1g" + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 8100 + name: http + - containerPort: 9600 + name: transport + # Need better readiness check policy than checking if port 9600 is active +# livenessProbe: +# tcpSocket: +# port: transport +# initialDelaySeconds: 20 +# periodSeconds: 10 +# failureThreshold: 5 +# readinessProbe: +# httpGet: +# path: /?pretty=true +# port: http +# initialDelaySeconds: 30 +# periodSeconds: 10 +# failureThreshold: 10 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/elasticsearch/es-client-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/elasticsearch/es-client-svc.yaml b/kubernetes/elasticsearch/es-client-svc.yaml deleted file mode 100644 index 0d0622e..0000000 --- a/kubernetes/elasticsearch/es-client-svc.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Discovery Service for Elasticsearch client nodes over port 9200. -# -# A Kubernetes Service is an abstraction which defines a logical set of -# Pods and a policy by which to access them. -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch-loadbalancer - labels: - component: elk - role: client -spec: - selector: - component: elk - role: client - ports: - - name: http - port: 9200 -# targetPort: 9200 - protocol: TCP - type: NodePort -# type: Loadbalancer only works with certain providers: -# AWS, Azure, OpenStack, CloudStack, and Google Compute Engine -# type: LoadBalancer http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/elasticsearch/es-client.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/elasticsearch/es-client.yaml b/kubernetes/elasticsearch/es-client.yaml deleted file mode 100644 index 032ba18..0000000 --- a/kubernetes/elasticsearch/es-client.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-client - labels: - component: elk - role: client -spec: - replicas: 2 - template: - metadata: - labels: - component: elk - role: client - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-client - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Client node - - name: NODE_MASTER - value: "false" - - name: NODE_DATA - value: "false" - - name: NODE_INGEST - value: "false" - - name: HTTP_ENABLE - value: "true" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: "NETWORK_HOST" - value: _site_,_lo_ - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9200 - name: http - - containerPort: 9300 - name: transport - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /_cluster/health - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 10 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/elasticsearch/es-data.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/elasticsearch/es-data.yaml b/kubernetes/elasticsearch/es-data.yaml deleted file mode 100644 index afe5617..0000000 --- a/kubernetes/elasticsearch/es-data.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-data - labels: - component: elk - role: data -spec: - replicas: 2 - template: - metadata: - labels: - component: elk - role: data - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-data - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Data node - - name: NODE_MASTER - value: "false" - - name: NODE_DATA - value: "true" - - name: HTTP_ENABLE - value: "false" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9300 - name: transport - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/elasticsearch/es-master-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/elasticsearch/es-master-svc.yaml b/kubernetes/elasticsearch/es-master-svc.yaml deleted file mode 100644 index 0cb1de8..0000000 --- a/kubernetes/elasticsearch/es-master-svc.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Discovery Service for Elasticsearch master nodes over port 9300. -# -# A Kubernetes Service is an abstraction which defines a logical set of -# Pods and a policy by which to access them. -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch-discovery - labels: - component: elk - role: master -spec: - selector: - component: elk - role: master - ports: - - name: transport - port: 9300 - protocol: TCP http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/elasticsearch/es-master.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/elasticsearch/es-master.yaml b/kubernetes/elasticsearch/es-master.yaml deleted file mode 100644 index 592a00c..0000000 --- a/kubernetes/elasticsearch/es-master.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-master - labels: - component: elk - role: master -spec: - replicas: 3 - template: - metadata: - labels: - component: elk - role: master - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-master - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Dedicated master-eligible node - - name: NODE_MASTER - value: "true" - - name: NODE_DATA - value: "false" - - name: NODE_INGEST - value: "false" - - name: NODE_COORDINATE - value: "false" - - name: NUMBER_OF_MASTERS - value: "2" - - name: HTTP_ENABLE - value: "false" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: DISCOVERY_TIMEOUT - value: "30s" - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9300 - name: transport -# Need better readiness check policy than checking if port 9300 is active - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/kibana/kibana-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/kibana/kibana-svc.yaml b/kubernetes/kibana/kibana-svc.yaml deleted file mode 100644 index fbefde0..0000000 --- a/kubernetes/kibana/kibana-svc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kibana - labels: - component: elk - role: kibana -spec: - selector: - component: elk - role: kibana - ports: - - name: http - port: 5601 - type: NodePort \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/kibana/kibana.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/kibana/kibana.yaml b/kubernetes/kibana/kibana.yaml deleted file mode 100644 index f0f1ed0..0000000 --- a/kubernetes/kibana/kibana.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: kibana - labels: - component: elk - role: kibana -spec: - replicas: 1 - selector: - matchLabels: - component: elk - role: kibana - template: - metadata: - labels: - component: elk - role: kibana - spec: - containers: - - name: kibana - image: senssoft/kibana:6.2.2 - imagePullPolicy: IfNotPresent - resources: - # need more cpu upon initialization, therefore burstable class - limits: - cpu: 1000m - requests: - cpu: 100m - env: - - name: CLUSTER_NAME - value: SensSoft - - name: ELASTICSEARCH_URL - value: "http://elasticsearch-loadbalancer:9200"; - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - ports: - - containerPort: 5601 - name: http \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/logstash/logstash-svc.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/logstash/logstash-svc.yaml b/kubernetes/logstash/logstash-svc.yaml deleted file mode 100644 index b77a032..0000000 --- a/kubernetes/logstash/logstash-svc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: logstash-discovery - labels: - component: elk - role: logstash -spec: - selector: - component: elk - role: logstash - ports: -# - name: transport -# port: 9600 -# protocol: TCP - - name: http - port: 8100 - protocol: TCP - type: NodePort http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/6c79f91a/kubernetes/logstash/logstash.yaml ---------------------------------------------------------------------- diff --git a/kubernetes/logstash/logstash.yaml b/kubernetes/logstash/logstash.yaml deleted file mode 100644 index 8756731..0000000 --- a/kubernetes/logstash/logstash.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: logstash - labels: - component: elk - role: logstash -spec: - replicas: 1 - template: - metadata: - labels: - component: elk - role: logstash - spec: - containers: - - name: logstash - image: senssoft/logstash:6.2.2 - imagePullPolicy: IfNotPresent - command: - - logstash - - -f - - /etc/logstash/conf.d - env: - - name: ELASTICSEARCH_URL - value: "elasticsearch-loadbalancer:9200" - - name: ES_JAVA_OPTS - value: "-Xms1g -Xmx1g" - resources: - limits: - cpu: 1000m - requests: - cpu: 100m - ports: - - containerPort: 8100 - name: http - - containerPort: 9600 - name: transport - # Need better readiness check policy than checking if port 9600 is active -# livenessProbe: -# tcpSocket: -# port: transport -# initialDelaySeconds: 20 -# periodSeconds: 10 -# failureThreshold: 5 -# readinessProbe: -# httpGet: -# path: /?pretty=true -# port: http -# initialDelaySeconds: 30 -# periodSeconds: 10 -# failureThreshold: 10 \ No newline at end of file
