http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/config/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/config/logstash-userale.conf b/docker/logstash/config/logstash-userale.conf new file mode 100644 index 0000000..0c30077 --- /dev/null +++ b/docker/logstash/config/logstash-userale.conf @@ -0,0 +1,55 @@ +input { + http { + codec => "json" + port => 8000 + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept" + } + } +} + +## Add your filters / logstash plugins configuration here +filter { + # mutate { + # remove_field => [ "message", "@version", "@timestamp"] + # } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } + + # dns { + # add_field => [ "hostname", "%{host}" ] + # } + + # dns { + # resolve => [ "host" ] + # action => [ "replace" ] + # } + + # geoip { + # source => "host" + # database => "/usr/share/logstash/GeoIP/GeoIP.dat" + # target => "geoip" + # } +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "elasticsearch:9200" + index => "userale" + document_type => "logs" + user => "elastic" + password => "changeme" + manage_template => true + template_overwrite => true + template => "/usr/share/logstash/templates/userale.json" + template_name => "userale" + } + + # Debug + # stdout { codec => rubydebug } +}
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/templates/apache.json ---------------------------------------------------------------------- diff --git a/docker/logstash/templates/apache.json b/docker/logstash/templates/apache.json new file mode 100644 index 0000000..10552e3 --- /dev/null +++ b/docker/logstash/templates/apache.json @@ -0,0 +1,59 @@ +{ + "template": "apache", + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "_default_": { + "dynamic_templates": [{ + "string_fields": { + "match_mapping_type": "string", + "match": "*", + "mapping": { + "omit_norms": true, + "type": "text", + "fields": { + "raw": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }], + "properties": { + "geoip": { + "dynamic": true, + "path": "full", + "properties": { + "location": { + "type": "geo_point" + } + }, + "type": "object" + }, + "@version": { + "type": "keyword" + }, + "referer": { + "type": "keyword" + }, + "request": { + "type": "keyword" + }, + "responsetime": { + "type": "long" + }, + "bytes": { + "type": "long" + } + }, + "_all": { + "enabled": true + } + } + }, + "aliases": { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/templates/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/templates/userale.json b/docker/logstash/templates/userale.json new file mode 100644 index 0000000..e5d8f91 --- /dev/null +++ b/docker/logstash/templates/userale.json @@ -0,0 +1,157 @@ +{ + "template" : "userale", + "order" : 1, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "index.refresh_interval" : "5s" + }, + "mappings" : { + "_default_" : { + "_all" : { + "enabled" : false + }, + "properties" : { + "geoip" : { + "properties" : { + "ip" : { + "type" : "ip" + }, + "country_code2" : { + + "type" : "keyword" + }, + "country_code3" : { + + "type" : "keyword" + }, + "country_name" : { + + "type" : "keyword" + }, + "continent_code" : { + + "type" : "keyword" + }, + "region_name" : { + + "type" : "keyword" + }, + "city_name" : { + + "type" : "keyword" + }, + "postal_code" : { + + "type" : "keyword" + }, + "latitude" : { + "type" : "float" + }, + "longitude" : { + "type" : "float" + }, + "dma_code" : { + "type" : "integer" + }, + "area_code" : { + "type" : "long" + }, + "timezone" : { + + "type" : "keyword" + }, + "real_region_name" : { + + "type" : "keyword" + }, + "location" : { + "doc_values" : true, + "type" : "geo_point" + } + } + } + } + }, + "logs" : { + "properties" : { + "userAction" : { + "type" : "boolean" + }, + "type" : { + + "type" : "keyword" + }, + "clientTime" : { + "type" : "date", + "format" : "strict_date_optional_time||epoch_millis" + }, + "target" : { + + "type" : "keyword" + }, + "path" : { + + "type" : "keyword" + }, + "details" : { + "type" : "object", + "properties" : { + "source" : { + "type" : "keyword" + } + } + }, + "location" : { + "type" : "object", + "properties" : { + "x" : { + "type" : "integer" + }, + "y" : { + "type" : "integer" + } + } + }, + "userId" : { + + "type" : "keyword" + }, + "session" : { + + "type" : "keyword" + }, + "toolName" : { + + "type" : "keyword" + }, + "toolVersion" : { + + "type" : "keyword" + }, + "useraleVersion" : { + + "type" : "keyword" + }, + "major_ver" : { + "type" : "integer" + }, + "minor_ver" : { + "type" : "integer" + }, + "patch_ver" : { + "type" : "integer" + }, + "tool_major_ver" : { + "type" : "integer" + }, + "tool_minor_ver" : { + "type" : "integer" + }, + "tool_patch_ver" : { + "type" : "integer" + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/tap/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/tap/Dockerfile b/docker/tap/Dockerfile new file mode 100644 index 0000000..24f7291 --- /dev/null +++ b/docker/tap/Dockerfile @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.4 +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Install system wide dependencies +RUN apt-get -yqq update && apt-get -yqq install \ + curl \ + sudo + +# Install NodeJS 4.x +RUN curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash - + +RUN sudo -E apt-get -yqq install \ + nodejs \ + build-essential + +# Set the work directory +RUN mkdir -p /usr/src +WORKDIR /usr/src + +# Install gulp +RUN npm install -g bower gulp + +# Install git +RUN sudo -E apt-get -yqq install \ + git + +# Clone TAP +RUN git clone -b master https://github.com/apache/incubator-senssoft-tap.git app +WORKDIR /usr/src/app +RUN git pull + +# Install packages +RUN npm install + +# Install TAP requirements +RUN pip install -r requirements.txt + +# Add application files +ADD secret.py /usr/src/app/tap/settings +ADD neon_counts.js /usr/src/app/public +ADD neon_graph.js /usr/src/app/public + + +# Startup Application +RUN gulp build + +# Export port +EXPOSE 8000 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/tap/README.md ---------------------------------------------------------------------- diff --git a/docker/tap/README.md b/docker/tap/README.md new file mode 100644 index 0000000..eaf43b2 --- /dev/null +++ b/docker/tap/README.md @@ -0,0 +1,2 @@ +Copy neon_*.json files here until we integrate w/Distill +