This is an automated email from the ASF dual-hosted git repository. nickallen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push: new 024aa20 METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395 024aa20 is described below commit 024aa201bc4cbe00af71c57b9e0fe27b51305030 Author: nickwallen <n...@nickallen.org> AuthorDate: Wed May 1 10:48:45 2019 -0400 METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395 --- .../ansible/playbooks/metron_full_install.yml | 3 - .../ansible/roles/ambari_common/defaults/main.yml | 3 +- ...ri-repo-centos.yml => ambari-repo-CentOS-6.yml} | 2 +- ...ri-repo-centos.yml => ambari-repo-CentOS-7.yml} | 2 +- ...i-repo-ubuntu.yml => ambari-repo-Ubuntu-14.yml} | 1 - .../{iptables-centos.yml => firewall-CentOS-6.yml} | 2 +- .../{iptables-centos.yml => firewall-CentOS-7.yml} | 4 +- ...{iptables-ubuntu.yml => firewall-Ubuntu-14.yml} | 0 .../ansible/roles/ambari_common/tasks/main.yml | 19 +--- .../{iptables-centos.yml => nodejs-CentOS-6.yml} | 8 +- .../{iptables-centos.yml => nodejs-CentOS-7.yml} | 8 +- .../{iptables-centos.yml => nodejs-Ubuntu-14.yml} | 8 +- .../ansible/roles/ambari_common/tasks/nodejs.yml | 36 ------- .../ansible/roles/ambari_config/defaults/main.yml | 5 + ...{dependencies.yml => dependencies-CentOS-6.yml} | 7 +- ...{dependencies.yml => dependencies-CentOS-7.yml} | 7 +- ...dependencies.yml => dependencies-Ubuntu-14.yml} | 5 +- .../ansible/roles/ambari_config/tasks/main.yml | 2 +- .../roles/ambari_gather_facts/defaults/main.yml | 4 + .../ambari_slave/tasks/{main.yml => hostname.yml} | 11 --- .../tasks/install-agent-CentOS-6.yml} | 4 +- ...nstall-agent.yml => install-agent-CentOS-7.yml} | 12 ++- .../tasks/install-agent-Ubuntu-14.yml} | 4 +- .../ansible/roles/ambari_slave/tasks/main.yml | 21 +--- .../ansible/roles/enable-swap/defaults/main.yml | 4 +- .../enable-swap/tasks/{main.yml => check-swap.yml} | 15 ++- .../ansible/roles/enable-swap/tasks/main.yml | 23 ++--- .../tasks/{enable-swap.yml => setup-swap.yml} | 9 +- .../tasks/install-service-CentOS-6.yml} | 8 +- .../tasks/install-service-CentOS-7.yml} | 20 ++-- .../tasks/install-service-Ubuntu-14.yml} | 8 +- .../ansible/roles/sensor-stubs/tasks/main.yml | 8 +- .../templates/sensor-stubs-bro.service | 23 +++++ .../templates/sensor-stubs-snort.service | 23 +++++ .../templates/sensor-stubs-yaf.service | 23 +++++ metron-deployment/development/centos7/Vagrantfile | 108 +++++++++++++++++++++ .../centos7/ansible.cfg} | 17 ++-- .../centos7/ansible/inventory/group_vars/all} | 12 +-- .../centos7/ansible/inventory/hosts} | 35 ++++++- .../centos7/ansible/playbook.yml} | 12 +-- 40 files changed, 346 insertions(+), 180 deletions(-) diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml index ab72b85..c6e8d18 100644 --- a/metron-deployment/ansible/playbooks/metron_full_install.yml +++ b/metron-deployment/ansible/playbooks/metron_full_install.yml @@ -33,9 +33,6 @@ - import_playbook: sensor_install.yml -# -# deployment report -# - hosts: metron become: false roles: diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml index e7968f5..344638e 100644 --- a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml +++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml @@ -17,5 +17,6 @@ --- hadoop_logrotate_frequency: daily hadoop_logrotate_retention: 30 -centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo +centos6_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo +centos7_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.1.0/ambari.repo ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.6.1.0 diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml similarity index 91% copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml index 8c1bc33..7e6d9d2 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml @@ -16,7 +16,7 @@ # --- - name: Setup Ambari repo on CentOS - get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo + get_url: url="{{ centos6_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo - name: Update package cache on CentOS yum: name=* update_cache=yes diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml similarity index 91% copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml index 8c1bc33..f26b39c 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml @@ -16,7 +16,7 @@ # --- - name: Setup Ambari repo on CentOS - get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo + get_url: url="{{ centos7_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo - name: Update package cache on CentOS yum: name=* update_cache=yes diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml similarity index 99% copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml index 23c4aca..1e0ca7f 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml @@ -15,7 +15,6 @@ # limitations under the License. # --- - - name: Setup Ambari repo on Ubuntu shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | tee /etc/apt/sources.list.d/ambari.list become: true diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml similarity index 96% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml index 27e67d5..8f01e41 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml @@ -15,6 +15,6 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS +- name: Stop iptables on CentOS 6 ignore_errors: yes service: name=iptables state=stopped enabled=no diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml similarity index 90% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml index 27e67d5..db77446 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml @@ -15,6 +15,6 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS +- name: Stop firewalld on CentOS 7 ignore_errors: yes - service: name=iptables state=stopped enabled=no + service: name=firewalld state=stopped enabled=no diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-Ubuntu-14.yml similarity index 100% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-Ubuntu-14.yml diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml index f92a57e..98f5e4d 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml @@ -15,21 +15,12 @@ # limitations under the License. # --- +- include_tasks: firewall-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml -- include_tasks: iptables-centos.yml - when: ansible_distribution == "CentOS" +- include_tasks: hostname.yml -- include_tasks: iptables-ubuntu.yml - when: ansible_distribution == "Ubuntu" +- include_tasks: ambari-repo-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml -- include: hostname.yml +- include_tasks: nodejs-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml -- include_tasks: ambari-repo-centos.yml - when: ansible_distribution == "CentOS" - -- include_tasks: ambari-repo-ubuntu.yml - when: ansible_distribution == "Ubuntu" - -- include: nodejs.yml - -- include: logrotate.yml +- include_tasks: logrotate.yml diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml similarity index 84% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml index 27e67d5..8ea7363 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml @@ -15,6 +15,8 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS - ignore_errors: yes - service: name=iptables state=stopped enabled=no +- name: Install Node.js repo on CentOS 6 + shell: curl -sL https://rpm.nodesource.com/setup_6.x | bash - + args: + warn: false + become: true diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml similarity index 84% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml index 27e67d5..b2a2cc0 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml @@ -15,6 +15,8 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS - ignore_errors: yes - service: name=iptables state=stopped enabled=no +- name: Install Node.js repo on CentOS 7 + shell: curl -sL https://rpm.nodesource.com/setup_7.x | bash - + args: + warn: false + become: true diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml similarity index 84% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml index 27e67d5..2bfdecd 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml @@ -15,6 +15,8 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS - ignore_errors: yes - service: name=iptables state=stopped enabled=no +- name: Install Node.js repo on Ubuntu 14 + shell: curl -sL https://deb.nodesource.com/setup_6.x | bash - + args: + warn: false + become: true diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml deleted file mode 100644 index ea2067b..0000000 --- a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# -# for CentOS... -# -- name: Install Nodejs repo on CentOS - shell: curl -sL https://rpm.nodesource.com/setup_6.x | bash - - args: - warn: false - when: ansible_distribution == "CentOS" - become: true - -# -# for Ubuntu... -# -- name: Install Nodejs repo on Ubuntu - shell: curl -sL https://deb.nodesource.com/setup_6.x | bash - - args: - warn: false - when: ansible_distribution == "Ubuntu" - become: true diff --git a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml index ad7ca9e..85d7e90 100644 --- a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml +++ b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml @@ -36,3 +36,8 @@ mapred_reduce_mem_mb : 1229 topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf' hdp_stack: "2.6" elasticsearch_network_interface: _site_ + +ambari_host: "{{ groups.ambari_master[0] }}" +ambari_port: 8080 +ambari_user: admin +ambari_password: admin diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml similarity index 88% copy from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml copy to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml index 52d6ca1..72a96d4 100644 --- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml +++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml @@ -15,7 +15,12 @@ # limitations under the License. # --- -- name: Install python-requests module +- name: Install urllib3 with pip + pip: + name: urllib3 + version: 1.10.2 + +- name: Install requests with pip pip: name: requests version: 2.6.1 diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml similarity index 88% copy from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml copy to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml index 52d6ca1..72a96d4 100644 --- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml +++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml @@ -15,7 +15,12 @@ # limitations under the License. # --- -- name: Install python-requests module +- name: Install urllib3 with pip + pip: + name: urllib3 + version: 1.10.2 + +- name: Install requests with pip pip: name: requests version: 2.6.1 diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml similarity index 90% rename from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml rename to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml index 52d6ca1..964acf2 100644 --- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml +++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml @@ -15,7 +15,4 @@ # limitations under the License. # --- -- name: Install python-requests module - pip: - name: requests - version: 2.6.1 +# nothing to do diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml index d5f38b1..d5ab637 100644 --- a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml +++ b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml @@ -17,7 +17,7 @@ --- - include_vars: "{{ cluster_type }}.yml" -- include: dependencies.yml +- include_tasks: "dependencies-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml" - name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }} wait_for : diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml index 5351a60..94bd95a 100644 --- a/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml +++ b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml @@ -17,3 +17,7 @@ --- curl: "curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H \"X-Requested-By: ambari\"" parse_json: "import sys, json; print json.load(sys.stdin)" +ambari_host: "{{ groups.ambari_master[0] }}" +ambari_port: 8080 +ambari_user: admin +ambari_password: admin diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml similarity index 88% copy from metron-deployment/ansible/roles/ambari_slave/tasks/main.yml copy to metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml index b78a2a7..4cfd115 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml @@ -15,8 +15,6 @@ # limitations under the License. # --- -- include: install-agent.yml - - name: Create ambari-agent hostname script template: src: "../roles/ambari_slave/files/hostname.sh" @@ -35,12 +33,3 @@ with_items: - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' } - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'} - -- name: Ensure ambari-agent is running - service: - name: ambari-agent - state: restarted - enabled: yes - -- name : Wait for agent to register - command : sleep 10 diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml similarity index 92% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml copy to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml index 9fdda7e..8f16b92 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -- name: Disable firewall on Ubuntu - shell: ufw disable +- name: Install ambari-agent on CentOS 6 + yum: name=ambari-agent diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml similarity index 76% copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml copy to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml index 62d0027..d64f21f 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml @@ -15,10 +15,12 @@ # limitations under the License. # --- -- name: Install ambari-agent on CentOS +- name: Install ambari-agent on CentOS 7 yum: name=ambari-agent - when: ansible_distribution == "CentOS" -- name: Install ambari-agent on Ubuntu - apt: name=ambari-agent force=yes - when: ansible_distribution == "Ubuntu" +- name: Force Ambari Agent to use TLSv1.2 + lineinfile: + dest: /etc/ambari-agent/conf/ambari-agent.ini + state: present + insertafter: '\[security\]' + line: "force_https_protocol=PROTOCOL_TLSv1_2" diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml similarity index 91% rename from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml rename to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml index 9fdda7e..ae48214 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -- name: Disable firewall on Ubuntu - shell: ufw disable +- name: Install ambari-agent on Ubuntu 14 + apt: name=ambari-agent force=yes diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml index b78a2a7..7d9bac0 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml @@ -15,26 +15,9 @@ # limitations under the License. # --- -- include: install-agent.yml +- include_tasks: install-agent-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml -- name: Create ambari-agent hostname script - template: - src: "../roles/ambari_slave/files/hostname.sh" - dest: "/var/lib/ambari-agent/hostname.sh" - mode: 0744 - owner: "{{ ambari_installation_user }}" - group: "{{ ambari_installation_user }}" - -- name: Configure ambari-server hostname in ambari-agent configuration - lineinfile: - dest: /etc/ambari-agent/conf/ambari-agent.ini - regexp: "{{ item.regexp }}" - line: "{{ item.line }}" - insertafter: "{{ item.insertafter }}" - backup: yes - with_items: - - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' } - - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'} +- include_tasks: hostname.yml - name: Ensure ambari-agent is running service: diff --git a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml index 7190606..94a63c2 100644 --- a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml +++ b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -swapfile: /swapfile -swapspace: 4G +swapfile: /swappy +swapspace: 8092 diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml similarity index 79% copy from metron-deployment/ansible/roles/enable-swap/tasks/main.yml copy to metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml index 4b159b4..3bd0146 100644 --- a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml +++ b/metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml @@ -14,17 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # ---- -- name: "Is swap space enabled?" - shell: swapon -s | grep {{ swapfile }} +--- +- shell: swapon -s | grep {{ swapfile }} register: swapcheck failed_when: swapcheck.rc != 0 and swapcheck.rc != 1 -- debug: msg="Swap space is already enabled" +- shell: free -m | grep Swap | awk -F" " '{ print $2}' + register: actual_swapspace + +- debug: msg="Swap is enabled with {{ actual_swapspace.stdout }} mb" when: swapcheck.rc == 0 -- debug: msg="Swap space is NOT yet enabled" +- debug: msg="Swap is NOT enabled" when: swapcheck.rc == 1 - -- include_tasks: enable-swap.yml - when: swapcheck.rc != 0 diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml index 4b159b4..8247ab1 100644 --- a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml +++ b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml @@ -14,17 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # ---- -- name: "Is swap space enabled?" - shell: swapon -s | grep {{ swapfile }} - register: swapcheck - failed_when: swapcheck.rc != 0 and swapcheck.rc != 1 +--- +- include_tasks: check-swap.yml -- debug: msg="Swap space is already enabled" - when: swapcheck.rc == 0 +# +# only enable swap if the swapfile does not already exist +# +- name: "Checking for {{ swapfile }}" + stat: path="{{ swapfile }}" + register: swap -- debug: msg="Swap space is NOT yet enabled" - when: swapcheck.rc == 1 +- include_tasks: setup-swap.yml + when: swap.stat.exists == False -- include_tasks: enable-swap.yml - when: swapcheck.rc != 0 +- debug: msg="No swap changes necessary" + when: swap.stat.exists == True diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml b/metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml similarity index 84% rename from metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml rename to metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml index fc6c094..648c208 100644 --- a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml +++ b/metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml @@ -15,8 +15,11 @@ # limitations under the License. # --- -- name: "Allocate {{ swapspace }} for swap space" - shell: fallocate -l {{ swapspace }} {{ swapfile }} +- name: "Disable swap space" + shell: swapoff -a + +- name: "Allocate {{ swapspace }} mb for swap space" + shell: dd if=/dev/zero of={{ swapfile }} count={{ swapspace }} bs=1MiB - name: "Permissioning {{ swapfile }}" file: @@ -33,3 +36,5 @@ lineinfile: dest: "/etc/fstab" line: "{{ swapfile }} none swap sw 0 0" + +- include_tasks: check-swap.yml diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml similarity index 82% copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml copy to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml index 27e67d5..022e49f 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml @@ -15,6 +15,8 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS - ignore_errors: yes - service: name=iptables state=stopped enabled=no +- name: Install init.d service + template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755 + +- name: Start sensor stubs + service: name=sensor-stubs state=restarted diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml similarity index 68% copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml copy to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml index 62d0027..76293ad 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml +++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml @@ -15,10 +15,18 @@ # limitations under the License. # --- -- name: Install ambari-agent on CentOS - yum: name=ambari-agent - when: ansible_distribution == "CentOS" +- name: Install systemd services + template: src={{ item }} dest=/etc/systemd/system/ mode=0755 + with_items: + - sensor-stubs-bro.service + - sensor-stubs-yaf.service + - sensor-stubs-snort.service -- name: Install ambari-agent on Ubuntu - apt: name=ambari-agent force=yes - when: ansible_distribution == "Ubuntu" +- name: Start sensor stubs + systemd: + name: "{{ item }}" + state: restarted + with_items: + - sensor-stubs-bro + - sensor-stubs-yaf + - sensor-stubs-snort diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml similarity index 82% rename from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml rename to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml index 27e67d5..022e49f 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml +++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml @@ -15,6 +15,8 @@ # limitations under the License. # --- -- name: Stop iptables on CentOS - ignore_errors: yes - service: name=iptables state=stopped enabled=no +- name: Install init.d service + template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755 + +- name: Start sensor stubs + service: name=sensor-stubs state=restarted diff --git a/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml index c333025..2e05d99 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml +++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml @@ -29,15 +29,11 @@ - snort.out - yaf.out -- name: Install service script - template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755 - -- name: Install sensor stubs +- name: Install sensor stubs scripts template: src={{ item }} dest={{ sensor_stubs_bin }}/ mode=0755 with_items: - start-bro-stub - start-snort-stub - start-yaf-stub -- name: Start sensor stubs - service: name=sensor-stubs state=restarted +- include_tasks: install-service-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service new file mode 100644 index 0000000..79094ae --- /dev/null +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[Unit] + Description=Sends canned Bro telemetry to a Kafka topic + +[Service] + ExecStart={{ sensor_stubs_bin }}/start-bro-stub diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service new file mode 100644 index 0000000..5f4ec25 --- /dev/null +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[Unit] + Description=Sends canned Snort telemetry to a Kafka topic + +[Service] + ExecStart={{ sensor_stubs_bin }}/start-snort-stub diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service new file mode 100644 index 0000000..aa4bebb --- /dev/null +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[Unit] + Description=Sends canned YAF telemetry to a Kafka topic + +[Service] + ExecStart={{ sensor_stubs_bin }}/start-yaf-stub diff --git a/metron-deployment/development/centos7/Vagrantfile b/metron-deployment/development/centos7/Vagrantfile new file mode 100644 index 0000000..4706e42 --- /dev/null +++ b/metron-deployment/development/centos7/Vagrantfile @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'getoptlong' + +ansibleTags='' +ansibleSkipTags='sensors' + +begin + opts = GetoptLong.new( + [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ], + [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ] + ) + + opts.quiet = true + + opts.each do |opt, arg| + case opt + when '--ansible-tags' + ansibleTags=arg + when '--ansible-skip-tags' + ansibleSkipTags=arg + end + end +rescue Exception => ignored +#Ignore to allow other opts to be passed to Vagrant +end + +puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != '' +puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != '' + +hosts = [{ + hostname: "node1", + ip: "192.168.66.121", + memory: "8192", + cpus: 4, + promisc: 2 # enables promisc on the 'Nth' network interface +}] + +Vagrant.configure(2) do |config| + + # host built on centos + config.vm.box = "centos/7" + config.ssh.insert_key = true + + # enable the hostmanager plugin + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + + # enable vagrant cachier if present + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.enable :yum + config.cache.scope = :box + + config.cache.synced_folder_opts = { + type: :nfs, + mount_options: ['rw', 'vers=3', 'tcp', 'nolock'] + } + end + + # host definition + hosts.each_with_index do |host, index| + config.vm.define host[:hostname] do |node| + + # host settings + node.vm.hostname = host[:hostname] + node.vm.network "private_network", ip: host[:ip] + + # vm settings + node.vm.provider "virtualbox" do |vb| + vb.memory = host[:memory] + vb.cpus = host[:cpus] + + # enable promisc mode on the network interface + if host.has_key?(:promisc) + vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}", "allow-all"] + end + # disable audio, so that the vm doesn't capture the sound / mic + vb.customize ["modifyvm", :id, "--audio", "none"] + end + end + end + + # provision the host with ansible + config.vm.provision :ansible do |ansible| + ansible.playbook = "ansible/playbook.yml" + ansible.become = true + ansible.tags = ansibleTags.split(",") if ansibleTags != '' + ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' + ansible.inventory_path = "ansible/inventory" + ansible.compatibility_mode = "auto" + ansible.raw_arguments = [ + ] + end +end diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/development/centos7/ansible.cfg similarity index 71% copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml copy to metron-deployment/development/centos7/ansible.cfg index 62d0027..66f0afd 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml +++ b/metron-deployment/development/centos7/ansible.cfg @@ -14,11 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # ---- -- name: Install ambari-agent on CentOS - yum: name=ambari-agent - when: ansible_distribution == "CentOS" +[defaults] +host_key_checking = false +library = ../../ansible/extra_modules +roles_path = ../../ansible/roles +pipelining = True +log_path = ./ansible.log +callback_plugins = ../../ansible/callback_plugins -- name: Install ambari-agent on Ubuntu - apt: name=ambari-agent force=yes - when: ansible_distribution == "Ubuntu" +# fix for "ssh throws 'unix domain socket too long' " problem +[ssh_connection] +control_path = %(directory)s/%%h-%%p-%%r diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/development/centos7/ansible/inventory/group_vars/all similarity index 71% rename from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml rename to metron-deployment/development/centos7/ansible/inventory/group_vars/all index 23c4aca..ba37eed 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml +++ b/metron-deployment/development/centos7/ansible/inventory/group_vars/all @@ -14,11 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ---- -- name: Setup Ambari repo on Ubuntu - shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | tee /etc/apt/sources.list.d/ambari.list - become: true +# only need to build the RPMs for CentOS +metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms" -- name: Update package cache on Ubuntu - apt: update_cache=yes +cluster_type: single_node_vm +elasticsearch_hosts: "{{ groups.search | join(',') }}" +hdp_host_group: "{{ groups.ambari_slave }}" +hdp_stack: "2.6" diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/development/centos7/ansible/inventory/hosts similarity index 79% rename from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml rename to metron-deployment/development/centos7/ansible/inventory/hosts index 8c1bc33..9bd9ea1 100644 --- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml +++ b/metron-deployment/development/centos7/ansible/inventory/hosts @@ -14,9 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. # ---- -- name: Setup Ambari repo on CentOS - get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo -- name: Update package cache on CentOS - yum: name=* update_cache=yes +[ambari_master] +node1 + +[ambari_slave] +node1 + +[metron] +node1 + +[search] +node1 + +[sensors] +node1 + +[pcap_server] +node1 + +[web] +node1 + +[zeppelin] +node1 + +[monit:children] +sensors +pcap_server + +[local] +127.0.0.1 diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/development/centos7/ansible/playbook.yml similarity index 75% rename from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml rename to metron-deployment/development/centos7/ansible/playbook.yml index 62d0027..433d3b5 100644 --- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml +++ b/metron-deployment/development/centos7/ansible/playbook.yml @@ -15,10 +15,10 @@ # limitations under the License. # --- -- name: Install ambari-agent on CentOS - yum: name=ambari-agent - when: ansible_distribution == "CentOS" +- hosts: all + roles: + - { role: libselinux-python, tags: libselinux-python } + - { role: enable-swap, tags: enable-swap } + - { role: enable-remote-ssh, tags: enable-remote-ssh } -- name: Install ambari-agent on Ubuntu - apt: name=ambari-agent force=yes - when: ansible_distribution == "Ubuntu" +- import_playbook: ../../../ansible/playbooks/metron_full_install.yml