http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh deleted file mode 100644 index bfe3371..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Environment setup script from Ignite -# ----------------------------------------------------------------------------------------------- - -JVM_OPTS="-Xms10g -Xmx10g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m" -JVM_OPTS="$JVM_OPTS -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseTLAB -XX:NewSize=128m -XX:MaxNewSize=768m" -#JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=0 -XX:SurvivorRatio=1024 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60" -JVM_OPTS="$JVM_OPTS -Xss16m" - -export JVM_OPTS
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh deleted file mode 100644 index f2c1557..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Script to start Ignite daemon (used by ignite-bootstrap.sh) -# ----------------------------------------------------------------------------------------------- - -#profile=/home/ignite/.bash_profile -profile=/root/.bash_profile - -. $profile -. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "ignite" - -# Setups Cassandra seeds for this Ignite node being able to connect to Cassandra. -# Looks for the information in S3 about already up and running Cassandra cluster nodes. -setupCassandraSeeds() -{ - setupClusterSeeds "cassandra" "true" - - CLUSTER_SEEDS=($CLUSTER_SEEDS) - count=${#CLUSTER_SEEDS[@]} - - CASSANDRA_SEEDS= - - for (( i=0; i<=$(( $count -1 )); i++ )) - do - seed=${CLUSTER_SEEDS[$i]} - CASSANDRA_SEEDS="${CASSANDRA_SEEDS}<value>$seed<\/value>" - done - - cat /opt/ignite/config/ignite-cassandra-server-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server.xml -} - -# Setups Ignite nodes which this EC2 Ignite node will use to send its metadata and join Ignite cluster -setupIgniteSeeds() -{ - if [ "$FIRST_NODE_LOCK" == "true" ]; then - echo "[INFO] Setting up Ignite seeds" - - CLUSTER_SEEDS="127.0.0.1:47500..47509" - - echo "[INFO] Using localhost address as a seed for the first Ignite node: $CLUSTER_SEEDS" - - aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} - if [ $? -ne 0 ]; then - terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" - fi - else - setupClusterSeeds "ignite" "true" - fi - - CLUSTER_SEEDS=($CLUSTER_SEEDS) - count=${#CLUSTER_SEEDS[@]} - - IGNITE_SEEDS= - - for (( i=0; i<=$(( $count -1 )); i++ )) - do - seed=${CLUSTER_SEEDS[$i]} - IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" - done - - cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml - mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml -} - -# Checks status of Ignite daemon -checkIgniteStatus() -{ - proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") - - nodeId= - nodeAddrs= - nodePorts= - topology= - metrics= - - logFile=$(ls /opt/ignite/work/log/ | grep "\.log$") - if [ -n "$logFile" ]; then - logFile=/opt/ignite/work/log/$logFile - nodeId=$(cat $logFile | grep "Local node \[ID") - nodeAddrs=$(cat $logFile | grep "Local node addresses:") - nodePorts=$(cat $logFile | grep "Local ports:") - topology=$(cat $logFile | grep "Topology snapshot") - metrics=$(cat $logFile | grep "Metrics for local node" | head -n 1) - fi - - if [ -n "$nodeId" ] && [ -n "$nodeAddrs" ] && [ -n "$nodePorts" ] && [ -n "$topology" ] && [ -n "$metrics" ] && [ -n "$proc" ]; then - sleep 30s - return 0 - fi - - return 1 -} - -# Gracefully starts Ignite daemon and waits until it joins Ignite cluster -startIgnite() -{ - echo "[INFO]-------------------------------------------------------------" - echo "[INFO] Trying attempt $START_ATTEMPT to start Ignite daemon" - echo "[INFO]-------------------------------------------------------------" - echo "" - - setupCassandraSeeds - setupIgniteSeeds - - waitToJoinCluster - - if [ "$FIRST_NODE_LOCK" == "true" ]; then - aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} - if [ $? -ne 0 ]; then - terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" - fi - fi - - proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") - proc=($proc) - - if [ -n "${proc[1]}" ]; then - echo "[INFO] Terminating existing Ignite process ${proc[1]}" - kill -9 ${proc[1]} - fi - - echo "[INFO] Starting Ignite" - rm -Rf /opt/ignite/work/* - /opt/ignite/bin/ignite.sh /opt/ignite/config/ignite-cassandra-server.xml & - - echo "[INFO] Ignite job id: $!" - - sleep 1m - - START_ATTEMPT=$(( $START_ATTEMPT+1 )) -} - -####################################################################################################### - -START_ATTEMPT=0 - -# Cleans all the previous metadata about this EC2 node -unregisterNode - -# Tries to get first-node lock -tryToGetFirstNodeLock - -echo "[INFO]-----------------------------------------------------------------" - -if [ "$FIRST_NODE_LOCK" == "true" ]; then - echo "[INFO] Starting first Ignite node" -else - echo "[INFO] Starting Ignite node" -fi - -echo "[INFO]-----------------------------------------------------------------" -printInstanceInfo -echo "[INFO]-----------------------------------------------------------------" - -if [ "$FIRST_NODE_LOCK" != "true" ]; then - waitFirstClusterNodeRegistered "true" -else - cleanupMetadata -fi - -# Applies Ignite environment settings from ignite-env.sh -envScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/ignite-env.sh) -if [ -f "$envScript" ]; then - . $envScript -fi - -# Start Ignite daemon -startIgnite - -startTime=$(date +%s) - -# Trying multiple attempts to start Ignite daemon -while true; do - proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") - - checkIgniteStatus - - if [ $? -eq 0 ]; then - sleep 1m - echo "[INFO]-----------------------------------------------------" - echo "[INFO] Ignite daemon successfully started" - echo "[INFO]-----------------------------------------------------" - echo $proc - echo "[INFO]-----------------------------------------------------" - - # Once node joined the cluster we need to remove cluster-join lock - # to allow other EC2 nodes to acquire it and join cluster sequentially - removeClusterJoinLock - - break - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $SERVICE_STARTUP_TIME ]; then - if [ "$FIRST_NODE_LOCK" == "true" ]; then - # If the first node of Ignite cluster failed to start Ignite daemon in SERVICE_STARTUP_TIME min, - # we will not try any other attempts and just terminate with error. Terminate function itself, will - # take care about removing all the locks holding by this node. - terminate "${SERVICE_STARTUP_TIME}min timeout expired, but first Ignite daemon is still not up and running" - else - # If node isn't the first node of Ignite cluster and it failed to start we need to - # remove cluster-join lock to allow other EC2 nodes to acquire it - removeClusterJoinLock - - # If node failed all SERVICE_START_ATTEMPTS attempts to start Ignite daemon we will not - # try anymore and terminate with error - if [ $START_ATTEMPT -gt $SERVICE_START_ATTEMPTS ]; then - terminate "${SERVICE_START_ATTEMPTS} attempts exceed, but Ignite daemon is still not up and running" - fi - - # New attempt to start Ignite daemon - startIgnite - fi - - continue - fi - - # Handling situation when Ignite daemon process abnormally terminated - if [ -z "$proc" ]; then - # If this is the first node of Ignite cluster just terminating with error - if [ "$FIRST_NODE_LOCK" == "true" ]; then - terminate "Failed to start Ignite daemon" - fi - - # Remove cluster-join lock to allow other EC2 nodes to acquire it - removeClusterJoinLock - - echo "[WARN] Failed to start Ignite daemon. Sleeping for extra 30sec" - sleep 30s - - # New attempt to start Ignite daemon - startIgnite - - continue - fi - - echo "[INFO] Waiting for Ignite daemon to start, time passed ${duration}min" - sleep 30s -done - -# Once Ignite daemon successfully started we registering new Ignite node in S3 -registerNode - -# Terminating script with zero exit code -terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh b/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh deleted file mode 100644 index 1634b89..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Logs collector daemon -# ----------------------------------------------------------------------------------------------- -# Script is launched in background by all EC2 nodes of all clusters (Cassandra, Ignite, Tests) and -# periodically (each 30 seconds) checks if specific S3 trigger file (specified by $S3_LOGS_TRIGGER_URL) -# was created or its timestamp was changed. Such an event serve as a trigger for the script -# to collect EC2 instance logs (from folder specified by $1) and upload them into specific -# S3 folder (specified by $S3_LOGS_FOLDER). -# ----------------------------------------------------------------------------------------------- - -uploadLogs() -{ - if [ ! -d "$1" ]; then - echo "[INFO] Logs directory doesn't exist: $1" - return 0 - fi - - echo "[INFO] Uploading logs from directory: $1" - - dirList=$(ls $1 | head -1) - - if [ -z "$dirList" ]; then - echo "[INFO] Directory is empty: $1" - fi - - for i in 0 9; - do - aws s3 sync --sse AES256 --delete "$1" "$S3_LOGS_FOLDER" - code=$? - - if [ $code -eq 0 ]; then - echo "[INFO] Successfully uploaded logs from directory: $1" - return 0 - fi - - echo "[WARN] Failed to upload logs from $i attempt, sleeping extra 30sec" - sleep 30s - done - - echo "[ERROR] All 10 attempts to upload logs are failed for the directory: $1" -} - -createNewLogsSnapshot() -{ - rm -f ~/logs-collector.snapshot.new - - for log_src in "$@" - do - if [ -d "$log_src" ] || [ -f "$log_src" ]; then - ls -alR $log_src >> ~/logs-collector.snapshot.new - - fi - done -} - -checkLogsChanged() -{ - createNewLogsSnapshot $@ - - if [ ! -f "~/logs-collector.snapshot" ]; then - return 1 - fi - - diff "~/logs-collector.snapshot" "~/logs-collector.snapshot.new" > /dev/null - - return $? -} - -updateLogsSnapshot() -{ - if [ ! -f "~/logs-collector.snapshot.new" ]; then - return 0 - fi - - rm -f "~/logs-collector.snapshot" - mv "~/logs-collector.snapshot.new" "~/logs-collector.snapshot" -} - -collectLogs() -{ - createNewLogsSnapshot - - rm -Rf ~/logs-collector-logs - mkdir -p ~/logs-collector-logs - - for log_src in "$@" - do - if [ -f "$log_src" ]; then - echo "[INFO] Collecting log file: $log_src" - cp -f $log_src ~/logs-collector-logs - elif [ -d "$log_src" ]; then - echo "[INFO] Collecting logs from folder: $log_src" - cp -Rf $log_src ~/logs-collector-logs - fi - done - - uploadLogs ~/logs-collector-logs - - rm -Rf ~/logs-collector-logs - - updateLogsSnapshot -} - -echo "[INFO] Running Logs collector service" - -if [ -z "$1" ]; then - echo "[ERROR] Logs collection S3 trigger URL doesn't specified" - exit 1 -fi - -S3_LOGS_TRIGGER_URL=$1 - -echo "[INFO] Logs collection S3 trigger URL: $S3_LOGS_TRIGGER_URL" - -if [ -z "$2" ]; then - echo "[ERROR] S3 folder where to upload logs doesn't specified" - exit 1 -fi - -S3_LOGS_FOLDER=$2 - -echo "[INFO] S3 logs upload folder: $S3_LOGS_FOLDER" - -shift 2 - -if [ -z "$1" ]; then - echo "[WARN] Local logs sources don't specified" -else - echo "[INFO] Local logs sources: $@" -fi - -echo "--------------------------------------------------------------------" - -TRIGGER_STATE= - -while true; do - sleep 30s - - STATE=$(aws s3 ls $S3_LOGS_TRIGGER_URL) - - if [ -z "$STATE" ] || [ "$STATE" == "$TRIGGER_STATE" ]; then - checkLogsChanged - - if [ $? -eq 0 ]; then - continue - fi - fi - - TRIGGER_STATE=$STATE - - collectLogs $@ /var/log/cloud-init.log /var/log/cloud-init-output.log - - echo "--------------------------------------------------------------------" -done http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml b/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml deleted file mode 100644 index 77ffb9e..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml +++ /dev/null @@ -1,192 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<beans xmlns="http://www.springframework.org/schema/beans" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util" - xsi:schemaLocation="http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans.xsd - http://www.springframework.org/schema/util - http://www.springframework.org/schema/util/spring-util.xsd"> - - <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.TokenAwarePolicy"> - <constructor-arg type="com.datastax.driver.core.policies.LoadBalancingPolicy"> - <bean class="com.datastax.driver.core.policies.RoundRobinPolicy"/> - </constructor-arg> - </bean> - - <util:list id="contactPoints" value-type="java.lang.String"> - ${CASSANDRA_SEEDS} - </util:list> - - <bean id="cassandraAdminDataSource" class="org.apache.ignite.cache.store.cassandra.datasource.DataSource"> - <property name="user" value="cassandra"/> - <property name="password" value="cassandra"/> - <property name="contactPoints" ref="contactPoints"/> - <property name="readConsistency" value="ONE"/> - <property name="writeConsistency" value="ONE"/> - <property name="loadBalancingPolicy" ref="loadBalancingPolicy"/> - </bean> - - <!-- Persistence settings for 'cache1' --> - <bean id="cache1_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> - <constructor-arg type="java.lang.String"> - <value><![CDATA[ -<persistence keyspace="test1" table="primitive_test1"> - <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" column="key"/> - <valuePersistence class="java.lang.Long" strategy="PRIMITIVE" column="value"/> -</persistence>]]> - </value> - </constructor-arg> - </bean> - - <!-- Persistence settings for 'cache2' --> - <bean id="cache2_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> - <constructor-arg type="java.lang.String"> - <value><![CDATA[ -<persistence keyspace="test1" table="blob_test1"> - <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" /> - <valuePersistence strategy="BLOB"/> -</persistence>]]> - </value> - </constructor-arg> - </bean> - - <!-- Persistence settings for 'cache3' --> - <bean id="cache3_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> - <constructor-arg type="java.lang.String"> - <value><![CDATA[ -<persistence keyspace="test1" table="pojo_test3" ttl="86400"> - <keyspaceOptions> - REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 3} - AND DURABLE_WRITES = true - </keyspaceOptions> - <tableOptions> - comment = 'A most excellent and useful table' - AND read_repair_chance = 0.2 - </tableOptions> - <keyPersistence class="org.apache.ignite.tests.pojos.PersonId" strategy="POJO"> - <partitionKey> - <field name="companyCode" column="company" /> - <field name="departmentCode" column="department" /> - </partitionKey> - <clusterKey> - <field name="personNumber" column="number" sort="desc"/> - </clusterKey> - </keyPersistence> - <valuePersistence class="org.apache.ignite.tests.pojos.Person" - strategy="POJO" - serializer="org.apache.ignite.cache.store.cassandra.serializer.KryoSerializer"> - <field name="firstName" column="first_name" /> - <field name="lastName" column="last_name" /> - <field name="age" /> - <field name="married" index="true"/> - <field name="height" /> - <field name="weight" /> - <field name="birthDate" column="birth_date" /> - <field name="phones" /> - </valuePersistence> -</persistence>]]> - </value> - </constructor-arg> - </bean> - - <!-- Ignite configuration --> - <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> - <!-- Turn on client mode --> - <property name="clientMode" value="true"/> - - <property name="metricsLogFrequency" value="0"/> - - <property name="cacheConfiguration"> - <list> - <!-- Configuring persistence for "cache1" cache --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="cache1"/> - <property name="startSize" value="1000000"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="backups" value="0"/> - <property name="offHeapMaxMemory" value="0"/> - <property name="swapEnabled" value="false"/> - <property name="readThrough" value="true"/> - <property name="writeThrough" value="true"/> - <property name="writeBehindEnabled" value="true"/> - <property name="cacheStoreFactory"> - <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> - <property name="dataSourceBean" value="cassandraAdminDataSource"/> - <property name="persistenceSettingsBean" value="cache1_settings"/> - </bean> - </property> - </bean> - - <!-- Configuring persistence for "cache2" cache --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="cache2"/> - <property name="startSize" value="1000000"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="backups" value="0"/> - <property name="offHeapMaxMemory" value="0"/> - <property name="swapEnabled" value="false"/> - <property name="readThrough" value="true"/> - <property name="writeThrough" value="true"/> - <property name="writeBehindEnabled" value="true"/> - <property name="cacheStoreFactory"> - <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> - <property name="dataSourceBean" value="cassandraAdminDataSource"/> - <property name="persistenceSettingsBean" value="cache2_settings"/> - </bean> - </property> - </bean> - - <!-- Configuring persistence for "cache3" cache --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="cache3"/> - <property name="startSize" value="1000000"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="backups" value="0"/> - <property name="offHeapMaxMemory" value="0"/> - <property name="swapEnabled" value="false"/> - <property name="readThrough" value="true"/> - <property name="writeThrough" value="true"/> - <property name="writeBehindEnabled" value="true"/> - <property name="cacheStoreFactory"> - <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> - <property name="dataSourceBean" value="cassandraAdminDataSource"/> - <property name="persistenceSettingsBean" value="cache3_settings"/> - </bean> - </property> - </bean> - </list> - </property> - - <!-- Configuring remote ignite cluster connections --> - <property name="discoverySpi"> - <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> - <property name="ipFinder"> - <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder"> - <property name="addresses"> - <list> - ${IGNITE_SEEDS} - </list> - </property> - </bean> - </property> - </bean> - </property> - </bean> -</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh deleted file mode 100644 index 8e6faff..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh +++ /dev/null @@ -1,317 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Bootstrap script to spin up Tests cluster -# ----------------------------------------------------------------------------------------------- - -# URL to download AWS CLI tools -AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip - -# URL to download JDK -JDK_DOWNLOAD_URL=http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz - -# URL to download Ignite-Cassandra tests package - you should previously package and upload it to this place -TESTS_PACKAGE_DONLOAD_URL=s3://<bucket>/<folder>/ignite-cassandra-tests-<version>.zip - -# Terminates script execution and upload logs to S3 -terminate() -{ - SUCCESS_URL=$S3_TESTS_SUCCESS - FAILURE_URL=$S3_TESTS_FAILURE - - if [ -n "$SUCCESS_URL" ] && [[ "$SUCCESS_URL" != */ ]]; then - SUCCESS_URL=${SUCCESS_URL}/ - fi - - if [ -n "$FAILURE_URL" ] && [[ "$FAILURE_URL" != */ ]]; then - FAILURE_URL=${FAILURE_URL}/ - fi - - host_name=$(hostname -f | tr '[:upper:]' '[:lower:]') - msg=$host_name - - if [ -n "$1" ]; then - echo "[ERROR] $1" - echo "[ERROR]-----------------------------------------------------" - echo "[ERROR] Test node bootstrap failed" - echo "[ERROR]-----------------------------------------------------" - msg=$1 - - if [ -z "$FAILURE_URL" ]; then - exit 1 - fi - - reportFolder=${FAILURE_URL}${host_name} - reportFile=$reportFolder/__error__ - else - echo "[INFO]-----------------------------------------------------" - echo "[INFO] Test node bootstrap successfully completed" - echo "[INFO]-----------------------------------------------------" - - if [ -z "$SUCCESS_URL" ]; then - exit 0 - fi - - reportFolder=${SUCCESS_URL}${host_name} - reportFile=$reportFolder/__success__ - fi - - echo $msg > /opt/bootstrap-result - - aws s3 rm --recursive $reportFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to drop report folder: $reportFolder" - fi - - aws s3 cp --sse AES256 /opt/bootstrap-result $reportFile - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to report bootstrap result to: $reportFile" - fi - - rm -f /opt/bootstrap-result - - if [ -n "$1" ]; then - exit 1 - fi - - exit 0 -} - -# Downloads specified package -downloadPackage() -{ - echo "[INFO] Downloading $3 package from $1 into $2" - - for i in 0 9; - do - if [[ "$1" == s3* ]]; then - aws s3 cp $1 $2 - code=$? - else - curl "$1" -o "$2" - code=$? - fi - - if [ $code -eq 0 ]; then - echo "[INFO] $3 package successfully downloaded from $1 into $2" - return 0 - fi - - echo "[WARN] Failed to download $3 package from $i attempt, sleeping extra 5sec" - sleep 5s - done - - terminate "All 10 attempts to download $3 package from $1 are failed" -} - -# Downloads and setup JDK -setupJava() -{ - rm -Rf /opt/java /opt/jdk.tar.gz - - echo "[INFO] Downloading 'jdk'" - wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "$JDK_DOWNLOAD_URL" -O /opt/jdk.tar.gz - if [ $? -ne 0 ]; then - terminate "Failed to download 'jdk'" - fi - - echo "[INFO] Untaring 'jdk'" - tar -xvzf /opt/jdk.tar.gz -C /opt - if [ $? -ne 0 ]; then - terminate "Failed to untar 'jdk'" - fi - - rm -Rf /opt/jdk.tar.gz - - unzipDir=$(ls /opt | grep "jdk") - if [ "$unzipDir" != "java" ]; then - mv /opt/$unzipDir /opt/java - fi -} - -# Downloads and setup AWS CLI -setupAWSCLI() -{ - echo "[INFO] Installing 'awscli'" - pip install --upgrade awscli - if [ $? -eq 0 ]; then - return 0 - fi - - echo "[ERROR] Failed to install 'awscli' using pip" - echo "[INFO] Trying to install awscli using zip archive" - echo "[INFO] Downloading awscli zip" - - downloadPackage "$AWS_CLI_DOWNLOAD_URL" "/opt/awscli-bundle.zip" "awscli" - - echo "[INFO] Unzipping awscli zip" - unzip /opt/awscli-bundle.zip -d /opt - if [ $? -ne 0 ]; then - terminate "Failed to unzip awscli zip" - fi - - rm -Rf /opt/awscli-bundle.zip - - echo "[INFO] Installing awscli" - /opt/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws - if [ $? -ne 0 ]; then - terminate "Failed to install awscli" - fi - - echo "[INFO] Successfully installed awscli from zip archive" -} - -# Setup all the pre-requisites (packages, settings and etc.) -setupPreRequisites() -{ - echo "[INFO] Installing 'wget' package" - yum -y install wget - if [ $? -ne 0 ]; then - terminate "Failed to install 'wget' package" - fi - - echo "[INFO] Installing 'net-tools' package" - yum -y install net-tools - if [ $? -ne 0 ]; then - terminate "Failed to install 'net-tools' package" - fi - - echo "[INFO] Installing 'python' package" - yum -y install python - if [ $? -ne 0 ]; then - terminate "Failed to install 'python' package" - fi - - echo "[INFO] Installing 'unzip' package" - yum -y install unzip - if [ $? -ne 0 ]; then - terminate "Failed to install 'unzip' package" - fi - - downloadPackage "https://bootstrap.pypa.io/get-pip.py" "/opt/get-pip.py" "get-pip.py" - - echo "[INFO] Installing 'pip'" - python /opt/get-pip.py - if [ $? -ne 0 ]; then - terminate "Failed to install 'pip'" - fi -} - -# Downloads and setup tests package -setupTestsPackage() -{ - downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/ignite-cassandra-tests.zip" "Tests" - - rm -Rf /opt/ignite-cassandra-tests - - unzip /opt/ignite-cassandra-tests.zip -d /opt - if [ $? -ne 0 ]; then - terminate "Failed to unzip tests package" - fi - - rm -f /opt/ignite-cassandra-tests.zip - - unzipDir=$(ls /opt | grep "ignite-cassandra") - if [ "$unzipDir" != "ignite-cassandra-tests" ]; then - mv /opt/$unzipDir /opt/ignite-cassandra-tests - fi - - find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \; - - . /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" - - setupNTP - - echo "[INFO] Starting logs collector daemon" - - HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') - /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh "$S3_LOGS_TRIGGER" "$S3_TESTS_LOGS/$HOST_NAME" "/opt/ignite-cassandra-tests/logs" > /opt/logs-collector.log & - - echo "[INFO] Logs collector daemon started: $!" - - echo "----------------------------------------------------------------------------------------" - printInstanceInfo - echo "----------------------------------------------------------------------------------------" - tagInstance - bootstrapGangliaAgent "test" 8643 - - ################################################### - # Extra configuration specific only for test node # - ################################################### - - echo "[INFO] Installing bc package" - - yum -y install bc - - if [ $? -ne 0 ]; then - terminate "Failed to install bc package" - fi - - echo "[INFO] Installing zip package" - - yum -y install zip - - if [ $? -ne 0 ]; then - terminate "Failed to install zip package" - fi - - echo "[INFO] Creating 'ignite' group" - exists=$(cat /etc/group | grep ignite) - if [ -z "$exists" ]; then - groupadd ignite - if [ $? -ne 0 ]; then - terminate "Failed to create 'ignite' group" - fi - fi - - echo "[INFO] Creating 'ignite' user" - exists=$(cat /etc/passwd | grep ignite) - if [ -z "$exists" ]; then - useradd -g ignite ignite - if [ $? -ne 0 ]; then - terminate "Failed to create 'ignite' user" - fi - fi - - mkdir -p /opt/ignite-cassandra-tests/logs - chown -R ignite:ignite /opt/ignite-cassandra-tests - - echo "export JAVA_HOME=/opt/java" >> $1 - echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> $1 -} - -################################################################################################################### - -echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Bootstrapping Tests node" -echo "[INFO]-----------------------------------------------------------------" - -setupPreRequisites -setupJava -setupAWSCLI -setupTestsPackage "/root/.bash_profile" - -cmd="/opt/ignite-cassandra-tests/bootstrap/aws/tests/tests-manager.sh" - -#sudo -u ignite -g ignite sh -c "$cmd > /opt/ignite-cassandra-tests/tests-manager" & - -$cmd > /opt/ignite-cassandra-tests/logs/tests-manager.log & - -terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh deleted file mode 100644 index c0f5d6b..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh +++ /dev/null @@ -1,458 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Tests manager daemon -# ----------------------------------------------------------------------------------------------- -# Script is launched in background by all nodes of Tests cluster and -# periodically (each 30 seconds) checks if specific S3 trigger file was created or -# its timestamp was changed. Such an event serve as a trigger for the script to start -# preparing to run load tests. -# ----------------------------------------------------------------------------------------------- - -#profile=/home/ignite/.bash_profile -profile=/root/.bash_profile - -. $profile -. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" - -# Switch test node to IDLE state -switchToIdleState() -{ - if [ "$NODE_STATE" != "IDLE" ]; then - echo "[INFO] Switching node to IDLE state" - dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" - createStateFlag "$S3_TESTS_IDLE" - NODE_STATE="IDLE" - echo "[INFO] Node was switched to IDLE state" - fi -} - -# Switch test node to PREPARING state -switchToPreparingState() -{ - if [ "$NODE_STATE" != "PREPARING" ]; then - echo "[INFO] Switching node to PREPARING state" - dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_IDLE" "$S3_TESTS_RUNNING" - createStateFlag "$S3_TESTS_PREPARING" - NODE_STATE="PREPARING" - echo "[INFO] Node was switched to PREPARING state" - fi -} - -# Switch test node to WAITING state -switchToWaitingState() -{ - if [ "$NODE_STATE" != "WAITING" ]; then - echo "[INFO] Switching node to WAITING state" - dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" - createStateFlag "$S3_TESTS_WAITING" - NODE_STATE="WAITING" - echo "[INFO] Node was switched to WAITING state" - fi -} - -# Switch test node to RUNNING state -switchToRunningState() -{ - if [ "$NODE_STATE" != "RUNNING" ]; then - echo "[INFO] Switching node to RUNNING state" - dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_WAITING" - createStateFlag "$S3_TESTS_RUNNING" - NODE_STATE="RUNNING" - echo "[INFO] Node was switched to RUNNING state" - fi -} - -# Creates appropriate state flag for the node in S3 -createStateFlag() -{ - HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') - - aws s3 cp --sse AES256 /etc/hosts ${1}${HOST_NAME} - if [ $? -ne 0 ]; then - terminate "Failed to create state flag: ${1}${HOST_NAME}" - fi -} - -# Drops appropriate state flag for the node in S3 -dropStateFlag() -{ - HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') - - for flagUrl in "$@" - do - exists=$(aws s3 ls ${flagUrl}${HOST_NAME}) - if [ -n "$exists" ]; then - aws s3 rm ${flagUrl}${HOST_NAME} - if [ $? -ne 0 ]; then - terminate "Failed to drop state flag: ${flagUrl}${HOST_NAME}" - fi - fi - done -} - -# Removes tests summary report from S3 -dropTestsSummary() -{ - exists=$(aws s3 ls $S3_TESTS_SUMMARY) - if [ -z "$exists" ]; then - return 0 - fi - - aws s3 rm $S3_TESTS_SUMMARY - if [ $? -ne 0 ]; then - terminate "Failed to drop tests summary info: $S3_TESTS_SUMMARY" - fi -} - -# Recreate all the necessary Cassandra artifacts before running Load tests -recreateCassandraArtifacts() -{ - /opt/ignite-cassandra-tests/recreate-cassandra-artifacts.sh - if [ $? -ne 0 ]; then - terminate "Failed to recreate Cassandra artifacts" - fi -} - -# Setups Cassandra seeds for this Tests node being able to connect to Cassandra. -# Looks for the information in S3 about already up and running Cassandra cluster nodes. -setupCassandraSeeds() -{ - if [ $CASSANDRA_NODES_COUNT -eq 0 ]; then - return 0 - fi - - setupClusterSeeds "cassandra" - - CASSANDRA_SEEDS1=$(echo $CLUSTER_SEEDS | sed -r "s/ /,/g") - CASSANDRA_SEEDS2= - - CLUSTER_SEEDS=($CLUSTER_SEEDS) - count=${#CLUSTER_SEEDS[@]} - - for (( i=0; i<=$(( $count -1 )); i++ )) - do - seed=${CLUSTER_SEEDS[$i]} - CASSANDRA_SEEDS2="${CASSANDRA_SEEDS2}<value>$seed<\/value>" - done - - echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS1" - - echo "contact.points=$CASSANDRA_SEEDS1" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/connection.properties - - cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS2/g" > /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml -} - -# Setups Ignite nodes for this Tests node being able to connect to Ignite. -# Looks for the information in S3 about already up and running Cassandra cluster nodes. -setupIgniteSeeds() -{ - if [ $IGNITE_NODES_COUNT -eq 0 ]; then - return 0 - fi - - setupClusterSeeds "ignite" - - CLUSTER_SEEDS=($CLUSTER_SEEDS) - count=${#CLUSTER_SEEDS[@]} - - IGNITE_SEEDS= - - for (( i=0; i<=$(( $count -1 )); i++ )) - do - seed=${CLUSTER_SEEDS[$i]} - IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" - done - - echo "[INFO] Using Ignite seeds: $IGNITE_SEEDS" - - cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/persistence/primitive/ignite-remote-client-config.xml - rm -f /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml -} - -# Setups Cassandra credentials to connect to Cassandra cluster -setupCassandraCredentials() -{ - echo "admin.user=cassandra" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties - echo "admin.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties - echo "regular.user=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties - echo "regular.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties -} - -# Triggering first time tests execution for all nodes in the Tests cluster -triggerFirstTimeTestsExecution() -{ - if [ -z "$TESTS_TYPE" ]; then - return 0 - fi - - tryToGetFirstNodeLock - if [ $? -ne 0 ]; then - return 0 - fi - - sleep 30s - - echo "[INFO] Triggering first time tests execution" - - echo "TESTS_TYPE=$TESTS_TYPE" > /opt/ignite-cassandra-tests/tests-trigger - echo "#--------------------------------------------------" >> /opt/ignite-cassandra-tests/tests-trigger - echo "" >> /opt/ignite-cassandra-tests/tests-trigger - cat /opt/ignite-cassandra-tests/settings/tests.properties >> /opt/ignite-cassandra-tests/tests-trigger - - aws s3 cp --sse AES256 /opt/ignite-cassandra-tests/tests-trigger $S3_TESTS_TRIGGER - code=$? - - rm -f /opt/ignite-cassandra-tests/tests-trigger - - if [ $code -ne 0 ]; then - terminate "Failed to create tests trigger: $S3_TESTS_TRIGGER" - fi -} - -# Cleans previously created logs from S3 -cleanPreviousLogs() -{ - for logFile in /opt/ignite-cassandra-tests/logs/* - do - managerLog=$(echo $logFile | grep "tests-manager") - if [ -z "$managerLog" ]; then - rm -Rf $logFile - fi - done - - HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') - - aws s3 rm --recursive ${S3_TESTS_FAILURE}${HOST_NAME} - aws s3 rm --recursive ${S3_TESTS_SUCCESS}${HOST_NAME} -} - -# Uploads tests logs to S3 -uploadTestsLogs() -{ - HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') - - if [ -f "/opt/ignite-cassandra-tests/logs/__success__" ]; then - logsFolder=${S3_TESTS_SUCCESS}${HOST_NAME} - else - logsFolder=${S3_TESTS_FAILURE}${HOST_NAME} - fi - - aws s3 rm --recursive $logsFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to drop logs folder: $logsFolder" - fi - - if [ -d "/opt/ignite-cassandra-tests/logs" ]; then - aws s3 sync --sse AES256 /opt/ignite-cassandra-tests/logs $logsFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to export tests logs to: $logsFolder" - fi - fi -} - -# Runs tests-report.sh to prepare tests summary report -buildTestsSummaryReport() -{ - reportScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/tests-report.sh) - $reportScript - - if [ -n "$S3_LOGS_TRIGGER" ]; then - aws s3 cp --sse AES256 /etc/hosts $S3_LOGS_TRIGGER - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to trigger logs collection" - fi - fi -} - -# Running load tests -runLoadTests() -{ - cd /opt/ignite-cassandra-tests - - if [ "$TESTS_TYPE" == "ignite" ]; then - echo "[INFO] Running Ignite load tests" - ./ignite-load-tests.sh & - else - echo "[INFO] Running Cassandra load tests" - ./cassandra-load-tests.sh & - fi - - testsJobId=$! - - echo "[INFO] Tests job id: $testsJobId" - - sleep 1m - - LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) - LOGS_SNAPSHOT_TIME=$(date +%s) - - TERMINATED= - - # tests monitoring - while true; do - proc=$(ps -ef | grep java | grep "org.apache.ignite.tests") - if [ -z "$proc" ]; then - break - fi - - NEW_LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) - NEW_LOGS_SNAPSHOT_TIME=$(date +%s) - - # if logs state updated it means that tests are running and not stuck - if [ "$LOGS_SNAPSHOT" != "$NEW_LOGS_SNAPSHOT" ]; then - LOGS_SNAPSHOT=$NEW_LOGS_SNAPSHOT - LOGS_SNAPSHOT_TIME=$NEW_LOGS_SNAPSHOT_TIME - continue - fi - - duration=$(( $NEW_LOGS_SNAPSHOT_TIME-$LOGS_SNAPSHOT_TIME )) - duration=$(( $duration/60 )) - - # if logs wasn't updated during 5min it means that load tests stuck - if [ $duration -gt 5 ]; then - proc=($proc) - kill -9 ${proc[1]} - TERMINATED="true" - break - fi - - echo "[INFO] Waiting extra 30sec for load tests to complete" - - sleep 30s - done - - rm -f /opt/ignite-cassandra-tests/logs/tests.properties - cp /opt/ignite-cassandra-tests/settings/tests.properties /opt/ignite-cassandra-tests/logs - - if [ "$TERMINATED" == "true" ]; then - echo "[ERROR] Load tests stuck, tests process terminated" - echo "Load tests stuck, tests process terminated" > /opt/ignite-cassandra-tests/logs/__error__ - return 0 - fi - - failed= - if [ "$TESTS_TYPE" == "cassandra" ]; then - failed=$(cat /opt/ignite-cassandra-tests/cassandra-load-tests.log | grep "load tests execution failed") - else - failed=$(cat /opt/ignite-cassandra-tests/ignite-load-tests.log | grep "load tests execution failed") - fi - - if [ -n "$failed" ]; then - echo "[ERROR] Load tests execution failed" - echo "Load tests execution failed" > /opt/ignite-cassandra-tests/logs/__error__ - else - echo "[INFO] Load tests execution successfully completed" - echo "Load tests execution successfully completed" > /opt/ignite-cassandra-tests/logs/__success__ - fi -} - -####################################################################################################### - -sleep 1m - -NODE_STATE= -TRIGGER_STATE= - -printInstanceInfo -setupCassandraCredentials -switchToIdleState - -triggerFirstTimeTestsExecution - -registerNode - -while true; do - # switching state to IDLE - switchToIdleState - - sleep 30s - - NEW_TRIGGER_STATE=$(aws s3 ls $S3_TESTS_TRIGGER | xargs) - if [ -z "$NEW_TRIGGER_STATE" ] || [ "$NEW_TRIGGER_STATE" == "$TRIGGER_STATE" ]; then - continue - fi - - echo "----------------------------------------------------------------------" - echo "[INFO] Tests trigger changed" - echo "----------------------------------------------------------------------" - echo "[INFO] Old trigger: $TRIGGER_STATE" - echo "----------------------------------------------------------------------" - echo "[INFO] New trigger: $NEW_TRIGGER_STATE" - echo "----------------------------------------------------------------------" - - TRIGGER_STATE=$NEW_TRIGGER_STATE - - aws s3 cp $S3_TESTS_TRIGGER /opt/ignite-cassandra-tests/tests-trigger - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to download tests trigger info from: $S3_TESTS_TRIGGER" - continue - fi - - TESTS_TYPE=$(cat /opt/ignite-cassandra-tests/tests-trigger | grep TESTS_TYPE | xargs | sed -r "s/TESTS_TYPE=//g") - if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then - rm -f /opt/ignite-cassandra-tests/tests-trigger - echo "[ERROR] Incorrect tests type specified in the trigger info: $S3_TESTS_TRIGGER" - continue - fi - - rm -f /opt/ignite-cassandra-tests/settings/tests.properties - mv -f /opt/ignite-cassandra-tests/tests-trigger /opt/ignite-cassandra-tests/settings/tests.properties - - waitAllTestNodesCompletedTests - - # switching state to PREPARING - switchToPreparingState - - waitAllClusterNodesReady "cassandra" - waitAllClusterNodesReady "ignite" - setupCassandraSeeds - setupIgniteSeeds - - cleanPreviousLogs - - tryToGetFirstNodeLock - if [ $? -eq 0 ]; then - dropTestsSummary - recreateCassandraArtifacts - fi - - # switching state to WAITING - switchToWaitingState - - waitAllClusterNodesReady "test" - - if [ "$FIRST_NODE_LOCK" == "true" ]; then - aws s3 rm $S3_TESTS_TRIGGER - fi - - # switching state to RUNNING - switchToRunningState - - runLoadTests - uploadTestsLogs - - tryToGetFirstNodeLock - if [ $? -eq 0 ]; then - waitAllTestNodesCompletedTests - buildTestsSummaryReport - removeFirstNodeLock - fi -done \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh deleted file mode 100644 index 1576d57..0000000 --- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh +++ /dev/null @@ -1,499 +0,0 @@ -#!/bin/sh - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------------------------- -# Tests report builder -# ----------------------------------------------------------------------------------------------- -# Script is used to analyze load tests logs collected from all 'Tests' cluster nodes and build -# summary report -# ----------------------------------------------------------------------------------------------- - -#profile=/home/ignite/.bash_profile -profile=/root/.bash_profile - -. $profile -. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" - -# Building tests summary report -reportTestsSummary() -{ - echo "[INFO] Preparing tests results summary" - - TESTS_SUMMARY_DIR=/opt/ignite-cassandra-tests/tests-summary - SUCCEED_NODES_FILE=$TESTS_SUMMARY_DIR/succeed-nodes - SUCCEED_NODES_DIR=$TESTS_SUMMARY_DIR/succeed - FAILED_NODES_FILE=$TESTS_SUMMARY_DIR/failed-nodes - FAILED_NODES_DIR=$TESTS_SUMMARY_DIR/failed - REPORT_FILE=$TESTS_SUMMARY_DIR/report.txt - - rm -Rf $TESTS_SUMMARY_DIR - mkdir -p $TESTS_SUMMARY_DIR - mkdir -p $SUCCEED_NODES_DIR - mkdir -p $FAILED_NODES_DIR - - aws s3 ls $S3_TESTS_SUCCESS | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $SUCCEED_NODES_FILE - aws s3 ls $S3_TESTS_FAILURE | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $FAILED_NODES_FILE - - succeedCount=$(cat $SUCCEED_NODES_FILE | wc -l) - failedCount=$(cat $FAILED_NODES_FILE | wc -l) - count=$(( $succeedCount+$failedCount )) - - echo "Test type : $TESTS_TYPE" > $REPORT_FILE - echo "Test nodes count : $count" >> $REPORT_FILE - echo "Test nodes succeed: $succeedCount" >> $REPORT_FILE - echo "Test nodes failed : $failedCount" >> $REPORT_FILE - echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE - - if [ $succeedCount -gt 0 ]; then - echo "Succeed test nodes |" >> $REPORT_FILE - echo "-------------------" >> $REPORT_FILE - cat $SUCCEED_NODES_FILE >> $REPORT_FILE - echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE - - aws s3 sync --delete $S3_TESTS_SUCCESS $SUCCEED_NODES_DIR - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to get succeed tests details" - else - reportSucceedTestsStatistics "$REPORT_FILE" "$SUCCEED_NODES_DIR" - fi - fi - - if [ $failedCount -gt 0 ]; then - echo "Failed test nodes |" >> $REPORT_FILE - echo "------------------" >> $REPORT_FILE - cat $FAILED_NODES_FILE >> $REPORT_FILE - echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE - - aws sync --delete $S3_TESTS_FAILURE $FAILED_NODES_DIR - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to get failed tests details" - else - reportFailedTestsDetailes "$REPORT_FILE" "$FAILED_NODES_DIR" - fi - fi - - rm -f $HOME/tests-summary.zip - - pushd $TESTS_SUMMARY_DIR - - zip -r -9 $HOME/tests-summary.zip . - code=$? - - rm -Rf $TESTS_SUMMARY_DIR - - popd - - if [ $code -ne 0 ]; then - echo "-------------------------------------------------------------------------------------" - echo "[ERROR] Failed to create tests summary zip archive $HOME/tests-summary.zip for $TESTS_SUMMARY_DIR" - echo "-------------------------------------------------------------------------------------" - return 1 - fi - - aws s3 cp --sse AES256 $HOME/tests-summary.zip $S3_TESTS_SUMMARY - if [ $? -ne 0 ]; then - echo "-------------------------------------------------------------------------------------" - echo "[ERROR] Failed to uploat tests summary archive to: $S3_TESTS_SUMMARY" - echo "-------------------------------------------------------------------------------------" - else - echo "-------------------------------------------------------------------------------------" - echo "[INFO] Tests results summary uploaded to: $S3_TESTS_SUMMARY" - echo "-------------------------------------------------------------------------------------" - fi - - rm -f $HOME/tests-summary.zip -} - -# Creates report for succeed tests -reportSucceedTestsStatistics() -{ - writeMsg="0" - writeErrors="0" - writeSpeed="0" - blkWriteMsg="0" - blkWriteErrors="0" - blkWriteSpeed="0" - readMsg="0" - readErrors="0" - readSpeed="0" - blkReadMsg="0" - blkReadErrors="0" - blkReadSpeed="0" - - writeErrNodes= - blkWriteErrNodes= - readErrNodes= - blkReadErrNodes= - - tmpFile=`mktemp` - - for dir in $2/* - do - node=$(echo $dir | sed -r "s/^.*\///g") - echo "-------------------------------------------------------------------------------------" - echo "[INFO] Gathering statistics from $node test node" - echo "-------------------------------------------------------------------------------------" - - logFile=$(ls $dir | grep "${TESTS_TYPE}-load-tests.log" | head -1) - if [ -z "$logFile" ]; then - echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Node $node marked as succeeded," >> $tmpFile - echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - continue - fi - - logFile=$dir/$logFile - if [ ! -f "$logFile" ]; then - echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Node $node marked as succeeded," >> $tmpFile - echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - continue - fi - - cnt=$(cat $logFile | grep "^WRITE messages" | sed -r "s/WRITE messages: //g" | xargs) - if [ -n "$cnt" ]; then - writeMsg=$(bc <<< "$writeMsg + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] WRITE messages: $cnt" - else - echo "[WARN] WRITE messages count is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect WRITE messages count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect WRITE messages count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^WRITE errors" | sed -r "s/WRITE errors: //g" | sed -r "s/,.*//g" | xargs) - if [ -n "$cnt" ]; then - echo "[INFO] WRITE errors: $cnt" - writeErrors=$(bc <<< "$writeErrors + $cnt") - if [ $cnt -ne 0 ]; then - if [ -n "$writeErrNodes" ]; then - writeErrNodes="${writeErrNodes}, " - fi - writeErrNodes="${writeErrNodes}${node}" - fi - else - echo "[WARN] Failed to detect WRITE errors count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect WRITE errors count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^WRITE speed" | sed -r "s/WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) - if [ -n "$cnt" ]; then - writeSpeed=$(bc <<< "$writeSpeed + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] WRITE speed: $cnt msg/sec" - else - echo "[WARN] WRITE speed is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "WRITE speed is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect WRITE speed for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect WRITE speed for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_WRITE messages" | sed -r "s/BULK_WRITE messages: //g" | xargs) - if [ -n "$cnt" ]; then - blkWriteMsg=$(bc <<< "$blkWriteMsg + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] BULK_WRITE messages: $cnt" - else - echo "[WARN] BULK_WRITE messages count is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "BULK_WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect BULK_WRITE messages count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_WRITE messages count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_WRITE errors" | sed -r "s/BULK_WRITE errors: //g" | sed -r "s/,.*//g" | xargs) - if [ -n "$cnt" ]; then - blkWriteErrors=$(bc <<< "$blkWriteErrors + $cnt") - echo "[INFO] BULK_WRITE errors: $cnt" - if [ $cnt -ne 0 ]; then - if [ -n "$blkWriteErrNodes" ]; then - blkWriteErrNodes="${blkWriteErrNodes}, " - fi - blkWriteErrNodes="${blkWriteErrNodes}${node}" - fi - else - echo "[WARN] Failed to detect BULK_WRITE errors count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_WRITE errors count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_WRITE speed" | sed -r "s/BULK_WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) - if [ -n "$cnt" ]; then - blkWriteSpeed=$(bc <<< "$blkWriteSpeed + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] BULK_WRITE speed: $cnt msg/sec" - else - echo "[WARN] BULK_WRITE speed is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "BULK_WRITE speed is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect BULK_WRITE speed for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_WRITE speed for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^READ messages" | sed -r "s/READ messages: //g" | xargs) - if [ -n "$cnt" ]; then - readMsg=$(bc <<< "$readMsg + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] READ messages: $cnt" - else - echo "[WARN] READ messages count is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "READ messages count is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect READ messages count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect READ messages count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^READ errors" | sed -r "s/READ errors: //g" | sed -r "s/,.*//g" | xargs) - if [ -n "$cnt" ]; then - readErrors=$(bc <<< "$readErrors + $cnt") - echo "[INFO] READ errors: $cnt" - if [ $cnt -ne 0 ]; then - if [ -n "$readErrNodes" ]; then - blkWriteErrNodes="${readErrNodes}, " - fi - readErrNodes="${readErrNodes}${node}" - fi - else - echo "[WARN] Failed to detect READ errors count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect READ errors count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^READ speed" | sed -r "s/READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) - if [ -n "$cnt" ]; then - readSpeed=$(bc <<< "$readSpeed + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] READ speed: $cnt msg/sec" - else - echo "[WARN] READ speed is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "READ speed is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect READ speed for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect READ speed for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_READ messages" | sed -r "s/BULK_READ messages: //g" | xargs) - if [ -n "$cnt" ]; then - blkReadMsg=$(bc <<< "$blkReadMsg + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] BULK_READ messages: $cnt" - else - echo "[WARN] BULK_READ messages count is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "BULK_READ messages count is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect BULK_READ messages count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_READ messages count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_READ errors" | sed -r "s/BULK_READ errors: //g" | sed -r "s/,.*//g" | xargs) - if [ -n "$cnt" ]; then - blkReadErrors=$(bc <<< "$blkReadErrors + $cnt") - echo "[INFO] BULK_READ errors: $cnt" - if [ $cnt -ne 0 ]; then - if [ -n "$blkReadErrNodes" ]; then - blkReadErrNodes="${blkReadErrNodes}, " - fi - blkReadErrNodes="${blkReadErrNodes}${node}" - fi - else - echo "[WARN] Failed to detect BULK_READ errors count for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_READ errors count for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - - cnt=$(cat $logFile | grep "^BULK_READ speed" | sed -r "s/BULK_READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) - if [ -n "$cnt" ]; then - blkReadSpeed=$(bc <<< "$blkReadSpeed + $cnt") - if [ $cnt -ne 0 ]; then - echo "[INFO] BULK_READ speed: $cnt msg/sec" - else - echo "[WARN] BULK_READ speed is zero for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "BULK_READ speed is zero for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - else - echo "[WARN] Failed to detect BULK_READ speed for $node node. This test probably failed." - echo "WARNING |" >> $tmpFile - echo "--------" >> $tmpFile - echo "Failed to detect BULK_READ speed for $node node. This test probably failed." >> $tmpFile - echo "----------------------------------------------------------------------------------------------" >> $tmpFile - fi - done - - echo "-------------------------------------------------------------------------------------" - - echo "WRITE test metrics |" >> $1 - echo "-------------------" >> $1 - echo "Messages: $writeMsg" >> $1 - echo "Speed : $writeSpeed msg/sec" >> $1 - echo "Errors : $writeErrors" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - - echo "BULK_WRITE test metrics |" >> $1 - echo "------------------------" >> $1 - echo "Messages: $blkWriteMsg" >> $1 - echo "Speed : $blkWriteSpeed msg/sec" >> $1 - echo "Errors : $blkWriteErrors" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - - echo "READ test metrics |" >> $1 - echo "------------------" >> $1 - echo "Messages: $readMsg" >> $1 - echo "Speed : $readSpeed msg/sec" >> $1 - echo "Errors : $readErrors" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - - echo "BULK_READ test metrics |" >> $1 - echo "-----------------------" >> $1 - echo "Messages: $blkReadMsg" >> $1 - echo "Speed : $blkReadSpeed msg/sec" >> $1 - echo "Errors : $blkReadErrors" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - - if [ -n "$writeErrNodes" ]; then - echo "Nodes having WRITE errors |" >> $1 - echo "-------------------------------" >> $1 - echo "$writeErrNodes" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - fi - - if [ -n "$blkWriteErrNodes" ]; then - echo "Nodes having BULK_WRITE errors |" >> $1 - echo "-------------------------------" >> $1 - echo "$blkWriteErrNodes" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - fi - - if [ -n "$readErrNodes" ]; then - echo "Nodes having READ errors |" >> $1 - echo "-------------------------------" >> $1 - echo "$readErrNodes" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - fi - - if [ -n "$blkReadErrNodes" ]; then - echo "Nodes having BULK_READ errors |" >> $1 - echo "-------------------------------" >> $1 - echo "$blkReadErrNodes" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - fi - - cat $tmpFile >> $1 - - rm -f $tmpFile -} - -# Creates report for failed tests -reportFailedTestsDetailes() -{ - for dir in $2/* - do - node=$(echo $dir | sed -r "s/^.*\///g") - if [ -z "$node" ]; then - continue - fi - - echo "----------------------------------------------------------------------------------------------" >> $1 - echo "Error details for node: $node" >> $1 - echo "----------------------------------------------------------------------------------------------" >> $1 - - if [ -f "$dir/__error__" ]; then - cat $dir/__error__ >> $1 - else - echo "N/A" >> $1 - fi - done -} - -####################################################################################################### - -if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then - terminate "Incorrect tests type specified: $TESTS_TYPE" -fi - -reportTestsSummary \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java deleted file mode 100644 index c031bfb..0000000 --- a/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.tests; - -import org.apache.ignite.tests.load.LoadTestDriver; -import org.apache.ignite.tests.load.cassandra.BulkReadWorker; -import org.apache.ignite.tests.load.cassandra.BulkWriteWorker; -import org.apache.ignite.tests.load.cassandra.ReadWorker; -import org.apache.ignite.tests.load.cassandra.WriteWorker; -import org.apache.ignite.tests.utils.CacheStoreHelper; -import org.apache.ignite.tests.utils.CassandraHelper; -import org.apache.ignite.tests.utils.TestsHelper; -import org.apache.log4j.Logger; - -/** - * Load tests for {@link org.apache.ignite.cache.store.cassandra.CassandraCacheStore} implementation of - * {@link org.apache.ignite.cache.store.CacheStore} which allows to store Ignite cache data into Cassandra tables. - */ -public class CassandraDirectPersistenceLoadTest extends LoadTestDriver { - /** */ - private static final Logger LOGGER = Logger.getLogger("CassandraLoadTests"); - - /** - * - * @param args Test arguments. - */ - public static void main(String[] args) { - try { - LOGGER.info("Cassandra load tests execution started"); - - LoadTestDriver driver = new CassandraDirectPersistenceLoadTest(); - - /** - * Load test scripts could be executed from several machines. Current implementation can correctly, - * handle situation when Cassandra keyspace/table was dropped - for example by the same load test - * started a bit later on another machine. Moreover there is a warm up period for each load test. - * Thus all the delays related to keyspaces/tables recreation actions will not affect performance metrics, - * but it will be produced lots of "trash" output in the logs (related to correct handling of such - * exceptional situation and keyspace/table recreation). - * - * Thus dropping test keyspaces at the beginning of the tests makes sense only for Unit tests, - * but not for Load tests. - **/ - - //CassandraHelper.dropTestKeyspaces(); - - driver.runTest("WRITE", WriteWorker.class, WriteWorker.LOGGER_NAME); - - driver.runTest("BULK_WRITE", BulkWriteWorker.class, BulkWriteWorker.LOGGER_NAME); - - driver.runTest("READ", ReadWorker.class, ReadWorker.LOGGER_NAME); - - driver.runTest("BULK_READ", BulkReadWorker.class, BulkReadWorker.LOGGER_NAME); - - /** - * Load test script executed on one machine could complete earlier that the same load test executed from - * another machine. Current implementation can correctly handle situation when Cassandra keyspace/table - * was dropped (simply recreate it). But dropping keyspace/table during load tests execution and subsequent - * recreation of such objects can have SIGNIFICANT EFFECT on final performance metrics. - * - * Thus dropping test keyspaces at the end of the tests makes sense only for Unit tests, - * but not for Load tests. - */ - - //CassandraHelper.dropTestKeyspaces(); // REVIEW This line is commented by purpose? - - LOGGER.info("Cassandra load tests execution completed"); - } - catch (Throwable e) { - LOGGER.error("Cassandra load tests execution failed", e); - throw new RuntimeException("Cassandra load tests execution failed", e); - } - finally { - CassandraHelper.releaseCassandraResources(); - } - } - - /** {@inheritDoc} */ - @Override protected Logger logger() { - return LOGGER; - } - - /** {@inheritDoc} */ - @Override protected Object setup(String logName) { - return CacheStoreHelper.createCacheStore( - TestsHelper.getLoadTestsCacheName(), - TestsHelper.getLoadTestsPersistenceSettings(), - CassandraHelper.getAdminDataSrc(), - Logger.getLogger(logName)); - } - -}