anton-vinogradov commented on a change in pull request #8294: URL: https://github.com/apache/ignite/pull/8294#discussion_r516674944
########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); Review comment: default value never used. ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; Review comment: I see only one usage, why this located at the root directory? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cacheCfg.setIndexedTypes(UUID.class, byte[].class); Review comment: do you need indexes? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cacheCfg.setIndexedTypes(UUID.class, byte[].class); + + ignite.getOrCreateCache(cacheCfg); + + long start = System.currentTimeMillis(); + + markInitialized(); + + workParallel(ignite, cacheName, iterSize, dataSize); + + recordResult("DURATION", System.currentTimeMillis() - start); + + markFinished(); + } + + /** */ + private void workParallel(Ignite ignite, String cacheName, long iterSize, int dataSize) + throws InterruptedException { + int threads = Runtime.getRuntime().availableProcessors() / 2; + + long iterThread = iterSize / threads; + + assert iterThread > 0; + + CountDownLatch latch = new CountDownLatch(threads); + + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("UuidDataStreamer-%d") + .build(); Review comment: Do we really need this overkill, why not just start threads with the blank name? ########## File path: modules/ducktests/tests/ignitetest/services/utils/control_utility.py ########## @@ -127,6 +131,70 @@ def tx_kill(self, **kwargs): res = self.__parse_tx_list(output) return res if res else output + def validate_indexes(self, check_assert: bool = None): + """ + Validate indexes. + If check is true, will be return + """ + data = self.__run("--cache validate_indexes") + + if check_assert is not None: + assert (('no issues found.' in data) == check_assert), data + + def idle_verify(self, check_assert: bool = None): + """ + Idle verify. + """ + data = self.__run("--cache idle_verify") + + if check_assert is not None: + assert (('idle_verify check has finished, no conflicts have been found.' in data) == check_assert), data + + def idle_verify_dump(self, node=None, return_path: bool = False): + """ + Idle verify dump. + """ + data = self.__run("--cache idle_verify --dump", node=node) + + if return_path & ('VisorIdleVerifyDumpTask successfully' in data): + match = re.search(r'/.*.txt', data) + return match[0] + + return data + + def snapshot_create(self, snapshot_name: str, sync_mode: bool = True, time_out: int = 60): + """ + Create snapshot. + """ + res = self.__run(f"--snapshot create {snapshot_name}") + + self.logger.info(res) + + if ("Command [SNAPSHOT] finished with code: 0" in res) & sync_mode: + self.await_snapshot(time_out) + + def await_snapshot(self, time_out=60): + """ + Waiting for the snapshot to complete. + """ + delta_time = datetime.now() + timedelta(seconds=time_out) + + while datetime.now() < delta_time: + for node in self._cluster.nodes: + mbean = JmxClient(node).find_mbean('snapshot') + start_time = int(list(mbean.__getattr__('LastSnapshotStartTime'))[0]) + end_time = int(list(mbean.__getattr__('LastSnapshotEndTime'))[0]) + err_msg = list(mbean.__getattr__('LastSnapshotErrorMessage'))[0] + + if (0 < start_time < end_time) & (err_msg == ''): + return Review comment: this snapshot is not related to the created, just a random one finished. Should we finish https://issues.apache.org/jira/browse/IGNITE-13510 before making this? ########## File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py ########## @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains discovery tests. +""" +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SnapshotTest(IgniteTest): + """ + Test Snapshot. + """ + NUM_NODES = 4 + + SNAPSHOT_NAME = "test_snap" + + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH)) + def snapshot_test(self, ignite_version): + """ + Test Snapshot. + 1. Start of ignite cluster. + 2. Activate cluster. + 3. Load. + 4. Idle verify dump_1. + 5. Snapshot. + 6. Load. + 7. Idle verify dump_2. + 8. Сhecking that dump_1 and dump_2 are different. + 9. Stop ignite and replace db from snapshot. + 10. Run ignite cluster. + 11. Idle verify dump_3. + 12. Checking the equality of dump_1 and dump_3. Review comment: should this be clear from code? this does not seem refactoring friendly. ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) Review comment: any reason to have param if only default used? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; Review comment: strange assert when you use Optional with default ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); Review comment: Why Transactional? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); Review comment: it is not the default value? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); Review comment: Why 2? ########## File path: modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 ########## @@ -19,37 +19,52 @@ limitations under the License. --> -<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"; debug="false"> - <appender name="CONSOLE_ERR" class="org.apache.log4j.ConsoleAppender"> - <param name="Target" value="System.err"/> - - <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="[%d{{ISO8601}}][%-5p][%t][%c{{1}}] %m%n"/> - </layout> - </appender> - - <category name="org.springframework"> - <level value="WARN"/> - </category> - - <category name="org.eclipse.jetty"> - <level value="WARN"/> - </category> - - <category name="org.eclipse.jetty.util.log"> - <level value="ERROR"/> - </category> - - <category name="org.eclipse.jetty.util.component"> - <level value="ERROR"/> - </category> - - <category name="com.amazonaws"> - <level value="WARN"/> - </category> - - <root> - <level value="INFO"/> - <appender-ref ref="CONSOLE_ERR"/> - </root> -</log4j:configuration> +<Configuration status="info" shutdownHook="disable"> + <Appenders> + <Console name="CONSOLE" target="SYSTEM_OUT"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + <ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/> + </Console> + + <Console name="CONSOLE_ERR" target="SYSTEM_ERR"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + </Console> + + <RollingFile name="FILE" fileName="{{ logs_dir }}/ignite.log" + filePattern="{{ logs_dir }}/ignite-%i-%d{yyyy-MM-dd_HH-mm}.log" + immediateFlush="true"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + <Policies> + <OnStartupTriggeringPolicy /> + <SizeBasedTriggeringPolicy size="10 MB" /> + </Policies> + </RollingFile> + </Appenders> + + <Loggers> + <Logger name="org.apache.ignite" level="INFO"/> + + <!-- + <Logger name="org.apache.ignite.CourtesyConfigNotice" level=OFF/> + --> Review comment: Commented code at PR? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cacheCfg.setIndexedTypes(UUID.class, byte[].class); + + ignite.getOrCreateCache(cacheCfg); + + long start = System.currentTimeMillis(); + + markInitialized(); + + workParallel(ignite, cacheName, iterSize, dataSize); + + recordResult("DURATION", System.currentTimeMillis() - start); + + markFinished(); + } + + /** */ + private void workParallel(Ignite ignite, String cacheName, long iterSize, int dataSize) Review comment: the bad method name, tells nothing about it's goal ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cacheCfg.setIndexedTypes(UUID.class, byte[].class); + + ignite.getOrCreateCache(cacheCfg); + + long start = System.currentTimeMillis(); + + markInitialized(); + + workParallel(ignite, cacheName, iterSize, dataSize); + + recordResult("DURATION", System.currentTimeMillis() - start); + + markFinished(); + } + + /** */ + private void workParallel(Ignite ignite, String cacheName, long iterSize, int dataSize) + throws InterruptedException { + int threads = Runtime.getRuntime().availableProcessors() / 2; + + long iterThread = iterSize / threads; Review comment: what this variable name means? ########## File path: modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Loading random uuids to cache. + */ +public class UuidDataStreamerApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) throws InterruptedException { + String cacheName = jNode.get("cacheName").asText(); + + int dataSize = Optional.ofNullable(jNode.get("dataSize")) + .map(JsonNode::asInt) + .orElse(1024); + + long iterSize = Optional.ofNullable(jNode.get("iterSize")) + .map(JsonNode::asLong) + .orElse(1024L); + + assert dataSize > 0; + assert iterSize > 0; + + CacheConfiguration<UUID, byte[]> cacheCfg = new CacheConfiguration<>(cacheName); + cacheCfg.setCacheMode(CacheMode.PARTITIONED); + cacheCfg.setBackups(2); + cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cacheCfg.setIndexedTypes(UUID.class, byte[].class); + + ignite.getOrCreateCache(cacheCfg); + + long start = System.currentTimeMillis(); + + markInitialized(); + + workParallel(ignite, cacheName, iterSize, dataSize); + + recordResult("DURATION", System.currentTimeMillis() - start); + + markFinished(); + } + + /** */ + private void workParallel(Ignite ignite, String cacheName, long iterSize, int dataSize) + throws InterruptedException { + int threads = Runtime.getRuntime().availableProcessors() / 2; + + long iterThread = iterSize / threads; + + assert iterThread > 0; + + CountDownLatch latch = new CountDownLatch(threads); + + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("UuidDataStreamer-%d") + .build(); + + for (int i = 0; i < threads; i++) + threadFactory.newThread(new UuidDataStreamer(ignite, cacheName, latch, iterThread, dataSize)) + .start(); + + latch.await(); + } + + /** */ + private static class UuidDataStreamer implements Runnable { + /** Ignite. */ + private final Ignite ignite; + + /** Cache name. */ + private final String cacheName; + + /** Latch. */ + private final CountDownLatch latch; + + /** Iteration size. */ + private final long iterSize; + + /** Data size. */ + private final int dataSize; + + /** */ + public UuidDataStreamer(Ignite ignite, String cacheName, CountDownLatch latch, long iterSize, int dataSize) { + this.ignite = ignite; + this.cacheName = cacheName; + this.latch = latch; + this.iterSize = iterSize; + this.dataSize = dataSize; + } + + /** {@inheritDoc} */ + @Override public void run() { + try (IgniteDataStreamer<UUID, byte[]> dataStreamer = ignite.dataStreamer(cacheName)) { + dataStreamer.autoFlushFrequency(100L); + + for (long i = 0L; i <= iterSize; i++) { + UUID uuid = UUID.randomUUID(); + + byte[] data = new byte[dataSize]; + + ThreadLocalRandom.current().nextBytes(data); + + dataStreamer.addData(uuid, data); Review comment: what the reason between sending UUID with the same value vs random value? ########## File path: modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 ########## @@ -19,37 +19,52 @@ limitations under the License. --> -<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"; debug="false"> - <appender name="CONSOLE_ERR" class="org.apache.log4j.ConsoleAppender"> - <param name="Target" value="System.err"/> - - <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="[%d{{ISO8601}}][%-5p][%t][%c{{1}}] %m%n"/> - </layout> - </appender> - - <category name="org.springframework"> - <level value="WARN"/> - </category> - - <category name="org.eclipse.jetty"> - <level value="WARN"/> - </category> - - <category name="org.eclipse.jetty.util.log"> - <level value="ERROR"/> - </category> - - <category name="org.eclipse.jetty.util.component"> - <level value="ERROR"/> - </category> - - <category name="com.amazonaws"> - <level value="WARN"/> - </category> - - <root> - <level value="INFO"/> - <appender-ref ref="CONSOLE_ERR"/> - </root> -</log4j:configuration> +<Configuration status="info" shutdownHook="disable"> + <Appenders> + <Console name="CONSOLE" target="SYSTEM_OUT"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + <ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/> + </Console> + + <Console name="CONSOLE_ERR" target="SYSTEM_ERR"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + </Console> + + <RollingFile name="FILE" fileName="{{ logs_dir }}/ignite.log" + filePattern="{{ logs_dir }}/ignite-%i-%d{yyyy-MM-dd_HH-mm}.log" + immediateFlush="true"> + <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/> + <Policies> + <OnStartupTriggeringPolicy /> + <SizeBasedTriggeringPolicy size="10 MB" /> + </Policies> + </RollingFile> + </Appenders> + + <Loggers> + <Logger name="org.apache.ignite" level="INFO"/> + + <!-- + <Logger name="org.apache.ignite.CourtesyConfigNotice" level=OFF/> + --> + + <Logger name="org.springframework" level="WARN"/> + <Logger name="org.eclipse.jetty" level="WARN"/> + + <!-- + Avoid warnings about failed bind attempt when multiple nodes running on the same host. + --> + <Logger name="org.eclipse.jetty.util.log" level="ERROR"/> + <Logger name="org.eclipse.jetty.util.component" level="ERROR"/> + + <Logger name="com.amazonaws" level="WARN"/> + + <Root level="INFO"> + <AppenderRef ref="CONSOLE" level="INFO"/> + + <AppenderRef ref="CONSOLE_ERR" level="ERROR"/> + + <AppenderRef ref="FILE" level="DEBUG"/> Review comment: debug? what do you expect to log here? ########## File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py ########## @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains discovery tests. +""" +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SnapshotTest(IgniteTest): + """ + Test Snapshot. + """ + NUM_NODES = 4 + + SNAPSHOT_NAME = "test_snap" + + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH)) + def snapshot_test(self, ignite_version): + """ + Test Snapshot. + 1. Start of ignite cluster. + 2. Activate cluster. + 3. Load. + 4. Idle verify dump_1. + 5. Snapshot. + 6. Load. + 7. Idle verify dump_2. + 8. Сhecking that dump_1 and dump_2 are different. + 9. Stop ignite and replace db from snapshot. + 10. Run ignite cluster. + 11. Idle verify dump_3. + 12. Checking the equality of dump_1 and dump_3. + """ + data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)) + + ignite_config = IgniteConfiguration( + version=IgniteVersion(ignite_version), + data_storage=data_storage + ) + + service = IgniteService(self.test_context, ignite_config, num_nodes=self.NUM_NODES - 1) + service.start() + + control_utility = ControlUtility(service, self.test_context) + control_utility.activate() + + client_config = IgniteConfiguration( + client_mode=True, + version=IgniteVersion(ignite_version), + discovery_spi=from_ignite_cluster(service), + ) + + streamer = IgniteApplicationService( + self.test_context, + client_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.UuidDataStreamerApplication", + params={ + "cacheName": "test-cache", + "iterSize": 512 * 1024 + } + ) + + load(streamer) + + node = service.nodes[0] + + control_utility.validate_indexes(check_assert=True) + control_utility.idle_verify(check_assert=True) + dump_1 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_1 on {node.account.externally_routable_ip}={dump_1}') Review comment: any reason to log this? is this a "phase logging"? ########## File path: modules/ducktests/tests/ignitetest/services/ignite.py ########## @@ -127,6 +141,36 @@ def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT, allow_fail=False) + def rename_db(self, new_db_name: str): Review comment: bad method naming, what is db? ########## File path: modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py ########## @@ -28,11 +28,13 @@ class PersistenceAware: """ # Root directory for persistent output PERSISTENT_ROOT = "/mnt/service" - STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") + PATH_TO_LOGS_DIR = os.path.join(PERSISTENT_ROOT, "logs") + STDOUT_STDERR_CAPTURE = os.path.join(PATH_TO_LOGS_DIR, "console.log") + CONSOLE_ALL_CAPTURE = os.path.join(PATH_TO_LOGS_DIR, "console_all.log") Review comment: the difference is not clear to me ########## File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py ########## @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains discovery tests. +""" +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SnapshotTest(IgniteTest): + """ + Test Snapshot. + """ + NUM_NODES = 4 + + SNAPSHOT_NAME = "test_snap" + + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH)) + def snapshot_test(self, ignite_version): + """ + Test Snapshot. + 1. Start of ignite cluster. + 2. Activate cluster. + 3. Load. + 4. Idle verify dump_1. + 5. Snapshot. + 6. Load. + 7. Idle verify dump_2. + 8. Сhecking that dump_1 and dump_2 are different. + 9. Stop ignite and replace db from snapshot. + 10. Run ignite cluster. + 11. Idle verify dump_3. + 12. Checking the equality of dump_1 and dump_3. + """ + data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)) + + ignite_config = IgniteConfiguration( + version=IgniteVersion(ignite_version), + data_storage=data_storage + ) + + service = IgniteService(self.test_context, ignite_config, num_nodes=self.NUM_NODES - 1) + service.start() + + control_utility = ControlUtility(service, self.test_context) + control_utility.activate() + + client_config = IgniteConfiguration( + client_mode=True, + version=IgniteVersion(ignite_version), + discovery_spi=from_ignite_cluster(service), + ) + + streamer = IgniteApplicationService( + self.test_context, + client_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.UuidDataStreamerApplication", + params={ + "cacheName": "test-cache", + "iterSize": 512 * 1024 + } + ) + + load(streamer) + + node = service.nodes[0] + + control_utility.validate_indexes(check_assert=True) + control_utility.idle_verify(check_assert=True) + dump_1 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_1 on {node.account.externally_routable_ip}={dump_1}') + + control_utility.snapshot_create(self.SNAPSHOT_NAME) + + load(streamer) + + dump_2 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_2 on {node.account.externally_routable_ip}={dump_2}') + + diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) + assert len(diff) != 0 + + service.stop() + + service.rename_db(new_db_name='old_db') + service.restore_from_snapshot(self.SNAPSHOT_NAME) + + service.restart() + + control_utility.activate() + + control_utility.validate_indexes(check_assert=True) + control_utility.idle_verify(check_assert=True) + dump_3 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_3 on {node.account.externally_routable_ip}={dump_3}') + + diff = node.account.ssh_output(f'diff {dump_1} {dump_3}', allow_fail=True) + assert len(diff) == 0, diff + + +def load(service_load: IgniteApplicationService, duration: int = 300): + """ + Load. + """ + service_load.start() + + service_load.await_stopped(duration) Review comment: why not just .run()? ########## File path: modules/ducktests/tests/ignitetest/services/ignite.py ########## @@ -127,6 +141,36 @@ def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT, allow_fail=False) + def rename_db(self, new_db_name: str): + """ + Rename db. + """ + for node in self.nodes: + self._rename_db(node, new_db_name) + + def _rename_db(self, node, new_db_name: str): Review comment: why not inlined? ########## File path: modules/ducktests/tests/ignitetest/utils/ignite_test.py ########## @@ -66,3 +68,37 @@ def monotonic(): so that only the difference between the results of consecutive calls is valid. """ return monotonic() + + def copy_ignite_work_dir(self): Review comment: too strange place to works with Ignite internals ########## File path: modules/ducktests/tests/ignitetest/services/ignite.py ########## @@ -52,6 +52,20 @@ def start(self, timeout_sec=180): for node in self.nodes: self.await_node_started(node, timeout_sec) + def restart(self, timeout_sec=180): Review comment: why not at IgniteAware? ########## File path: modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py ########## @@ -102,12 +102,13 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware): """ @property def command(self): - cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ + cmd = "%s %s %s %s 2>&1 | tee %s >> %s &" % \ (self._envs(), self.path.script("ignite.sh"), self._jvm_opts(), self.CONFIG_FILE, - self.STDOUT_STDERR_CAPTURE) + self.STDOUT_STDERR_CAPTURE, + self.CONSOLE_ALL_CAPTURE) Review comment: why not just add some separator on the restart and continue to use the same log? more logs to the god of logs? %) ########## File path: modules/ducktests/tests/ignitetest/services/ignite.py ########## @@ -127,6 +141,36 @@ def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT, allow_fail=False) + def rename_db(self, new_db_name: str): + """ + Rename db. + """ + for node in self.nodes: + self._rename_db(node, new_db_name) + + def _rename_db(self, node, new_db_name: str): + """ + Rename db. + """ + assert len(self.pids(node)) == 0 + + node.account.ssh(f"mv {self.WORK_DIR}/db {self.WORK_DIR}/{new_db_name}") + + def restore_from_snapshot(self, snapshot_name: str): + """ + Copy from snapshot to db. + """ + for node in self.nodes: + self._copy_snap_to_db(node, snapshot_name) + + def _copy_snap_to_db(self, node, snapshot_name: str): Review comment: snap? why not cp_a_to_b? why not inlined? ########## File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py ########## @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains discovery tests. +""" +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SnapshotTest(IgniteTest): + """ + Test Snapshot. + """ + NUM_NODES = 4 + + SNAPSHOT_NAME = "test_snap" + + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH)) + def snapshot_test(self, ignite_version): + """ + Test Snapshot. + 1. Start of ignite cluster. + 2. Activate cluster. + 3. Load. + 4. Idle verify dump_1. + 5. Snapshot. + 6. Load. + 7. Idle verify dump_2. + 8. Сhecking that dump_1 and dump_2 are different. + 9. Stop ignite and replace db from snapshot. + 10. Run ignite cluster. + 11. Idle verify dump_3. + 12. Checking the equality of dump_1 and dump_3. + """ + data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)) + + ignite_config = IgniteConfiguration( + version=IgniteVersion(ignite_version), + data_storage=data_storage + ) + + service = IgniteService(self.test_context, ignite_config, num_nodes=self.NUM_NODES - 1) + service.start() + + control_utility = ControlUtility(service, self.test_context) + control_utility.activate() + + client_config = IgniteConfiguration( + client_mode=True, + version=IgniteVersion(ignite_version), + discovery_spi=from_ignite_cluster(service), + ) + + streamer = IgniteApplicationService( + self.test_context, + client_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.UuidDataStreamerApplication", + params={ + "cacheName": "test-cache", + "iterSize": 512 * 1024 + } + ) + + load(streamer) + + node = service.nodes[0] + + control_utility.validate_indexes(check_assert=True) + control_utility.idle_verify(check_assert=True) + dump_1 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_1 on {node.account.externally_routable_ip}={dump_1}') + + control_utility.snapshot_create(self.SNAPSHOT_NAME) + + load(streamer) + + dump_2 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_2 on {node.account.externally_routable_ip}={dump_2}') + + diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) + assert len(diff) != 0 + + service.stop() + + service.rename_db(new_db_name='old_db') + service.restore_from_snapshot(self.SNAPSHOT_NAME) + + service.restart() + + control_utility.activate() + + control_utility.validate_indexes(check_assert=True) + control_utility.idle_verify(check_assert=True) + dump_3 = control_utility.idle_verify_dump(node, return_path=True) + + self.logger.info(f'Path to dump_3 on {node.account.externally_routable_ip}={dump_3}') + + diff = node.account.ssh_output(f'diff {dump_1} {dump_3}', allow_fail=True) + assert len(diff) == 0, diff + + +def load(service_load: IgniteApplicationService, duration: int = 300): Review comment: should be inlined? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
