date:20210415

Re: Cannot start ignite nodes with shared memory - Ignite version 2.10.0

2021-04-15 Thread sarahsamji

Sample

import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.*;
import org.apache.ignite.events.EventType;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import
org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.logging.log4j.ThreadContext;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.nio.file.Paths;
import java.util.List;
import java.util.Random;
import static org.junit.jupiter.api.Assertions.*;

public class SampleTest {
static String localhost;
private static Random random;
final static int discPort = 36101;
final static int commPort = 37601;
final static int sharedMemPort = 37235;
final static int clusterSize = 2;
private static final int messageQueueLimit = 8192;
private static final long tcpCommTimeout = 10_000L;
private static final int igniteCommPortRange = 5;
final static int igniteDiscPortRange = clusterSize * 2;
final static  int igniteDiscMaxPort = discPort + igniteDiscPortRange;
private static final long discoveryNetworkTimeout = 20_000L;
private static final int default_threadpool_size = 8;
private static final int igniteLongRunningThreadPoolSize = 8;
private static final int igniteStreamerThreadPoolSize = 8;

private static final String igniteDiscHost = "127.0.0.1";
final static String tmpdir = System.getProperty("java.io.tmpdir");
public static final int[] EVENT_TYPES = {
EventType.EVT_CACHE_OBJECT_EXPIRED,
EventType.EVT_NODE_JOINED,
EventType.EVT_NODE_LEFT,
EventType.EVT_NODE_SEGMENTED,
EventType.EVT_CLIENT_NODE_DISCONNECTED,
EventType.EVT_CLIENT_NODE_RECONNECTED,
EventType.EVT_NODE_FAILED,
EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST};


@BeforeAll
static void beforeAll() {
IgniteUtils.setCurrentIgniteName("");
ThreadContext.clearAll();
System.setProperty("ignite.cluster.encryption.disabled", "true");
}

@Test
void test() {
int httpPort = 37500;
localhost = "localhost:" + httpPort;
try {
startCluster();
final Ignite ignite = Ignition.ignite("NODE_I1");
assertNotNull(ignite);
} finally {
IgniteUtils.setCurrentIgniteName(null);
ThreadContext.clearAll();
}
}

private static void startCluster() {
random = new Random();
Thread.interrupted();

startNode("NODE_1");

for (int i = 2; i <= clusterSize; i++) {
startNode("NODE" + i);
}

Ignite i1 = Ignition.ignite("NODE_1");
assertEquals(clusterSize, i1.cluster().nodes().size());
}

private static void startNode(String instanceName) {
IgniteConfiguration igniteConfig = new
IgniteConfiguration().setIgniteInstanceName(instanceName)
.setPeerClassLoadingEnabled(false)
.setWorkDirectory(Paths.get(tmpdir,"ignite", "work",
String.valueOf(random.nextInt(100))).toString())
.setFailureDetectionTimeout(30_000L)
.setMetricsLogFrequency(30L)
   
.setDataStorageConfiguration(createDataStorageConfiguration())
.setIncludeEventTypes(EVENT_TYPES)
.setCommunicationSpi(createCommunicationSpi())
.setDiscoverySpi(createDiscoverySpi())
.setPublicThreadPoolSize(default_threadpool_size)
.setDataStreamerThreadPoolSize(igniteStreamerThreadPoolSize)
.setSystemThreadPoolSize(default_threadpool_size)
.setServiceThreadPoolSize(default_threadpool_size)
.setStripedPoolSize(default_threadpool_size)
   
.setExecutorConfiguration(createExecutorConfig("IgniteLongRunning",
igniteLongRunningThreadPoolSize))
.setConnectorConfiguration(null)
.setClientConnectorConfiguration(null)
.setBinaryConfiguration(createBinaryConfiguration())
.setFailureHandler(new StopNodeOrHaltFailureHandler(false,
0));;
Ignite ignite = Ignition.getOrStart(igniteConfig);
}

static DataStorageConfiguration createDataStorageConfiguration() {

int evictionThreshold = 90;
String DEFAULT_MEMORY_REGION = "Default_Region";
long memSize = 200 * 1024 * 1024;;

DataRegionConfiguration regionConfig = new DataRegionConfiguration()
.setName(DEFAULT_MEMORY_REGION)

Re: Cannot start ignite nodes with shared memory - Ignite version 2.10.0

2021-04-15 Thread sarahsamji



public class SampleTest {
static String localhost;
private static Random random;
final static int discPort = 36101;
final static int commPort = 37601;
final static int sharedMemPort = 37235;
final static int clusterSize = 2;
private static final int messageQueueLimit = 8192;
private static final long tcpCommTimeout = 10_000L;
private static final int igniteCommPortRange = 5;
final static int igniteDiscPortRange = clusterSize * 2;
final static  int igniteDiscMaxPort = discPort + igniteDiscPortRange;
private static final long discoveryNetworkTimeout = 20_000L;
private static final int default_threadpool_size = 8;
private static final int igniteLongRunningThreadPoolSize = 8;
private static final int igniteStreamerThreadPoolSize = 8;

private static final String igniteDiscHost = "127.0.0.1";
final static String tmpdir = System.getProperty("java.io.tmpdir");
public static final int[] EVENT_TYPES = {
EventType.EVT_CACHE_OBJECT_EXPIRED,
EventType.EVT_NODE_JOINED,
EventType.EVT_NODE_LEFT,
EventType.EVT_NODE_SEGMENTED,
EventType.EVT_CLIENT_NODE_DISCONNECTED,
EventType.EVT_CLIENT_NODE_RECONNECTED,
EventType.EVT_NODE_FAILED,
EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST};


@BeforeAll
static void beforeAll() {
IgniteUtils.setCurrentIgniteName("");
ThreadContext.clearAll();
System.setProperty("ignite.cluster.encryption.disabled", "true");
}

@Test
void test() {
int httpPort = PortFinder.getRandomPort(37000, 37500);
localhost = "localhost:" + httpPort;
try {
startCluster();
final Ignite ignite = Ignition.ignite("NODE_I1");
assertNotNull(ignite);
TestDomainSvcImpl.resetCounters();
} finally {
IgniteUtils.setCurrentIgniteName(null);
ThreadContext.clearAll();
}
}

private static void startCluster() {
random = new Random();
Thread.interrupted();

startNode("NODE_1");

for (int i = 2; i <= clusterSize; i++) {
startNode("NODE" + i);
}

Ignite i1 = Ignition.ignite("NODE_1");
assertEquals(clusterSize, i1.cluster().nodes().size());
}

private static void startNode(String instanceName) {
IgniteConfiguration igniteConfig = buildIgniteConfig(instanceName);
Ignite ignite = Ignition.start(igniteConfig);
}

private static IgniteConfiguration buildIgniteConfig(String
instanceName) {
IgniteConfiguration cfg = new IgniteConfigBuilder().build();
return cfg
.setIgniteInstanceName(instanceName)
.setPeerClassLoadingEnabled(false)
.setWorkDirectory(Paths.get(tmpdir, "firecracker", "ignite",
"work", String.valueOf(random.nextInt(100))).toString())
.setFailureDetectionTimeout(30_000L)
.setMetricsLogFrequency(30L)
   
.setDataStorageConfiguration(createDataStorageConfiguration())
.setIncludeEventTypes(EVENT_TYPES)
.setCommunicationSpi(createCommunicationSpi())
.setDiscoverySpi(createDiscoverySpi())
.setPublicThreadPoolSize(default_threadpool_size)
.setDataStreamerThreadPoolSize(igniteStreamerThreadPoolSize)
.setSystemThreadPoolSize(default_threadpool_size)
.setServiceThreadPoolSize(default_threadpool_size)
.setStripedPoolSize(default_threadpool_size)
   
.setExecutorConfiguration(createExecutorConfig("IgniteLongRunning",
igniteLongRunningThreadPoolSize))
.setConnectorConfiguration(null)
.setClientConnectorConfiguration(null)
.setBinaryConfiguration(createBinaryConfiguration())
.setFailureHandler(new StopNodeOrHaltFailureHandler(false,
0));
}

static DataStorageConfiguration createDataStorageConfiguration() {

int evictionThreshold = 90;
String DEFAULT_MEMORY_REGION = "Default_Region";
long memSize = 200 * 1024 * 1024;;

DataRegionConfiguration regionConfig = new DataRegionConfiguration()
.setName(DEFAULT_MEMORY_REGION)
.setInitialSize(memSize)
.setMaxSize(memSize)
.setMetricsEnabled(true)
.setPageEvictionMode(DataPageEvictionMode.DISABLED) //
Only enable eviction if we can find a way to control it
.setEvictionThreshold(evictionThreshold / 100);

DataStorageConfiguration dataStorageConfiguration = new
DataStorageConfiguration()
.setDefaultDataRegionConfiguration(regionConfig);

dataStorageConfiguration.setMetricsEnabled(true);

return dataStorageConfiguration;
}

static CommunicationSpi createCommunicationSpi() {

Re: Fastest way to iterate over a persistence cache

2021-04-15 Thread Sebastian Macke

Hi,

of course we are using SSDs, but 4kB is still a tiny amount, when you have
to wait for the SSD read roundtrip every time. The CPU is idle at about
10%.. The Linux kernel does not seem to perform a readahead, probably
because of the random access structure.

For some reason the QueryParallelism feature does not work either. No change
in the speed.

Regards

Sebastian






--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

help with control.sh script

2021-04-15 Thread facundo.maldonado

I'm testing a 32 nodes cluster with a partitioned cache with one backup.
If 2 of them crashed (not if, when) I have the lost partitions problem.

Now I ssh to one of the nodes and execute *control.sh --baseline.*
>From every node other than the one marked as "coordinator" (?) I get this
output:


Failed to execute baseline command='collect'
Failed to communicate with grid nodes (maximum count of retries reached).
Connection to cluster failed. Failed to communicate with grid nodes (maximum
count of retries reached).

Ok, I went to every node and do the same until I found the 'coordinator'.
Once I made the failing nodes get online again I execute:
*control.sh --cache reset_lost_partitions mycache*

To my surprise, I'm getting 

Connection to cluster failed. Failed to communicate with grid nodes (maximum
count of retries reached).

So, started again looking for the nodes where that command actually works.

I'm sure I'm doing something wrong. Could someone help me?




--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

Is there any way to get cache and node status programatically?

2021-04-15 Thread facundo.maldonado

Is there any API or recommended way to get if a node is Online or Offline
programmatically the same way control script does?

Also, once a cache is moved to read-only state, I can't find a way to get
that status? 





--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

Re: Fastest way to iterate over a persistence cache

2021-04-15 Thread Stephen Darlington

You may need to do the parallelism “manually.” Something like:

for (int p = 0; p < ignite.affinity("TEST").partitions(); p++) {
ignite.compute().affinityRunAsync(Arrays.asList("TEST"), p, () -> {
ScanQuery q = new ScanQuery()
.setPartition(p);
System.out.println(ignite.cache("TEST").query(q).getAll());
});
}

Regards,
Stephen

> On 15 Apr 2021, at 17:52, Sebastian Macke  wrote:
> 
> Hi,
> 
> of course we are using SSDs, but 4kB is still a tiny amount, when you have
> to wait for the SSD read roundtrip every time. The CPU is idle at about
> 10%.. The Linux kernel does not seem to perform a readahead, probably
> because of the random access structure.
> 
> For some reason the QueryParallelism feature does not work either. No change
> in the speed.
> 
> Regards
> 
> Sebastian
> 
> 
> 
> 
> 
> 
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/

List all available distributed locks in cluster

2021-04-15 Thread Paolo Bazzi

Hi all,

Is there a way to list all available distributed locks within the cluster?
Would be nice to allow some sort of monitoring about used locks and their
correct disposal by our application.

In fact I'm looking for something similar to:
org.apache.ignite.Ignite#cacheNames but for locks.

Looking at the source code, I found the following map containing some of the
desired information:
org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor#dsMap

-> Any way to expose this content by API?

Regards,
Paolo



--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

Re: Abrupt stop of application on using sql

2021-04-15 Thread rakshita04

Hi Ilya,

How is decreasing maxSize of data region related to "Buffer Overflow"
problem?
I mean we dont get any Out of memory log in DeMsg logs and also "top"
command shows enough available RAM.
We tried decreasing maxSize to 50mb now but the problem still persists.
Can this also be related to checkpointing?

regards,
Rakshita Chaudhary




--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

Re: Is Geo redundancy supported in Ignite

2021-04-15 Thread Ilya Kasnacheev

Hello!

The one I know of, yes.

Regards,
-- 
Ilya Kasnacheev


чт, 15 апр. 2021 г. в 09:30, Venkata Bhagavatula :

> Hi Ilya Kasnacheev,
>
> We are also looking for this.
> Can you please highlight the third party solution?  I know that GridGain
> supports this in their entrerprise solution.  Is this you are referring to ?
>
> Thanks n Regards,
> Chalapathi
>
> On Tue, Apr 13, 2021 at 6:29 PM Ilya Kasnacheev 
> wrote:
>
>> Hello!
>>
>> There is a third-party solution for geo-redundancy based on Apache
>> Ignite. But as X Guest said, no native support in Apache Ignite.
>>
>> Regards,
>> --
>> Ilya Kasnacheev
>>
>>
>> пн, 12 апр. 2021 г. в 16:20, vbm :
>>
>>> Hi,
>>>
>>> Is geo-redundancy supported in Ignite ?
>>>
>>> Can we  bring up 2 clusters in different location(geo) and make one
>>> cluster
>>> as backup for another ?
>>> Basically can ignite cluster in different location act as a active/
>>> passive
>>> setup ?
>>>
>>> If one of the cluster goes down due to some issue in one location, can
>>> another cluster across geo become active ?
>>>
>>>
>>> Regards,
>>> Vishwas
>>>
>>>
>>>
>>> --
>>> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>>>
>>

Re: Ignite unable to understand recursive query.

2021-04-15 Thread Ilya Kasnacheev

Hello!

As far as my understanding goes, Ignite does not support recursive CTE.

You may still try running it as a local query by setting local=true in your
connection/query settings.

Regards,
-- 
Ilya Kasnacheev


чт, 15 апр. 2021 г. в 02:33, PunxsutawneyPhil3 :

> I am having a problem with the following query which results in this error:
>
> javax.cache.CacheException: Failed to parse query. Column "S.TAGID" not
> found; SQL statement:
>
> From reading it looks like the reason for this is explained in  this
> 
> documentation,
> and is that Ignite cannot resolve the columns of reportIdList after the
> Inner Join.
>
> Is there any way to restructure this query so it can be understood by
> Ignite?
>
>
>
> WITH RECURSIVE reportIdList AS
> (
> SELECT reportId, tagId, owner
> FROM "MyReportPojoCache".MyReportPojo
> WHERE id = ANY (SELECT id
> FROM "MyOtherReportPojoCache".MyOtherReportPojo
> WHERE owner IS NOT NULL
>   AND isManage IS TRUE
>   AND type = 'tag-group')
> UNION
> SELECT m.reportId, m.tagId, m.owner
> FROM "MyReportPojoCache".MyReportPojo m
> INNER JOIN reportIdList s ON s.tagId = m.reportId
>   AND s.owner IS NOT NULL
>   AND s.owner != 'admin'
>   AND s.owner = m.owner
> )
> SELECT qpIntId
> FROM "MyReportPojoCache".MyReportPojo
> WHERE (report_id, owner) IN (SELECT report_id, owner FROM reportIdList)
>
>
> link StackOverflow post
>
> https://stackoverflow.com/questions/67098705/how-to-rewrite-recursive-sql-query-to-work-with-ignite-sql-queries
> <
> https://stackoverflow.com/questions/67098705/how-to-rewrite-recursive-sql-query-to-work-with-ignite-sql-queries>
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>

Re: Cannot start ignite nodes with shared memory - Ignite version 2.10.0

Re: Cannot start ignite nodes with shared memory - Ignite version 2.10.0

Re: Fastest way to iterate over a persistence cache

help with control.sh script

Is there any way to get cache and node status programatically?

Re: Fastest way to iterate over a persistence cache

List all available distributed locks in cluster

Re: Abrupt stop of application on using sql

Re: Is Geo redundancy supported in Ignite

Re: Ignite unable to understand recursive query.

10 matches

Site Navigation

Mail list logo

Footer information