[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336441#comment-15336441 ] ASF GitHub Bot commented on RYA-51: --- Github user isper3at commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67540581 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { +log.error("Unable to add: " + statement.toString()); +throw new RyaDAOException(e); +} +} + +@Override +public void add(final Iterator statement) throws RyaDAOException { +final List dbInserts = new ArrayList(); +while (statement.hasNext()){ +final RyaStatement ryaStatement = statement.next(); +final DBObject insert = storageStrategy.serialize(ryaStatement); +dbInserts.add(insert); + try { -for (RyaSecondaryIndexer index : secondaryIndexers) { +for (final RyaSecondaryIndexer index : secondaryIndexers) { index.storeStatement(ryaStatement); } -} catch (IOException e) { -throw new RyaDAOException(e); +} catch (final IOException e) { +log.error("Failed to add: " + ryaStatement.toString() + " to the indexer"); } - - } - coll.insert(dbInserts, new InsertOptions().continueOnError(true)); - } - - public void delete(RyaStatement statement, MongoDBRdfConfiguration conf) - throws RyaDAOException { - DBObject obj = storageStrategy.getQuery(statement); - coll.remove(obj); - } - - public void dropGraph(MongoDBRdfConfiguration conf, RyaURI... graphs) - throws RyaDAOException { - - } - - public void delete(Iterator statements, - MongoDBRdfConfiguration conf) throws RyaDAOException { - while (statements.hasNext()){ - RyaStatement ryaStatement = statements.next(); - coll.remove(storageStrategy.getQuery(ryaStatement)); - } - - } - - public String getVersion() throws RyaDAOException { - return "1.0"; - } - - public RyaQueryEngine getQueryEngine() { - return queryEngine; - } - - public RyaNamespaceManager getNamespaceManager() { - return nameSpaceManager; - } - - public void purge(RdfCloudTripleStoreConfiguration configuration) { - // TODO Auto-generated method stub - - } - - public void dropAndDestroy() throws RyaDAOException { - db.dropDatabase(); // this is dangerous! - } +} +coll.insert(dbInserts, new InsertOptions().continueOnError(true)); +} + +@Override +public void delete(final
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336426#comment-15336426 ] ASF GitHub Bot commented on RYA-51: --- Github user isper3at commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67539468 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { +log.error("Unable to add: " + statement.toString()); +throw new RyaDAOException(e); +} +} + +@Override +public void add(final Iterator statement) throws RyaDAOException { +final List dbInserts = new ArrayList(); +while (statement.hasNext()){ +final RyaStatement ryaStatement = statement.next(); +final DBObject insert = storageStrategy.serialize(ryaStatement); +dbInserts.add(insert); + try { -for (RyaSecondaryIndexer index : secondaryIndexers) { +for (final RyaSecondaryIndexer index : secondaryIndexers) { index.storeStatement(ryaStatement); } -} catch (IOException e) { -throw new RyaDAOException(e); +} catch (final IOException e) { +log.error("Failed to add: " + ryaStatement.toString() + " to the indexer"); } - - } - coll.insert(dbInserts, new InsertOptions().continueOnError(true)); - } - - public void delete(RyaStatement statement, MongoDBRdfConfiguration conf) - throws RyaDAOException { - DBObject obj = storageStrategy.getQuery(statement); - coll.remove(obj); - } - - public void dropGraph(MongoDBRdfConfiguration conf, RyaURI... graphs) - throws RyaDAOException { - - } - - public void delete(Iterator statements, - MongoDBRdfConfiguration conf) throws RyaDAOException { - while (statements.hasNext()){ - RyaStatement ryaStatement = statements.next(); - coll.remove(storageStrategy.getQuery(ryaStatement)); - } - - } - - public String getVersion() throws RyaDAOException { - return "1.0"; - } - - public RyaQueryEngine getQueryEngine() { - return queryEngine; - } - - public RyaNamespaceManager getNamespaceManager() { - return nameSpaceManager; - } - - public void purge(RdfCloudTripleStoreConfiguration configuration) { - // TODO Auto-generated method stub - - } - - public void dropAndDestroy() throws RyaDAOException { - db.dropDatabase(); // this is dangerous! - } +} +coll.insert(dbInserts, new InsertOptions().continueOnError(true)); --- End diff -- yeah, I was following how the mongo client
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336392#comment-15336392 ] ASF GitHub Bot commented on RYA-51: --- Github user isper3at commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67537827 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { --- End diff -- since both are just ignored and the actual exception that is caught is a type of IOException it makes it more clear where the exception is coming from. Its dangerous to catch exceptions just to ignore... > Temporal indexing support for MongoDB > - > > Key: RYA-51 > URL: https://issues.apache.org/jira/browse/RYA-51 > Project: Rya > Issue Type: New Feature > Components: dao >Reporter: Puja Valiyil >Assignee: Puja Valiyil > > There should be temporal indexing support for Mongo DB backed Rya. i imagine > this would be implemented using another collection, similar to how temporal > indexing is implemented in Rya with a supplementary table. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336389#comment-15336389 ] ASF GitHub Bot commented on RYA-51: --- Github user isper3at commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67537519 --- Diff: common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java --- @@ -42,4 +44,14 @@ public void dropGraph(RyaURI... graphs); +/** + * @return the set of predicates indexed by the indexer. + */ +public abstract Set getIndexablePredicates(); --- End diff -- All secondary indexers have one, PCJ's just return an empty HashSet > Temporal indexing support for MongoDB > - > > Key: RYA-51 > URL: https://issues.apache.org/jira/browse/RYA-51 > Project: Rya > Issue Type: New Feature > Components: dao >Reporter: Puja Valiyil >Assignee: Puja Valiyil > > There should be temporal indexing support for Mongo DB backed Rya. i imagine > this would be implemented using another collection, similar to how temporal > indexing is implemented in Rya with a supplementary table. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336319#comment-15336319 ] ASF GitHub Bot commented on RYA-51: --- Github user pujav65 commented on the issue: https://github.com/apache/incubator-rya/pull/48 ok other than needing to put in the delete call in the dao, it looks good. please adjust your eclipse settings so you're not auto-adding the final statements > Temporal indexing support for MongoDB > - > > Key: RYA-51 > URL: https://issues.apache.org/jira/browse/RYA-51 > Project: Rya > Issue Type: New Feature > Components: dao >Reporter: Puja Valiyil >Assignee: Puja Valiyil > > There should be temporal indexing support for Mongo DB backed Rya. i imagine > this would be implemented using another collection, similar to how temporal > indexing is implemented in Rya with a supplementary table. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336318#comment-15336318 ] ASF GitHub Bot commented on RYA-51: --- Github user pujav65 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67529573 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { +log.error("Unable to add: " + statement.toString()); +throw new RyaDAOException(e); +} +} + +@Override +public void add(final Iterator statement) throws RyaDAOException { +final List dbInserts = new ArrayList(); +while (statement.hasNext()){ +final RyaStatement ryaStatement = statement.next(); +final DBObject insert = storageStrategy.serialize(ryaStatement); +dbInserts.add(insert); + try { -for (RyaSecondaryIndexer index : secondaryIndexers) { +for (final RyaSecondaryIndexer index : secondaryIndexers) { index.storeStatement(ryaStatement); } -} catch (IOException e) { -throw new RyaDAOException(e); +} catch (final IOException e) { +log.error("Failed to add: " + ryaStatement.toString() + " to the indexer"); } - - } - coll.insert(dbInserts, new InsertOptions().continueOnError(true)); - } - - public void delete(RyaStatement statement, MongoDBRdfConfiguration conf) - throws RyaDAOException { - DBObject obj = storageStrategy.getQuery(statement); - coll.remove(obj); - } - - public void dropGraph(MongoDBRdfConfiguration conf, RyaURI... graphs) - throws RyaDAOException { - - } - - public void delete(Iterator statements, - MongoDBRdfConfiguration conf) throws RyaDAOException { - while (statements.hasNext()){ - RyaStatement ryaStatement = statements.next(); - coll.remove(storageStrategy.getQuery(ryaStatement)); - } - - } - - public String getVersion() throws RyaDAOException { - return "1.0"; - } - - public RyaQueryEngine getQueryEngine() { - return queryEngine; - } - - public RyaNamespaceManager getNamespaceManager() { - return nameSpaceManager; - } - - public void purge(RdfCloudTripleStoreConfiguration configuration) { - // TODO Auto-generated method stub - - } - - public void dropAndDestroy() throws RyaDAOException { - db.dropDatabase(); // this is dangerous! - } +} +coll.insert(dbInserts, new InsertOptions().continueOnError(true)); +} + +@Override +public void delete(final
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336299#comment-15336299 ] ASF GitHub Bot commented on RYA-51: --- Github user pujav65 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67527705 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { +log.error("Unable to add: " + statement.toString()); +throw new RyaDAOException(e); +} +} + +@Override +public void add(final Iterator statement) throws RyaDAOException { +final List dbInserts = new ArrayList(); +while (statement.hasNext()){ +final RyaStatement ryaStatement = statement.next(); +final DBObject insert = storageStrategy.serialize(ryaStatement); +dbInserts.add(insert); + try { -for (RyaSecondaryIndexer index : secondaryIndexers) { +for (final RyaSecondaryIndexer index : secondaryIndexers) { index.storeStatement(ryaStatement); } -} catch (IOException e) { -throw new RyaDAOException(e); +} catch (final IOException e) { +log.error("Failed to add: " + ryaStatement.toString() + " to the indexer"); } - - } - coll.insert(dbInserts, new InsertOptions().continueOnError(true)); - } - - public void delete(RyaStatement statement, MongoDBRdfConfiguration conf) - throws RyaDAOException { - DBObject obj = storageStrategy.getQuery(statement); - coll.remove(obj); - } - - public void dropGraph(MongoDBRdfConfiguration conf, RyaURI... graphs) - throws RyaDAOException { - - } - - public void delete(Iterator statements, - MongoDBRdfConfiguration conf) throws RyaDAOException { - while (statements.hasNext()){ - RyaStatement ryaStatement = statements.next(); - coll.remove(storageStrategy.getQuery(ryaStatement)); - } - - } - - public String getVersion() throws RyaDAOException { - return "1.0"; - } - - public RyaQueryEngine getQueryEngine() { - return queryEngine; - } - - public RyaNamespaceManager getNamespaceManager() { - return nameSpaceManager; - } - - public void purge(RdfCloudTripleStoreConfiguration configuration) { - // TODO Auto-generated method stub - - } - - public void dropAndDestroy() throws RyaDAOException { - db.dropDatabase(); // this is dangerous! - } +} +coll.insert(dbInserts, new InsertOptions().continueOnError(true)); --- End diff -- ok does the continue on error catch them? >
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15336294#comment-15336294 ] ASF GitHub Bot commented on RYA-51: --- Github user pujav65 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/48#discussion_r67527603 --- Diff: dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java --- @@ -179,85 +145,85 @@ public void destroy() throws RyaDAOException { IOUtils.closeQuietly(queryEngine); } - public void add(RyaStatement statement) throws RyaDAOException { - // add it to the collection - try { - coll.insert(storageStrategy.serialize(statement)); - for(RyaSecondaryIndexer index: secondaryIndexers) { - index.storeStatement(statement); - } - } - catch (com.mongodb.MongoException.DuplicateKey exception){ - // ignore - } - catch (com.mongodb.DuplicateKeyException exception){ - // ignore - } - catch (Exception ex){ - // ignore single exceptions - ex.printStackTrace(); - } - } - - public void add(Iterator statement) throws RyaDAOException { - List dbInserts = new ArrayList(); - while (statement.hasNext()){ - RyaStatement ryaStatement = statement.next(); - DBObject insert = storageStrategy.serialize(ryaStatement); - dbInserts.add(insert); - +@Override +public void add(final RyaStatement statement) throws RyaDAOException { +// add it to the collection +try { +coll.insert(storageStrategy.serialize(statement)); +for(final RyaSecondaryIndexer index: secondaryIndexers) { +index.storeStatement(statement); +} +} catch (final IOException e) { --- End diff -- what happened to the duplicate key exceptions? > Temporal indexing support for MongoDB > - > > Key: RYA-51 > URL: https://issues.apache.org/jira/browse/RYA-51 > Project: Rya > Issue Type: New Feature > Components: dao >Reporter: Puja Valiyil >Assignee: Puja Valiyil > > There should be temporal indexing support for Mongo DB backed Rya. i imagine > this would be implemented using another collection, similar to how temporal > indexing is implemented in Rya with a supplementary table. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (RYA-51) Temporal indexing support for MongoDB
[ https://issues.apache.org/jira/browse/RYA-51?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15314352#comment-15314352 ] ASF GitHub Bot commented on RYA-51: --- GitHub user isper3at opened a pull request: https://github.com/apache/incubator-rya/pull/48 RYA-51 Temporal Indexing mongo support Added support for mongo backed rya temporal indexing. Added integration tests Repackaged mongo secondary indexing Cleaned up the storage strategies. Indexing is now consistent across all secondary indexers (mongo only). renamed StatementContraints to be spelled correctly. [Jira](https://issues.apache.org/jira/browse/RYA-51) You can merge this pull request into a Git repository by running: $ git pull https://github.com/isper3at/incubator-rya RYA-51_temporal Alternatively you can review and apply these changes as the patch at: https://github.com/apache/incubator-rya/pull/48.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #48 commit e9a227bb2173b3f92216a4b04126182c2bd3dda0 Author: isper3atDate: 2016-04-05T19:42:13Z RYA-51 Temporal Indexing mongo support Added support for mongo backed rya temporal indexing. Added integration tests Repackaged mongo secondary indexing Cleaned up the storage strategies. Indexing is now consistent across all secondary indexers (mongo only). renamed StatementContraints to be spelled correctly. > Temporal indexing support for MongoDB > - > > Key: RYA-51 > URL: https://issues.apache.org/jira/browse/RYA-51 > Project: Rya > Issue Type: New Feature > Components: dao >Reporter: Puja Valiyil >Assignee: Puja Valiyil > > There should be temporal indexing support for Mongo DB backed Rya. i imagine > this would be implemented using another collection, similar to how temporal > indexing is implemented in Rya with a supplementary table. -- This message was sent by Atlassian JIRA (v6.3.4#6332)