RYA-104 LoadStatementFile is not reading for some reason, todo: MongoExecSparql.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/82d421b2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/82d421b2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/82d421b2 Branch: refs/heads/master Commit: 82d421b29c1ce20cb793f5c12a6601f71c5f01ce Parents: e60da4c Author: David W. Lotts <david.lo...@parsons.com> Authored: Fri Dec 15 17:19:45 2017 -0500 Committer: kchilton2 <kevin.e.chil...@gmail.com> Committed: Fri Jan 5 16:48:40 2018 -0500 ---------------------------------------------------------------------- .../rya/mongodb/EmbeddedMongoFactory.java | 2 +- .../client/mongo/MongoConnectionDetails.java | 10 +- .../client/mongo/MongoLoadStatementsFile.java | 6 +- .../client/mongo/MongoExecuteSparqlQueryIT.java | 26 ++- .../client/mongo/MongoLoadStatementsFileIT.java | 189 ++++++++++--------- 5 files changed, 141 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/82d421b2/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java index b3cfaa4..a2c25ed 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java @@ -101,4 +101,4 @@ public class EmbeddedMongoFactory { mongodProcess.stop(); mongodExecutable.stop(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/82d421b2/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index f05b38d..81106f9 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -24,6 +24,8 @@ import org.apache.rya.api.RdfCloudTripleStoreConfiguration; import org.apache.rya.indexing.accumulo.ConfigUtils; import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import com.mongodb.MongoClient; + import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -92,6 +94,10 @@ public class MongoConnectionDetails { * @return Constructs a new {@link MongoDBRdfConfiguration} object with values from this object. */ public MongoDBRdfConfiguration build(final String ryaInstanceName) { + return build(ryaInstanceName, null); + } + + public MongoDBRdfConfiguration build(final String ryaInstanceName, MongoClient mongoClient) { // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. @@ -106,7 +112,9 @@ public class MongoConnectionDetails { // Both of these are ways to configure the collection prefixes. conf.setCollectionName(ryaInstanceName); conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, ryaInstanceName); - + if (mongoClient != null) { + conf.setMongoClient(mongoClient); + } return conf; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/82d421b2/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java index bf525c1..6229a3f 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -67,12 +67,12 @@ public class MongoLoadStatementsFile extends MongoCommand implements LoadStateme SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName); + final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. try { sail = RyaSailFactory.getInstance(ryaConf); - } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { - throw new RyaClientException("While getting an sail instance.", e); + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { + throw new RyaClientException("While getting a sail instance.", e); } // Load the file. http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/82d421b2/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index 787db2b..153992c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -18,9 +18,33 @@ */ package org.apache.rya.api.client.mongo; +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Test; + +import com.mongodb.MongoException; + /** * Integration tests the methods of {@link }. */ -public class MongoExecuteSparqlQueryIT { +public class MongoExecuteSparqlQueryIT extends MongoTestBase { + @Test + public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { + MongoConnectionDetails connectionDetails = getConnectionDetails(); + // Install a few instances of Rya using the install command. + final Install install = new MongoInstall(connectionDetails, getMongoClient()); + install.install("instanceExec", InstallConfiguration.builder().build()); + MongoExecuteSparqlQuery executeSparql = new MongoExecuteSparqlQuery(connectionDetails, getMongoClient()); + // TODO executeSparql. + } + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails(conf.getMongoUser(), conf.getMongoPassword().toCharArray(), conf.getMongoInstance(), Integer.parseInt(conf.getMongoPort())); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/82d421b2/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index 500bf31..c3b7cd0 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -18,97 +18,114 @@ */ package org.apache.rya.api.client.mongo; +import static org.junit.Assert.assertEquals; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.mongodb.MongoTestBase; +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.rio.RDFFormat; +import com.mongodb.client.MongoCursor; /** * Integration tests the methods of {@link MongoLoadStatementsFile}. */ public class MongoLoadStatementsFileIT extends MongoTestBase { -// @Test(expected = InstanceDoesNotExistException.class) -// public void instanceDoesNotExist() throws Exception { -// -// final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); -// ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getInstance(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); -// } -// -// @Test -// public void loadTurtleFile() throws Exception { -// // Install an instance of Rya. -// final InstallConfiguration installConfig = InstallConfiguration.builder() -// .setEnableTableHashPrefix(false) -// .setEnableEntityCentricIndex(false) -// .setEnableFreeTextIndex(false) -// .setEnableTemporalIndex(false) -// .setEnablePcjIndex(false) -// .setEnableGeoIndex(false) -// .build(); -// MongoConnectionDetails connectionDetails = getConnectionDetails(); -// final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); -// final Install install = ryaClient.getInstall(); -// install.install(connectionDetails.getInstance(), installConfig); -// -// // Load the test statement file. -// ryaClient.getLoadStatementsFile().loadStatements( // -// connectionDetails.getInstance(), // -// Paths.get("src/test/resources/example.ttl"), // -// RDFFormat.TURTLE); -// -// // Verify that the statements were loaded. -// final ValueFactory vf = new ValueFactoryImpl(); -// -// final List<Statement> expected = new ArrayList<>(); -// expected.add( vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob")) ); -// expected.add( vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie")) ); -// expected.add( vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream")) ); -// -// final List<Statement> statements = new ArrayList<>(); -// MongoCursor<Document> x = getRyaCollection().find().iterator(); -// System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); -// while (x.hasNext()) { -// Document y = x.next(); -// System.out.println("getRyaCollection()=" + y); -// } -// assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); -// // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); -// // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); -// // final Iterator<Entry<Key, Value>> it = scanner.iterator(); -// // while(it.hasNext()) { -// // final Entry<Key, Value> next = it.next(); -// // -// // final Key key = next.getKey(); -// // final byte[] row = key.getRow().getBytes(); -// // final byte[] columnFamily = key.getColumnFamily().getBytes(); -// // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); -// // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); -// // -// // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); -// // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); -// // -// // // Filter out the rya version statement if it is present. -// // if(!isRyaMetadataStatement(vf, statement)) { -// // statements.add( statement ); -// // } -// // } -// // -// // assertEquals(expected, statements); -// } -// -// private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { -// return statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#version") ) || -// statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#rts") ); -// } -// /** -// * @return copy from conf to MongoConnectionDetails -// */ -// private MongoConnectionDetails getConnectionDetails() { -// final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(// -// conf.getMongoUser(), // -// conf.getMongoPassword().toCharArray(), // -// conf.getMongoDBName(), // aka instance -// conf.getMongoInstance(), // aka hostname -// conf.getCollectionName() -// ); -// return connectionDetails; -// } + @Test(expected = InstanceDoesNotExistException.class) + public void instanceDoesNotExist() throws Exception { + + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getHostname(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); + } + + @Test + public void loadTurtleFile() throws Exception { + // Install an instance of Rya. + final InstallConfiguration installConfig = InstallConfiguration.builder()// + .setEnableTableHashPrefix(false)// + .setEnableEntityCentricIndex(false)// + .setEnableFreeTextIndex(false)// + .setEnableTemporalIndex(false)// + .setEnablePcjIndex(false)// + .setEnableGeoIndex(false)// + .build(); + MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + install.install(conf.getCollectionName(), installConfig); + + // Load the test statement file. + ryaClient.getLoadStatementsFile() + // LoadStatementsFile loadStatementsFile = new MongoLoadStatementsFile(connectionDetails, getMongoClient()); + // loadStatementsFile + .loadStatements( // + conf.getCollectionName(), // + Paths.get("src/test/resources/example.ttl"), // + RDFFormat.TURTLE); + + // Verify that the statements were loaded. + final ValueFactory vf = new ValueFactoryImpl(); + + final List<Statement> expected = new ArrayList<>(); + expected.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob"))); + expected.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie"))); + expected.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream"))); + + final List<Statement> statements = new ArrayList<>(); + MongoCursor<Document> x = getRyaCollection().find().iterator(); + System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); + while (x.hasNext()) { + Document y = x.next(); + System.out.println("getRyaCollection()=" + y); + } + assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); + // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); + // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); + // final Iterator<Entry<Key, Value>> it = scanner.iterator(); + // while(it.hasNext()) { + // final Entry<Key, Value> next = it.next(); + // + // final Key key = next.getKey(); + // final byte[] row = key.getRow().getBytes(); + // final byte[] columnFamily = key.getColumnFamily().getBytes(); + // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); + // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); + // + // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); + // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); + // + // // Filter out the rya version statement if it is present. + // if(!isRyaMetadataStatement(vf, statement)) { + // statements.add( statement ); + // } + // } + // + // assertEquals(expected, statements); + } + + private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { + return statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#version")) || statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#rts")); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails(// + conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoInstance(), // + Integer.parseInt(conf.getMongoPort())); + } }