First round of update of MongoStore after GORA_94 merge
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/ab2102a2 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/ab2102a2 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/ab2102a2 Branch: refs/heads/master Commit: ab2102a29ced91101c8d4987115996071efd02f9 Parents: 30c671b Author: Damien Raude-Morvan <dam...@dictanova.com> Authored: Wed Apr 16 00:18:47 2014 +0200 Committer: Damien Raude-Morvan <dam...@dictanova.com> Committed: Wed Apr 16 00:18:55 2014 +0200 ---------------------------------------------------------------------- gora-mongodb/pom.xml | 2 +- .../apache/gora/mongodb/query/MongoDBQuery.java | 5 ++- .../gora/mongodb/query/MongoDBResult.java | 1 - .../apache/gora/mongodb/store/MongoStore.java | 46 ++++++++------------ .../src/test/conf/gora-mongodb-mapping.xml | 1 + 5 files changed, 24 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/pom.xml ---------------------------------------------------------------------- diff --git a/gora-mongodb/pom.xml b/gora-mongodb/pom.xml index 5ff0b21..f48270c 100644 --- a/gora-mongodb/pom.xml +++ b/gora-mongodb/pom.xml @@ -119,7 +119,7 @@ </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java index ec440da..81fe765 100644 --- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java +++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java @@ -71,7 +71,10 @@ public class MongoDBQuery<K, T extends PersistentBase> extends QueryBase<K, T> { BasicDBObject proj = new BasicDBObject(); for (String k : fields) { - proj.put(mapping.getDocumentField(k), true); + String dbFieldName = mapping.getDocumentField(k); + if (dbFieldName != null && dbFieldName.length() > 0) { + proj.put(dbFieldName, true); + } } return proj; http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java index b3550ef..d1c53e7 100644 --- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java +++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java @@ -75,7 +75,6 @@ public class MongoDBResult<K, T extends PersistentBase> extends ResultBase<K, T> persistent = ((MongoStore<K, T>) getDataStore()).newInstance(obj, getQuery().getFields()); if (persistent != null) { - persistent.clearNew(); persistent.clearDirty(); } http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java index e94113a..3313469 100644 --- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java +++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java @@ -37,13 +37,11 @@ import org.apache.gora.mongodb.utils.BSONDecorator; import org.apache.gora.mongodb.utils.GoraDBEncoder; import org.apache.gora.mongodb.query.MongoDBQuery; import org.apache.gora.mongodb.query.MongoDBResult; -import org.apache.gora.persistency.ListGenericArray; +import org.apache.gora.persistency.impl.DirtyListWrapper; import org.apache.gora.persistency.Persistent; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.StatefulHashMap; +import org.apache.gora.persistency.impl.DirtyMapWrapper; import org.apache.gora.persistency.impl.BeanFactoryImpl; import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; import org.apache.gora.query.PartitionQuery; import org.apache.gora.query.Query; import org.apache.gora.query.Result; @@ -353,7 +351,6 @@ public class MongoStore<K, T extends PersistentBase> extends // Build the corresponding persistent and clears its states final T persistent = newInstance(res, fields); if (persistent != null) { - persistent.clearNew(); persistent.clearDirty(); } // Done @@ -371,7 +368,7 @@ public class MongoStore<K, T extends PersistentBase> extends @Override public void put(K key, T obj) { // Save the object in the database - if (obj.isNew() || obj.isDirty()) { + if (obj.isDirty()) { putUpdate(key, obj); // TODO: why is Nutch marking objects as new ? // putInsert(key, obj); @@ -381,7 +378,6 @@ public class MongoStore<K, T extends PersistentBase> extends LOG.info("Ignored putting object {} in the store as it is neither " + "new, neither dirty.", new Object[] { obj }); // Clear its state - obj.clearNew(); obj.clearDirty(); } } @@ -524,7 +520,6 @@ public class MongoStore<K, T extends PersistentBase> extends BSONDecorator easybson = new BSONDecorator(obj); // Create new empty persistent bean instance T persistent = newPersistent(); - StateManager stateManager = persistent.getStateManager(); fields = getFieldsToQuery(fields); // Populate each field @@ -546,7 +541,7 @@ public class MongoStore<K, T extends PersistentBase> extends Object result = fromDBObject(fieldSchema, storeType, field, docf, easybson); persistent.put(field.pos(), result); } - stateManager.clearDirty(persistent); + persistent.clearDirty(); return persistent; } @@ -555,7 +550,7 @@ public class MongoStore<K, T extends PersistentBase> extends switch (fieldSchema.getType()) { case MAP: BasicDBObject map = easybson.getDBObject(docf); - StatefulHashMap<Utf8, Object> rmap = new StatefulHashMap<Utf8, Object>(); + Map<Utf8, Object> rmap = new HashMap<Utf8, Object>(); for (Entry<String, Object> e : map.entrySet()) { // ensure Key decoding -> middle dots replaced with dots // FIXME: better approach ? @@ -573,31 +568,28 @@ public class MongoStore<K, T extends PersistentBase> extends break; } } - rmap.clearStates(); - result = rmap; + result = new DirtyMapWrapper(rmap); break; case ARRAY: List<Object> list = easybson.getDBList(docf); switch (fieldSchema.getElementType().getType()) { case STRING: - ListGenericArray<Utf8> arrS = new ListGenericArray<Utf8>(fieldSchema); + List<Utf8> arrS = new ArrayList<Utf8>(); for (Object o : list) arrS.add(new Utf8((String) o)); - result = arrS; + result = new DirtyListWrapper<Utf8>(arrS); break; case BYTES: - ListGenericArray<ByteBuffer> arrB = new ListGenericArray<ByteBuffer>( - fieldSchema); + List<ByteBuffer> arrB = new ArrayList<ByteBuffer>(); for (Object o : list) arrB.add(ByteBuffer.wrap((byte[]) o)); - result = arrB; + result = new DirtyListWrapper<ByteBuffer>(arrB); break; default: - ListGenericArray<Object> arrT = new ListGenericArray<Object>( - fieldSchema); + List<Object> arrT = new ArrayList<Object>(); for (Object o : list) arrT.add(o); - result = arrT; + result = new DirtyListWrapper<Object>(arrT); break; } break; @@ -756,8 +748,7 @@ public class MongoStore<K, T extends PersistentBase> extends private BasicDBObject newUpdateSetInstance(T persistent) { BasicDBObject result = new BasicDBObject(); for (Field f : persistent.getSchema().getFields()) { - if (persistent.isReadable(f.pos()) && persistent.isDirty(f.pos()) - && (persistent.get(f.pos()) != null)) { + if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) != null)) { String docf = mapping.getDocumentField(f.name()); Object value = persistent.get(f.pos()); DocumentFieldType storeType = mapping.getDocumentFieldType(docf); @@ -791,8 +782,7 @@ public class MongoStore<K, T extends PersistentBase> extends private BasicDBObject newUpdateUnsetInstance(T persistent) { BasicDBObject result = new BasicDBObject(); for (Field f : persistent.getSchema().getFields()) { - if (persistent.isReadable(f.pos()) && persistent.isDirty(f.pos()) - && (persistent.get(f.pos()) == null)) { + if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) == null)) { String docf = mapping.getDocumentField(f.name()); Object value = persistent.get(f.pos()); DocumentFieldType storeType = mapping.getDocumentFieldType(docf); @@ -825,7 +815,7 @@ public class MongoStore<K, T extends PersistentBase> extends + ": To store a Gora 'array', target Mongo mapping have to be of 'list' type"); } Schema elementSchema = fieldSchema.getElementType(); - result = toMongoList((GenericArray<?>) value, elementSchema.getType()); + result = toMongoList((List<?>) value, elementSchema.getType()); break; case BYTES: // Beware of ByteBuffer not being safely serialized @@ -957,7 +947,7 @@ public class MongoStore<K, T extends PersistentBase> extends case ARRAY: easybson.put( key, - toMongoList((GenericArray<?>) value, field.schema().getElementType() + toMongoList((List<?>) value, field.schema().getElementType() .getType())); break; case BYTES: @@ -990,7 +980,7 @@ public class MongoStore<K, T extends PersistentBase> extends case ARRAY: record.put( member.name(), - toMongoList((GenericArray<?>) recValue, member.schema() + toMongoList((List<?>) recValue, member.schema() .getElementType().getType())); break; case LONG: @@ -1077,7 +1067,7 @@ public class MongoStore<K, T extends PersistentBase> extends * @return a {@link BasicDBList} version of the {@link GenericArray} that can * be safely serialized into MongoDB. */ - private BasicDBList toMongoList(GenericArray<?> array, Type type) { + private BasicDBList toMongoList(Collection<?> array, Type type) { // Handle null case if (array == null) return null; http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml b/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml index a99a12b..b89c888 100644 --- a/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml +++ b/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml @@ -31,6 +31,7 @@ <field name="content" docfield="content" type="binary"/> <field name="parsedContent" docfield="pContent" type="list"/> <field name="outlinks" docfield="links.out" type="document"/> + <field name="headers" docfield="headers" type="document"/> <field name="metadata" docfield="metadata" type="document"/> </class>