Yingyi Bu has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/334
Change subject: 1. Fix the memory bloat issue introduced by https://github.com/apache/incubator-asterixdb/commit/c66d23a5ac65ec5218ee47134aea423fd62a32cc Recycle object pools in ADMDataParser before parsing each top-level ADM record. ...................................................................... 1. Fix the memory bloat issue introduced by https://github.com/apache/incubator-asterixdb/commit/c66d23a5ac65ec5218ee47134aea423fd62a32cc Recycle object pools in ADMDataParser before parsing each top-level ADM record. 2. Avoid using Strings as parameters of object pools because string comparison is slow and several comparisons done for parsing each ADM record. Change-Id: I31848d8bd42a07f72035aefa2d671fdb2fce037d --- M asterix-om/src/main/java/edu/uci/ics/asterix/builders/AbvsBuilderFactory.java M asterix-om/src/main/java/edu/uci/ics/asterix/builders/ListBuilderFactory.java M asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilderFactory.java M asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/AListPointable.java M asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java M asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java M asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java 7 files changed, 80 insertions(+), 63 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/34/334/1 diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/AbvsBuilderFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/AbvsBuilderFactory.java index 9d0e434..46b29a5 100644 --- a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/AbvsBuilderFactory.java +++ b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/AbvsBuilderFactory.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,14 +14,15 @@ */ package edu.uci.ics.asterix.builders; +import edu.uci.ics.asterix.om.types.ATypeTag; import edu.uci.ics.asterix.om.util.container.IObjectFactory; import edu.uci.ics.hyracks.data.std.api.IMutableValueStorage; import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage; -public class AbvsBuilderFactory implements IObjectFactory<IMutableValueStorage, String> { +public class AbvsBuilderFactory implements IObjectFactory<IMutableValueStorage, ATypeTag> { @Override - public IMutableValueStorage create(String type) { + public IMutableValueStorage create(ATypeTag type) { return new ArrayBackedValueStorage(); } } diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/ListBuilderFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/ListBuilderFactory.java index da098e5..5c70022 100644 --- a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/ListBuilderFactory.java +++ b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/ListBuilderFactory.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,13 +14,14 @@ */ package edu.uci.ics.asterix.builders; +import edu.uci.ics.asterix.om.types.ATypeTag; import edu.uci.ics.asterix.om.util.container.IObjectFactory; -public class ListBuilderFactory implements IObjectFactory<IAsterixListBuilder, String> { +public class ListBuilderFactory implements IObjectFactory<IAsterixListBuilder, ATypeTag> { @Override - public IAsterixListBuilder create(String type) { - if (type.equals("ordered")) { + public IAsterixListBuilder create(ATypeTag typeTag) { + if (typeTag == ATypeTag.ORDEREDLIST) { return new OrderedListBuilder(); } else { return new UnorderedListBuilder(); diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilderFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilderFactory.java index 55ab2d0..d6c7cac 100644 --- a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilderFactory.java +++ b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilderFactory.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,12 +14,13 @@ */ package edu.uci.ics.asterix.builders; +import edu.uci.ics.asterix.om.types.ATypeTag; import edu.uci.ics.asterix.om.util.container.IObjectFactory; -public class RecordBuilderFactory implements IObjectFactory<IARecordBuilder, String> { +public class RecordBuilderFactory implements IObjectFactory<IARecordBuilder, ATypeTag> { @Override - public IARecordBuilder create(String type) { + public IARecordBuilder create(ATypeTag type) { return new RecordBuilder(); } } diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/AListPointable.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/AListPointable.java index d14443d..7e7f380 100644 --- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/AListPointable.java +++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/AListPointable.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,7 +33,7 @@ /* * This class interprets the binary data representation of a list. - * + * * List { * byte type; * int length; @@ -71,12 +71,13 @@ } }; - public static final IObjectFactory<IPointable, String> ALLOCATOR = new IObjectFactory<IPointable, String>() { - public IPointable create(String id) { + public static final IObjectFactory<IPointable, ATypeTag> ALLOCATOR = new IObjectFactory<IPointable, ATypeTag>() { + @Override + public IPointable create(ATypeTag type) { return new AListPointable(); } }; - + private static final int TAG_SIZE = 1; private static final int TYPE_SIZE = 1; private static final int LENGTH_SIZE = 4; @@ -129,6 +130,7 @@ return false; } + @Override public int getLength() { return IntegerPointable.getInteger(bytes, getLengthOffset()); } diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java index cb0e136..7e97efc 100644 --- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java +++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,7 +38,7 @@ /* * This class interprets the binary data representation of a record. - * + * * Record { * byte tag; * int length; @@ -52,12 +52,12 @@ * OpenFieldLookup[numberOfOpenFields] lookup; * OpenField[numberOfOpenFields] openFields; * } - * + * * OpenFieldLookup { * int hashCode; * int Offset; * } - * + * * OpenField { * StringPointable fieldName; * IPointable fieldValue; @@ -92,13 +92,13 @@ return TYPE_TRAITS; } }; - - public static final IObjectFactory<IPointable, String> ALLOCATOR = new IObjectFactory<IPointable, String>() { - public IPointable create(String id) { + + public static final IObjectFactory<IPointable, ATypeTag> ALLOCATOR = new IObjectFactory<IPointable, ATypeTag>() { + @Override + public IPointable create(ATypeTag type) { return new ARecordPointable(); } }; - private static final int TAG_SIZE = 1; private static final int RECORD_LENGTH_SIZE = 4; @@ -116,7 +116,7 @@ private static boolean isOpen(ARecordType recordType) { return recordType == null || recordType.isOpen(); } - + public int getSchemeFieldCount(ARecordType recordType) { return recordType.getFieldNames().length; } @@ -133,6 +133,7 @@ return TAG_SIZE; } + @Override public int getLength() { return IntegerPointable.getInteger(bytes, getLengthOffset()); } diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java index fbb9a1b..9855192 100644 --- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java +++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java @@ -22,7 +22,6 @@ import edu.uci.ics.asterix.builders.IAsterixListBuilder; import edu.uci.ics.asterix.builders.ListBuilderFactory; import edu.uci.ics.asterix.builders.OrderedListBuilder; -import edu.uci.ics.asterix.builders.RecordBuilder; import edu.uci.ics.asterix.builders.RecordBuilderFactory; import edu.uci.ics.asterix.common.exceptions.AsterixException; import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider; @@ -61,15 +60,15 @@ private final static AString nestedName = new AString("nested"); private final static AString listName = new AString("list"); - private IObjectPool<IARecordBuilder, String> recordBuilderPool = new ListObjectPool<IARecordBuilder, String>( + private IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool = new ListObjectPool<IARecordBuilder, ATypeTag>( new RecordBuilderFactory()); - private IObjectPool<IAsterixListBuilder, String> listBuilderPool = new ListObjectPool<IAsterixListBuilder, String>( + private IObjectPool<IAsterixListBuilder, ATypeTag> listBuilderPool = new ListObjectPool<IAsterixListBuilder, ATypeTag>( new ListBuilderFactory()); - private IObjectPool<IMutableValueStorage, String> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, String>( + private IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, ATypeTag>( new AbvsBuilderFactory()); - private IObjectPool<IPointable, String> recordPointablePool = new ListObjectPool<IPointable, String>( + private IObjectPool<IPointable, ATypeTag> recordPointablePool = new ListObjectPool<IPointable, ATypeTag>( ARecordPointable.ALLOCATOR); - private IObjectPool<IPointable, String> listPointablePool = new ListObjectPool<IPointable, String>( + private IObjectPool<IPointable, ATypeTag> listPointablePool = new ListObjectPool<IPointable, ATypeTag>( AListPointable.ALLOCATOR); private final static AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "fields"); @@ -82,7 +81,7 @@ private final static ARecordType openType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE; - public void processRecord(ARecordPointable recordAccessor, ARecordType recType, DataOutput out, int level) + public void processRecord(ARecordPointable recordAccessor, ARecordType recType, DataOutput out, int level) throws IOException, AsterixException, AlgebricksException { ArrayBackedValueStorage itemValue = getTempBuffer(); ArrayBackedValueStorage fieldName = getTempBuffer(); @@ -168,8 +167,8 @@ orderedListBuilder.write(out, true); } - public void addNameField(IValueReference nameArg, IARecordBuilder fieldRecordBuilder) - throws HyracksDataException, AsterixException { + public void addNameField(IValueReference nameArg, IARecordBuilder fieldRecordBuilder) throws HyracksDataException, + AsterixException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); fieldAbvs.reset(); @@ -177,7 +176,7 @@ fieldRecordBuilder.addField(fieldAbvs, nameArg); } - public void addFieldType(byte tagId, IARecordBuilder fieldRecordBuilder) throws HyracksDataException, + public void addFieldType(byte tagId, IARecordBuilder fieldRecordBuilder) throws HyracksDataException, AsterixException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); ArrayBackedValueStorage valueAbvs = getTempBuffer(); @@ -194,7 +193,7 @@ fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void addIsOpenField(boolean isOpen, IARecordBuilder fieldRecordBuilder) throws HyracksDataException, + public void addIsOpenField(boolean isOpen, IARecordBuilder fieldRecordBuilder) throws HyracksDataException, AsterixException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); ArrayBackedValueStorage valueAbvs = getTempBuffer(); @@ -212,8 +211,8 @@ fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void addListField(IValueReference listArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, - int level) throws AsterixException, IOException, AlgebricksException { + public void addListField(IValueReference listArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, int level) + throws AsterixException, IOException, AlgebricksException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); ArrayBackedValueStorage valueAbvs = getTempBuffer(); @@ -226,7 +225,7 @@ fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void addNestedField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, + public void addNestedField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, int level) throws HyracksDataException, AlgebricksException, IOException, AsterixException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); ArrayBackedValueStorage valueAbvs = getTempBuffer(); @@ -244,11 +243,11 @@ } ARecordPointable recordP = getRecordPointable(); recordP.set(recordArg); - processRecord(recordP, (ARecordType) newType, valueAbvs.getDataOutput(), level); + processRecord(recordP, newType, valueAbvs.getDataOutput(), level); fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void processListValue(IValueReference listArg, IAType fieldType, DataOutput out, int level) + public void processListValue(IValueReference listArg, IAType fieldType, DataOutput out, int level) throws AsterixException, IOException, AlgebricksException { ArrayBackedValueStorage itemValue = getTempBuffer(); IARecordBuilder listRecordBuilder = getRecordBuilder(); @@ -281,23 +280,23 @@ innerListBuilder.write(out, true); } - private ARecordPointable getRecordPointable() { - return (ARecordPointable) recordPointablePool.allocate("record"); + private ARecordPointable getRecordPointable() { + return (ARecordPointable) recordPointablePool.allocate(ATypeTag.RECORD); } - private AListPointable getListPointable() { - return (AListPointable) listPointablePool.allocate("list"); + private AListPointable getListPointable() { + return (AListPointable) listPointablePool.allocate(ATypeTag.ORDEREDLIST); } - private IARecordBuilder getRecordBuilder() { - return (RecordBuilder) recordBuilderPool.allocate("record"); + private IARecordBuilder getRecordBuilder() { + return recordBuilderPool.allocate(ATypeTag.RECORD); } - private OrderedListBuilder getOrderedListBuilder() { - return (OrderedListBuilder) listBuilderPool.allocate("ordered"); + private OrderedListBuilder getOrderedListBuilder() { + return (OrderedListBuilder) listBuilderPool.allocate(ATypeTag.ORDEREDLIST); } - private ArrayBackedValueStorage getTempBuffer() { - return (ArrayBackedValueStorage) abvsBuilderPool.allocate("buffer"); + private ArrayBackedValueStorage getTempBuffer() { + return (ArrayBackedValueStorage) abvsBuilderPool.allocate(ATypeTag.BINARY); } } diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java index 1bdd264..e7198d3 100644 --- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java +++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java @@ -62,11 +62,11 @@ private int nullableFieldId = 0; private ArrayBackedValueStorage castBuffer = new ArrayBackedValueStorage(); - private IObjectPool<IARecordBuilder, String> recordBuilderPool = new ListObjectPool<IARecordBuilder, String>( + private IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool = new ListObjectPool<IARecordBuilder, ATypeTag>( new RecordBuilderFactory()); - private IObjectPool<IAsterixListBuilder, String> listBuilderPool = new ListObjectPool<IAsterixListBuilder, String>( + private IObjectPool<IAsterixListBuilder, ATypeTag> listBuilderPool = new ListObjectPool<IAsterixListBuilder, ATypeTag>( new ListBuilderFactory()); - private IObjectPool<IMutableValueStorage, String> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, String>( + private IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, ATypeTag>( new AbvsBuilderFactory()); private String mismatchErrorMessage = "Mismatch Type, expecting a value of type "; @@ -101,6 +101,7 @@ this.column = column; } + @Override public String getMessage() { StringBuilder msg = new StringBuilder("Parse error"); if (filename != null) { @@ -128,7 +129,8 @@ @Override public boolean parse(DataOutput out) throws AsterixException { try { - return parseAdmInstance((IAType) recordType, datasetRec, out); + resetPools(); + return parseAdmInstance(recordType, datasetRec, out); } catch (IOException e) { throw new ParseException(e, filename, admLexer.getLine(), admLexer.getColumn()); } catch (AdmLexerException e) { @@ -760,19 +762,19 @@ } private IARecordBuilder getRecordBuilder() { - return (RecordBuilder) recordBuilderPool.allocate("record"); + return recordBuilderPool.allocate(ATypeTag.RECORD); } private IAsterixListBuilder getOrderedListBuilder() { - return listBuilderPool.allocate("ordered"); + return listBuilderPool.allocate(ATypeTag.ORDEREDLIST); } private IAsterixListBuilder getUnorderedListBuilder() { - return listBuilderPool.allocate("unordered"); + return listBuilderPool.allocate(ATypeTag.UNORDEREDLIST); } private ArrayBackedValueStorage getTempBuffer() { - return (ArrayBackedValueStorage) abvsBuilderPool.allocate("buffer"); + return (ArrayBackedValueStorage) abvsBuilderPool.allocate(ATypeTag.BINARY); } private void parseToBinaryTarget(int lexerToken, String tokenImage, DataOutput out) throws ParseException, @@ -1086,4 +1088,14 @@ aInt64.setValue(value); int64Serde.serialize(aInt64, out); } + + /** + * Resets the pools before parsing a top-level record. + * In this way the elements in those pools can be re-used. + */ + private void resetPools() { + listBuilderPool.reset(); + recordBuilderPool.reset(); + abvsBuilderPool.reset(); + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/334 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I31848d8bd42a07f72035aefa2d671fdb2fce037d Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <[email protected]>
