Update based on feedback from code review.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/b5ce0a21 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/b5ce0a21 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/b5ce0a21 Branch: refs/heads/site Commit: b5ce0a21d8b5dc32930b9030a1cd40b40b39379e Parents: ef6b20c Author: Preston Carman <[email protected]> Authored: Thu Jun 12 21:41:11 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Thu Jun 12 21:41:11 2014 -0700 ---------------------------------------------------------------------- src/site/apt/user_running_tests.apt | 15 +- .../scripts/run_benchmark_cluster.sh | 7 +- .../accessors/nodes/AttributeNodePointable.java | 6 + .../exceptions/VXQueryDataException.java | 1 - .../VXQueryCollectionOperatorDescriptor.java | 6 +- .../FnDocAvailableScalarEvaluatorFactory.java | 3 +- .../node/FnDocScalarEvaluatorFactory.java | 3 +- .../runtime/functions/util/FunctionHelper.java | 2 +- .../vxquery/xmlparser/SAXContentHandler.java | 372 +++++++++---------- .../org/apache/vxquery/xmlparser/XMLParser.java | 38 +- .../xmlquery/query/XMLQueryCompiler.java | 3 +- 11 files changed, 237 insertions(+), 219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/src/site/apt/user_running_tests.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user_running_tests.apt b/src/site/apt/user_running_tests.apt index 0e41ab5..ef9748b 100644 --- a/src/site/apt/user_running_tests.apt +++ b/src/site/apt/user_running_tests.apt @@ -19,7 +19,7 @@ Running the Test Suite * Command ---------------------------------------- -sh ./apache-vxquery-X.Y-incubating/vxquery/vxquery-xtest/target/appassembler/bin/xtest +sh ./vxquery/vxquery-xtest/target/appassembler/bin/xtest ---------------------------------------- * Command Line Options @@ -70,7 +70,18 @@ $ cd .. * run the tests, and ---------------------------------------- -JAVA_OPTS="-Xmx1024m -Djava.util.logging.config.file=/path/to/logging.properties" sh ./apache-vxquery-X.Y-incubating/vxquery-xtest/target/appassembler/bin/xtest -catalog xqts/XQTSCatalog.xml -htmlreport /tmp/full_report.html +sh ./vxquery-xtest/target/appassembler/bin/xtest -catalog xqts/XQTSCatalog.xml -htmlreport /tmp/full_report.html ---------------------------------------- * view the results at {{{file:///tmp/full_report.html}file:///tmp/full_report.html}}. + * Add JAVA_OPTS for additional java parameters. + +* VXQuery Test Suite + + The VXQuery test suite focuses on parallel processing tests used in our weather benchmark. + +---------------------------------------- +sh ./vxquery-xtest/target/appassembler/bin/xtest -catalog vxquery-xtest/src/test/resources/VXQueryCatalog.xml -htmlreport /tmp/full_report.html +---------------------------------------- + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_benchmark_cluster.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_benchmark_cluster.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_benchmark_cluster.sh index 74bb088..f31f9e7 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_benchmark_cluster.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_benchmark_cluster.sh @@ -43,9 +43,12 @@ fi # Run queries for the specified number of nodes. echo "Starting ${2} cluster nodes" python vxquery-server/src/main/resources/scripts/cluster_cli.py -c vxquery-server/src/main/resources/conf/${2}nodes.xml -a start - + +# wait for cluster to finish setting up +sleep 5 + export JAVA_OPTS="$JAVA_OPTS -server -Xmx8G -XX:+HeapDumpOnOutOfMemoryError -Djava.util.logging.config.file=./vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/benchmark_logging.properties" - + for j in $(find ${1} -name '*q??.xq') do # Only work with i nodes. http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/AttributeNodePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/AttributeNodePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/AttributeNodePointable.java index 264f5ed..22d651e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/AttributeNodePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/AttributeNodePointable.java @@ -32,6 +32,12 @@ import edu.uci.ics.hyracks.data.std.primitive.VoidPointable; * NodeId nodeId?; * TaggedValue value; * } + * + * NamePtr { + * Integer PrefixCode; + * Integer NamespaceCode; + * Integer LocalCode; + * } */ public class AttributeNodePointable extends AbstractPointable { private static final int LOCAL_NODE_ID_SIZE = 4; http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryDataException.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryDataException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryDataException.java index c05a023..79a37af 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryDataException.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryDataException.java @@ -29,7 +29,6 @@ public class VXQueryDataException extends HyracksDataException { public VXQueryDataException(String message, Exception ex, File file) { super(message, ex); this.file = file; - setStackTrace(ex.getStackTrace()); } @Override http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index ad8e19a..89968d5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -74,7 +74,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; - final XMLParser parser = new XMLParser(false, nodeIdProvider, frame, appender, childSeq, + final XMLParser parser = new XMLParser(false, nodeIdProvider, nodeId, frame, appender, childSeq, dCtx.getStaticContext()); return new AbstractUnaryInputUnaryOutputOperatorNodePushable() { @@ -92,7 +92,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO // Go through each tuple. if (collectionDirectory.isDirectory()) { - for (int t = 0; t < fta.getTupleCount(); ++t) { + for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { @SuppressWarnings("unchecked") Iterator<File> it = FileUtils.iterateFiles(collectionDirectory, new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE); @@ -101,7 +101,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Starting to read XML document: " + xmlDocument.getAbsolutePath()); } - parser.parseOutElements(xmlDocument, writer, fta, t); + parser.parseElements(xmlDocument, writer, fta, tupleIndex); } } } else { http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java index ad4b1f0..3f74a4f 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java @@ -57,6 +57,7 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); + final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override @@ -76,7 +77,7 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg } tvp.getValue(stringp); try { - XMLParser parser = new XMLParser(false, nodeIdProvider); + XMLParser parser = new XMLParser(false, nodeIdProvider, nodeId); FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); XDMConstants.setTrue(result); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 2c9fc03..2d957b6 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -57,6 +57,7 @@ public class FnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); + final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override @@ -77,7 +78,7 @@ public class FnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal tvp.getValue(stringp); try { // Only one document should be parsed so its ok to have a unique parser. - XMLParser parser = new XMLParser(false, nodeIdProvider); + XMLParser parser = new XMLParser(false, nodeIdProvider, nodeId); FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 2d1c239..799d9c7 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -1190,7 +1190,7 @@ public class FunctionHelper { throw new HyracksDataException(e); } File file = new File(fName); - parser.parseFile(file, abvs); + parser.parseDocument(file, abvs); } public static boolean transformThenCompareMinMaxTaggedValues(AbstractValueComparisonOperation aOp, http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java index 53e0cda..5057aaf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java @@ -52,64 +52,43 @@ import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender; import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils; public class SAXContentHandler implements ContentHandler, LexicalHandler { + // XML node builders private final AttributeNodeBuilder anb; + private final CommentNodeBuilder cnb; + private final DictionaryBuilder db; + private final DocumentNodeBuilder docb; + private final PINodeBuilder pinb; + private final TextNodeBuilder tnb; + private final List<ElementNodeBuilder> enbStack; + private final List<ElementNodeBuilder> freeENBList; + // Frame writing variables private FrameTupleAppender appender; + private ByteBuffer frame; + private FrameTupleAccessor fta; + private int tupleIndex; + private IFrameWriter writer; - private final boolean attachTypes; - - private final StringBuilder buffer; - + // Element writing and path step variables + private boolean skipping; private String[] childLocalName = null; - private String[] childUri = null; + private boolean[] subElement = null; + private final TaggedValuePointable tvp; - private final CommentNodeBuilder cnb; - + // Basic tracking and setting variables + private final boolean attachTypes; + private final StringBuilder buffer; private final boolean createNodeIds; - - private final DictionaryBuilder db; - private int depth = 0; - private final ArrayBackedValueStorage docABVS; - - private final DocumentNodeBuilder docb; - private final ArrayBackedValueStorage elementABVS; - - private final List<ElementNodeBuilder> enbStack; - - private ByteBuffer frame; - - private final List<ElementNodeBuilder> freeENBList; - - private FrameTupleAccessor fta; - + private boolean pendingText; private int nodeIdCounter; - private final ITreeNodeIdProvider nodeIdProvider; - - private boolean pendingText; - - private final PINodeBuilder pinb; - private final ArrayBackedValueStorage resultABVS; - - private boolean skipping; - - private boolean[] subElement = null; - - private int t; - private final ArrayBackedValueStorage tempABVS; - private final TextNodeBuilder tnb; - - private final TaggedValuePointable tvp; - - private IFrameWriter writer; - public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider) { docABVS = new ArrayBackedValueStorage(); elementABVS = new ArrayBackedValueStorage(); @@ -140,56 +119,80 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { setChildPathSteps(childSequenceTypes); } - @Override - public void characters(char[] ch, int start, int length) throws SAXException { - if (!skipping) { - buffer.append(ch, start, length); - pendingText = true; + private void setChildPathSteps(List<SequenceType> childSeq) { + if (!childSeq.isEmpty()) { + subElement = new boolean[childSeq.size()]; + childUri = new String[childSeq.size()]; + childLocalName = new String[childSeq.size()]; + } + + int index = 0; + for (SequenceType sType : childSeq) { + NodeType nodeType = (NodeType) sType.getItemType(); + ElementType eType = (ElementType) nodeType; + NameTest nameTest = eType.getNameTest(); + childUri[index] = getStringFromBytes(nameTest.getUri()); + childLocalName[index] = getStringFromBytes(nameTest.getLocalName());; + ++index; } } + public void setupElementWriter(IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex) { + this.writer = writer; + this.fta = fta; + this.tupleIndex = tupleIndex; + } + @Override - public void endDocument() throws SAXException { - if (!skipping) { - try { - flushText(); - docb.endChildrenChunk(); - docb.finish(); - if (frame != null && appender != null) { - writeElement(); - } - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); - } + public void characters(char[] ch, int start, int length) throws SAXException { + if (skipping) { + return; } + buffer.append(ch, start, length); + pendingText = true; } - private void endElementChildPathStep() throws IOException { - if (foundFirstNonSkippedElement()) { - writeElement(); + @Override + public void endDocument() throws SAXException { + if (skipping) { + return; } - if (subElement != null && depth <= subElement.length) { - subElement[depth - 1] = false; + try { + flushText(); + docb.endChildrenChunk(); + docb.finish(); + if (frame != null && appender != null) { + writeElement(); + } + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } } @Override public void endElement(String uri, String localName, String name) throws SAXException { - if (!skipping) { - try { - flushText(); - ElementNodeBuilder enb = enbStack.remove(enbStack.size() - 1); - enb.endChildrenChunk(); - endChildInParent(enb, foundFirstNonSkippedElement()); - freeENB(enb); - endElementChildPathStep(); - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); + if (skipping) { + return; + } + try { + boolean nonSkipped = foundFirstNonSkippedElement(); + flushText(); + ElementNodeBuilder enb = enbStack.remove(enbStack.size() - 1); + enb.endChildrenChunk(); + endChildInParent(enb, nonSkipped); + freeENB(enb); + if (nonSkipped) { + writeElement(); + } + if (subElement != null && depth <= subElement.length) { + subElement[depth - 1] = false; } + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } - depth--; + --depth; } @Override @@ -202,24 +205,25 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { @Override public void processingInstruction(String target, String data) throws SAXException { - if (!skipping) { - try { - flushText(); - startChildInParent(pinb); - tempABVS.reset(); - tempABVS.getDataOutput().writeUTF(target); - if (createNodeIds) { - pinb.setLocalNodeId(nodeIdCounter++); - } - pinb.setTarget(tempABVS); - tempABVS.reset(); - tempABVS.getDataOutput().writeUTF(data); - pinb.setContent(tempABVS); - endChildInParent(pinb); - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); + if (skipping) { + return; + } + try { + flushText(); + startChildInParent(pinb); + tempABVS.reset(); + tempABVS.getDataOutput().writeUTF(target); + if (createNodeIds) { + pinb.setLocalNodeId(nodeIdCounter++); } + pinb.setTarget(tempABVS); + tempABVS.reset(); + tempABVS.getDataOutput().writeUTF(data); + pinb.setContent(tempABVS); + endChildInParent(pinb); + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } } @@ -238,18 +242,19 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { } db.reset(); docABVS.reset(); - if (!skipping) { - try { - docb.reset(docABVS); - if (createNodeIds) { - docb.setLocalNodeId(nodeIdCounter++); - } - docb.startChildrenChunk(); - flushText(); - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); + if (skipping) { + return; + } + try { + docb.reset(docABVS); + if (createNodeIds) { + docb.setLocalNodeId(nodeIdCounter++); } + docb.startChildrenChunk(); + flushText(); + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } } @@ -271,63 +276,64 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { @Override public void startElement(String uri, String localName, String name, Attributes atts) throws SAXException { - depth++; + ++depth; boolean start = startElementChildPathStep(uri, localName); - if (!skipping) { - try { - flushText(); - int idx = name.indexOf(':'); - String prefix = idx < 0 ? "" : name.substring(0, idx); - ElementNodeBuilder enb = createENB(); - startChildInParent(enb, start); - int uriCode = db.lookup(uri); - int localNameCode = db.lookup(localName); - int prefixCode = db.lookup(prefix); - enb.setName(uriCode, localNameCode, prefixCode); + if (skipping) { + return; + } + try { + flushText(); + int idx = name.indexOf(':'); + String prefix = idx < 0 ? "" : name.substring(0, idx); + ElementNodeBuilder enb = createENB(); + startChildInParent(enb, start); + int uriCode = db.lookup(uri); + int localNameCode = db.lookup(localName); + int prefixCode = db.lookup(prefix); + enb.setName(uriCode, localNameCode, prefixCode); + if (attachTypes) { + int typeUriCode = db.lookup(XQueryConstants.XS_NSURI); + int typeLocalNameCode = db.lookup(BuiltinTypeQNames.UNTYPED_STR); + int typePrefixCode = db.lookup(XQueryConstants.XS_PREFIX); + enb.setType(typeUriCode, typeLocalNameCode, typePrefixCode); + } + if (createNodeIds) { + enb.setLocalNodeId(nodeIdCounter++); + } + enb.startAttributeChunk(); + final int nAttrs = atts.getLength(); + for (int i = 0; i < nAttrs; ++i) { + String aName = atts.getQName(i); + int aIdx = aName.indexOf(':'); + int aPrefixCode = db.lookup(aIdx < 0 ? "" : aName.substring(0, aIdx)); + int aLocalNameCode = db.lookup(atts.getLocalName(i)); + int aUriCode = db.lookup(atts.getURI(i)); + String aValue = atts.getValue(i); + tempABVS.reset(); + DataOutput tempOut = tempABVS.getDataOutput(); + tempOut.write(ValueTag.XS_UNTYPED_ATOMIC_TAG); + tempOut.writeUTF(aValue); + enb.startAttribute(anb); + anb.setName(aUriCode, aLocalNameCode, aPrefixCode); if (attachTypes) { int typeUriCode = db.lookup(XQueryConstants.XS_NSURI); - int typeLocalNameCode = db.lookup(BuiltinTypeQNames.UNTYPED_STR); + int typeLocalNameCode = db.lookup(BuiltinTypeQNames.UNTYPED_ATOMIC_STR); int typePrefixCode = db.lookup(XQueryConstants.XS_PREFIX); - enb.setType(typeUriCode, typeLocalNameCode, typePrefixCode); + anb.setType(typeUriCode, typeLocalNameCode, typePrefixCode); } if (createNodeIds) { - enb.setLocalNodeId(nodeIdCounter++); + anb.setLocalNodeId(nodeIdCounter++); } - enb.startAttributeChunk(); - final int nAttrs = atts.getLength(); - for (int i = 0; i < nAttrs; ++i) { - String aName = atts.getQName(i); - int aIdx = aName.indexOf(':'); - int aPrefixCode = db.lookup(aIdx < 0 ? "" : aName.substring(0, aIdx)); - int aLocalNameCode = db.lookup(atts.getLocalName(i)); - int aUriCode = db.lookup(atts.getURI(i)); - String aValue = atts.getValue(i); - tempABVS.reset(); - DataOutput tempOut = tempABVS.getDataOutput(); - tempOut.write(ValueTag.XS_UNTYPED_ATOMIC_TAG); - tempOut.writeUTF(aValue); - enb.startAttribute(anb); - anb.setName(aUriCode, aLocalNameCode, aPrefixCode); - if (attachTypes) { - int typeUriCode = db.lookup(XQueryConstants.XS_NSURI); - int typeLocalNameCode = db.lookup(BuiltinTypeQNames.UNTYPED_ATOMIC_STR); - int typePrefixCode = db.lookup(XQueryConstants.XS_PREFIX); - anb.setType(typeUriCode, typeLocalNameCode, typePrefixCode); - } - if (createNodeIds) { - anb.setLocalNodeId(nodeIdCounter++); - } - anb.setValue(tempABVS); - enb.endAttribute(anb); - } - enb.endAttributeChunk(); - enb.startChildrenChunk(); - enbStack.add(enb); - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); + anb.setValue(tempABVS); + enb.endAttribute(anb); } + enb.endAttributeChunk(); + enb.startChildrenChunk(); + enbStack.add(enb); + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } } @@ -337,23 +343,24 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { @Override public void comment(char[] ch, int start, int length) throws SAXException { - if (!skipping) { - try { - flushText(); - startChildInParent(cnb); - buffer.append(ch, start, length); - tempABVS.reset(); - tempABVS.getDataOutput().writeUTF(buffer.toString()); - if (createNodeIds) { - cnb.setLocalNodeId(nodeIdCounter++); - } - cnb.setValue(tempABVS); - endChildInParent(cnb); - buffer.delete(0, buffer.length()); - } catch (IOException e) { - e.printStackTrace(); - throw new SAXException(e); + if (skipping) { + return; + } + try { + flushText(); + startChildInParent(cnb); + buffer.append(ch, start, length); + tempABVS.reset(); + tempABVS.getDataOutput().writeUTF(buffer.toString()); + if (createNodeIds) { + cnb.setLocalNodeId(nodeIdCounter++); } + cnb.setValue(tempABVS); + endChildInParent(cnb); + buffer.delete(0, buffer.length()); + } catch (IOException e) { + e.printStackTrace(); + throw new SAXException(e); } } @@ -396,31 +403,6 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { public void startEntity(String name) throws SAXException { } - private void setChildPathSteps(List<SequenceType> childSeq) { - // this.childSeq = childSeq; - if (!childSeq.isEmpty()) { - subElement = new boolean[childSeq.size()]; - childUri = new String[childSeq.size()]; - childLocalName = new String[childSeq.size()]; - } - - int index = 0; - for (SequenceType sType : childSeq) { - NodeType nodeType = (NodeType) sType.getItemType(); - ElementType eType = (ElementType) nodeType; - NameTest nameTest = eType.getNameTest(); - childUri[index] = getStringFromBytes(nameTest.getUri()); - childLocalName[index] = getStringFromBytes(nameTest.getLocalName());; - index++; - } - } - - public void setupElementWriter(IFrameWriter writer, FrameTupleAccessor fta, int t) { - this.writer = writer; - this.fta = fta; - this.t = t; - } - public void writeElement() throws IOException { resultABVS.reset(); DataOutput out = resultABVS.getDataOutput(); @@ -443,7 +425,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler { out.write(elementABVS.getByteArray(), elementABVS.getStartOffset(), elementABVS.getLength()); } tvp.set(resultABVS.getByteArray(), resultABVS.getStartOffset(), resultABVS.getLength()); - addNodeToTuple(tvp, t); + addNodeToTuple(tvp, tupleIndex); skipping = true; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java index c1fd6f0..0c31d27 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java @@ -43,14 +43,16 @@ public class XMLParser { final XMLReader parser; final SAXContentHandler handler; final InputSource in; + final String nodeId; - public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider) throws HyracksDataException { - this(attachTypes, idProvider, null, null, null, null); + public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, String nodeId) throws HyracksDataException { + this(attachTypes, idProvider, nodeId, null, null, null, null); } - public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, ByteBuffer frame, + public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, String nodeId, ByteBuffer frame, FrameTupleAppender appender, List<Integer> childSeq, StaticContext staticContext) throws HyracksDataException { + this.nodeId = nodeId; try { parser = XMLReaderFactory.createXMLReader(); if (frame == null || appender == null) { @@ -70,7 +72,7 @@ public class XMLParser { } } - public void parseFile(File file, ArrayBackedValueStorage abvs) throws HyracksDataException { + public void parseDocument(File file, ArrayBackedValueStorage abvs) throws HyracksDataException { try { if (file.getName().toLowerCase().endsWith(".xml.gz")) { in.setCharacterStream(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)))); @@ -80,15 +82,21 @@ public class XMLParser { parser.parse(in); handler.writeDocument(abvs); } catch (FileNotFoundException e) { - throw new VXQueryFileNotFoundException(e, file); + HyracksDataException hde = new VXQueryFileNotFoundException(e, file); + hde.setNodeId(nodeId); + throw hde; } catch (SAXException e) { - throw new VXQueryParseException(e, file); + HyracksDataException hde = new VXQueryParseException(e, file); + hde.setNodeId(nodeId); + throw hde; } catch (IOException e) { - throw new HyracksDataException(e); + HyracksDataException hde = new HyracksDataException(e); + hde.setNodeId(nodeId); + throw hde; } } - public void parseOutElements(File file, IFrameWriter writer, FrameTupleAccessor fta, int t) + public void parseElements(File file, IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex) throws HyracksDataException { try { if (file.getName().toLowerCase().endsWith(".xml.gz")) { @@ -96,14 +104,20 @@ public class XMLParser { } else { in.setCharacterStream(new InputStreamReader(new FileInputStream(file))); } - handler.setupElementWriter(writer, fta, t); + handler.setupElementWriter(writer, fta, tupleIndex); parser.parse(in); } catch (FileNotFoundException e) { - throw new VXQueryFileNotFoundException(e, file); + HyracksDataException hde = new VXQueryFileNotFoundException(e, file); + hde.setNodeId(nodeId); + throw hde; } catch (SAXException e) { - throw new VXQueryParseException(e, file); + HyracksDataException hde = new VXQueryParseException(e, file); + hde.setNodeId(nodeId); + throw hde; } catch (IOException e) { - throw new HyracksDataException(e); + HyracksDataException hde = new HyracksDataException(e); + hde.setNodeId(nodeId); + throw hde; } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/b5ce0a21/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java index 15443a9..2fdfbea 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java @@ -41,6 +41,7 @@ import org.apache.vxquery.xmlquery.ast.ModuleNode; import org.apache.vxquery.xmlquery.translator.XMLQueryTranslator; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException; import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder; import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler; @@ -152,7 +153,7 @@ public class XMLQueryCompiler { builder.setNullableTypeComputer(new INullableTypeComputer() { @Override public Object makeNullableType(Object type) throws AlgebricksException { - return new VXQueryNullableTypeComputer(); + throw new NotImplementedException("NullableTypeComputer is not implented"); } }); builder.setNullWriterFactory(new VXQueryNullWriterFactory());
