This is an automated email from the ASF dual-hosted git repository. jinterrante pushed a commit to branch runtime2-2202 in repository https://gitbox.apache.org/repos/asf/daffodil.git
commit 3f96471b75412a966257dedac92fc239c33f15e7 Author: John Interrante <[email protected]> AuthorDate: Wed Dec 9 17:16:16 2020 -0500 Expand C code generator to handle more integer types Add another example schema to runtime2 tests and make C code generator able to handle more integer types. Starting to run into alignment issues currently causing different behavior between runtime1 and runtime2 and will need to handle these issues later. In xml_reader.c, rename xmlInt32Elem to xmlIntegerElem and expand it to read 8, 16, 32, and 64-bit signed/unsigned integer numbers from XML elements. Have not tested all cases yet. In xml_writer.c, rename xmlInt32Elem to xmlIntegerElem and expand it to write 8, 16, 32, and 64-bit signed/unsigned integers as XML element values. Have not tested all cases yet. In infoset.c, expand walkInfosetNode to call visitIntegerElem method for 8, 16, 32, and 64-bit signed/unsigned integer type codes. In infoset.h, rename VisitInt32Elem type to VisitIntegerElem and visitInteger32Elem field to visitIntegerElem. Add 8, 16, 32, and 64-bit signed/unsigned integer type codes to TypeCode. In ex_int32.c, update to reflect changes to generated code actually generated from ex_int32 root element in TestRuntime2.tdml. In ex_int32.h, update to reflect changes to generated code actually generated from ex_int32 root element in TestRuntime2.tdml. In Runtime2DataProcessor.scala, make parse and unparse methods remove temporary directories after using them and make ParseResult convert output file to XML immediately instead of waiting until called later since output file will no longer be around. In BinaryIntegerKnownLengthCodeGenerator.scala, remove some restrictions from when code generator could handle only 32-bit signed integers. Modify generated code more flexible and capable of handling different integer lengths. In CodeGeneratorState.scala, expand CodeGeneratorState to handle 8, 16, 32, and 64-bit signed/unsigned integers. Modify generated code to define dummy be8toh and htobe8 functions since <endian.h> doesn't define any endian byte-swapping functions for 8-bit integers. In ElementParseAndUnspecifiedLengthCodeGenerator.scala, allow alignment fills to precede elements in order to allow code to be generated, although we still need to generate some extra code to adjust for alignment fills later. In TestRuntime2.dfdl.xsd, expand test schema to add another schema example coming from somewhere else (an orion_command element using 8-bit and 16-bit signed/unsigned integers). This element has inadvertently required an alignment fill of 1 byte before the checksum field that probably should be removed, but we may need to generate code to handle alignment fills anyway. In TestRuntime2.tdml, rename parse_int32 & unparse_int32 tests to ex_int32_parse & ex_int32_unparse and add two new tests orion_command_parse & orion_command_unparse. Specify explicit root elements for every test since we have multiple root elements now. Rename parse_int32 & unparse_int32 data files to ex_int32_parse & ex_int32_unparse. Generated code doesn't expect XML elements to have an ex prefix anymore (need to check why) so remove ex prefixes from ex_int32_unparse elements. Add orion_command_parse && orion_command_unparse data files for new tests. In TestRuntime2.scala, call ex_int32 & orion_command tests. In Rat.scala, exclude renamed & new binary data files from checks. --- .../src/main/resources/c/libcli/xml_reader.c | 52 ++++++++++++++-- .../src/main/resources/c/libcli/xml_writer.c | 49 ++++++++++++--- .../src/main/resources/c/libruntime/infoset.c | 16 +++-- .../src/main/resources/c/libruntime/infoset.h | 16 +++-- .../src/main/resources/examples/ex_int32.c | 67 +++++++++++---------- .../src/main/resources/examples/ex_int32.h | 4 +- .../daffodil/runtime2/Runtime2DataProcessor.scala | 9 +-- .../BinaryIntegerKnownLengthCodeGenerator.scala | 33 +++++----- .../runtime2/generators/CodeGeneratorState.scala | 17 +++++- ...entParseAndUnspecifiedLengthCodeGenerator.scala | 4 +- .../apache/daffodil/runtime2/TestRuntime2.dfdl.xsd | 66 +++++++++++++++++--- .../org/apache/daffodil/runtime2/TestRuntime2.tdml | 44 +++++++++++--- .../runtime2/{parse_int32 => ex_int32_parse} | Bin .../runtime2/{unparse_int32 => ex_int32_unparse} | 16 ++--- .../apache/daffodil/runtime2/orion_command_parse | Bin 0 -> 13 bytes .../{unparse_int32 => orion_command_unparse} | 21 ++++--- .../apache/daffodil/runtime2/TestRuntime2.scala | 6 +- project/Rat.scala | 3 +- 18 files changed, 308 insertions(+), 115 deletions(-) diff --git a/daffodil-runtime2/src/main/resources/c/libcli/xml_reader.c b/daffodil-runtime2/src/main/resources/c/libcli/xml_reader.c index 34035d3..763c5e3 100644 --- a/daffodil-runtime2/src/main/resources/c/libcli/xml_reader.c +++ b/daffodil-runtime2/src/main/resources/c/libcli/xml_reader.c @@ -162,10 +162,10 @@ xmlEndComplex(XMLReader *reader, const InfosetBase *base) return NULL; } -// Read 32-bit integer value from XML data +// Read 8, 16, 32, or 64-bit signed/unsigned integer number from XML data static const char * -xmlInt32Elem(XMLReader *reader, const ERD *erd, int32_t *location) +xmlIntegerElem(XMLReader *reader, const ERD *erd, void *intLocation) { // Consume any newlines or whitespace before the element while (mxmlGetType(reader->node) == MXML_OPAQUE) @@ -184,10 +184,50 @@ xmlInt32Elem(XMLReader *reader, const ERD *erd, int32_t *location) { if (strcmp(name_from_xml, name_from_erd) == 0) { - // Check for any errors getting the 32-bit integer + // Check for any errors getting the integer number const char *errstr = NULL; - *location = (int32_t)strtonum(number_from_xml, INT32_MIN, INT32_MAX, - &errstr); + + // Need to handle varying bit lengths and signedness + const enum TypeCode typeCode = erd->typeCode; + switch (typeCode) + { + case PRIMITIVE_UINT64: + *(uint64_t *)intLocation = + (uint64_t)strtonum(number_from_xml, 0, UINT64_MAX, &errstr); + break; + case PRIMITIVE_UINT32: + *(uint32_t *)intLocation = + (uint32_t)strtonum(number_from_xml, 0, UINT32_MAX, &errstr); + break; + case PRIMITIVE_UINT16: + *(uint16_t *)intLocation = + (uint16_t)strtonum(number_from_xml, 0, UINT16_MAX, &errstr); + break; + case PRIMITIVE_UINT8: + *(uint8_t *)intLocation = + (uint8_t)strtonum(number_from_xml, 0, UINT8_MAX, &errstr); + break; + case PRIMITIVE_INT64: + *(int64_t *)intLocation = (int64_t)strtonum( + number_from_xml, INT64_MIN, INT64_MAX, &errstr); + break; + case PRIMITIVE_INT32: + *(int32_t *)intLocation = (int32_t)strtonum( + number_from_xml, INT32_MIN, INT32_MAX, &errstr); + break; + case PRIMITIVE_INT16: + *(int16_t *)intLocation = (int16_t)strtonum( + number_from_xml, INT16_MIN, INT16_MAX, &errstr); + break; + case PRIMITIVE_INT8: + *(int8_t *)intLocation = (int8_t)strtonum( + number_from_xml, INT8_MIN, INT8_MAX, &errstr); + break; + default: + errstr = "Unexpected ERD typeCode while reading integer from XML data"; + break; + } + return errstr; } else @@ -206,5 +246,5 @@ xmlInt32Elem(XMLReader *reader, const ERD *erd, int32_t *location) const VisitEventHandler xmlReaderMethods = { (VisitStartDocument)&xmlStartDocument, (VisitEndDocument)&xmlEndDocument, (VisitStartComplex)&xmlStartComplex, (VisitEndComplex)&xmlEndComplex, - (VisitInt32Elem)&xmlInt32Elem, + (VisitIntegerElem)&xmlIntegerElem, }; diff --git a/daffodil-runtime2/src/main/resources/c/libcli/xml_writer.c b/daffodil-runtime2/src/main/resources/c/libcli/xml_writer.c index 2db0ce4..791968b 100644 --- a/daffodil-runtime2/src/main/resources/c/libcli/xml_writer.c +++ b/daffodil-runtime2/src/main/resources/c/libcli/xml_writer.c @@ -90,23 +90,58 @@ xmlEndComplex(XMLWriter *writer, const InfosetBase *base) return complex ? NULL : "Underflowed the XML stack"; } -// Write 32-bit integer value as element +// Write 8, 16, 32, or 64-bit signed/unsigned integer as element static const char * -xmlInt32Elem(XMLWriter *writer, const ERD *erd, const int32_t *location) +xmlIntegerElem(XMLWriter *writer, const ERD *erd, const void *intLocation) { mxml_node_t *parent = stack_top(&writer->stack); const char * name = get_erd_name(erd); - const char * xmlns = get_erd_xmlns(erd); mxml_node_t *simple = mxmlNewElement(parent, name); - int32_t value = *location; - mxml_node_t *text = mxmlNewOpaquef(simple, "%i", value); + + // Set namespace declaration if necessary + const char *xmlns = get_erd_xmlns(erd); if (xmlns) { const char *ns = get_erd_ns(erd); mxmlElementSetAttr(simple, xmlns, ns); } - return (simple && text) ? NULL : "Error making new int32 simple element"; + + // Need to handle varying bit lengths and signedness + const enum TypeCode typeCode = erd->typeCode; + mxml_node_t * text = NULL; + switch (typeCode) + { + case PRIMITIVE_UINT64: + text = mxmlNewOpaquef(simple, "%lu", *(const uint64_t *)intLocation); + break; + case PRIMITIVE_UINT32: + text = mxmlNewOpaquef(simple, "%u", *(const uint32_t *)intLocation); + break; + case PRIMITIVE_UINT16: + text = mxmlNewOpaquef(simple, "%hu", *(const uint16_t *)intLocation); + break; + case PRIMITIVE_UINT8: + text = mxmlNewOpaquef(simple, "%hhu", *(const uint8_t *)intLocation); + break; + case PRIMITIVE_INT64: + text = mxmlNewOpaquef(simple, "%li", *(const int64_t *)intLocation); + break; + case PRIMITIVE_INT32: + text = mxmlNewOpaquef(simple, "%i", *(const int32_t *)intLocation); + break; + case PRIMITIVE_INT16: + text = mxmlNewOpaquef(simple, "%hi", *(const int16_t *)intLocation); + break; + case PRIMITIVE_INT8: + text = mxmlNewOpaquef(simple, "%hhi", *(const int8_t *)intLocation); + break; + default: + // Let text remain NULL and report error below + break; + } + + return (simple && text) ? NULL : "Error making new simple integer element"; } // Initialize a struct with our visitor event handler methods @@ -114,5 +149,5 @@ xmlInt32Elem(XMLWriter *writer, const ERD *erd, const int32_t *location) const VisitEventHandler xmlWriterMethods = { (VisitStartDocument)&xmlStartDocument, (VisitEndDocument)&xmlEndDocument, (VisitStartComplex)&xmlStartComplex, (VisitEndComplex)&xmlEndComplex, - (VisitInt32Elem)&xmlInt32Elem, + (VisitIntegerElem)&xmlIntegerElem, }; diff --git a/daffodil-runtime2/src/main/resources/c/libruntime/infoset.c b/daffodil-runtime2/src/main/resources/c/libruntime/infoset.c index f0c8bff..b30f92e 100644 --- a/daffodil-runtime2/src/main/resources/c/libruntime/infoset.c +++ b/daffodil-runtime2/src/main/resources/c/libruntime/infoset.c @@ -136,18 +136,26 @@ walkInfosetNode(const VisitEventHandler *handler, const InfosetBase *infoNode) // We use only one of these variables below depending on typeCode const InfosetBase *childNode = (const InfosetBase *)((const char *)infoNode + offset); - const int32_t *intLocation = - (const int32_t *)((const char *)infoNode + offset); + const void *intLocation = + (const void *)((const char *)infoNode + offset); // Need to handle more element types - enum TypeCode typeCode = childERD->typeCode; + const enum TypeCode typeCode = childERD->typeCode; switch (typeCode) { case COMPLEX: error_msg = walkInfosetNode(handler, childNode); break; + case PRIMITIVE_UINT64: + case PRIMITIVE_UINT32: + case PRIMITIVE_UINT16: + case PRIMITIVE_UINT8: + case PRIMITIVE_INT64: case PRIMITIVE_INT32: - error_msg = handler->visitInt32Elem(handler, childERD, intLocation); + case PRIMITIVE_INT16: + case PRIMITIVE_INT8: + error_msg = + handler->visitIntegerElem(handler, childERD, intLocation); break; } } diff --git a/daffodil-runtime2/src/main/resources/c/libruntime/infoset.h b/daffodil-runtime2/src/main/resources/c/libruntime/infoset.h index 459557c..523a0d6 100644 --- a/daffodil-runtime2/src/main/resources/c/libruntime/infoset.h +++ b/daffodil-runtime2/src/main/resources/c/libruntime/infoset.h @@ -48,8 +48,9 @@ typedef const char *(*VisitStartComplex)(const VisitEventHandler *handler, const InfosetBase * base); typedef const char *(*VisitEndComplex)(const VisitEventHandler *handler, const InfosetBase * base); -typedef const char *(*VisitInt32Elem)(const VisitEventHandler *handler, - const ERD *erd, const int32_t *location); +typedef const char *(*VisitIntegerElem)(const VisitEventHandler *handler, + const ERD * erd, + const void * intLocation); // NamedQName - name of an infoset element @@ -65,7 +66,14 @@ typedef struct NamedQName enum TypeCode { COMPLEX, - PRIMITIVE_INT32 + PRIMITIVE_UINT64, + PRIMITIVE_UINT32, + PRIMITIVE_UINT16, + PRIMITIVE_UINT8, + PRIMITIVE_INT64, + PRIMITIVE_INT32, + PRIMITIVE_INT16, + PRIMITIVE_INT8 }; // ERD - element runtime data needed to parse/unparse objects @@ -112,7 +120,7 @@ typedef struct VisitEventHandler const VisitEndDocument visitEndDocument; const VisitStartComplex visitStartComplex; const VisitEndComplex visitEndComplex; - const VisitInt32Elem visitInt32Elem; + const VisitIntegerElem visitIntegerElem; } VisitEventHandler; // get_erd_name, get_erd_xmlns, get_erd_ns - get name and xmlns diff --git a/daffodil-runtime2/src/main/resources/examples/ex_int32.c b/daffodil-runtime2/src/main/resources/examples/ex_int32.c index d2b9217..5e38ae0 100644 --- a/daffodil-runtime2/src/main/resources/examples/ex_int32.c +++ b/daffodil-runtime2/src/main/resources/examples/ex_int32.c @@ -25,15 +25,15 @@ static void c2_initSelf(c2 *instance); static const char *c2_parseSelf(c2 *instance, const PState *pstate); static const char *c2_unparseSelf(const c2 *instance, const UState *ustate); -static void c1_initSelf(c1 *instance); -static const char *c1_parseSelf(c1 *instance, const PState *pstate); -static const char *c1_unparseSelf(const c1 *instance, const UState *ustate); +static void ex_int32_initSelf(ex_int32 *instance); +static const char *ex_int32_parseSelf(ex_int32 *instance, const PState *pstate); +static const char *ex_int32_unparseSelf(const ex_int32 *instance, const UState *ustate); // Metadata singletons static const ERD e1_ERD = { { - "ex", // namedQName.prefix + NULL, // namedQName.prefix "e1", // namedQName.local NULL, // namedQName.ns }, @@ -48,7 +48,7 @@ static const ERD e1_ERD = { static const ERD e2_ERD = { { - "ex", // namedQName.prefix + NULL, // namedQName.prefix "e2", // namedQName.local NULL, // namedQName.ns }, @@ -63,7 +63,7 @@ static const ERD e2_ERD = { static const ERD e3_ERD = { { - "ex", // namedQName.prefix + NULL, // namedQName.prefix "e3", // namedQName.local NULL, // namedQName.ns }, @@ -86,7 +86,7 @@ static const ERD *c2_childrenERDs[2] = {&e2_ERD, &e3_ERD}; static const ERD c2_ERD = { { - "ex", // namedQName.prefix + NULL, // namedQName.prefix "c2", // namedQName.local NULL, // namedQName.ns }, @@ -99,27 +99,27 @@ static const ERD c2_ERD = { (ERDUnparseSelf)&c2_unparseSelf, // unparseSelf }; -static const c1 c1_compute_ERD_offsets; +static const ex_int32 ex_int32_compute_ERD_offsets; -static const ptrdiff_t c1_offsets[2] = { - (char *)&c1_compute_ERD_offsets.e1 - (char *)&c1_compute_ERD_offsets, - (char *)&c1_compute_ERD_offsets.c2 - (char *)&c1_compute_ERD_offsets}; +static const ptrdiff_t ex_int32_offsets[2] = { + (char *)&ex_int32_compute_ERD_offsets.e1 - (char *)&ex_int32_compute_ERD_offsets, + (char *)&ex_int32_compute_ERD_offsets.c2 - (char *)&ex_int32_compute_ERD_offsets}; -static const ERD *c1_childrenERDs[2] = {&e1_ERD, &c2_ERD}; +static const ERD *ex_int32_childrenERDs[2] = {&e1_ERD, &c2_ERD}; -static const ERD c1_ERD = { +static const ERD ex_int32_ERD = { { - "ex", // namedQName.prefix - "c1", // namedQName.local + NULL, // namedQName.prefix + "ex_int32", // namedQName.local "http://example.com";, // namedQName.ns }, COMPLEX, // typeCode 2, // numChildren - c1_offsets, // offsets - c1_childrenERDs, // childrenERDs - (ERDInitSelf)&c1_initSelf, // initSelf - (ERDParseSelf)&c1_parseSelf, // parseSelf - (ERDUnparseSelf)&c1_unparseSelf, // unparseSelf + ex_int32_offsets, // offsets + ex_int32_childrenERDs, // childrenERDs + (ERDInitSelf)&ex_int32_initSelf, // initSelf + (ERDParseSelf)&ex_int32_parseSelf, // parseSelf + (ERDUnparseSelf)&ex_int32_unparseSelf, // unparseSelf }; // Return a root element to be used for parsing or unparsing @@ -127,14 +127,17 @@ static const ERD c1_ERD = { InfosetBase * rootElement() { - static c1 instance; + static ex_int32 instance; InfosetBase *root = &instance._base; - c1_ERD.initSelf(root); + ex_int32_ERD.initSelf(root); return root; } // Methods to initialize, parse, and unparse infoset nodes +static inline uint8_t be8toh(uint8_t be8b) { return be8b; } +static inline uint8_t htobe8(uint8_t h8b) { return h8b; } + static void c2_initSelf(c2 *instance) { @@ -149,7 +152,7 @@ c2_parseSelf(c2 *instance, const PState *pstate) const char *error_msg = NULL; if (!error_msg) { - char buffer[4]; + char buffer[sizeof(uint32_t)]; size_t count = fread(&buffer, 1, sizeof(buffer), pstate->stream); if (count < sizeof(buffer)) { @@ -159,7 +162,7 @@ c2_parseSelf(c2 *instance, const PState *pstate) } if (!error_msg) { - char buffer[4]; + char buffer[sizeof(uint32_t)]; size_t count = fread(&buffer, 1, sizeof(buffer), pstate->stream); if (count < sizeof(buffer)) { @@ -178,7 +181,7 @@ c2_unparseSelf(const c2 *instance, const UState *ustate) { union { - char c_val[4]; + char c_val[sizeof(uint32_t)]; uint32_t i_val; } buffer; buffer.i_val = htobe32(instance->e2); @@ -192,7 +195,7 @@ c2_unparseSelf(const c2 *instance, const UState *ustate) { union { - char c_val[4]; + char c_val[sizeof(uint32_t)]; uint32_t i_val; } buffer; buffer.i_val = htobe32(instance->e3); @@ -206,20 +209,20 @@ c2_unparseSelf(const c2 *instance, const UState *ustate) } static void -c1_initSelf(c1 *instance) +ex_int32_initSelf(ex_int32 *instance) { instance->e1 = 0xCDCDCDCD; c2_initSelf(&instance->c2); - instance->_base.erd = &c1_ERD; + instance->_base.erd = &ex_int32_ERD; } static const char * -c1_parseSelf(c1 *instance, const PState *pstate) +ex_int32_parseSelf(ex_int32 *instance, const PState *pstate) { const char *error_msg = NULL; if (!error_msg) { - char buffer[4]; + char buffer[sizeof(uint32_t)]; size_t count = fread(&buffer, 1, sizeof(buffer), pstate->stream); if (count < sizeof(buffer)) { @@ -235,14 +238,14 @@ c1_parseSelf(c1 *instance, const PState *pstate) } static const char * -c1_unparseSelf(const c1 *instance, const UState *ustate) +ex_int32_unparseSelf(const ex_int32 *instance, const UState *ustate) { const char *error_msg = NULL; if (!error_msg) { union { - char c_val[4]; + char c_val[sizeof(uint32_t)]; uint32_t i_val; } buffer; buffer.i_val = htobe32(instance->e1); diff --git a/daffodil-runtime2/src/main/resources/examples/ex_int32.h b/daffodil-runtime2/src/main/resources/examples/ex_int32.h index affd8b4..b46c30c 100644 --- a/daffodil-runtime2/src/main/resources/examples/ex_int32.h +++ b/daffodil-runtime2/src/main/resources/examples/ex_int32.h @@ -30,11 +30,11 @@ typedef struct c2 int32_t e3; } c2; -typedef struct c1 +typedef struct ex_int32 { InfosetBase _base; int32_t e1; c2 c2; -} c1; +} ex_int32; #endif // GENERATED_CODE_H diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala index 7dd8f68..9e47349 100644 --- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala +++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala @@ -108,6 +108,8 @@ class Runtime2DataProcessor(executableFile: os.Path) extends DFDL.DataProcessorB val parseResult = new ParseResult(outfile, Failure(parseError)) parseResult.addDiagnostic(parseError) parseResult + } finally { + os.remove.all(tempDir) } } @@ -145,6 +147,8 @@ class Runtime2DataProcessor(executableFile: os.Path) extends DFDL.DataProcessorB val unparseResult = new UnparseResult(finalBitPos0b, Failure(unparseError)) unparseResult.addDiagnostic(unparseError) unparseResult + } finally { + os.remove.all(tempDir) } } } @@ -178,10 +182,7 @@ final class ParseResult(outfile: os.Path, override def currentLocation: DataLocation = loc - def infosetAsXML : scala.xml.Elem = { - val xml = scala.xml.XML.loadFile(outfile.toIO) - xml - } + val infosetAsXML : scala.xml.Elem = scala.xml.XML.loadFile(outfile.toIO) } final class UnparseResult(val finalBitPos0b: Long, diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/BinaryIntegerKnownLengthCodeGenerator.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/BinaryIntegerKnownLengthCodeGenerator.scala index e1c399d..2936ea6 100644 --- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/BinaryIntegerKnownLengthCodeGenerator.scala +++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/BinaryIntegerKnownLengthCodeGenerator.scala @@ -23,6 +23,8 @@ import org.apache.daffodil.schema.annotation.props.gen.{ BitOrder, ByteOrder } trait BinaryIntegerKnownLengthCodeGenerator { def binaryIntegerKnownLengthGenerateCode(g: BinaryIntegerKnownLength, cgState: CodeGeneratorState): Unit = { + // For the time being this is a very limited back end. + // So there are numerous restrictions to enforce. val e = g.e val fieldName = e.namedQName.local val lengthInBits: Long = { @@ -36,50 +38,47 @@ trait BinaryIntegerKnownLengthCodeGenerator { val bo = e.byteOrderEv.constValue bo } - // For the time being this is a very limited back end. - // So there are numerous restrictions to enforce. - e.schemaDefinitionUnless(lengthInBits <= 64, "Length must be 64 bits or less, but was: %s", lengthInBits) - if (lengthInBits == 64 && !g.signed) - e.SDE("Integers of 64 bits length must be signed.") // This insures we can use regular java.io library calls. if (e.alignmentValueInBits.intValue() % 8 != 0) e.SDE("Only alignment to 8-bit (1 byte) boundaries is supported.") // The restrictions below are ones we want to eventually lift. - if (lengthInBits != 32) - e.SDE("Lengths other than 32 bits are not supported.") - if (byteOrder ne ByteOrder.BigEndian) e.SDE("Only dfdl:byteOrder 'bigEndian' is supported.") if (e.bitOrder ne BitOrder.MostSignificantBitFirst) e.SDE("Only dfdl:bitOrder 'mostSignificantBitFirst' is supported.") - if (!g.signed) - e.SDE("Only signed integers are supported.") - - val initStatement = s" instance->$fieldName = 0xCDCDCDCD;" + // Start generating code snippets + val initialValue = lengthInBits match { + case 8 => "0xCD" + case 16 => "0xCDCD" + case 32 => "0xCDCDCDCD" + case 64 => "0xCDCDCDCDCDCDCDCD" + case _ => e.SDE("Lengths other than 8, 16, 32, or 64 bits are not supported.") + } + val initStatement = s" instance->$fieldName = $initialValue;" val parseStatement = s""" if (!error_msg) | { - | char buffer[4]; + | char buffer[sizeof(uint${lengthInBits}_t)]; | size_t count = fread(&buffer, 1, sizeof(buffer), pstate->stream); | if (count < sizeof(buffer)) | { | error_msg = eof_or_error_msg(pstate->stream); | } - | instance->$fieldName = be32toh(*((uint32_t *)(&buffer))); + | instance->$fieldName = be${lengthInBits}toh(*((uint${lengthInBits}_t *)(&buffer))); | }""".stripMargin val unparseStatement = s""" if (!error_msg) | { | union | { - | char c_val[4]; - | uint32_t i_val; + | char c_val[sizeof(uint${lengthInBits}_t)]; + | uint${lengthInBits}_t i_val; | } buffer; - | buffer.i_val = htobe32(instance->$fieldName); + | buffer.i_val = htobe${lengthInBits}(instance->$fieldName); | size_t count = fwrite(buffer.c_val, 1, sizeof(buffer), ustate->stream); | if (count < sizeof(buffer)) | { diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/CodeGeneratorState.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/CodeGeneratorState.scala index bf97636..0255111 100644 --- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/CodeGeneratorState.scala +++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/CodeGeneratorState.scala @@ -170,8 +170,14 @@ class CodeGeneratorState { val e = context.namedQName.local val qnameInit = defineQNameInit(context) val typeCode = context.optPrimType.get match { + case PrimType.UnsignedLong => "PRIMITIVE_UINT64" + case PrimType.UnsignedInt => "PRIMITIVE_UINT32" + case PrimType.UnsignedShort => "PRIMITIVE_UINT16" + case PrimType.UnsignedByte => "PRIMITIVE_UINT8" + case PrimType.Long => "PRIMITIVE_INT64" case PrimType.Int => "PRIMITIVE_INT32" - case PrimType.String => "PRIMITIVE_STRING" + case PrimType.Short => "PRIMITIVE_INT16" + case PrimType.Byte => "PRIMITIVE_INT8" case p: PrimType => context.SDE("PrimType %s not supported yet.", p.toString) } val erd = @@ -203,8 +209,14 @@ class CodeGeneratorState { val definition = if (child.isSimpleType) { import NodeInfo.PrimType child.optPrimType.get match { + case PrimType.UnsignedLong => "uint64_t " + case PrimType.UnsignedInt => "uint32_t " + case PrimType.UnsignedShort => "uint16_t " + case PrimType.UnsignedByte => "uint8_t " case PrimType.Long => "int64_t " case PrimType.Int => "int32_t " + case PrimType.Short => "int16_t " + case PrimType.Byte => "int8_t " case x => context.SDE("Unsupported primitive type: " + x) } } else { @@ -260,6 +272,9 @@ class CodeGeneratorState { | |// Methods to initialize, parse, and unparse infoset nodes | + |static inline uint8_t be8toh(uint8_t be8b) { return be8b; } + |static inline uint8_t htobe8(uint8_t h8b) { return h8b; } + | |$finalImplementation |""".stripMargin code diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/ElementParseAndUnspecifiedLengthCodeGenerator.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/ElementParseAndUnspecifiedLengthCodeGenerator.scala index d13be31..b1157d1 100644 --- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/ElementParseAndUnspecifiedLengthCodeGenerator.scala +++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/generators/ElementParseAndUnspecifiedLengthCodeGenerator.scala @@ -17,6 +17,7 @@ package org.apache.daffodil.runtime2.generators +import org.apache.daffodil.grammar.primitives.AlignmentFill import org.apache.daffodil.grammar.primitives.ElementParseAndUnspecifiedLength import org.apache.daffodil.runtime2.Runtime2CodeGenerator @@ -29,7 +30,8 @@ trait ElementParseAndUnspecifiedLengthCodeGenerator { context.schemaDefinitionWhen(context.inputValueCalcOption.isDefined, "Elements with inputValueCalc are not supported.") context.schemaDefinitionWhen(context.outputValueCalcOption.isDefined, "Elements with outputValueCalc are not supported.") - context.schemaDefinitionUnless(g.eBeforeGram.isEmpty, "Statements associated with elements are not supported.") + context.schemaDefinitionUnless(g.eBeforeGram.isEmpty + || g.eBeforeGram == AlignmentFill(context), "Statements associated with elements are not supported.") context.schemaDefinitionUnless(g.eAfterGram.isEmpty, "Statements associated with elements are not supported.") context.schemaDefinitionUnless(g.repTypeElementGram.isEmpty, "dfdlx:repType is not supported.") diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.dfdl.xsd b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.dfdl.xsd index 81cade0..88fdf56 100644 --- a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.dfdl.xsd +++ b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.dfdl.xsd @@ -16,30 +16,37 @@ limitations under the License. --> -<xs:schema elementFormDefault="qualified" - targetNamespace="http://example.com"; - xmlns="http://example.com"; - xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; - xmlns:ex="http://example.com"; - xmlns:xs="http://www.w3.org/2001/XMLSchema";> +<xs:schema + elementFormDefault="qualified" + targetNamespace="http://example.com"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:ex="http://example.com"; + xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns="http://example.com";> <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/> <xs:annotation> <xs:appinfo source="http://www.ogf.org/dfdl/";> - <dfdl:format ref="GeneralFormat" representation="binary"/> + <dfdl:format + alignment="8" + alignmentUnits="bits" + encoding="UTF-8" + lengthUnits="bits" + representation="binary" + ref="GeneralFormat"/> </xs:appinfo> </xs:annotation> <xs:element name="ex_int32"> <xs:complexType> <xs:sequence> - <xs:element name="e1" type="xs:int"/> + <xs:element name="e1" type="xs:int" /> <xs:element name="c2"> <xs:complexType> <xs:sequence> - <xs:element name="e2" type="xs:int"/> - <xs:element name="e3" type="xs:int"/> + <xs:element name="e2" type="xs:int" /> + <xs:element name="e3" type="xs:int" /> </xs:sequence> </xs:complexType> </xs:element> @@ -47,4 +54,43 @@ </xs:complexType> </xs:element> + <xs:simpleType + name="int16" + dfdl:alignment="16" + dfdl:length="16" + dfdl:lengthKind="explicit"> + <xs:restriction base="xs:short"/> + </xs:simpleType> + <xs:simpleType + name="uint8" + dfdl:alignment="8" + dfdl:length="8" + dfdl:lengthKind="explicit"> + <xs:restriction base="xs:unsignedByte"/> + </xs:simpleType> + <xs:simpleType + name="uint16" + dfdl:alignment="16" + dfdl:length="16" + dfdl:lengthKind="explicit"> + <xs:restriction base="xs:unsignedShort"/> + </xs:simpleType> + + <xs:element name="orion_command"> + <xs:complexType> + <xs:sequence> + <xs:element name="sync0" type="uint8" fixed="208"/> + <xs:element name="sync1" type="uint8" fixed="13"/> + <xs:element name="id" type="uint8" fixed="1"/> + <xs:element name="length" type="uint8" fixed="7"/> + <xs:element name="pan" type="int16"/> + <xs:element name="tilt" type="int16"/> + <xs:element name="mode" type="uint8" fixed="80"/> + <xs:element name="stabilized" type="uint8" fixed="0"/> + <xs:element name="impulse" type="uint8" fixed="0"/> + <xs:element name="checksum" type="uint16"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:schema> diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.tdml b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.tdml index 7a5d08b..7b85711 100644 --- a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.tdml +++ b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/TestRuntime2.tdml @@ -41,26 +41,54 @@ limitations under the License. </tdml:defineConfig> <tdml:parserTestCase - description="Parse binary int32 integers" + description="Parse example ex_int32" model="TestRuntime2.dfdl.xsd" - name="parse_int32"> + name="ex_int32_parse" + root="ex_int32"> <tdml:document> - <tdml:documentPart type="file">parse_int32</tdml:documentPart> + <tdml:documentPart type="file">ex_int32_parse</tdml:documentPart> </tdml:document> <tdml:infoset> - <tdml:dfdlInfoset type="file">unparse_int32</tdml:dfdlInfoset> + <tdml:dfdlInfoset type="file">ex_int32_unparse</tdml:dfdlInfoset> </tdml:infoset> </tdml:parserTestCase> <tdml:unparserTestCase - description="Unparse binary int32 numbers" + description="Unparse example ex_int32" model="TestRuntime2.dfdl.xsd" - name="unparse_int32"> + name="ex_int32_unparse" + root="ex_int32"> <tdml:infoset> - <tdml:dfdlInfoset type="file">unparse_int32</tdml:dfdlInfoset> + <tdml:dfdlInfoset type="file">ex_int32_unparse</tdml:dfdlInfoset> </tdml:infoset> <tdml:document> - <tdml:documentPart type="file">parse_int32</tdml:documentPart> + <tdml:documentPart type="file">ex_int32_parse</tdml:documentPart> + </tdml:document> + </tdml:unparserTestCase> + + <tdml:parserTestCase + description="Parse example orion_command" + model="TestRuntime2.dfdl.xsd" + name="orion_command_parse" + root="orion_command"> + <tdml:document> + <tdml:documentPart type="file">orion_command_parse</tdml:documentPart> + </tdml:document> + <tdml:infoset> + <tdml:dfdlInfoset type="file">orion_command_unparse</tdml:dfdlInfoset> + </tdml:infoset> + </tdml:parserTestCase> + + <tdml:unparserTestCase + description="Unparse example orion_command" + model="TestRuntime2.dfdl.xsd" + name="orion_command_unparse" + root="orion_command"> + <tdml:infoset> + <tdml:dfdlInfoset type="file">orion_command_unparse</tdml:dfdlInfoset> + </tdml:infoset> + <tdml:document> + <tdml:documentPart type="file">orion_command_parse</tdml:documentPart> </tdml:document> </tdml:unparserTestCase> diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/parse_int32 b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_parse similarity index 100% rename from daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/parse_int32 rename to daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_parse diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_unparse similarity index 81% copy from daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 copy to daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_unparse index ea15640..cad2f06 100644 --- a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 +++ b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_unparse @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="utf-8"?> +<?xml version="1.0" encoding="UTF-8"?> <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -16,10 +16,10 @@ limitations under the License. --> -<ex:ex_int32 xmlns:ex="http://example.com";> - <ex:e1>1</ex:e1> - <ex:c2> - <ex:e2>2</ex:e2> - <ex:e3>3</ex:e3> - </ex:c2> -</ex:ex_int32> +<ex_int32 xmlns="http://example.com";> + <e1>1</e1> + <c2> + <e2>2</e2> + <e3>3</e3> + </c2> +</ex_int32> diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_parse b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_parse new file mode 100644 index 0000000..4484b44 Binary files /dev/null and b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_parse differ diff --git a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_unparse similarity index 71% rename from daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 rename to daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_unparse index ea15640..8cda886 100644 --- a/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/unparse_int32 +++ b/daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_unparse @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="utf-8"?> +<?xml version="1.0" encoding="UTF-8"?> <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -16,10 +16,15 @@ limitations under the License. --> -<ex:ex_int32 xmlns:ex="http://example.com";> - <ex:e1>1</ex:e1> - <ex:c2> - <ex:e2>2</ex:e2> - <ex:e3>3</ex:e3> - </ex:c2> -</ex:ex_int32> +<orion_command xmlns="http://example.com";> + <sync0>208</sync0> + <sync1>12</sync1> + <id>1</id> + <length>7</length> + <pan>-999</pan> + <tilt>-21231</tilt> + <mode>80</mode> + <stabilized>0</stabilized> + <impulse>0</impulse> + <checksum>30149</checksum> +</orion_command> diff --git a/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2.scala b/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2.scala index 9bada7c..5a5b7e7 100644 --- a/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2.scala +++ b/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2.scala @@ -31,6 +31,8 @@ object TestRuntime2 { class TestRuntime2 { import TestRuntime2._ - @Test def test_parse_int32(): Unit = { runner.runOneTest("parse_int32") } - @Test def test_unparse_int32(): Unit = { runner.runOneTest("unparse_int32") } + @Test def test_ex_int32_parse(): Unit = { runner.runOneTest("ex_int32_parse") } + @Test def test_ex_int32_unparse(): Unit = { runner.runOneTest("ex_int32_unparse") } + @Test def test_orion_command_parse(): Unit = { runner.runOneTest("orion_command_parse") } + @Test def test_orion_command_unparse(): Unit = { runner.runOneTest("orion_command_unparse") } } diff --git a/project/Rat.scala b/project/Rat.scala index f9742a1..aa3b78d 100644 --- a/project/Rat.scala +++ b/project/Rat.scala @@ -116,7 +116,8 @@ object Rat { file("daffodil-sapi/src/test/resources/test/sapi/myData16.dat"), file("daffodil-sapi/src/test/resources/test/sapi/myData19.dat"), file("daffodil-sapi/src/test/resources/test/sapi/myDataBroken.dat"), - file("daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/parse_int32"), + file("daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/ex_int32_parse"), + file("daffodil-runtime2/src/test/resources/org/apache/daffodil/runtime2/orion_command_parse"), file("daffodil-tdml-lib/src/test/resources/test/tdml/test.bin"), file("daffodil-tdml-lib/src/test/resources/test/tdml/test.txt"), file("daffodil-tdml-processor/src/test/resources/test/tdml/test.bin"),
