bsloane1650 commented on a change in pull request #210: Add midBitsToEnd layer transform URL: https://github.com/apache/incubator-daffodil/pull/210#discussion_r277730375
########## File path: daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/midBitsToEndTransformer.scala ########## @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.daffodil.layers + +import org.apache.daffodil.processors.TermRuntimeData +import org.apache.daffodil.util.Maybe +import org.apache.daffodil.processors.LayerCharsetEv +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind +import org.apache.daffodil.processors.LayerLengthInBytesEv +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits +import org.apache.daffodil.processors.LayerBoundaryMarkEv +import org.apache.daffodil.processors.LayerTransformArgsEv +import java.io.InputStream +import java.io.OutputStream +import org.apache.daffodil.processors.unparsers.UState +import org.apache.daffodil.processors.parsers.PState +import org.apache.daffodil.exceptions.Assert +import org.apache.daffodil.processors.ParseOrUnparseState +import org.apache.daffodil.processors.parsers.Parser + +import org.apache.daffodil.processors.ExplicitLengthEv +import org.apache.daffodil.io.ExplicitLengthLimitingStream +import java.io.ByteArrayOutputStream +import java.io.ByteArrayInputStream +import org.apache.daffodil.util.MaybeULong +import org.apache.daffodil.processors.Failure + +class midBitsToEndTransformer( + layerLengthInBytesEv: LayerLengthInBytesEv, + layerTransformArgsEv: LayerTransformArgsEv) + extends LayerTransformer { + /* + * For unparsing, it is more or less unavoidable that we cannot stream, as we need to reach + * the end of the input before we can provide the mid-bits that were moved. + * In practice, it is not expected that this transform will be used on "large" blocks, + * so it is not worth the effort to figure out how to implement some form of streaming. + */ + + var length: Int = -1 + var firstBitIndex: Int = -1 + var numBits: Int = -1 + + /* + * Note the extra padding bytes. One of these bytes is just to accomodate rounding from integer division. + * The other one is because, to simplify the code, a portion of the encoder may attempt to read up to 7 extra bits, + * then discard them. (This would happen when the last bit that gets moved is the first bit of a byte in the encoded form; + * the encoder will attempt to read enough bits to be able to fully replace the entire byte, then mask out the lower 7 bits) + */ + def tmpByteCount = numBits / 8 + 2 + def buffSize = length + tmpByteCount + + def byteShift = numBits / 8 + def bitShift = numBits % 8 + + def firstBitByteIndex = (firstBitIndex / 8) + def lastBitByteIndex = ((firstBitIndex + numBits) / 8) + def firstBitBitIndex = firstBitIndex % 8 + def lastBitBitIndex = (firstBitIndex + numBits) % 8 + + private def buffToHex(buff: Array[Byte]): String = { + buff.map(b => String.format("%02x", Byte.box(b))).mkString(" ") + } + + private def initialize(state: ParseOrUnparseState) { + length = layerLengthInBytesEv.evaluate(state).toInt + val argsString = layerTransformArgsEv.evaluate(state) + .trim() + .split("\\ +") + if (argsString.length != 2) { + state.SDE(s"""midBitsToEnd layer transform requires exactly 2 arguements, recieved ${argsString.length}: "${argsString.mkString(",")}" """) + } + try { + firstBitIndex = argsString(0).toInt + numBits = argsString(1).toInt + } catch { + case _: NumberFormatException => state.SDE(s"""midBitsToEnd layer transform requires Integer arguements, recieved: "${argsString.mkString(",")}"""") + } + + } + + /* + * Scala does not seem to support bitwise operations on bytes; and instead silently upcasts to Int. + * Spamming toByte is insufficient since negative numbers will get padded with 1 instead of 0 + * For this reason, we do most of our internal logic in Int, and mask out the upper 24 bits + */ + + val bitMask: Int = 0x000000ff + + private def doDecode(buff: Array[Byte]): Unit = { + + /*First, move the middle bits to the end. + * + * Since the layer is byte aligned, we know we are writing them a byte aligned location + * We will end up moving a multiple of 8 bits. This is okay, since, after we shift, + * any extra bytes will be left passed the length byte, and so will be ignored + */ + var dstByteIndex = length + var srcByteIndex = firstBitByteIndex + while (dstByteIndex < buffSize) { + val srcByte = buff(srcByteIndex) & bitMask + val srcByteNext = buff(srcByteIndex + 1) & bitMask + + val byteUpper = (srcByte << firstBitBitIndex) & bitMask Review comment: I believe the case implemented is MSBF (as you point out, elsewhere, that seems to be the case tested). Is the endianness actually relevent to this transform? My understanding is that endianness refers to how bits are interpereted as atomic fields, but broadly speaking the the lower indexed bytes always come first. Eg. if we have 2 16-byte fields in the following datastream: ``` 0000 0001 0000 0000 0000 0000 0000 0000 ``` Endianness will determine if the first field is 1 or 256, but the first field is always represented by the first 2 bytes. Given that the transform has no knowledge of where field boundaries are, it doesn't make sense to respond to endianess. As an aside, I believe our motivating case is actually MSBF, but it is probably best we discuss that in a more private forum. Regardless, it is probably best I add the LSBF support now, while this shifting logic is still fresh in my mind. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
