From dev-return-2382-archive-asf-public=cust-asf.ponee.io@daffodil.apache.org Tue Aug 7 23:43:13 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id E1F0F180657 for ; Tue, 7 Aug 2018 23:43:12 +0200 (CEST) Received: (qmail 98242 invoked by uid 500); 7 Aug 2018 21:43:12 -0000 Mailing-List: contact dev-help@daffodil.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@daffodil.apache.org Delivered-To: mailing list dev@daffodil.apache.org Received: (qmail 98231 invoked by uid 99); 7 Aug 2018 21:43:12 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Aug 2018 21:43:12 +0000 From: GitBox To: dev@daffodil.apache.org Subject: [GitHub] mbeckerle commented on a change in pull request #88: Daffodil 1919 separators Message-ID: <153367819140.28550.6942807402851738784.gitbox@gitbox.apache.org> Date: Tue, 07 Aug 2018 21:43:11 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit mbeckerle commented on a change in pull request #88: Daffodil 1919 separators URL: https://github.com/apache/incubator-daffodil/pull/88#discussion_r208395373 ########## File path: daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SeparatedSequenceUnparsers.scala ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.daffodil.processors.unparsers + +import org.apache.daffodil.equality.TypeEqual +import org.apache.daffodil.exceptions.Assert +import org.apache.daffodil.processors.{ ElementRuntimeData, SequenceRuntimeData, TermRuntimeData } +import org.apache.daffodil.schema.annotation.props.SeparatorSuppressionPolicy +import org.apache.daffodil.schema.annotation.props.gen.{ SeparatorPosition } +import org.apache.daffodil.infoset.DIElement + +/** + * DFDL Spec. section 14.2.3 specifies only a few different behaviors + * for separator suppression. Each has an algorithm. + */ +sealed trait SeparatorSuppressionAlgorithm extends Serializable { + + /** + * Determines if we should suppress a zero length element and + * separator. + * + * Checks if the length is zero if necessary to decide. + * + * Returns Suppress or IfTrailing when the element is, in fact, zero length, + * and the algorithm wants those to be suppressed. + * Returns DoNotSuppress otherwise. + */ + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement): SeparatorSuppressionAction + + def shouldDoExtraSeparators = false + + def checkArrayPos(state: UState, maxRepeats: Long): Boolean = true + +} + +object SeparatorSuppressionAlgorithm { + type Type = SeparatorSuppressionAlgorithm + import SeparatorSuppressionAction._ + + object None extends Type { + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement) = Assert.usageError("not to be called.") + } + object FixedOrExpression extends Type { + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement) = DoNotSuppress + } + + class SuppressAnyEmpty(zeroLengthDetector: ZeroLengthDetector) extends Type { + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement) = { + val isZL = zeroLengthDetector.isZeroLength(infosetElement) + if (isZL) Suppress + else DoNotSuppress + } + } + + trait CheckArrayPosMixin { self: SeparatorSuppressionAlgorithm => + override def checkArrayPos(state: UState, maxRepeats: Long): Boolean = { + state.arrayPos <= maxRepeats + } + } + + class ImplicitSuppressAnyEmpty(zeroLengthDetector: ZeroLengthDetector) + extends SuppressAnyEmpty(zeroLengthDetector) + with CheckArrayPosMixin + + object ImplicitNeverOrNotPotentiallyTrailing extends Type + with CheckArrayPosMixin { + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement) = DoNotSuppress + + override def shouldDoExtraSeparators = true + + } + + class ImplicitPotentiallyTrailing(zeroLengthDetector: ZeroLengthDetector) extends Type + with CheckArrayPosMixin { + def shouldSuppressIfZeroLength(state: UState, infosetElement: DIElement) = { + val isZL = zeroLengthDetector.isZeroLength(infosetElement) + if (isZL) IfTrailing + else DoNotSuppress + } + } +} + +sealed trait SeparatorSuppressionAction extends Serializable +object SeparatorSuppressionAction { + type Type = SeparatorSuppressionAction + case object Suppress extends Type + case object DoNotSuppress extends Type + case object IfTrailing extends Type +} + +trait Separated { self: SequenceChildUnparser => + + def sep: Unparser + def spos: SeparatorPosition + def ssp: SeparatorSuppressionPolicy + def ssAlgorithm: SeparatorSuppressionAlgorithm + + val childProcessors = Seq(childUnparser, sep) +} + +class ScalarOrderedSeparatedSequenceChildUnparser( + childUnparser: Unparser, + srd: SequenceRuntimeData, + trd: TermRuntimeData, + val sep: Unparser, + val spos: SeparatorPosition, + val ssp: SeparatorSuppressionPolicy, + val ssAlgorithm: SeparatorSuppressionAlgorithm) + extends SequenceChildUnparser(childUnparser, srd, trd) + with Separated { + + override def unparse(state: UState) = childUnparser.unparse1(state) +} + +class RepOrderedSeparatedSequenceChildUnparser( + childUnparser: Unparser, + srd: SequenceRuntimeData, + erd: ElementRuntimeData, + val sep: Unparser, + val spos: SeparatorPosition, + val ssp: SeparatorSuppressionPolicy, // need for diagnostics perhaps + val ssAlgorithm: SeparatorSuppressionAlgorithm) + extends RepeatingChildUnparser(childUnparser, srd, erd) + with Separated { + + override def checkArrayPos(state: UState) = ssAlgorithm.checkArrayPos(state, maxRepeats(state)) +} + +class OrderedSeparatedSequenceUnparser( + rd: SequenceRuntimeData, + ssp: SeparatorSuppressionPolicy, + spos: SeparatorPosition, + sep: Unparser, + childUnparsersArg: Seq[SequenceChildUnparser]) + extends OrderedSequenceUnparserBase(rd, childUnparsersArg) { + + private val childUnparsers = childUnparsersArg.asInstanceOf[Seq[SequenceChildUnparser with Separated]] + + /** + * True if requires special treatment in the unparse processing loop, as occurrences + * of later sequence children can influence whether possible trailing separators + * from earlier sequence children are actually trailing or not. + */ + private type IPT = SeparatorSuppressionAlgorithm.ImplicitPotentiallyTrailing + + private val hasTrailingSeparatorSuppression = { + childUnparsers.last.ssAlgorithm.isInstanceOf[IPT] + } + + /** + * Unparses one occurrence. + */ + protected def unparseOne( + unparser: SequenceChildUnparser, + trd: TermRuntimeData, + state: UState): Unit = { + + if (trd.isRepresented) { + if ((spos eq SeparatorPosition.Prefix)) { + sep.unparse1(state) + } else if ((spos eq SeparatorPosition.Infix) && state.groupPos > 1) { + sep.unparse1(state) + } + } + + unparser.unparse1(state) + + if ((spos eq SeparatorPosition.Postfix) && trd.isRepresented) { + sep.unparse1(state) + } + } + + /** + * Unparses a zero-length occurrence, without the separator. This is so that + * any statements or other side-effects (discriminators, setVariable, etc.) + * will occur. + */ + private def unparseZeroLengthWithoutSeparatorForSideEffect( + unparser: SequenceChildUnparser, + trd: TermRuntimeData, + state: UState): Unit = { + // + // Unfortunately there's no way to confirm that this produced zero length + // because of the possible buffering going on in the unparser. + // We'd have to depend on intricate details of the unparser behavior to do this, + // and that's unwise from separation-of-concerns perspective. + // + unparser.unparse1(state) + } + + /** + * Unparses the separator only. + * + * Does not deals with infix boundary condition, because + * the counting of the potential trailing separators takes + * this into account. + */ + private def unparseJustSeparator(state: UState): Unit = { + sep.unparse1(state) + } + + /** + * Returns 1, or 0 if infix separator, and this is the first thing + * in the sequence meaning there is no separator for it. + * + * However, if we're not doing trailing separator suppression, always + * returns 0. + */ + private def suppressedTrailingSeparatorIncrement(unparser: SequenceChildUnparser with Separated, state: UState): Int = { + Assert.usage(unparser.trd.isRepresented) + val notIPT = !unparser.ssAlgorithm.isInstanceOf[IPT] + val result = + if (notIPT) + 0 + else { + val infixAndFirst = (spos eq SeparatorPosition.Infix) && state.groupPos == 1 + if (infixAndFirst) + 0 + else + 1 + } + result + } + + /** + * Unparses an entire sequence, including both scalar and array/optional children. + */ + protected def unparse(state: UState): Unit = { + + state.groupIndexStack.push(1L) // one-based indexing + + var index = 0 + var doUnparser = false + val limit = childUnparsers.length + + var potentialTrailingSeparatorCount: Int = 0 + + // This state var just lets us check some important + // invariants about potentially trailing, e.g., once + // you hit it, it sticks until the end of the group. + // + var haveSeenPotentiallyTrailingSeparators = false + + while (index < limit) { + val childUnparser = childUnparsers(index) + val trd = childUnparser.trd + + // + // Unparsing an ordered sequence depends on the incoming + // stream of infoset events matching up with the order that + // they are expected as the unparser recurses through the + // child term unparsers. + // + childUnparser match { + case unparser: RepOrderedSeparatedSequenceChildUnparser => { + val erd = unparser.erd + var numOccurrences = 0 + val maxReps = unparser.maxRepeats(state) + // + // The number of occurrances we unparse is always exactly driven + // by the number of infoset events for the repeating/optional element. + // + // For RepUnparser - array/optional case - in all cases we should get a + // startArray event. If we don't then + // the element must be entirely optional, so we get no events for it + // at all. + // + val ssAlgorithm = unparser.ssAlgorithm + + if (state.inspect) { + val ev = state.inspectAccessor + val isArray = erd.isArray + if (ev.isStart && (isArray || erd.isOptional)) { + val eventNQN = ev.node.namedQName + if (eventNQN =:= erd.namedQName) { + + // + // Note: leaving in some ofthese println, since debugger for unparsing is so inadequate currently. + // This is the only way to figure out what is going on. + // + // System.err.println("Starting unparse of array/opt %s. Array Index Stack is: %s".format( + // erd.namedQName, state.arrayIndexStack)) + // + + // StartArray for this unparser's array element + // + unparser.startArray(state) + while ({ + doUnparser = unparser.shouldDoUnparser(unparser, state) + doUnparser + }) { + val suppressionAction = + ssAlgorithm.shouldSuppressIfZeroLength( + state, + state.inspectAccessor.asElement) + import SeparatorSuppressionAction._ + suppressionAction match { + case DoNotSuppress => { + // + // If there are pending potentially trailing separators, + // then we've just proven that they are NOT actually trailing + // So we output them all. + // + while (potentialTrailingSeparatorCount > 0) { + Assert.invariant(haveSeenPotentiallyTrailingSeparators) + Assert.invariant(ssAlgorithm.isInstanceOf[IPT]) // sticks once we hit one + unparseJustSeparator(state) + potentialTrailingSeparatorCount -= 1 + } + + if (state.dataProc.isDefined) state.dataProc.get.beforeRepetition(state, this) + // System.err.println("Starting unparse of occurrence of %s. Array Index Stack is: %s".format( + // erd.namedQName, state.arrayIndexStack)) + + unparseOne(unparser, erd, state) + numOccurrences += 1 + + state.moveOverOneArrayIndexOnly() + // System.err.println("Finished unparse of occurrence of %s. Array Index Stack is: %s".format( + // erd.namedQName, state.arrayIndexStack)) + + state.moveOverOneGroupIndexOnly() // array elements are always represented. + + if (state.dataProc.isDefined) state.dataProc.get.afterRepetition(state, this) + } + case Suppress => { + unparseZeroLengthWithoutSeparatorForSideEffect(unparser, trd, state) + state.moveOverOneArrayIndexOnly() + state.moveOverOneGroupIndexOnly() // array elements are always represented. + } + case IfTrailing => { + Assert.invariant(hasTrailingSeparatorSuppression) + haveSeenPotentiallyTrailingSeparators = true // sticks once we've hit one. + unparseZeroLengthWithoutSeparatorForSideEffect(unparser, trd, state) + state.moveOverOneArrayIndexOnly() + state.moveOverOneGroupIndexOnly() // array elements are always represented + potentialTrailingSeparatorCount += suppressedTrailingSeparatorIncrement(unparser, state) + } + } + } + // + // For maxOccurs bounded, and never or not potentially trailing + // + if (ssAlgorithm.shouldDoExtraSeparators && maxReps > numOccurrences) { + var numExtraSeps = erd.maxOccurs - numOccurrences + while (numExtraSeps > 0) { + unparseJustSeparator(state) + numExtraSeps -= 1 + } + } + // System.err.println("Finished unparse of array/opt %s. Array Index Stack is: %s, maxReps %s, numOccurrences %s".format( + // erd.namedQName, state.arrayIndexStack, maxReps, numOccurrences)) + unparser.checkOccursCount(state, unparser, numOccurrences, maxReps) + unparser.endArray(erd, state) + } else { + // + // start array for some other array. Not this one. So we + // don't unparse anything here, and we'll go on to the next + // sequence child, which hopefully will be a matching array. + // + Assert.invariant(erd.minOccurs == 0L) + } + + } else if (ev.isStart) { + Assert.invariant(!ev.erd.isArray && !erd.isOptional) + // + // start of scalar. + // That has to be for a different element later in the sequence + // since this one has a RepUnparser (i.e., is NOT scalar) + val eventNQN = ev.node.namedQName + Assert.invariant(eventNQN != erd.namedQName) + } else { + // if (!(ev.isEnd && ev.isComplex)) + // Assert.invariantFailed("Expected end of complex. Was " + ev) + // System.err.println("End event: " + ev) + } + } else { + // no event (state.inspect returned false) + Assert.invariantFailed("No event for unparsing.") + } + } + // + case scalarUnparser => { Review comment: Yes. Will add assertion. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services