Use Exificient to support EXI format for infosets
Demonstrates how to use Exificient's EXI library with Daffodil by using
SAX. Adds support to the daffodil-cli tool to parse and unparse EXI
infosets.
diff --git a/build.sbt b/build.sbt
index 5c24814..53610bf 100644
--- a/build.sbt
+++ b/build.sbt
@@ -112,6 +112,7 @@
.dependsOn(tdmlProc, runtime2, sapi, japi, schematron % Runtime, udf % "it->test") // causes runtime2/sapi/japi to be pulled into the helper zip/tar
.settings(commonSettings, nopublish)
.settings(libraryDependencies ++= Dependencies.cli)
+ .settings(libraryDependencies ++= Dependencies.exi)
lazy val udf = Project("daffodil-udf", file("daffodil-udf")).configs(IntegrationTest)
.settings(commonSettings)
diff --git a/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.exi b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.exi
new file mode 100644
index 0000000..bba9aa4
--- /dev/null
+++ b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.exi
@@ -0,0 +1 @@
+ ËÙ^[\KÛÛ@ÙLèæc¨æe
\ No newline at end of file
diff --git a/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.exi b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.exi
new file mode 100644
index 0000000..68e20e8
--- /dev/null
+++ b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.exi
@@ -0,0 +1 @@
+ ËÙ^[\KÛÛ@ÙLptVÆÆð
\ No newline at end of file
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
index 38ba43c..2da7dd9 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
@@ -1300,6 +1300,28 @@
}
}
+ @Test def test_XXX_CLI_Parsing_SimpleParse_exi(): Unit = {
+
+ val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section00/general/generalSchema.dfdl.xsd")
+ val (testSchemaFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile)) else (schemaFile)
+
+ val shell = Util.start("")
+
+ try {
+ val cmd = String.format(Util.echoN("Hello") + "| %s parse -I exi -s %s -r e1 | md5sum", Util.binPath, testSchemaFile)
+
+ shell.sendLine(cmd)
+ shell.expect(contains("937b3f96ee0b5cd1ac9f537cf8ddc580"))
+
+ Util.expectExitCode(ExitCode.Success, shell)
+
+ shell.send("exit\n")
+ shell.expect(eof)
+ } finally {
+ shell.close()
+ }
+ }
+
@Test def test_CLI_Error_Return_Codes(): Unit = {
val shell = Util.start("")
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/performance/TestCLIPerformance.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/performance/TestCLIPerformance.scala
index 3673dd5..9d78e67 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/performance/TestCLIPerformance.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/performance/TestCLIPerformance.scala
@@ -68,6 +68,27 @@
}
}
+ @Test def test_XXX_CLI_Performance_2_Threads_2_Times_exi(): Unit = {
+ val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section06/entities/charClassEntities.dfdl.xsd")
+ val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt")
+ val (testSchemaFile, testInputFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile), Util.cmdConvert(inputFile)) else (schemaFile, inputFile)
+
+ val shell = Util.start("")
+
+ try {
+ val cmd = String.format("%s performance -I exi -N 2 -t 2 -s %s -r matrix %s", Util.binPath, testSchemaFile, testInputFile)
+ shell.sendLine(cmd)
+ shell.expect(contains("total parse time (sec):"))
+ shell.expect(contains("avg rate (files/sec):"))
+
+ Util.expectExitCode(ExitCode.Success, shell)
+ shell.sendLine("exit")
+ shell.expect(eof())
+ } finally {
+ shell.close()
+ }
+ }
+
@Test def test_3394_CLI_Performance_3_Threads_20_Times(): Unit = {
val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section06/entities/charClassEntities.dfdl.xsd")
val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt")
@@ -175,6 +196,27 @@
}
}
+ @Test def test_XXX_CLI_Performance_Unparse_2_Threads_2_Times_exi(): Unit = {
+ val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section00/general/generalSchema.dfdl.xsd")
+ val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.exi")
+ val (testSchemaFile, testInputFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile), Util.cmdConvert(inputFile)) else (schemaFile, inputFile)
+
+ val shell = Util.start("")
+
+ try {
+ val cmd = String.format("%s performance --unparse -I exi -N 2 -t 2 -s %s -r e3 %s", Util.binPath, testSchemaFile, testInputFile)
+ shell.sendLine(cmd)
+ shell.expect(contains("total unparse time (sec):"))
+ shell.expect(contains("avg rate (files/sec):"))
+
+ Util.expectExitCode(ExitCode.Success, shell)
+ shell.sendLine("exit")
+ shell.expect(eof())
+ } finally {
+ shell.close()
+ }
+ }
+
@Test def test_XXX_CLI_Performance_Unparse_2_Threads_2_Times_null(): Unit = {
val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section00/general/generalSchema.dfdl.xsd")
val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.txt")
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/unparsing/TestCLIUnparsing.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/unparsing/TestCLIUnparsing.scala
index 1c96980..d56ee4b 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/unparsing/TestCLIUnparsing.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/unparsing/TestCLIUnparsing.scala
@@ -606,6 +606,28 @@
}
}
+ @Test def test_xxxx_CLI_Unparsing_SimpleUnparse_exi(): Unit = {
+
+ val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section00/general/generalSchema.dfdl.xsd")
+ val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.exi")
+ val (testSchemaFile, testInputFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile), Util.cmdConvert(inputFile)) else (schemaFile, inputFile)
+
+ val shell = Util.start("")
+
+ try {
+ val cmd = String.format("%s unparse -I exi -s %s --root e1 %s", Util.binPath, testSchemaFile, testInputFile)
+ shell.sendLine(cmd)
+ shell.expect(contains("Hello"))
+
+ Util.expectExitCode(ExitCode.Success, shell)
+ shell.send("exit\n")
+ shell.expect(eof)
+ shell.close()
+ } finally {
+ shell.close()
+ }
+ }
+
@Test def test_xxxx_CLI_Unparsing_SimpleUnparse_null(): Unit = {
val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section00/general/generalSchema.dfdl.xsd")
diff --git a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
index 8f97d35..08c5479 100644
--- a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
+++ b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
@@ -46,10 +46,15 @@
import org.rogach.scallop.ScallopOption
import org.rogach.scallop.ValueConverter
import org.xml.sax.XMLReader
+import org.xml.sax.ContentHandler
import org.apache.logging.log4j.Level
import org.apache.logging.log4j.core.config.Configurator
import org.apache.commons.io.IOUtils
import org.apache.commons.io.output.NullOutputStream
+import com.siemens.ct.exi.core.helpers.DefaultEXIFactory
+import com.siemens.ct.exi.core.FidelityOptions
+import com.siemens.ct.exi.main.api.sax.EXIResult
+import com.siemens.ct.exi.main.api.sax.EXISource
import org.apache.daffodil.Main.ExitCode
import org.apache.daffodil.api.DFDL
import org.apache.daffodil.api.DFDL.DaffodilUnparseErrorSAXException
@@ -119,6 +124,7 @@
val SCALA_XML = Value("scala-xml")
val W3CDOM = Value("w3cdom")
val XML = Value("xml")
+ val EXI = Value("exi")
val NULL = Value("null")
}
@@ -304,7 +310,7 @@
val config = opt[File](short = 'c', argName = "file", descr = "XML file containing configuration items")
val vars = props[String](name = 'D', keyName = "variable", valueName = "value", descr = "Variables to be used when parsing. Can be prefixed with {namespace}.")
- val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to output. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
+ val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to output. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', 'exi', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
val output = opt[String](argName = "file", descr = "Output file to write infoset to. If not given or is -, infoset is written to stdout.")
val parser = opt[File](short = 'P', argName = "file", descr = "Previously saved parser to reuse")
val path = opt[String](argName = "path", descr = "Path from root element to node from which to start parsing", hidden = true)
@@ -330,6 +336,10 @@
case _ => Right(Unit)
}
+ validateOpt(infosetType, stream) {
+ case (Some(InfosetType.EXI), Some(true)) => Left("Streaming mode is not currently supported with EXI infosets.")
+ case _ => Right(Unit)
+ }
}
// Unparse Subcommand Options
@@ -348,7 +358,7 @@
val config = opt[File](short = 'c', argName = "file", descr = "XML file containing configuration items")
val vars = props[String](name = 'D', keyName = "variable", valueName = "value", descr = "Variables to be used when parsing. Can be prefixed with {namespace}.")
- val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to unparse. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
+ val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to unparse. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', 'exi', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
val output = opt[String](argName = "file", descr = "Output file to write data to. If not given or is -, data is written to stdout.")
val parser = opt[File](short = 'P', argName = "file", descr = "Previously saved parser to reuse")
val path = opt[String](argName = "path", descr = "Path from root element to node from which to start unparsing", hidden = true)
@@ -435,7 +445,7 @@
val config = opt[File](short = 'c', argName = "file", descr = "XML file containing configuration items")
val vars = props[String](name = 'D', keyName = "variable", valueName = "value", descr = "Variables to be used when parsing. Can be prefixed with {namespace}.")
- val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to output or unparse. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
+ val infosetType = opt[InfosetType.Type](short = 'I', argName = "infoset_type", descr = "Infoset type to output or unparse. Use 'xml', 'scala-xml', 'json', 'jdom', 'w3cdom', 'sax', 'exi', or 'null'. Defaults to 'xml'.", default = Some(InfosetType.XML))
val number = opt[Int](short = 'N', argName = "number", default = Some(1), descr = "Total number of files to process. Defaults to 1.")
val parser = opt[File](short = 'P', argName = "file", descr = "Previously saved parser to reuse")
val path = opt[String](argName = "path", descr = "Path from root element to node from which to start parsing or unparsing", hidden = true)
@@ -676,14 +686,21 @@
val blobSuffix = ".bin"
def getInfosetOutputter(infosetType: InfosetType.Type, os: java.io.OutputStream)
- : Either[InfosetOutputter, DaffodilParseOutputStreamContentHandler] = {
+ : Either[InfosetOutputter, ContentHandler] = {
val outputter = infosetType match {
case InfosetType.XML => Left(new XMLTextInfosetOutputter(os, pretty = true))
case InfosetType.SCALA_XML => Left(new ScalaXMLInfosetOutputter())
case InfosetType.JSON => Left(new JsonInfosetOutputter(os, pretty = true))
case InfosetType.JDOM => Left(new JDOMInfosetOutputter())
case InfosetType.W3CDOM => Left(new W3CDOMInfosetOutputter())
- case InfosetType.SAX => Right(new DaffodilParseOutputStreamContentHandler(os, pretty = true))
+ case InfosetType.SAX => Right(new DaffodilParseOutputStreamContentHandler(os, pretty=true))
+ case InfosetType.EXI => {
+ val exiFactory = DefaultEXIFactory.newInstance()
+ exiFactory.getFidelityOptions.setFidelity(FidelityOptions.FEATURE_PREFIX, true)
+ val exiResult = new EXIResult()
+ exiResult.setOutputStream(os)
+ Right(exiResult.getHandler)
+ }
case InfosetType.NULL => Left(new NullInfosetOutputter())
}
if (outputter.isLeft) {
@@ -723,7 +740,7 @@
*/
def infosetDataToInputterData(infosetType: InfosetType.Type, data: Either[Array[Byte],InputStream]): AnyRef = {
infosetType match {
- case InfosetType.XML | InfosetType.JSON | InfosetType.SAX => data match {
+ case InfosetType.XML | InfosetType.JSON | InfosetType.SAX | InfosetType.EXI => data match {
case Left(bytes) => bytes
case Right(is) => is
}
@@ -810,14 +827,14 @@
val tl = anyRef.asInstanceOf[ThreadLocal[org.w3c.dom.Document]]
Left(new W3CDOMInfosetInputter(tl.get))
}
+ case InfosetType.EXI | InfosetType.SAX => {
+ val dp = processor
+ Right(dp.newContentHandlerInstance(outChannel))
+ }
case InfosetType.NULL => {
val events = anyRef.asInstanceOf[Array[NullInfosetInputter.Event]]
Left(new NullInfosetInputter(events))
}
- case InfosetType.SAX => {
- val dp = processor
- Right(dp.newContentHandlerInstance(outChannel))
- }
}
}
@@ -887,8 +904,6 @@
val parseResult = eitherOutputterOrHandler match {
case Right(saxContentHandler) =>
- // reset in case we are streaming
- saxContentHandler.reset()
Timer.getResult("parsing",
parseWithSAX(processor, inStream, saxContentHandler))
case Left(outputter) =>
@@ -1089,7 +1104,7 @@
case bytes: Array[Byte] => new ByteArrayInputStream(bytes)
case is: InputStream => is
}
- unparseWithSAX(is, contentHandler)
+ unparseWithSAX(is, contentHandler, infosetType)
}
})
case Right(data) => Timer.getTimeResult({
@@ -1218,7 +1233,7 @@
case bytes: Array[Byte] => new ByteArrayInputStream(bytes)
case is: InputStream => is
}
- Timer.getResult("unparsing", unparseWithSAX(is, contentHandler))
+ Timer.getResult("unparsing", unparseWithSAX(is, contentHandler, unparseOpts.infosetType.toOption.get))
}
displayDiagnostics(unparseResult)
@@ -1416,8 +1431,15 @@
private def unparseWithSAX(
is: InputStream,
- contentHandler: DFDL.DaffodilUnparseContentHandler): UnparseResult = {
- val xmlReader = DaffodilSAXParserFactory().newSAXParser.getXMLReader
+ contentHandler: DFDL.DaffodilUnparseContentHandler,
+ infosetType: InfosetType.Type): UnparseResult = {
+ val xmlReader = infosetType match {
+ case InfosetType.EXI => {
+ val exiSource = new EXISource()
+ exiSource.getXMLReader
+ }
+ case _ => DaffodilSAXParserFactory().newSAXParser.getXMLReader
+ }
xmlReader.setContentHandler(contentHandler)
xmlReader.setFeature(XMLUtils.SAX_NAMESPACES_FEATURE, true)
xmlReader.setFeature(XMLUtils.SAX_NAMESPACE_PREFIXES_FEATURE, true)
@@ -1434,7 +1456,7 @@
private def parseWithSAX(
processor: DFDL.DataProcessor,
data: InputSourceDataInputStream,
- saxContentHandler: DaffodilParseOutputStreamContentHandler): ParseResult = {
+ saxContentHandler: ContentHandler): ParseResult = {
val saxXmlRdr = processor.newXMLReaderInstance
saxXmlRdr.setContentHandler(saxContentHandler)
saxXmlRdr.setProperty(XMLUtils.DAFFODIL_SAX_URN_BLOBDIRECTORY, blobDir)
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
index 1e06291..0e7bc3b 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
@@ -20,11 +20,11 @@
import java.net.URI
import scala.xml.Node
-import java.io.FileInputStream
import org.apache.daffodil.xml.XMLUtils
import org.apache.commons.io.input.XmlStreamReader
import java.io.File
+import java.io.FileInputStream
import java.nio.file.Paths
import org.apache.daffodil.exceptions.Assert
import org.apache.daffodil.equality._
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DaffodilParseOutputStreamContentHandler.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DaffodilParseOutputStreamContentHandler.scala
index 5121483..0f7a0b4 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DaffodilParseOutputStreamContentHandler.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DaffodilParseOutputStreamContentHandler.scala
@@ -68,28 +68,24 @@
// if the top of the stack is true, we have guessed we should output a newline
private def outputNewline: Boolean = outputNewlineStack.top
- def reset(): Unit = {
- resetIndentation()
- writer.flush()
- activePrefixMapping = null
- currentElementPrefixMapping = null
- activePrefixMappingContextStack.clear()
- outputNewlineStack.clear()
- outputNewlineStack.push(false) //to match initialization state
- out.flush()
- }
-
override def setDocumentLocator(locator: Locator): Unit = {
// do nothing
}
override def startDocument(): Unit = {
+ resetIndentation()
+ activePrefixMapping = null
+ currentElementPrefixMapping = null
+ activePrefixMappingContextStack.clear()
+ outputNewlineStack.clear()
+ outputNewlineStack.push(false) //to match initialization state
writer.write("""<?xml version="1.0" encoding="UTF-8"?>""")
}
override def endDocument(): Unit = {
writer.write(System.lineSeparator())
writer.flush()
+ out.flush()
}
override def startPrefixMapping(prefix: String, uri: String): Unit = {
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index fe5bfb6..278efe4 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -58,4 +58,8 @@
lazy val schematron = Seq(
"net.sf.saxon" % "Saxon-HE" % "11.3",
)
+
+ lazy val exi = Seq(
+ "com.siemens.ct.exi" % "exificient" % "1.0.4",
+ )
}
diff --git a/project/Rat.scala b/project/Rat.scala
index 8213a79..c7eafbc 100644
--- a/project/Rat.scala
+++ b/project/Rat.scala
@@ -70,10 +70,12 @@
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input12.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input13.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.txt"),
+ file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input14.exi"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input15.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input16.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.json"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.txt"),
+ file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input18.exi"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input19.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/inputBig1M.txt"),
file("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/prefix.txt"),
