Improve memory effeciency (#22)

#20 Use s9api only for internal processing; 
Improve memory effeciency
This commit is contained in:
apenski 2019-02-13 08:19:02 +01:00 committed by GitHub
parent aacbce522b
commit 34d79d5e2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 326 additions and 202 deletions

13
pom.xml
View file

@ -27,7 +27,7 @@
<name>KoSIT XML Prüftool Implementierung</name>
<groupId>de.kosit</groupId>
<version>1.0.3-SNAPSHOT</version>
<version>1.1.0-SNAPSHOT</version>
<artifactId>validationtool</artifactId>
<description>KoSIT XML Prüftool zur Prüfung von XML Dateien gegenüber definierten Szenarien.</description>
@ -49,7 +49,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<version.jacoco>0.7.9</version.jacoco>
<version.lombok>1.16.16</version.lombok>
<version.saxon-he>9.7.0-15</version.saxon-he>
<version.saxon-he>9.9.1-1</version.saxon-he>
<version.slf4j>1.7.25</version.slf4j>
</properties>
@ -128,15 +128,6 @@
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<configuration>
<forkMode>always</forkMode>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.1</version>

View file

@ -24,6 +24,9 @@ import java.util.stream.Collectors;
import org.w3c.dom.Document;
import net.sf.saxon.dom.NodeOverNodeInfo;
import net.sf.saxon.s9api.XdmNode;
/**
* Zentrale Schnittstellendefinition für das Prüf-Tool.
*
@ -33,21 +36,46 @@ public interface Check {
/**
* Führt die konfigurierte Prüfung für die übergebene Resource aus.
*
*
* @param input die Resource / XML-Datei, die geprüft werden soll.
* @return ein Ergebnis-{@link Document}
* @deprecated use {@link #checkInput(Input)}
*/
@Deprecated
default Document check(Input input) {
final XdmNode node = checkInput(input);
// readonly view of the document!!!
return (Document) NodeOverNodeInfo.wrap(node.getUnderlyingNode());
}
/**
* Führt die konfigurierte Prüfung für die übergebene Resource aus.
*
* @param input die Resource / XML-Datei, die geprüft werden soll.
* @return ein Ergebnis-{@link Document}
*/
Document check(Input input);
XdmNode checkInput(Input input);
/**
* Führt eine Prüfung im Batch-Mode durch. Die Default-Implementierung führt die Prüfung sequentiell aus.
*
* @param input die Eingabe
* @return Liste mit Ergebnis-Dokumenten
* @deprecated use {@link #checkInput(List)}
*/
@Deprecated
default List<Document> check(List<Input> input) {
return input.stream().map(i -> check(i)).collect(Collectors.toList());
return input.stream().map(this::check).collect(Collectors.toList());
}
/**
* Führt eine Prüfung im Batch-Mode durch. Die Default-Implementierung führt die Prüfung sequentiell aus.
*
* @param input die Eingabe
* @return Liste mit Ergebnis-Dokumenten
*/
default List<XdmNode> checkInput(List<Input> input) {
return input.stream().map(this::checkInput).collect(Collectors.toList());
}
}

View file

@ -21,7 +21,12 @@ package de.kosit.validationtool.api;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import java.io.*;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
@ -51,7 +56,7 @@ public class InputFactory {
private static final int DEFAULT_BUFFER_SIZE = 4096;
public static final String MESSAGE_OPEN_STREAM_ERROR = "Can not open stream from";
private static final String MESSAGE_OPEN_STREAM_ERROR = "Can not open stream from";
@Getter
private final String algorithm;

View file

@ -19,24 +19,29 @@
package de.kosit.validationtool.cmd;
import de.kosit.validationtool.cmd.assertions.AssertionType;
import de.kosit.validationtool.cmd.assertions.Assertions;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.impl.tasks.CheckAction;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.sf.saxon.s9api.*;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Document;
import javax.xml.transform.dom.DOMSource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.cmd.assertions.AssertionType;
import de.kosit.validationtool.cmd.assertions.Assertions;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.impl.tasks.CheckAction;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XPathCompiler;
import net.sf.saxon.s9api.XPathSelector;
import net.sf.saxon.s9api.XdmNode;
/**
* Überprüft den Report mittels bereitgestellter Assertions. Diese {@link CheckAction} dient der Überprüfung der von der
* KoSIT bereitgestellten Prüfszenarien und den darin enthaltenen Artefakten.
@ -64,7 +69,7 @@ public class CheckAssertionAction implements CheckAction {
final List<AssertionType> toCheck = findAssertions(results.getName());
final List<String> errors = new ArrayList<>();
if (toCheck != null && !toCheck.isEmpty()) {
final XdmNode node = loadDocument(results.getReport());
final XdmNode node = results.getReport();
toCheck.forEach(a -> {
if (!check(node, a)) {
log.error("Assertion mismatch: {}", a.getValue());
@ -86,15 +91,6 @@ public class CheckAssertionAction implements CheckAction {
return getMapped().entrySet().stream().filter(e -> matches(e.getKey(), name)).map(Map.Entry::getValue).findFirst().orElse(null);
}
private XdmNode loadDocument(Document d) {
DocumentBuilder documentBuilder = getProcessor().newDocumentBuilder();
try {
return documentBuilder.build(new DOMSource(d));
} catch (SaxonApiException e) {
log.error("Can not load result document. Therefore can not run defined assertions", e);
}
return null;
}
private boolean check(XdmNode document, AssertionType assertion) {
try {

View file

@ -77,6 +77,8 @@ public class CommandLineApplication {
private static final Option CHECK_ASSERTIONS = Option.builder("c").longOpt("check-assertions").hasArg()
.desc("Check the result using defined assertions").argName("assertions-file").build();
private static final Option PRINT_MEM_STATS = Option.builder("m").longOpt("memory-stats").desc("Prints some memory stats").build();
private CommandLineApplication() {
// main class -> hide constructor
}
@ -151,6 +153,9 @@ public class CommandLineApplication {
Assertions assertions = loadAssertions(cmd.getOptionValue(CHECK_ASSERTIONS.getOpt()));
check.getCheckSteps().add(new CheckAssertionAction(assertions, ObjectFactory.createProcessor()));
}
if (cmd.hasOption(PRINT_MEM_STATS.getOpt())) {
check.getCheckSteps().add(new PrintMemoryStats());
}
log.info("Setup completed in {}ms\n", System.currentTimeMillis() - start);
@ -226,7 +231,7 @@ public class CommandLineApplication {
try {
return Files.list(d).filter(path -> path.toString().endsWith(".xml")).collect(Collectors.toList());
} catch (IOException e) {
throw new IllegalStateException("IOException while liste directory content. Can not determine test targets.", e);
throw new IllegalStateException("IOException while list directory content. Can not determine test targets.", e);
}
}
@ -293,6 +298,7 @@ public class CommandLineApplication {
options.addOption(EXTRACT_HTML);
options.addOption(DEBUG);
options.addOption(CHECK_ASSERTIONS);
options.addOption(PRINT_MEM_STATS);
return options;
}
}

View file

@ -23,15 +23,19 @@ import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import javax.xml.transform.dom.DOMSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.impl.ContentRepository;
import de.kosit.validationtool.impl.tasks.CheckAction;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.Serializer;
import net.sf.saxon.s9api.XPathExecutable;
import net.sf.saxon.s9api.XPathSelector;
import net.sf.saxon.s9api.XdmItem;
import net.sf.saxon.s9api.XdmNode;
/**
* Extrahiert HTML-Dokumente aus dem Report und persistiert diese im konfigurierten Ausgabe-Verzeichnis.
@ -54,9 +58,7 @@ public class ExtractHtmlContentAction implements CheckAction {
public void check(Bag results) {
try {
final XPathSelector selector = getSelector();
DocumentBuilder documentBuilder = repository.getProcessor().newDocumentBuilder();
final XdmNode xdmSource = documentBuilder.build(new DOMSource(results.getReport()));
final XdmNode xdmSource = results.getReport();
selector.setContextItem(xdmSource);
selector.forEach(m -> print(results.getName(), m));

View file

@ -26,6 +26,8 @@ import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.DefaultCheck;
import de.kosit.validationtool.impl.tasks.CheckAction;
import net.sf.saxon.s9api.XdmNode;
/**
* Simple Erweiterung der Klasse {@link DefaultCheck} um das Ergebnis der Assertion-Prüfung auszwerten und auszugeben.
* Diese Klasse stellt keine fachlicher Erweiterung des eigentlichen Prüfvorganges dar!
@ -54,16 +56,17 @@ class InternalCheck extends DefaultCheck {
* @param input die Prüflinge
* @return false wenn es Assertion-Fehler gibt, sonst true
*/
void checkInput(Input input) {
public XdmNode checkInput(Input input) {
CheckAction.Bag bag = new CheckAction.Bag(input, createReport());
runCheckInternal(bag);
XdmNode result = runCheckInternal(bag);
if (bag.getAssertionResult() != null) {
checkAssertions += bag.getAssertionResult().getObject();
failedAssertions += bag.getAssertionResult().getErrors().size();
}
return result;
}
public boolean printAndEvaluate() {
boolean printAndEvaluate() {
if (failedAssertions > 0) {
log.error("Assertion check failed.\n\nAssertions run: {}, Assertions failed: {}\n", checkAssertions, failedAssertions);
} else if (checkAssertions > 0) {

View file

@ -0,0 +1,52 @@
/*
* Licensed to the Koordinierungsstelle für IT-Standards (KoSIT) under
* one or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. KoSIT licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package de.kosit.validationtool.cmd;
import java.text.NumberFormat;
import lombok.extern.slf4j.Slf4j;
/**
*
* Prints some memory usage information for debugging purposes.
*
* @author Andreas Penski
*/
@Slf4j
public class PrintMemoryStats implements de.kosit.validationtool.impl.tasks.CheckAction {
private static final int BYTES_PER_K = 1024;
@Override
public void check(final Bag results) {
final Runtime runtime = Runtime.getRuntime();
long maxMemory = runtime.maxMemory();
long allocatedMemory = runtime.totalMemory();
long freeMemory = runtime.freeMemory();
NumberFormat format = NumberFormat.getInstance();
final String freeStr = format.format(freeMemory / BYTES_PER_K);
final String allocStr = format.format(allocatedMemory / BYTES_PER_K);
final String maxStr = format.format(maxMemory / BYTES_PER_K);
final String totalFreeStr = format.format((freeMemory + (maxMemory - allocatedMemory)) / BYTES_PER_K);
log.info("free memory: {}MB; allocated memory: {}MB", freeStr, allocStr);
log.info("max memory: {}MB; total free memory: {}MB", maxStr, totalFreeStr);
}
}

View file

@ -21,18 +21,14 @@ package de.kosit.validationtool.cmd;
import java.io.StringWriter;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.tasks.CheckAction;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.Serializer;
/**
* Gibt das Ergebnis-Document auf std-out aus.
*
@ -44,13 +40,11 @@ public class PrintReportAction implements CheckAction {
@Override
public void check(Bag results) {
try {
Transformer transformer = ObjectFactory.createTransformer(true);
final StringWriter writer = new StringWriter();
Result output = new StreamResult(writer);
Source input = new DOMSource(results.getReport());
transformer.transform(input, output);
final Serializer serializer = ObjectFactory.createProcessor().newSerializer(writer);
serializer.serializeNode(results.getReport());
System.out.print(writer.toString());
} catch (TransformerException e) {
} catch (SaxonApiException e) {
log.error("Error while printing result to stdout", e);
}
}

View file

@ -19,18 +19,16 @@
package de.kosit.validationtool.cmd;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.tasks.CheckAction;
import java.nio.file.Path;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.nio.file.Path;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.tasks.CheckAction;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.Serializer;
/**
* Schreibt das Prüfergebnis als XML-Dokument an eine definierte Stelle.
@ -48,11 +46,9 @@ public class SerializeReportAction implements CheckAction {
final Path file = outputDirectory.resolve(results.getName() + "-report.xml");
try {
log.info("Serializing result to {}", file.toAbsolutePath());
Transformer transformer = ObjectFactory.createTransformer(true);
Result output = new StreamResult(file.toFile());
Source input = new DOMSource(results.getReport());
transformer.transform(input, output);
} catch (TransformerException e) {
final Serializer serializer = ObjectFactory.createProcessor().newSerializer(file.toFile());
serializer.serializeNode(results.getReport());
} catch (SaxonApiException e) {
log.error("Can not serialize result report to {}", file.toAbsolutePath(), e);
}
}

View file

@ -20,11 +20,7 @@
package de.kosit.validationtool.impl;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.w3c.dom.Document;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@ -32,13 +28,20 @@ import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.api.Check;
import de.kosit.validationtool.api.CheckConfiguration;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.tasks.*;
import de.kosit.validationtool.impl.tasks.CheckAction;
import de.kosit.validationtool.impl.tasks.CreateReportAction;
import de.kosit.validationtool.impl.tasks.DocumentParseAction;
import de.kosit.validationtool.impl.tasks.ScenarioSelectionAction;
import de.kosit.validationtool.impl.tasks.SchemaValidationAction;
import de.kosit.validationtool.impl.tasks.SchematronValidationAction;
import de.kosit.validationtool.impl.tasks.ValidateReportInputAction;
import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.DocumentIdentificationType;
import de.kosit.validationtool.model.reportInput.EngineType;
import de.kosit.validationtool.model.reportInput.ProcessingError;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.XdmNode;
/**
* Die Referenz-Implementierung für den Prüfprozess. Nach initialer Konfiguration ist diese Klasse threadsafe und kann
@ -95,26 +98,25 @@ public class DefaultCheck implements Check {
}
@Override
public Document check(Input input) {
public XdmNode checkInput(Input input) {
CheckAction.Bag t = new CheckAction.Bag(input, createReport());
return runCheckInternal(t);
}
protected Document runCheckInternal(CheckAction.Bag t) {
protected XdmNode runCheckInternal(CheckAction.Bag t) {
long started = System.currentTimeMillis();
log.info("Checking content of {}", t.getInput().getName());
Iterator<CheckAction> it = checkSteps.iterator();
while (it.hasNext()) {
final CheckAction action = it.next();
for (final CheckAction action : checkSteps) {
long start = System.currentTimeMillis();
if (!action.isSkipped(t)) {
action.check(t);
}
log.info("Step {} finished in {}ms", action.getClass().getSimpleName(), System.currentTimeMillis() - start);
if (t.isStopped()) {
final ProcessingError processingError = t.getReportInput().getProcessingError();
log.error("Error processing input {}: {}", t.getInput().getName(),
processingError != null ? processingError.getError().stream().collect(Collectors.joining("\n")) : "");
processingError != null ? String.join("\n", processingError.getError()) : "");
break;
}
}

View file

@ -19,16 +19,14 @@
package de.kosit.validationtool.impl;
import static org.apache.commons.lang3.StringUtils.startsWith;
import java.net.MalformedURLException;
import java.net.URI;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import javax.xml.transform.dom.DOMSource;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@ -42,7 +40,13 @@ import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import de.kosit.validationtool.model.scenarios.Scenarios;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XPathSelector;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XdmNodeKind;
import net.sf.saxon.s9api.XsltExecutable;
/**
* Repository for die aktiven Szenario einer Prüfinstanz.
@ -69,16 +73,28 @@ public class ScenarioRepository {
@Getter(value = AccessLevel.PACKAGE)
private Scenarios scenarios;
private static boolean isSupportedDocument(Document doc) {
final Element root = doc.getDocumentElement();
return root.hasAttribute("frameworkVersion") && root.getAttribute("frameworkVersion").startsWith(SUPPORTED_MAJOR_VERSION)
&& doc.getDocumentElement().getNamespaceURI().equals(SUPPORTED_MAJOR_VERSION_SCHEMA);
private static boolean isSupportedDocument(XdmNode doc) {
final XdmNode root = findRoot(doc);
final String frameworkVersion = root.getAttributeValue(new QName("frameworkVersion"));
return startsWith(frameworkVersion, SUPPORTED_MAJOR_VERSION)
&& root.getNodeName().getNamespaceURI().equals(SUPPORTED_MAJOR_VERSION_SCHEMA);
}
private static XdmNode findRoot(final XdmNode doc) {
final Iterator<XdmNode> it = doc.children().iterator();
while (it.hasNext()) {
final XdmNode node = it.next();
if (node.getNodeKind() == XdmNodeKind.ELEMENT) {
return node;
}
}
throw new IllegalArgumentException("Kein root element gefunden");
}
private static void checkVersion(URI scenarioDefinition) {
DocumentParseAction p = new DocumentParseAction();
try {
final Result<Document, XMLSyntaxError> result = p.parseDocument(InputFactory.read(scenarioDefinition.toURL()));
final Result<XdmNode, XMLSyntaxError> result = p.parseDocument(InputFactory.read(scenarioDefinition.toURL()));
if (result.isValid() && !isSupportedDocument(result.getObject())) {
throw new IllegalStateException(String.format(
"Specified scenario configuration %s is not supported.%nThis version only supports definitions of '%s'",
@ -138,7 +154,7 @@ public class ScenarioRepository {
* @param document das Eingabedokument
* @return ein Ergebnis-Objekt zur weiteren Verarbeitung
*/
public Result<ScenarioType, String> selectScenario(Document document) {
public Result<ScenarioType, String> selectScenario(XdmNode document) {
Result<ScenarioType, String> result = new Result<>();
final List<ScenarioType> collect = scenarios.getScenario().stream().filter(s -> match(document, s)).collect(Collectors.toList());
if (collect.size() == 1) {
@ -152,13 +168,10 @@ public class ScenarioRepository {
}
private boolean match(Document document, ScenarioType scenario) {
private boolean match(XdmNode document, ScenarioType scenario) {
try {
final XPathSelector selector = scenario.getSelector();
DocumentBuilder documentBuilder = getProcessor().newDocumentBuilder();
final XdmNode xdmSource = documentBuilder.build(new DOMSource(document));
selector.setContextItem(xdmSource);
selector.setContextItem(document);
return selector.effectiveBooleanValue();
} catch (SaxonApiException e) {
log.error("Error evaluating xpath expression", e);

View file

@ -19,17 +19,19 @@
package de.kosit.validationtool.impl.tasks;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.Getter;
import lombok.Setter;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import lombok.Getter;
import lombok.Setter;
import org.w3c.dom.Document;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sf.saxon.s9api.XdmNode;
/**
* Interface, welches von allen Prüfschritten implementiert wird. Der Parameter vom Typ {@link Bag} dient dabei sowohl
@ -72,7 +74,7 @@ public interface CheckAction {
private CreateReportInput reportInput;
/** Das finale Ergebnis */
private Document report;
private XdmNode report;
private boolean finished;
@ -81,7 +83,7 @@ public interface CheckAction {
/** Das zu prüfende Dokument */
private Input input;
private Result<Document, XMLSyntaxError> parserResult;
private Result<XdmNode, XMLSyntaxError> parserResult;
private Result<Integer, String> assertionResult;

View file

@ -27,11 +27,22 @@ import org.w3c.dom.Document;
import lombok.RequiredArgsConstructor;
import de.kosit.validationtool.impl.*;
import de.kosit.validationtool.impl.CollectingErrorEventHandler;
import de.kosit.validationtool.impl.ConversionService;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.RelativeUriResolver;
import de.kosit.validationtool.impl.ScenarioRepository;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.DocumentBuilder;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmDestination;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;
/**
* Erzeugt den Report auf Basis der gesammelten Informationen über den Prüfling. Sollte kein Szenario identifiziert
@ -59,10 +70,9 @@ public class CreateReportAction implements CheckAction {
final DocumentBuilder documentBuilder = processor.newDocumentBuilder();
try {
final Document inputDoc = results.getParserResult().isValid() ? results.getParserResult().getObject()
: ObjectFactory.createDocumentBuilder(true).newDocument();
final XdmNode parsedDocument = results.getParserResult().isValid() ? results.getParserResult().getObject()
: ObjectFactory.createProcessor().newDocumentBuilder().newBuildingContentHandler().getDocumentNode();
final XdmNode parsedDocument = documentBuilder.build(new DOMSource(inputDoc));
final Document reportInput = conversionService.writeDocument(results.getReportInput());
final XdmNode root = documentBuilder.build(new DOMSource(reportInput));
final XsltTransformer transformer = getTransformation(results).load();
@ -73,10 +83,10 @@ public class CreateReportAction implements CheckAction {
transformer.setURIResolver(resolver);
transformer.getUnderlyingController().setUnparsedTextURIResolver(resolver);
transformer.setParameter(new QName("input-document"), parsedDocument);
Document result = ObjectFactory.createDocumentBuilder(false).newDocument();
transformer.setDestination(new DOMDestination(result));
final XdmDestination destination = new XdmDestination();
transformer.setDestination(destination);
transformer.transform();
results.setReport(result);
results.setReport(destination.getXdmNode());
} catch (SaxonApiException e) {
throw new IllegalStateException("Can not create final report", e);

View file

@ -19,23 +19,26 @@
package de.kosit.validationtool.impl.tasks;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import javax.xml.transform.stream.StreamSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.CollectingErrorEventHandler;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.reportInput.ValidationResultsWellformedness;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.reportInput.XMLSyntaxErrorSeverity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import net.sf.saxon.s9api.DocumentBuilder;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmNode;
/**
* Setzt Parsing-Funktionalitäten um. Prüft auf well-formedness
@ -53,22 +56,18 @@ public class DocumentParseAction implements CheckAction {
* @param content ein Dokument
* @return Ergebnis des Parsings inklusive etwaiger Fehler
*/
public Result<Document, XMLSyntaxError> parseDocument(Input content) {
public Result<XdmNode, XMLSyntaxError> parseDocument(Input content) {
if (content == null) {
throw new IllegalArgumentException("Url may not be null");
throw new IllegalArgumentException("Input may not be null");
}
Result<Document, XMLSyntaxError> result;
CollectingErrorEventHandler errorHandler = new CollectingErrorEventHandler();
Result<XdmNode, XMLSyntaxError> result;
try ( InputStream input = new ByteArrayInputStream(content.getContent()) ) {
DocumentBuilder db = ObjectFactory.createDocumentBuilder(false);
db.setErrorHandler(errorHandler);
Document doc = db.parse(input);
result = new Result<>(doc, errorHandler.getErrors());
} catch (SAXException e) {
log.debug("SAXException while parsing {}", content.getName(), e);
result = new Result<>(errorHandler.getErrors());
} catch (IOException e) {
log.debug("IOException while parsing {}", content, e);
final DocumentBuilder builder = ObjectFactory.createProcessor().newDocumentBuilder();
builder.setLineNumbering(true);
XdmNode doc = builder.build(new StreamSource(input));
result = new Result<>(doc, Collections.emptyList());
} catch (SaxonApiException | IOException e) {
log.debug("Exception while parsing {}", content.getName(), e);
XMLSyntaxError error = new XMLSyntaxError();
error.setSeverity(XMLSyntaxErrorSeverity.SEVERITY_FATAL_ERROR);
error.setMessage(String.format("IOException while reading resource %s", content.getName()));
@ -80,7 +79,7 @@ public class DocumentParseAction implements CheckAction {
@Override
public void check(Bag results) {
Result<Document, XMLSyntaxError> parserResult = parseDocument(results.getInput());
Result<XdmNode, XMLSyntaxError> parserResult = parseDocument(results.getInput());
ValidationResultsWellformedness v = new ValidationResultsWellformedness();
results.setParserResult(parserResult);
v.getXmlSyntaxError().addAll(parserResult.getErrors());

View file

@ -23,8 +23,6 @@ import java.net.URI;
import java.util.List;
import java.util.stream.Collectors;
import javax.xml.transform.dom.DOMSource;
import org.w3c.dom.Document;
import lombok.RequiredArgsConstructor;
@ -37,7 +35,11 @@ import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.ValidationResultsSchematron;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.DOMDestination;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XsltTransformer;
/**
* Ausführung von konfigurierten Schematron Validierungen eines Szenarios.
@ -51,16 +53,14 @@ public class SchematronValidationAction implements CheckAction {
private final URI repository;
private List<ValidationResultsSchematron> validate(Document document, ScenarioType scenario) {
private List<ValidationResultsSchematron> validate(XdmNode document, ScenarioType scenario) {
return scenario.getSchematronValidations().stream().map(v -> validate(document, v)).collect(Collectors.toList());
}
private ValidationResultsSchematron validate(Document document, BaseScenario.Transformation validation) {
final DocumentBuilder documentBuilder = processor.newDocumentBuilder();
private ValidationResultsSchematron validate(XdmNode document, BaseScenario.Transformation validation) {
try {
final XdmNode root = documentBuilder.build(new DOMSource(document));
final XsltTransformer transformer = validation.getExecutable().load();
//resolving nur relative zum Repository
// resolving nur relative zum Repository
final RelativeUriResolver resolver = new RelativeUriResolver(repository);
transformer.setURIResolver(resolver);
CollectingErrorEventHandler e = new CollectingErrorEventHandler();
@ -68,7 +68,7 @@ public class SchematronValidationAction implements CheckAction {
Document result = ObjectFactory.createDocumentBuilder(false).newDocument();
transformer.setDestination(new DOMDestination(result));
transformer.setInitialContextNode(root);
transformer.setInitialContextNode(document);
transformer.transform();
ValidationResultsSchematron s = new ValidationResultsSchematron();
s.setResource(validation.getResourceType());
@ -85,9 +85,9 @@ public class SchematronValidationAction implements CheckAction {
@Override
public void check(Bag results) {
final CreateReportInput report = results.getReportInput();
final List<ValidationResultsSchematron> bla = validate(results.getParserResult().getObject(),
final List<ValidationResultsSchematron> validationResult = validate(results.getParserResult().getObject(),
results.getScenarioSelectionResult().getObject());
report.getValidationResultsSchematron().addAll(bla);
report.getValidationResultsSchematron().addAll(validationResult);
}
@Override

View file

@ -66,7 +66,7 @@ public class CheckAssertionActionTest {
@Test
public void testSimple() throws URISyntaxException {
final CheckAction.Bag bag = new CheckAction.Bag(InputFactory.read(SAMPLE), new CreateReportInput());
bag.setReport(Helper.load(SAMPLE_REPORT).getObject());
bag.setReport(Helper.load(SAMPLE_REPORT));
final Assertions assertions = Helper.load(SAMPLE_ASSERTIONS, Assertions.class);
CheckAssertionAction a = new CheckAssertionAction(assertions, ObjectFactory.createProcessor());

View file

@ -19,8 +19,14 @@
package de.kosit.validationtool.cmd;
import org.junit.Before;
import org.junit.Test;
import static de.kosit.validationtool.impl.Helper.ASSERTIONS;
import static de.kosit.validationtool.impl.Helper.NOT_EXISTING;
import static de.kosit.validationtool.impl.Helper.REPOSITORY;
import static de.kosit.validationtool.impl.Helper.SAMPLE;
import static de.kosit.validationtool.impl.Helper.SAMPLE2;
import static de.kosit.validationtool.impl.Helper.SAMPLE_DIR;
import static de.kosit.validationtool.impl.Helper.SCENARIO_FILE;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.nio.file.Files;
@ -28,8 +34,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import static de.kosit.validationtool.impl.Helper.*;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.Before;
import org.junit.Test;
/**
* Testet die Parameter des Kommandozeilen-Tools.
@ -146,7 +152,7 @@ public class CommandlineApplicationTest {
Paths.get(SAMPLE).toString() };
CommandLineApplication.mainProgram(args);
assertThat(commandLine.getErrorOutput()).contains(RESULT_OUTPUT);
assertThat(commandLine.getOutputLines()).contains("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
assertThat(commandLine.getOutputLines().get(0)).contains("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
}
@Test

View file

@ -64,7 +64,7 @@ public class ExtractHtmlActionTest {
public void testSimple() throws IOException {
CheckAction.Bag b = new CheckAction.Bag(InputFactory.read(REPORT));
assertThat(action.isSkipped(b)).isTrue();
b.setReport(Helper.load(REPORT).getObject());
b.setReport(Helper.load(REPORT));
action.check(b);
assertThat(action.isSkipped(b)).isFalse();
action.check(b);

View file

@ -58,7 +58,7 @@ public class PrintReportActionTest {
@Test
public void testSimpleSerialize() {
CheckAction.Bag b = new CheckAction.Bag(InputFactory.read(REPORT));
b.setReport(Helper.load(REPORT).getObject());
b.setReport(Helper.load(REPORT));
assertThat(action.isSkipped(b)).isFalse();
action.check(b);
assertThat(b.isStopped()).isFalse();

View file

@ -19,20 +19,21 @@
package de.kosit.validationtool.cmd;
import de.kosit.validationtool.api.InputFactory;
import de.kosit.validationtool.impl.Helper;
import de.kosit.validationtool.impl.tasks.CheckAction;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import de.kosit.validationtool.api.InputFactory;
import de.kosit.validationtool.impl.Helper;
import de.kosit.validationtool.impl.tasks.CheckAction;
/**
* @author Andreas Penski
@ -60,7 +61,7 @@ public class SerializeReportActionTest {
public void testSimpleSerialize() {
CheckAction.Bag b = new CheckAction.Bag(InputFactory.read(REPORT));
assertThat(action.isSkipped(b)).isTrue();
b.setReport(Helper.load(REPORT).getObject());
b.setReport(Helper.load(REPORT));
assertThat(action.isSkipped(b)).isFalse();
action.check(b);
assertThat(b.isStopped()).isFalse();

View file

@ -19,11 +19,8 @@
package de.kosit.validationtool.impl;
import de.kosit.validationtool.api.CheckConfiguration;
import de.kosit.validationtool.api.Input;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
import static de.kosit.validationtool.api.InputFactory.read;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.net.URISyntaxException;
@ -32,8 +29,14 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static de.kosit.validationtool.api.InputFactory.read;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
import de.kosit.validationtool.api.CheckConfiguration;
import de.kosit.validationtool.api.Input;
import net.sf.saxon.s9api.XdmNode;
/**
* Test das Check-Interface
@ -60,12 +63,26 @@ public class DefaultCheckTest {
@Test
public void testHappyCase() throws Exception {
final XdmNode doc = implementation.checkInput(read(VALID_EXAMPLE));
assertThat(doc).isNotNull();
}
@Test
public void testHappyCaseDocument() throws Exception {
final Document doc = implementation.check(read(VALID_EXAMPLE));
assertThat(doc).isNotNull();
}
@Test
public void testMultipleCase() throws Exception {
final List<Input> input = IntStream.range(0, MULTI_COUNT).mapToObj(i -> read(VALID_EXAMPLE)).collect(Collectors.toList());
final List<XdmNode> docs = implementation.checkInput(input);
assertThat(docs).isNotNull();
assertThat(docs).hasSize(MULTI_COUNT);
}
@Test
public void testMultipleCaseDocument() throws Exception {
final List<Input> input = IntStream.range(0, MULTI_COUNT).mapToObj(i -> read(VALID_EXAMPLE)).collect(Collectors.toList());
final List<Document> docs = implementation.check(input);
assertThat(docs).isNotNull();

View file

@ -29,12 +29,13 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.w3c.dom.Document;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.impl.tasks.DocumentParseAction;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import net.sf.saxon.s9api.XdmNode;
/**
* @author Andreas Penski
*/
@ -60,7 +61,7 @@ public class DocumentParserTest {
@Test
public void testSimple() throws IOException {
final Result<Document, XMLSyntaxError> result = parser.parseDocument(read(CONTENT));
final Result<XdmNode, XMLSyntaxError> result = parser.parseDocument(read(CONTENT));
assertThat(result).isNotNull();
assertThat(result.getObject()).isNotNull();
assertThat(result.getErrors()).isEmpty();
@ -70,7 +71,7 @@ public class DocumentParserTest {
@Test
public void testIllformed() throws IOException {
final Result<Document, XMLSyntaxError> result = parser.parseDocument(read(ILLFORMED));
final Result<XdmNode, XMLSyntaxError> result = parser.parseDocument(read(ILLFORMED));
assertThat(result).isNotNull();
assertThat(result.getErrors()).isNotEmpty();
assertThat(result.getObject()).isNull();

View file

@ -20,18 +20,17 @@
package de.kosit.validationtool.impl;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;
import org.w3c.dom.Document;
import javax.xml.transform.stream.StreamSource;
import de.kosit.validationtool.api.InputFactory;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.impl.tasks.CheckAction;
import de.kosit.validationtool.impl.tasks.DocumentParseAction;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmNode;
/**
* Helferlein für Test-Artefakte
@ -50,36 +49,38 @@ public class Helper {
public static final URI TEST_ROOT = Paths.get("src/test/resources").toUri();
public static final URI EXAMPLES_DIR = TEST_ROOT.resolve("examples/");
public static final URI ASSERTIONS = EXAMPLES_DIR.resolve("assertions/tests-xrechnung.xml");
public static final URI SCENARIO_FILE = EXAMPLES_DIR.resolve("UBLReady/scenarios-2.xml");
public static final URI REPOSITORY = EXAMPLES_DIR.resolve("repository/");
public static final URI NOT_EXISTING = EXAMPLES_DIR.resolve("doesnotexist");
public static final URI SAMPLE_DIR = EXAMPLES_DIR.resolve("UBLReady/");
public static final URI SAMPLE_XSLT = EXAMPLES_DIR.resolve("repository/resources/eRechnung/report.xsl");
public static final URI SAMPLE = SAMPLE_DIR.resolve("UBLReady_EU_UBL-NL_20170102_FULL.xml");
public static final URI SAMPLE2 = SAMPLE_DIR.resolve("UBLReady_EU_UBL-NL_20170102_FULL-invalid.xml");
/**
* Lädt ein XML-Dokument von der gegebenen URL
*
* @param url die url die geladen werden soll
* @return ein result objekt mit Dokument
*/
public static Result<Document, XMLSyntaxError> load(URL url) {
DocumentParseAction a = new DocumentParseAction();
CheckAction.Bag b = new CheckAction.Bag(InputFactory.read(url));
a.check(b);
return b.getParserResult();
public static XdmNode load(URL url) {
try ( InputStream input = url.openStream() ) {
return ObjectFactory.createProcessor().newDocumentBuilder().build(new StreamSource(input));
} catch (SaxonApiException | IOException e) {
throw new IllegalStateException("Fehler beim Laden der XML-Datei", e);
}
}
public static <T> T load(URL url, Class<T> type) throws URISyntaxException {
@ -98,5 +99,5 @@ public class Helper {
public static ContentRepository loadTestRepository() {
return new ContentRepository(ObjectFactory.createProcessor(), new File("src/test/resources/examples/repository").toURI());
}
}
}

View file

@ -30,13 +30,14 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.w3c.dom.Document;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.impl.tasks.DocumentParseAction;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import de.kosit.validationtool.model.scenarios.Scenarios;
import net.sf.saxon.s9api.XdmNode;
/**
* Testet das {@link ScenarioRepository}.
*
@ -94,7 +95,7 @@ public class ScenarioRepositoryTest {
assertThat(scenario.isValid()).isFalse();
}
private Document load(URL url) throws IOException {
private XdmNode load(URL url) throws IOException {
DocumentParseAction p = new DocumentParseAction();
return p.parseDocument(read(url)).getObject();
}

View file

@ -69,7 +69,6 @@ public class SchemaValidatorActionTest {
@Test
public void testSimple() {
CheckAction.Bag bag = new CheckAction.Bag(InputFactory.read(VALID_EXAMPLE), new CreateReportInput());
bag.setParserResult(Helper.load(VALID_EXAMPLE));
ScenarioType t = new ScenarioType();
ValidateWithXmlSchema v = new ValidateWithXmlSchema();
ResourceType r = new ResourceType();
@ -88,7 +87,6 @@ public class SchemaValidatorActionTest {
@Test
public void testValidationFailure() throws MalformedURLException {
CheckAction.Bag bag = new CheckAction.Bag(InputFactory.read(INVALID_EXAMPLE.toURL()), new CreateReportInput());
bag.setParserResult(Helper.load(INVALID_EXAMPLE.toURL()));
ScenarioType t = new ScenarioType();
ValidateWithXmlSchema v = new ValidateWithXmlSchema();
ResourceType r = new ResourceType();