demo/validator
1
0
Fork 0
mirror of https://github.com/itplr-kosit/validator.git synced 2026-05-26 01:05:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Improve memory effeciency (#22)

Browse source 
#20 Use s9api only for internal processing; 
Improve memory effeciency
This commit is contained in:
apenski 2019-02-13 08:19:02 +01:00 • committed by GitHub
parent aacbce522b
commit 34d79d5e2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 326 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/main/java/de/kosit/validationtool/impl/DefaultCheck.java
View file

@ -20,11 +20,7 @@
package de.kosit.validationtool.impl;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.w3c.dom.Document;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@ -32,13 +28,20 @@ import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.api.Check;
import de.kosit.validationtool.api.CheckConfiguration;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.tasks.*;
import de.kosit.validationtool.impl.tasks.CheckAction;
import de.kosit.validationtool.impl.tasks.CreateReportAction;
import de.kosit.validationtool.impl.tasks.DocumentParseAction;
import de.kosit.validationtool.impl.tasks.ScenarioSelectionAction;
import de.kosit.validationtool.impl.tasks.SchemaValidationAction;
import de.kosit.validationtool.impl.tasks.SchematronValidationAction;
import de.kosit.validationtool.impl.tasks.ValidateReportInputAction;
import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.DocumentIdentificationType;
import de.kosit.validationtool.model.reportInput.EngineType;
import de.kosit.validationtool.model.reportInput.ProcessingError;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.XdmNode;
/**
* Die Referenz-Implementierung für den Prüfprozess. Nach initialer Konfiguration ist diese Klasse threadsafe und kann
@ -95,26 +98,25 @@ public class DefaultCheck implements Check {
}
@Override
public Document check(Input input) {
public XdmNode checkInput(Input input) {
CheckAction.Bag t = new CheckAction.Bag(input, createReport());
return runCheckInternal(t);
}
protected Document runCheckInternal(CheckAction.Bag t) {
protected XdmNode runCheckInternal(CheckAction.Bag t) {
long started = System.currentTimeMillis();
log.info("Checking content of {}", t.getInput().getName());
Iterator<CheckAction> it = checkSteps.iterator();
while (it.hasNext()) {
final CheckAction action = it.next();
for (final CheckAction action : checkSteps) {
long start = System.currentTimeMillis();
if (!action.isSkipped(t)) {
action.check(t);
}
log.info("Step {} finished in {}ms", action.getClass().getSimpleName(), System.currentTimeMillis() - start);
if (t.isStopped()) {
final ProcessingError processingError = t.getReportInput().getProcessingError();
log.error("Error processing input {}: {}", t.getInput().getName(),
processingError != null ? processingError.getError().stream().collect(Collectors.joining("\n")) : "");
processingError != null ? String.join("\n", processingError.getError()) : "");
break;
}
}

47
src/main/java/de/kosit/validationtool/impl/ScenarioRepository.java
View file

@ -19,16 +19,14 @@
package de.kosit.validationtool.impl;
import static org.apache.commons.lang3.StringUtils.startsWith;
import java.net.MalformedURLException;
import java.net.URI;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import javax.xml.transform.dom.DOMSource;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@ -42,7 +40,13 @@ import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import de.kosit.validationtool.model.scenarios.Scenarios;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XPathSelector;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XdmNodeKind;
import net.sf.saxon.s9api.XsltExecutable;
/**
* Repository for die aktiven Szenario einer Prüfinstanz.
@ -69,16 +73,28 @@ public class ScenarioRepository {
@Getter(value = AccessLevel.PACKAGE)
private Scenarios scenarios;
private static boolean isSupportedDocument(Document doc) {
final Element root = doc.getDocumentElement();
return root.hasAttribute("frameworkVersion") && root.getAttribute("frameworkVersion").startsWith(SUPPORTED_MAJOR_VERSION)
&& doc.getDocumentElement().getNamespaceURI().equals(SUPPORTED_MAJOR_VERSION_SCHEMA);
private static boolean isSupportedDocument(XdmNode doc) {
final XdmNode root = findRoot(doc);
final String frameworkVersion = root.getAttributeValue(new QName("frameworkVersion"));
return startsWith(frameworkVersion, SUPPORTED_MAJOR_VERSION)
&& root.getNodeName().getNamespaceURI().equals(SUPPORTED_MAJOR_VERSION_SCHEMA);
}
private static XdmNode findRoot(final XdmNode doc) {
final Iterator<XdmNode> it = doc.children().iterator();
while (it.hasNext()) {
final XdmNode node = it.next();
if (node.getNodeKind() == XdmNodeKind.ELEMENT) {
return node;
}
}
throw new IllegalArgumentException("Kein root element gefunden");
}
private static void checkVersion(URI scenarioDefinition) {
DocumentParseAction p = new DocumentParseAction();
try {
final Result<Document, XMLSyntaxError> result = p.parseDocument(InputFactory.read(scenarioDefinition.toURL()));
final Result<XdmNode, XMLSyntaxError> result = p.parseDocument(InputFactory.read(scenarioDefinition.toURL()));
if (result.isValid() && !isSupportedDocument(result.getObject())) {
throw new IllegalStateException(String.format(
"Specified scenario configuration %s is not supported.%nThis version only supports definitions of '%s'",
@ -138,7 +154,7 @@ public class ScenarioRepository {
* @param document das Eingabedokument
* @return ein Ergebnis-Objekt zur weiteren Verarbeitung
*/
public Result<ScenarioType, String> selectScenario(Document document) {
public Result<ScenarioType, String> selectScenario(XdmNode document) {
Result<ScenarioType, String> result = new Result<>();
final List<ScenarioType> collect = scenarios.getScenario().stream().filter(s -> match(document, s)).collect(Collectors.toList());
if (collect.size() == 1) {
@ -152,13 +168,10 @@ public class ScenarioRepository {
}
private boolean match(Document document, ScenarioType scenario) {
private boolean match(XdmNode document, ScenarioType scenario) {
try {
final XPathSelector selector = scenario.getSelector();
DocumentBuilder documentBuilder = getProcessor().newDocumentBuilder();
final XdmNode xdmSource = documentBuilder.build(new DOMSource(document));
selector.setContextItem(xdmSource);
selector.setContextItem(document);
return selector.effectiveBooleanValue();
} catch (SaxonApiException e) {
log.error("Error evaluating xpath expression", e);

16
src/main/java/de/kosit/validationtool/impl/tasks/CheckAction.java
View file

@ -19,17 +19,19 @@
package de.kosit.validationtool.impl.tasks;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.Getter;
import lombok.Setter;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import lombok.Getter;
import lombok.Setter;
import org.w3c.dom.Document;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sf.saxon.s9api.XdmNode;
/**
* Interface, welches von allen Prüfschritten implementiert wird. Der Parameter vom Typ {@link Bag} dient dabei sowohl
@ -72,7 +74,7 @@ public interface CheckAction {
private CreateReportInput reportInput;
/** Das finale Ergebnis */
private Document report;
private XdmNode report;
private boolean finished;
@ -81,7 +83,7 @@ public interface CheckAction {
/** Das zu prüfende Dokument */
private Input input;
private Result<Document, XMLSyntaxError> parserResult;
private Result<XdmNode, XMLSyntaxError> parserResult;
private Result<Integer, String> assertionResult;

26
src/main/java/de/kosit/validationtool/impl/tasks/CreateReportAction.java
View file

@ -27,11 +27,22 @@ import org.w3c.dom.Document;
import lombok.RequiredArgsConstructor;
import de.kosit.validationtool.impl.*;
import de.kosit.validationtool.impl.CollectingErrorEventHandler;
import de.kosit.validationtool.impl.ConversionService;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.RelativeUriResolver;
import de.kosit.validationtool.impl.ScenarioRepository;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.DocumentBuilder;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmDestination;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;
/**
* Erzeugt den Report auf Basis der gesammelten Informationen über den Prüfling. Sollte kein Szenario identifiziert
@ -59,10 +70,9 @@ public class CreateReportAction implements CheckAction {
final DocumentBuilder documentBuilder = processor.newDocumentBuilder();
try {
final Document inputDoc = results.getParserResult().isValid() ? results.getParserResult().getObject()
: ObjectFactory.createDocumentBuilder(true).newDocument();
final XdmNode parsedDocument = results.getParserResult().isValid() ? results.getParserResult().getObject()
: ObjectFactory.createProcessor().newDocumentBuilder().newBuildingContentHandler().getDocumentNode();
final XdmNode parsedDocument = documentBuilder.build(new DOMSource(inputDoc));
final Document reportInput = conversionService.writeDocument(results.getReportInput());
final XdmNode root = documentBuilder.build(new DOMSource(reportInput));
final XsltTransformer transformer = getTransformation(results).load();
@ -73,10 +83,10 @@ public class CreateReportAction implements CheckAction {
transformer.setURIResolver(resolver);
transformer.getUnderlyingController().setUnparsedTextURIResolver(resolver);
transformer.setParameter(new QName("input-document"), parsedDocument);
Document result = ObjectFactory.createDocumentBuilder(false).newDocument();
transformer.setDestination(new DOMDestination(result));
final XdmDestination destination = new XdmDestination();
transformer.setDestination(destination);
transformer.transform();
results.setReport(result);
results.setReport(destination.getXdmNode());
} catch (SaxonApiException e) {
throw new IllegalStateException("Can not create final report", e);

47
src/main/java/de/kosit/validationtool/impl/tasks/DocumentParseAction.java
View file

@ -19,23 +19,26 @@
package de.kosit.validationtool.impl.tasks;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import javax.xml.transform.stream.StreamSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import de.kosit.validationtool.api.Input;
import de.kosit.validationtool.impl.CollectingErrorEventHandler;
import de.kosit.validationtool.impl.ObjectFactory;
import de.kosit.validationtool.impl.model.Result;
import de.kosit.validationtool.model.reportInput.ValidationResultsWellformedness;
import de.kosit.validationtool.model.reportInput.XMLSyntaxError;
import de.kosit.validationtool.model.reportInput.XMLSyntaxErrorSeverity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import net.sf.saxon.s9api.DocumentBuilder;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmNode;
/**
* Setzt Parsing-Funktionalitäten um. Prüft auf well-formedness
@ -53,22 +56,18 @@ public class DocumentParseAction implements CheckAction {
* @param content ein Dokument
* @return Ergebnis des Parsings inklusive etwaiger Fehler
*/
public Result<Document, XMLSyntaxError> parseDocument(Input content) {
public Result<XdmNode, XMLSyntaxError> parseDocument(Input content) {
if (content == null) {
throw new IllegalArgumentException("Url may not be null");
throw new IllegalArgumentException("Input may not be null");
}
Result<Document, XMLSyntaxError> result;
CollectingErrorEventHandler errorHandler = new CollectingErrorEventHandler();
Result<XdmNode, XMLSyntaxError> result;
try ( InputStream input = new ByteArrayInputStream(content.getContent()) ) {
DocumentBuilder db = ObjectFactory.createDocumentBuilder(false);
db.setErrorHandler(errorHandler);
Document doc = db.parse(input);
result = new Result<>(doc, errorHandler.getErrors());
} catch (SAXException e) {
log.debug("SAXException while parsing {}", content.getName(), e);
result = new Result<>(errorHandler.getErrors());
} catch (IOException e) {
log.debug("IOException while parsing {}", content, e);
final DocumentBuilder builder = ObjectFactory.createProcessor().newDocumentBuilder();
builder.setLineNumbering(true);
XdmNode doc = builder.build(new StreamSource(input));
result = new Result<>(doc, Collections.emptyList());
} catch (SaxonApiException | IOException e) {
log.debug("Exception while parsing {}", content.getName(), e);
XMLSyntaxError error = new XMLSyntaxError();
error.setSeverity(XMLSyntaxErrorSeverity.SEVERITY_FATAL_ERROR);
error.setMessage(String.format("IOException while reading resource %s", content.getName()));
@ -80,7 +79,7 @@ public class DocumentParseAction implements CheckAction {
@Override
public void check(Bag results) {
Result<Document, XMLSyntaxError> parserResult = parseDocument(results.getInput());
Result<XdmNode, XMLSyntaxError> parserResult = parseDocument(results.getInput());
ValidationResultsWellformedness v = new ValidationResultsWellformedness();
results.setParserResult(parserResult);
v.getXmlSyntaxError().addAll(parserResult.getErrors());

22
src/main/java/de/kosit/validationtool/impl/tasks/SchematronValidationAction.java
View file

@ -23,8 +23,6 @@ import java.net.URI;
import java.util.List;
import java.util.stream.Collectors;
import javax.xml.transform.dom.DOMSource;
import org.w3c.dom.Document;
import lombok.RequiredArgsConstructor;
@ -37,7 +35,11 @@ import de.kosit.validationtool.model.reportInput.CreateReportInput;
import de.kosit.validationtool.model.reportInput.ValidationResultsSchematron;
import de.kosit.validationtool.model.scenarios.ScenarioType;
import net.sf.saxon.s9api.*;
import net.sf.saxon.s9api.DOMDestination;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XsltTransformer;
/**
* Ausführung von konfigurierten Schematron Validierungen eines Szenarios.
@ -51,16 +53,14 @@ public class SchematronValidationAction implements CheckAction {
private final URI repository;
private List<ValidationResultsSchematron> validate(Document document, ScenarioType scenario) {
private List<ValidationResultsSchematron> validate(XdmNode document, ScenarioType scenario) {
return scenario.getSchematronValidations().stream().map(v -> validate(document, v)).collect(Collectors.toList());
}
private ValidationResultsSchematron validate(Document document, BaseScenario.Transformation validation) {
final DocumentBuilder documentBuilder = processor.newDocumentBuilder();
private ValidationResultsSchematron validate(XdmNode document, BaseScenario.Transformation validation) {
try {
final XdmNode root = documentBuilder.build(new DOMSource(document));
final XsltTransformer transformer = validation.getExecutable().load();
//resolving nur relative zum Repository
// resolving nur relative zum Repository
final RelativeUriResolver resolver = new RelativeUriResolver(repository);
transformer.setURIResolver(resolver);
CollectingErrorEventHandler e = new CollectingErrorEventHandler();
@ -68,7 +68,7 @@ public class SchematronValidationAction implements CheckAction {
Document result = ObjectFactory.createDocumentBuilder(false).newDocument();
transformer.setDestination(new DOMDestination(result));
transformer.setInitialContextNode(root);
transformer.setInitialContextNode(document);
transformer.transform();
ValidationResultsSchematron s = new ValidationResultsSchematron();
s.setResource(validation.getResourceType());
@ -85,9 +85,9 @@ public class SchematronValidationAction implements CheckAction {
@Override
public void check(Bag results) {
final CreateReportInput report = results.getReportInput();
final List<ValidationResultsSchematron> bla = validate(results.getParserResult().getObject(),
final List<ValidationResultsSchematron> validationResult = validate(results.getParserResult().getObject(),
results.getScenarioSelectionResult().getObject());
report.getValidationResultsSchematron().addAll(bla);
report.getValidationResultsSchematron().addAll(validationResult);
}
@Override