RDFquality stage first try

Author: mikel-egana-aranguren

Jenkinsfile

@@ -1,24 +1,30 @@
 #!/usr/bin/env groovy
 
 def SPARQLendpoint = "http://172.16.0.81:58080/blazegraph/sparql"
-def NamedGraph = "http://lod.eurohelp.es/dataset/parkings"
 def URLParkingsDonosti = "http://www.donostia.eus/info/ciudadano/camaras_trafico.nsf/dameParkings?OpenAgent&idioma=cas"
 def CSVParkingsClean = "donostiaparkingsclean.csv"
-def RDFParkingsClean = "donostiaparkingsclean.nquads"
+def RDFParkingsClean = "donostiaparkingsclean.nq"
+def NamedGraph = "http://lod.eurohelp.es/dataset/parkings"
+def SHACLfile = "shacl/parkings.ttl"
+def SHACLReportCheckingQuery = "shacl/report.sparql"
+def SHACLReportFile = "shacl/report.ttl"
 
 node {
- stage('Clean blazegraph named graph'){
+ stage('Remove data from blazegraph'){
  sh 'curl --get -X DELETE -H "Accept: application/xml" ' + SPARQLendpoint + ' --data-urlencode "?c=<' + NamedGraph + '>"'
  }
  stage('Checkout pipeline') {
- git branch: 'develop', url: 'https://github.com/Eurohelp/Replicate-LinkedOpenData-Datasets.git'
+ git branch: 'RDFTests', url: 'https://github.com/Eurohelp/Replicate-LinkedOpenData-Datasets.git'
  }
  stage('Obtain and clean data') {
  sh 'java -jar loadparkingdata/cleanparkingdata.jar "' + URLParkingsDonosti + '" ' + CSVParkingsClean + ' temp_parkingsclean temp_parkingsclean.json'
  }
  stage('Convert CSV to RDF') {
  sh 'java -jar csv2rdf/csvparkingsclean2rdf.jar ' + CSVParkingsClean + ' ' + RDFParkingsClean + ' ' + NamedGraph
  }
+ stage('RDF quality') {
+ sh 'java -jar rdfquality/SHACLValidator.jar ' + RDFParkingsClean + ' ' + NamedGraph + ' ' + SHACLfile + ' ' + SHACLReportCheckingQuery + ' ' + SHACLfile
+ }
  stage('Upload RDF to blazegraph') {
  sh 'curl -X POST -H Content-Type:text/x-nquads --data-binary @' + RDFParkingsClean + ' ' + SPARQLendpoint
  }

rdfquality/SHACLValidator.jar

@@ -2,7 +2,6 @@
 
 import java.io.File;
 import java.io.FileWriter;
-import java.io.IOException;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.jena.query.Query;
@@ -12,39 +11,44 @@
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.apache.jena.rdf.model.Resource;
-import org.topbraid.shacl.util.ModelPrinter;
+import org.apache.jena.riot.RDFDataMgr;
 import org.topbraid.shacl.validation.ValidationUtil;
 
 public class SHACLValidator {
 
 public static void main(String[] args) throws Exception {
 String targetFile = args[0];
-String SHACLFile = args[1];
-String reportCheckingQueryFile = args[2];
-String reportFile = args[3];
-
-Model model = ModelFactory.createDefaultModel();
-model.read(targetFile);
-
+String namedGraphURI = args [1];
+String SHACLFile = args[2];
+String reportCheckingQueryFile = args[3];
+String reportFile = args[4];
+
+// Load the data to validate
+Model model = RDFDataMgr.loadDataset(targetFile).getNamedModel(namedGraphURI);
+
+// Load the quality tests
 Model shacl = ModelFactory.createDefaultModel();
 shacl.read(SHACLFile);
 
+// Create a validation report (execute the tests)
 Resource report = ValidationUtil.validateModel(model, shacl, true);
 
-//	System.out.println(ModelPrinter.get().print(report.getModel()));
-
+// Write report to disk
 FileWriter out = new FileWriter(reportFile);
-
 report.getModel().write(out, "TURTLE");
 
+// Query report to check if data is conformant
 String reportCheckingQuery = FileUtils.readFileToString(new File(reportCheckingQueryFile));
 Query query = QueryFactory.create(reportCheckingQuery);
 QueryExecution qexec = QueryExecutionFactory.create(query, report.getModel());
 boolean result = qexec.execAsk();
 qexec.close();
+
+// Data is not conformant
 if (result) {
-throw new Exception("SHACL violation: non-conformant RDF");
+throw new Exception("SHACL violation: non-conformant RDF, see report at " + reportFile);
 }
+// Conformant data
 else{
 System.out.println("Valid RDF");
 }

rdfquality/src/main/java/donosti/lod/replicate/eurohelp/es/SHACLValidator.java

@@ -0,0 +1,34 @@
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix sh:<http://www.w3.org/ns/shacl#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix schema: <http://schema.org/> .
+@prefix geo: <http://www.w3.org/2003/01/geo/wgs84_pos#> .
+@prefix ex:<http://lod.eurohelp.es/replicate/shacl/parkings> .
+
+# A parking should have a latitude with xsd:double value between 43.2 and 43.4
+
+ex:ParkingShape
+a sh:NodeShape ;
+sh:targetClass schema:ParkingFacility ;
+sh:property [
+sh:path geo:lat ;
+sh:datatype xsd:double ;
+ sh:minInclusive 43.2 ;
+sh:maxInclusive 43.4 ;
+] .
+
+# Since all the data is conformant, this shape can be used to make sure the thing works,
+# since it will flag valid data as invalid (the sh:minInclusive value is 43.3 instead of 43.2)
+# To activate it comment out the line sh:deactivated true ;
+
+ex:ParkingShapeDebug
+a sh:NodeShape ;
+sh:targetClass schema:ParkingFacility ;
+#sh:deactivated true ;
+sh:property [
+sh:path geo:lat ;
+sh:datatype xsd:double ;
+ sh:minInclusive 43.3 ;
+sh:maxInclusive 43.4 ;
+] .

shacl/parkings.ttl

@@ -0,0 +1,8 @@
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX sh: <http://www.w3.org/ns/shacl#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+
+ASK WHERE { 
+ ?ValidationReport rdf:type sh:ValidationReport .
+ ?ValidationReport sh:conforms "false"^^xsd:boolean .
+}