Skip to content

Commit 7659354

Browse files
author
TortugaAttack
committed
Merge branch 'develop'
2 parents b2865f3 + 8bf9478 commit 7659354

File tree

9 files changed

+328
-13
lines changed

9 files changed

+328
-13
lines changed

‎fuseki_example.ttl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88

99
[] rdf:type fuseki:Server ;
1010
ja:loadClass "org.dice_group.grp.fuseki.KD2GraphAssembler" ;
11+
ja:loadClass "org.dice_group.grp.fuseki.MutlipleKD2GraphAssembler" ;
1112

1213
fuseki:services (
1314
<#service1>
1415
) .
1516

1617
dice:KD2Graph rdfs:subClassOf ja:Graph .
18+
dice:MutlipleKD2Graph rdfs:subClassOf ja:Graph .
1719

1820
## ---------------------------------------------------------------
1921
## Currently only Read only is possible, writing in memory is theoretically possible, but persistence is not
@@ -36,3 +38,8 @@ dice:KD2Graph rdfs:subClassOf ja:Graph .
3638
<#graph1> rdfs:label "KD2 compressed file containing the dataset" ;
3739
rdf:type dice:KD2Graph ;
3840
dice:fileName "file.grp" .
41+
#rdf:type dice:KD2Graph ; #If you want several compressed files to be loaded in one graph
42+
#dice:folder "./kd2_files/" . #If you want several compressed files to be loaded in one graph
43+
44+
45+

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>org.dice-group</groupId>
66
<artifactId>RDFRePair</artifactId>
7-
<version>1.0</version>
7+
<version>1.0.1</version>
88

99
<properties>
1010
<java.version>1.11</java.version>

‎src/main/java/org/dice_group/grp/fuseki/ExtKD2JenaIterator.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package org.dice_group.grp.fuseki;
2+
3+
import org.apache.jena.graph.Triple;
4+
import org.apache.jena.util.iterator.ExtendedIterator;
5+
import org.apache.jena.util.iterator.NiceIterator;
6+
import org.rdfhdt.hdt.enums.TripleComponentRole;
7+
import org.rdfhdt.hdt.triples.TripleID;
8+
9+
import java.util.ArrayList;
10+
import java.util.List;
11+
12+
public class ExtKD2JenaIterator extends NiceIterator<Triple> {
13+
14+
private List<ExtendedIterator<Triple>> its = new ArrayList<ExtendedIterator<Triple>>();
15+
16+
private int currentIt=0;
17+
18+
public void addIterator(ExtendedIterator<Triple> it ){
19+
its.add(it);
20+
}
21+
22+
@Override
23+
public boolean hasNext(){
24+
if(its.size()>0 && its.size()>currentIt){
25+
if(its.get(currentIt).hasNext()){
26+
return true;
27+
}
28+
currentIt++;
29+
return hasNext();
30+
}
31+
return false;
32+
}
33+
34+
35+
@Override
36+
public Triple next() {
37+
38+
return its.get(currentIt).next();
39+
40+
}
41+
42+
43+
}

‎src/main/java/org/dice_group/grp/fuseki/KD2Graph.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,7 @@ protected ExtendedIterator<Triple> graphBaseFind(Triple triplePattern) {
4141
// :s ?p ?o -> get all points for all matrices with row
4242
// :s ?p :o -> get specific point for all matrices
4343
TripleID ids = getTriplePatID(triplePattern);
44-
System.out.println(triplePattern);
45-
System.out.println(ids);
44+
4645
KD2JenaIterator it = new KD2JenaIterator(dict);
4746
//Node does not exist in Dict
4847
if(ids.getPredicate()==-1|| ids.getSubject()==-1 || ids.getObject()==-1){

‎src/main/java/org/dice_group/grp/fuseki/MultipleKD2Graph.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package org.dice_group.grp.fuseki;
2+
3+
import org.apache.jena.graph.Triple;
4+
import org.apache.jena.graph.impl.GraphBase;
5+
import org.apache.jena.util.iterator.ExtendedIterator;
6+
import org.rdfhdt.hdt.triples.TripleID;
7+
8+
import java.util.ArrayList;
9+
import java.util.List;
10+
11+
public class MultipleKD2Graph extends GraphBase {
12+
13+
private List<KD2Graph> graphs = new ArrayList<KD2Graph>();
14+
15+
public void addGraph(KD2Graph graph){
16+
this.graphs.add(graph);
17+
}
18+
19+
@Override
20+
protected ExtendedIterator<Triple> graphBaseFind(Triple triplePattern) {
21+
ExtKD2JenaIterator ret = new ExtKD2JenaIterator();
22+
for(KD2Graph graph : graphs){
23+
ret.addIterator(graph.graphBaseFind(triplePattern));
24+
}
25+
return ret;
26+
}
27+
}

‎src/main/java/org/dice_group/grp/fuseki/MultipleKD2GraphAssembler.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package org.dice_group.grp.fuseki;
2+
3+
import org.apache.jena.assembler.Assembler;
4+
import org.apache.jena.assembler.Mode;
5+
import org.apache.jena.assembler.assemblers.AssemblerBase;
6+
import org.apache.jena.assembler.exceptions.AssemblerException;
7+
import org.apache.jena.rdf.model.ModelFactory;
8+
import org.apache.jena.rdf.model.Resource;
9+
import org.apache.jena.rdf.model.ResourceFactory;
10+
import org.apache.jena.sparql.util.graph.GraphUtils;
11+
import org.dice_group.grp.decompression.GRPReader;
12+
import org.dice_group.grp.serialization.impl.KD2TreeDeserializer;
13+
import org.dice_group.grp.util.LabledMatrix;
14+
import org.rdfhdt.hdt.dictionary.impl.PSFCFourSectionDictionary;
15+
import org.rdfhdt.hdt.hdt.HDTVocabulary;
16+
import org.rdfhdt.hdt.options.HDTSpecification;
17+
18+
import java.io.File;
19+
import java.nio.ByteBuffer;
20+
import java.util.List;
21+
22+
public class MultipleKD2GraphAssembler extends AssemblerBase implements Assembler {
23+
24+
25+
26+
private static boolean initialized;
27+
28+
public static void init() {
29+
if(initialized) {
30+
return;
31+
}
32+
33+
initialized = true;
34+
35+
Assembler.general.implementWith(ResourceFactory.createProperty("https://dice-research.org/fuseki/MultipleKD2Graph"), new MultipleKD2GraphAssembler());
36+
}
37+
38+
39+
@Override
40+
public Object open(Assembler a, Resource root, Mode mode) {
41+
String folder = GraphUtils.getStringValue(root, ResourceFactory.createProperty("https://dice-research.org/fuseki/folder"));
42+
File dir = new File(folder);
43+
MultipleKD2Graph mgraph = new MultipleKD2Graph();
44+
for (File file : dir.listFiles()) {
45+
HDTSpecification spec = new HDTSpecification();
46+
spec.set("dictionary.type", HDTVocabulary.DICTIONARY_TYPE_FOUR_PSFC_SECTION);
47+
48+
PSFCFourSectionDictionary dict = new PSFCFourSectionDictionary(spec);
49+
try {
50+
51+
byte[] kd2ser = GRPReader.load(file.getAbsolutePath(), dict);
52+
//read head
53+
ByteBuffer bb = ByteBuffer.wrap(kd2ser);
54+
byte[] startBytes = new byte[4];
55+
bb.get(startBytes);
56+
int startSize = ByteBuffer.wrap(startBytes).getInt();
57+
//2. X bytes = start Graph
58+
byte[] start = new byte[startSize];
59+
bb = bb.slice();
60+
bb.get(start);
61+
62+
KD2TreeDeserializer desr = new KD2TreeDeserializer();
63+
List<LabledMatrix> matrices = desr.deserialize(start);
64+
System.out.println("Created " + matrices.size() + " matrices");
65+
KD2Graph graph = new KD2Graph(matrices, dict);
66+
mgraph.addGraph(graph);
67+
} catch (Exception e) {
68+
System.err.println("Reading KD2 file " + file + " caused following error");
69+
e.printStackTrace();
70+
throw new AssemblerException(root, "Reading KD2 file " + file + " caused following error :\n " + e.toString());
71+
}
72+
}
73+
return ModelFactory.createModelForGraph(mgraph);
74+
}
75+
76+
77+
static {
78+
init();
79+
}
80+
81+
}

‎src/test/java/org/dice_group/grp/fuseki/ExtIteratorTest.java

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package org.dice_group.grp.fuseki;
2+
3+
import org.apache.jena.query.QueryExecution;
4+
import org.apache.jena.query.QueryExecutionFactory;
5+
import org.apache.jena.query.QuerySolution;
6+
import org.apache.jena.query.ResultSet;
7+
import org.apache.jena.rdf.model.Model;
8+
import org.apache.jena.rdf.model.ModelFactory;
9+
import org.apache.jena.rdf.model.RDFNode;
10+
import org.apache.jena.rdf.model.ResourceFactory;
11+
import org.apache.jena.riot.Lang;
12+
import org.apache.jena.riot.RDFDataMgr;
13+
import org.dice_group.grp.exceptions.NotAllowedInRDFException;
14+
import org.dice_group.grp.exceptions.NotSupportedException;
15+
import org.dice_group.grp.main.Main;
16+
import org.junit.Test;
17+
import org.junit.runner.RunWith;
18+
import org.junit.runners.Parameterized;
19+
20+
import java.io.File;
21+
import java.io.FileInputStream;
22+
import java.io.FileOutputStream;
23+
import java.io.IOException;
24+
import java.util.ArrayList;
25+
import java.util.Arrays;
26+
import java.util.Collection;
27+
import java.util.List;
28+
import java.util.concurrent.ExecutionException;
29+
30+
import static org.junit.Assert.assertTrue;
31+
32+
@RunWith(Parameterized.class)
33+
public class ExtIteratorTest {
34+
35+
36+
private final String query;
37+
private final List<String[]> expectedRows = new ArrayList<String[]>();
38+
39+
@Parameterized.Parameters
40+
public static Collection<Object[]> data() {
41+
return Arrays.asList(new Object[][] {
42+
{"SELECT * {?s <http://example.com/d> ?o}",
43+
new String[][]{{"http://example.com/a", "_:"}, {"_:", "http://example.com/a"},
44+
{"http://example.com/rt", "_:"}, {"_:", "http://example.com/ac"}}},
45+
{"SELECT * {?s <http://example.com/b> ?o}",
46+
new String[][]{{"http://example.com/rt", "http://example.com/c"}, {"_:", "_:"},
47+
{"http://example.com/a", "http://example.com/c"}, {"_:", "_:"}}},
48+
{"SELECT * {<http://example.com/a> <http://example.com/b> ?o}",
49+
new String[][]{{"http://example.com/c"}}},
50+
{"SELECT * {?s <http://example.com/b> <http://example.com/c>}",
51+
new String[][]{{"http://example.com/rt"}, {"http://example.com/a"}}},
52+
{"SELECT * {?s ?p <http://example.com/c>}",
53+
new String[][]{{"http://example.com/rt", "http://example.com/b"}, {"http://example.com/a", "http://example.com/b"}}}
54+
});
55+
}
56+
57+
public ExtIteratorTest(String query, String[][] expectedRows){
58+
this.query = query;
59+
for(String[] row : expectedRows){
60+
this.expectedRows.add(row);
61+
}
62+
63+
}
64+
65+
@Test
66+
public void testExtendedIterator() throws InterruptedException, NotAllowedInRDFException, NotSupportedException, ExecutionException, IOException {
67+
File dir = new File("src/test/resources/fuseki/");
68+
dir.mkdir();
69+
ExtKD2JenaIterator it = new ExtKD2JenaIterator();
70+
Main.compress("src/test/resources/matrix.nt", "src/test/resources/fuseki/matrix.kd2", true, true, true);
71+
Main.compress("src/test/resources/bn.nt", "src/test/resources/fuseki/bn.kd2", true, true, true);
72+
MultipleKD2GraphAssembler assembler = new MultipleKD2GraphAssembler();
73+
Model meta = ModelFactory.createDefaultModel();
74+
RDFDataMgr.read(meta, new FileInputStream("src/test/resources/multifilesexa.ttl"), Lang.TURTLE);
75+
76+
RDFDataMgr.write(new FileOutputStream(new File("hae.nt")), meta, Lang.NTRIPLES);
77+
78+
Model m = (Model) assembler.open(null, meta.getResource("file://"+new File("").getAbsolutePath()+"/#graph1"), null);
79+
QueryExecution qexec = QueryExecutionFactory.create(query, m);
80+
81+
ResultSet res = qexec.execSelect();
82+
83+
while (res.hasNext()){
84+
QuerySolution qsol = res.next();
85+
boolean contains = contains(qsol, expectedRows);
86+
assertTrue(contains);
87+
}
88+
assertTrue(expectedRows.size()==0);
89+
new File("src/test/resources/fuseki/matrix.kd2").delete();
90+
new File("src/test/resources/fuseki/bn.kd2").delete();
91+
expectedRows.clear();
92+
}
93+
94+
private boolean contains(QuerySolution qsol, List<String[]> expectedRows) {
95+
List<RDFNode> actual = new ArrayList<RDFNode>();
96+
if(qsol.contains("s")){
97+
actual.add(qsol.get("s"));
98+
}
99+
if(qsol.contains("p")){
100+
actual.add(qsol.get("p"));
101+
}
102+
if(qsol.contains("o")){
103+
actual.add(qsol.get("o"));
104+
}
105+
for(int j=0;j<expectedRows.size();j++){
106+
String[] row = expectedRows.get(j);
107+
boolean contains=true;
108+
if(actual.size()!=row.length){
109+
return false;
110+
}
111+
for(int i=0;i<actual.size();i++){
112+
if(!(actual.get(i).isAnon() && row[i].startsWith("_:"))){
113+
contains &= actual.get(i).toString().equals(row[i]);
114+
}
115+
}
116+
if(contains){
117+
expectedRows.remove(row);
118+
return true;
119+
}
120+
}
121+
return false;
122+
}
123+
124+
}

‎src/test/resources/matrix.nt

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
<:s1> <:p0> <:o7> .
2-
<:s2> <:p1> <:o8> .
3-
<:s3> <:p1> <:o9> .
4-
<:s3> <:p1> <:s3> .
5-
<:s4> <:p2> <:o10> .
6-
<:o5> <:p2> <:o6> .
7-
<:o6> <:p0> <:o11> .
8-
<:o11> <:p3> <:s1> .
9-
<:o11> <:p3> <:s4> .
10-
<:o11> <:p3> <:s5> .
1+
<http://example.com/rt> <http://example.com/b> <http://example.com/c> .
2+
<http://example.com/rt> <http://example.com/d> _:abcd .
3+
_:abcd <http://example.com/q> "abdd" .
4+
_:abcd <http://example.com/d> <http://example.com/ac> .
5+
_:abcd <http://example.com/b> _:abdd .
6+
_:abdd <http://example.com/u> <http://example.com/oc> .

‎src/test/resources/multifilesexa.ttl

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
@prefix : <#> .
2+
@prefix fuseki: <http://jena.apache.org/fuseki#> .
3+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
4+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
5+
@prefix dice: <https://dice-research.org/fuseki/> .
6+
@prefix tdb: <http://jena.hpl.hp.com/2008/tdb#> .
7+
@prefix ja: <http://jena.hpl.hp.com/2005/11/Assembler#> .
8+
9+
[] rdf:type fuseki:Server ;
10+
ja:loadClass "org.dice_group.grp.fuseki.MultipleKD2GraphAssembler" ;
11+
12+
fuseki:services (
13+
<#service1>
14+
) .
15+
16+
dice:MultipleKD2Graph rdfs:subClassOf ja:Graph .
17+
18+
## ---------------------------------------------------------------
19+
## Currently only Read only is possible, writing in memory is theoretically possible, but persistence is not
20+
21+
<#service1> rdf:type fuseki:Service ;
22+
rdfs:label "KD2" ;
23+
fuseki:name "kd2" ;
24+
fuseki:serviceQuery "" ;
25+
fuseki:serviceQuery "sparql" ;
26+
fuseki:serviceQuery "query" ;
27+
fuseki:serviceReadGraphStore "get" ;
28+
fuseki:serviceReadQuads "" ;
29+
fuseki:dataset <#dataset> .
30+
31+
<#dataset> rdf:type ja:RDFDataset ;
32+
rdfs:label "Dataset" ;
33+
ja:defaultGraph <#graph1> .
34+
35+
## You can add several graphs
36+
<#graph1> rdfs:label "KD2 compressed file containing the dataset" ;
37+
rdf:type dice:MultipleKD2Graph ;
38+
dice:folder "src/test/resources/fuseki/" .

0 commit comments

Comments
 (0)