Skip to content

Commit f5bd1a0

Browse files
hvarghvarg
committed
csv upload for wiki endpoints
1 parent 0eea382 commit f5bd1a0

File tree

5 files changed

+75
-77
lines changed

5 files changed

+75
-77
lines changed

‎server/src/main/java/org/diskproject/server/adapters/SparqlAdapter.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.io.ByteArrayOutputStream;
44
import java.io.IOException;
55
import java.util.ArrayList;
6+
import java.util.Base64;
67
import java.util.HashMap;
78
import java.util.HashSet;
89
import java.util.List;
@@ -69,14 +70,22 @@ public static Set<String> interceptVariables(final String queryA, final String q
6970

7071
@Override
7172
public byte[] queryCSV(String queryString) throws Exception, QueryParseException, QueryExceptionHTTP {
72-
String url = getEndpointUrl() + "sparql";
73+
String url = getEndpointUrl() + "/query";
7374
try {
7475
URIBuilder builder = new URIBuilder(url);
7576
builder.setParameter("query", queryString);
7677
HttpGet get = new HttpGet(builder.build());
77-
get.setHeader("Content-Type", "text/csv");
78+
get.setHeader("Content-Type", "application/x-www-form-urlencoded");
79+
get.setHeader("Accept", "text/csv");
80+
if (this.getUsername() != null && this.getPassword() != null) {
81+
String pwd = Base64.getEncoder().encodeToString((this.getUsername() + ":" + this.getPassword()).getBytes());
82+
get.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + pwd);
83+
}
84+
7885
HttpResponse response = httpClient.execute(get);
7986
HttpEntity entity = response.getEntity();
87+
System.out.println("URL> " + url);
88+
System.out.println("SL> " + response.getStatusLine());
8089
ByteArrayOutputStream rawBytes = new ByteArrayOutputStream();
8190
entity.writeTo(rawBytes);
8291
return rawBytes.toByteArray();
@@ -99,6 +108,7 @@ public List<DataResult> query(String queryString) throws Exception, QueryParseEx
99108
}
100109
} catch (Exception e) {
101110
System.err.println("Error querying SPARQL endpoint: " + e.getMessage());
111+
System.err.println(" query: " + queryString);
102112
throw e;
103113
}
104114
List<DataResult> results = new ArrayList<DataResult>();

‎server/src/main/java/org/diskproject/server/adapters/StorageManager.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
import java.security.InvalidKeyException;
66
import java.security.NoSuchAlgorithmException;
77

8-
import org.apache.commons.net.imap.IMAPClient.STATUS_DATA_ITEMS;
9-
108
import io.minio.BucketExistsArgs;
119
import io.minio.MakeBucketArgs;
1210
import io.minio.MinioClient;

‎server/src/main/java/org/diskproject/server/api/impl/DiskResource.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -547,15 +547,12 @@ public Map<String, String> getNarratives(
547547
@Override
548548
public Response getOutputData(@JsonProperty("request") ExternalDataRequest r) {
549549
FileAndMeta result = this.repo.getOutputData(r.getSource(), r.getDataId());
550-
System.out.println("<< " + r.getSource() + " -- " + r.getDataId());
551550
if (result == null) {
552551
ResponseBuilder rBuilder = Response.status(Response.Status.NOT_FOUND);
553-
System.out.println("ERROR !!");
554552
return rBuilder.type(MediaType.TEXT_PLAIN)
555553
.entity("Could not find file")
556554
.build();
557555
}
558-
System.out.println(">> " + result.contentType + " -- " + result.data.length);
559556

560557
ResponseBuilder rBuild = Response.ok(result.data, result.contentType);
561558
return rBuild.build();

‎server/src/main/java/org/diskproject/server/repository/DiskRepository.java‎

Lines changed: 62 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -756,43 +756,11 @@ public List<Question> listHypothesesQuestions() {
756756
return new ArrayList<Question>(this.allQuestions.values());
757757
}
758758

759-
public List<VariableOption> listVariableOptions(String sid) throws Exception {
760-
if (!optionsCache.containsKey(sid)) {
761-
optionsCache.put(sid, this.loadVariableOptions(sid));
762-
}
763-
return optionsCache.get(sid);
764-
}
765-
766-
private List<VariableOption> loadVariableOptions(String sid) throws Exception {
767-
QuestionVariable variable = null;
768-
// FIXME: Find a better way to handle url prefix or change the request to
769-
// include the full URI
770-
if (allVariables.containsKey("http://disk-project.org/resources/enigma/variable/" + sid)) {
771-
variable = allVariables.get("http://disk-project.org/resources/enigma/variable/" + sid);
772-
} else if (allVariables.containsKey("http://disk-project.org/resources/question/" + sid)) {
773-
variable = allVariables.get("http://disk-project.org/resources/question/" + sid);
774-
} else if (allVariables.containsKey("https://w3id.org/sqo/resource/" + sid)) {
775-
variable = allVariables.get("https://w3id.org/sqo/resource/" + sid);
776-
}
777-
// -----
778-
//QuestionVariable variable = allVariables.containsKey(sid) ? allVariables.get(sid) : null;
779-
if (variable != null) {
780-
String varname = variable.getVariableName();
781-
if (variable.getSubType() == QuestionVariable.QuestionSubtype.DYNAMIC_OPTIONS) {
782-
String optionsQuery = ((DynamicOptionsQuestionVariable) variable).getOptionsQuery();
783-
if (optionsQuery != null)
784-
return queryForOptions(varname, optionsQuery);
785-
}
786-
if (variable.getSubType() == QuestionVariable.QuestionSubtype.STATIC_OPTIONS) {
787-
List<VariableOption> fixedOptions = ((StaticOptionsQuestionVariable) variable).getOptions();
788-
if (fixedOptions != null)
789-
return fixedOptions;
790-
}
791-
}
792-
return null;
793-
}
794-
759+
// OPTIONS:
795760
private List<VariableOption> queryForOptions (String varName, String query) throws Exception {
761+
if (optionsCache.containsKey(varName + query))
762+
return optionsCache.get(varName + query);
763+
796764
List<VariableOption> options = new ArrayList<VariableOption>();
797765
// If there is a constraint query, send it to all data providers;
798766
Map<String, List<DataResult>> solutions = new HashMap<String, List<DataResult>>();
@@ -856,14 +824,15 @@ private List<VariableOption> queryForOptions (String varName, String query) thro
856824
}
857825
}
858826
}
827+
optionsCache.put(varName + query, options);
859828
return options;
860829
}
861830

862-
public String createQuestionOptionsQuery (Question q) {
831+
public String createQuestionOptionsQuery (Question q, List<QuestionVariable> includedVariables) {
863832
if (q != null) {
864833
String queryConstraint = q.getConstraint();
865834
String query = queryConstraint != null ? queryConstraint : "";
866-
for (QuestionVariable qv: q.getVariables()) {
835+
for (QuestionVariable qv: includedVariables) {
867836
QuestionSubtype t = qv.getSubType();
868837
if (t == QuestionSubtype.DYNAMIC_OPTIONS || t == QuestionSubtype.BOUNDING_BOX || t == QuestionSubtype.TIME_INTERVAL) {
869838
String queryFragment = ((DynamicOptionsQuestionVariable) qv).getOptionsQuery();
@@ -877,58 +846,78 @@ public String createQuestionOptionsQuery (Question q) {
877846
}
878847

879848
public Map<String,List<VariableOption>> listDynamicOptions (QuestionOptionsRequest cfg) throws Exception {
849+
// Returns a map[var.name] -> [option, option2, ...]
880850
Map<String, String> bindings = cfg.getBindings();
881851
Question q = allQuestions.get(cfg.getId());
882-
String query = createQuestionOptionsQuery(q);
883-
if (q == null) return null;
884-
885-
// Create map variableName -> filter
886-
Map<String, String> filters = new HashMap<String, String>();
887-
if (bindings != null && query != null) {
888-
for (String varUrl: bindings.keySet()) {
889-
QuestionVariable curVar = allVariables.get(varUrl);
852+
if (q == null) {
853+
System.err.println("Question not found: " + cfg.getId());
854+
return null;
855+
}
856+
857+
Map<String, String> queries = new HashMap<String, String>();
858+
Map<String, List<VariableOption>> options = new HashMap<String, List<VariableOption>>();
859+
if (bindings == null || bindings.size() == 0) {
860+
// If there are no bindings, we can just send the base query + the constraint for this variable
861+
for (QuestionVariable qv: q.getVariables()) {
862+
QuestionSubtype t = qv.getSubType();
863+
if (t == QuestionSubtype.DYNAMIC_OPTIONS || t == QuestionSubtype.BOUNDING_BOX || t == QuestionSubtype.TIME_INTERVAL ) {
864+
String curQuery = (q.getConstraint() != null ? q.getConstraint() : "") + ((DynamicOptionsQuestionVariable) qv).getOptionsQuery();
865+
queries.put(qv.getId(), curQuery);
866+
}
867+
}
868+
} else {
869+
// If we have at leas one binding, we need to create filters for all queries:
870+
Map<String, String> filters = new HashMap<String, String>();
871+
for (String varId: bindings.keySet()) {
872+
QuestionVariable curVar = allVariables.get(varId); // Should be the same as question.getVariables.
890873
if (curVar != null) {
891-
String value = bindings.get(varUrl);
892-
String name = curVar.getVariableName();
874+
String value = bindings.get(varId);
893875
String sparqlValue = value.startsWith("http") ? "<" + value + ">" : "\"" + value + "\"";
894-
String line = "VALUES " + name + " { " + sparqlValue + " }";
895-
filters.put(name, line);
876+
String line = "VALUES " + curVar.getVariableName() + " { " + sparqlValue + " }";
877+
filters.put(varId, line);
896878
} else {
897-
System.err.println("Cannot find variable ID: " + varUrl);
879+
System.err.println("Cannot find variable with ID: " + varId);
880+
}
881+
}
882+
883+
String baseQuery = createQuestionOptionsQuery(q, q.getVariables());
884+
for (QuestionVariable qv: q.getVariables()) {
885+
QuestionSubtype t = qv.getSubType();
886+
if (t == QuestionSubtype.DYNAMIC_OPTIONS || t == QuestionSubtype.BOUNDING_BOX || t == QuestionSubtype.TIME_INTERVAL ) {
887+
String varId = qv.getId();
888+
String curQuery = baseQuery;
889+
// We need to add all filters that ARE NOT THIS ONE
890+
for (String filterId: filters.keySet()) {
891+
if (!filterId.equals(varId)) {
892+
curQuery += "\n" + filters.get(filterId);
893+
}
894+
}
895+
queries.put(varId, curQuery);
898896
}
899897
}
900898
}
901899

902-
Map<String, List<VariableOption>> varNameToOptions = new HashMap<String, List<VariableOption>>();
900+
// Now run the queries and load results.
903901
for (QuestionVariable qv: q.getVariables()) {
904902
QuestionSubtype t = qv.getSubType();
905903
String varName = qv.getVariableName();
906904
if (t == QuestionSubtype.STATIC_OPTIONS) {
907-
varNameToOptions.put(varName, ((StaticOptionsQuestionVariable) qv).getOptions());
905+
options.put(varName, ((StaticOptionsQuestionVariable) qv).getOptions());
908906
} else if (t == QuestionSubtype.DYNAMIC_OPTIONS) {
909907
// We add all the filter except the value for the queried variable
910-
if (query == null) {
911-
System.err.println("WARN: Could not find suitable query for " + qv.getId());
912-
} else {
913-
String curQuery = query;
914-
if (filters != null && filters.size() > 0) {
915-
for (String filterVarName: filters.keySet()) {
916-
if (!filterVarName.equals(qv.getVariableName())) {
917-
curQuery += "\n" + filters.get(filterVarName);
918-
}
919-
}
920-
}
921-
varNameToOptions.put(varName, queryForOptions(varName, curQuery));
922-
if (varNameToOptions.get(varName).size() == 0) {
908+
if (queries.containsKey(qv.getId())) {
909+
String curQuery = queries.get(qv.getId());
910+
options.put(varName, queryForOptions(varName, curQuery));
911+
if (options.get(varName).size() == 0) {
923912
System.out.println(qv.getId() + " got 0 results:");
924913
System.out.println(curQuery);
925914
} else {
926-
System.out.println(qv.getId() + " got " + varNameToOptions.get(varName).size() + " results.");
915+
System.out.println(qv.getId() + " got " + options.get(varName).size() + " results.");
927916
}
928917
}
929918
}
930919
}
931-
return varNameToOptions;
920+
return options;
932921
}
933922

934923
/*
@@ -1310,7 +1299,9 @@ else if (qVar.equals("_CSV_")) {
13101299
}
13111300

13121301
// Attach CSV
1313-
if (attachCSV && this.externalStorage != null) {
1302+
if (this.externalStorage == null) {
1303+
System.out.println("Warning: External storage not found. Can not upload file.");
1304+
} else if (attachCSV) {
13141305
//run the query again, this time get the bits to create csv file.
13151306
byte[] csvFile = dataAdapter.queryCSV(query);
13161307
String csvHash = KBUtils.SHAsum(csvFile);
@@ -1437,6 +1428,7 @@ private List<WorkflowBindings> getTLOIBindings(String username, List<WorkflowBin
14371428
sparqlVar = "_CSV_";
14381429
}
14391430

1431+
System.out.println("keys:" + dataVarBindings.keySet());
14401432
if (sparqlVar == null)
14411433
continue;
14421434

@@ -2116,6 +2108,7 @@ public void run() {
21162108
}
21172109
}
21182110

2111+
// All of this needs to be reworked to clear caches and re run queries.
21192112
public class DataMonitor implements Runnable {
21202113
boolean stop;
21212114
ScheduledFuture<?> scheduledFuture;

‎server/src/main/java/org/diskproject/server/repository/WingsAdapter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1251,7 +1251,7 @@ private String toPlanAcceptableFormat(String wfName, List<VariableBinding> vbl,
12511251
String curBinding = "\"" + wfName + v.getName() + "\":[";
12521252
String bindingValue = vb.getBinding();
12531253

1254-
if (v.getDimensionality() == 0) { // && !bindingValue.startsWith("[")) {
1254+
if (v.getDimensionality() == 0) { // && !bindingValue.startsWith("[")) { FIXME!
12551255
curBinding += "\"" + bindingValue + "\"";
12561256
} else {
12571257
if (v.getDimensionality() == 0) {

0 commit comments

Comments
 (0)