Skip to content

Commit 0eea382

Browse files
hvarghvarg
committed
upload files to minio
1 parent 07c3c30 commit 0eea382

File tree

9 files changed

+54
-74
lines changed

9 files changed

+54
-74
lines changed

‎server/src/main/java/org/diskproject/server/adapters/GraphDBAdapter.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.diskproject.server.adapters;
22

3+
import java.io.ByteArrayOutputStream;
34
import java.io.IOException;
45
import java.util.ArrayList;
56
import java.util.HashMap;
@@ -207,7 +208,7 @@ public String getFileETag(String url){
207208
}
208209

209210
@Override
210-
public void queryCSV(String queryString) throws Exception {
211+
public byte[] queryCSV(String queryString) throws Exception {
211212
String url = getEndpointUrl() + "repositories/" + this.repository;
212213
URIBuilder builder = new URIBuilder(url);
213214
builder.setParameter("query", queryString);
@@ -219,14 +220,12 @@ public void queryCSV(String queryString) throws Exception {
219220

220221
try (CloseableHttpResponse response = httpClient.execute(get)) {
221222
HttpEntity entity = response.getEntity();
222-
String strResponse = EntityUtils.toString(entity);
223-
System.out.println(get.getURI());
224-
System.out.println(entity.getContentType());
225-
System.out.println(entity.getContentLength());
226-
EntityUtils.consume(entity);
227-
System.out.println(strResponse);
223+
ByteArrayOutputStream rawBytes = new ByteArrayOutputStream();
224+
entity.writeTo(rawBytes);
225+
return rawBytes.toByteArray();
228226
} catch (IOException e) {
229227
e.printStackTrace();
230228
}
229+
return null;
231230
}
232231
}

‎server/src/main/java/org/diskproject/server/adapters/SparqlAdapter.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
package org.diskproject.server.adapters;
22

3+
import java.io.ByteArrayOutputStream;
34
import java.io.IOException;
4-
import java.io.InputStream;
5-
import java.nio.charset.StandardCharsets;
65
import java.util.ArrayList;
76
import java.util.HashMap;
87
import java.util.HashSet;
@@ -12,7 +11,6 @@
1211
import java.util.regex.Matcher;
1312
import java.util.regex.Pattern;
1413

15-
import org.apache.commons.io.IOUtils;
1614
import org.apache.http.Header;
1715
import org.apache.http.HttpEntity;
1816
import org.apache.http.HttpHeaders;
@@ -70,23 +68,22 @@ public static Set<String> interceptVariables(final String queryA, final String q
7068
}
7169

7270
@Override
73-
public void queryCSV(String queryString) throws Exception, QueryParseException, QueryExceptionHTTP {
71+
public byte[] queryCSV(String queryString) throws Exception, QueryParseException, QueryExceptionHTTP {
7472
String url = getEndpointUrl() + "sparql";
7573
try {
7674
URIBuilder builder = new URIBuilder(url);
7775
builder.setParameter("query", queryString);
78-
7976
HttpGet get = new HttpGet(builder.build());
8077
get.setHeader("Content-Type", "text/csv");
8178
HttpResponse response = httpClient.execute(get);
8279
HttpEntity entity = response.getEntity();
83-
System.out.println(entity.getContentType());
84-
InputStream io = entity.getContent();
85-
String result = IOUtils.toString(io, StandardCharsets.UTF_8);
86-
System.out.println(result);
80+
ByteArrayOutputStream rawBytes = new ByteArrayOutputStream();
81+
entity.writeTo(rawBytes);
82+
return rawBytes.toByteArray();
8783
} catch (IOException e) {
8884
e.printStackTrace();
8985
}
86+
return null;
9087
}
9188

9289
@Override

‎server/src/main/java/org/diskproject/server/adapters/StorageManager.java‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@
55
import java.security.InvalidKeyException;
66
import java.security.NoSuchAlgorithmException;
77

8+
import org.apache.commons.net.imap.IMAPClient.STATUS_DATA_ITEMS;
9+
810
import io.minio.BucketExistsArgs;
911
import io.minio.MakeBucketArgs;
1012
import io.minio.MinioClient;
13+
import io.minio.ObjectWriteResponse;
1114
import io.minio.PutObjectArgs;
12-
import io.minio.UploadObjectArgs;
1315
import io.minio.errors.ErrorResponseException;
1416
import io.minio.errors.InsufficientDataException;
1517
import io.minio.errors.InternalException;
@@ -19,6 +21,7 @@
1921
import io.minio.errors.XmlParserException;
2022

2123
public class StorageManager {
24+
static String BUCKET_NAME = "disk";
2225
String url, username, password;
2326
MinioClient client;
2427

@@ -39,12 +42,12 @@ public boolean init () {
3942

4043
boolean found = false;
4144
try {
42-
found = client.bucketExists(BucketExistsArgs.builder().bucket("disk").build());
45+
found = client.bucketExists(BucketExistsArgs.builder().bucket(StorageManager.BUCKET_NAME).build());
4346
if (!found) {
4447
// Make a new bucket called 'disk'.
45-
client.makeBucket(MakeBucketArgs.builder().bucket("disk").build());
48+
client.makeBucket(MakeBucketArgs.builder().bucket(StorageManager.BUCKET_NAME).build());
4649
} else {
47-
System.out.println("Bucket 'disk' already exists.");
50+
System.out.println("Bucket '" + StorageManager.BUCKET_NAME + "' already exists.");
4851
}
4952
} catch (InvalidKeyException | NoSuchAlgorithmException | IllegalArgumentException | IOException e) {
5053
System.out.println("ERROR 1: " + e.toString());
@@ -59,24 +62,27 @@ public boolean init () {
5962
return true;
6063
}
6164

62-
public void upload (String filename, String datatype, byte[] bytes) {
63-
if (client == null) return;
65+
public String upload (String filename, String datatype, byte[] bytes) {
66+
System.out.println("Trying to upload " + filename + " -- " + bytes.length);
67+
if (client == null) return null;
6468
try {
65-
client.putObject(
69+
ObjectWriteResponse resp = client.putObject(
6670
PutObjectArgs.builder()
6771
.bucket("disk")
6872
.contentType(datatype)
6973
.object(filename)
7074
.stream(new ByteArrayInputStream(bytes), bytes.length, -1)
7175
.build()
7276
);
77+
System.out.println("Upload complete - ETAG: " + resp.etag());
78+
return this.url + "/" + StorageManager.BUCKET_NAME + "/" + filename;
7379
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException
7480
| InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException
7581
| IllegalArgumentException | IOException e) {
7682
System.err.println("ERROR 2: " + e.toString());
7783
e.printStackTrace();
7884
}
79-
85+
return null;
8086
}
8187

8288
}

‎server/src/main/java/org/diskproject/server/api/impl/DiskResource.java‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -435,36 +435,6 @@ public List<Question> listQuestions() {
435435
return this.repo.listHypothesesQuestions();
436436
}
437437

438-
@GET
439-
@Path("question/{id}/options")
440-
@Override
441-
public List<VariableOption> listOptions(
442-
@PathParam("id") String id) {
443-
try {
444-
// return WingsAdapter.get().getWorkflowList();
445-
return this.repo.listVariableOptions(id);
446-
} catch (Exception e) {
447-
try {
448-
// Create Json error response
449-
Gson gson = new Gson();
450-
ErrorMessage error = new ErrorMessage(e.getMessage());
451-
String jsonData = gson.toJson(error);
452-
453-
// Prepare the response
454-
response.setContentType("application/json");
455-
response.setCharacterEncoding("utf-8");
456-
response.setStatus(500);
457-
458-
// Send the response
459-
response.getWriter().print(jsonData.toString());
460-
response.getWriter().flush();
461-
} catch (IOException e1) {
462-
e1.printStackTrace();
463-
}
464-
}
465-
return null;
466-
};
467-
468438
@POST
469439
@Path("question/options")
470440
public Map<String, List<VariableOption>> listDynamicOptions(

‎server/src/main/java/org/diskproject/server/repository/DiskRepository.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ private void initializeExternalStorage() throws Exception {
177177
this.externalStorage = null;
178178
if (cfg.external != null && cfg.username !=null&& cfg.password!= null) {
179179
this.externalStorage = new StorageManager(cfg.external, cfg.username, cfg.password);
180-
//this.externalStorage.init();
180+
this.externalStorage.init();
181181
}
182182
}
183183

@@ -1310,12 +1310,15 @@ else if (qVar.equals("_CSV_")) {
13101310
}
13111311

13121312
// Attach CSV
1313-
if (attachCSV) {
1314-
//send query again
1313+
if (attachCSV && this.externalStorage != null) {
1314+
//run the query again, this time get the bits to create csv file.
1315+
byte[] csvFile = dataAdapter.queryCSV(query);
1316+
String csvHash = KBUtils.SHAsum(csvFile);
1317+
String csvUri = this.externalStorage.upload(csvHash, "text/csv", csvFile);
1318+
13151319
dataVarBindings.put("_CSV_", new ArrayList<String>());
1316-
//dataVarBindings.get("_CSV_").add(createCSVFromQuerySolutions(solutions));
1317-
//System.out.println("CSV: " + dataVarBindings.get("_CSV_").get(0));
1318-
dataAdapter.queryCSV(query);
1320+
dataVarBindings.get("_CSV_").add(csvUri);
1321+
System.out.println("CSV: " + csvUri);
13191322
}
13201323

13211324
TriggeredLOI tloi = new TriggeredLOI(loi, id);
@@ -1344,8 +1347,6 @@ private Set<String> getDataQueryVariables(String wherePart) {
13441347
String var = a.group();
13451348
if (var.charAt(1) != '_')
13461349
l.add(var);
1347-
else
1348-
System.out.println("DISCARTED: " + var);
13491350
}
13501351
return l;
13511352
}

‎server/src/main/java/org/diskproject/server/util/Config.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -204,15 +204,10 @@ private void loadDISKConfiguration() {
204204
String storageLocal = props.getString("storage.local");
205205
String storageTdb = props.getString("storage.tdb");
206206
String storageDb = props.getString("storage.db");
207-
208207
String external = props.getString("storage.external");
209208
String ex_user = props.getString("storage.username");
210209
String ex_pass = props.getString("storage.password");
211-
212210
this.storage = new StorageConfig(storageLocal, storageTdb, storageDb, external, ex_user, ex_pass);
213-
System.out.println(external);
214-
System.out.println(ex_user);
215-
System.out.println(ex_pass);
216211

217212
String keycloakUrl = props.getString("keycloak.url");
218213
String keycloakRealm = props.getString("keycloak.realm");

‎server/src/main/java/org/diskproject/server/util/KBUtils.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
import org.diskproject.shared.classes.vocabulary.Type;
77
import org.diskproject.shared.classes.vocabulary.Vocabulary;
88

9+
import java.security.MessageDigest;
10+
import java.security.NoSuchAlgorithmException;
11+
import java.util.Formatter;
12+
913
import edu.isi.kcap.ontapi.KBAPI;
1014
import edu.isi.kcap.ontapi.KBObject;
1115
import edu.isi.kcap.ontapi.KBTriple;
@@ -125,4 +129,17 @@ public static void fetchTypesAndIndividualsFromKB(KBAPI kb, Vocabulary vocabular
125129
}
126130
}
127131

128-
}
132+
public static String SHAsum(byte[] data) throws NoSuchAlgorithmException {
133+
MessageDigest md = MessageDigest.getInstance("SHA-1");
134+
return byteArray2Hex(md.digest(data));
135+
}
136+
137+
private static String byteArray2Hex(final byte[] hash) {
138+
try (Formatter formatter = new Formatter()) {
139+
for (byte b : hash) {
140+
formatter.format("%02x", b);
141+
}
142+
return formatter.toString();
143+
}
144+
}
145+
}

‎shared/src/main/java/org/diskproject/shared/api/DiskService.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -179,11 +179,6 @@ public WorkflowRun monitorWorkflow(
179179
@GET
180180
@Path("questions")
181181
public List<Question> listQuestions();
182-
183-
@GET
184-
@Path("question/{id}/options")
185-
public List<VariableOption> listOptions(
186-
@PathParam("id") String id);
187182

188183
@POST
189184
@Path("question/options")

‎shared/src/main/java/org/diskproject/shared/classes/adapters/DataAdapter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public String toString () {
7171
}
7272

7373
public abstract List<DataResult> query (String queryString) throws Exception;
74-
public abstract void queryCSV(String queryString) throws Exception;
74+
public abstract byte[] queryCSV(String queryString) throws Exception;
7575

7676
//This data query must return two variable names:
7777
static public String VARURI = "uri";

0 commit comments

Comments
 (0)