Merge pull request #86 from octaavio/master

Feature to specify ES pipeline for a given spreadsheet

Author: codingchili

README.md

@@ -13,6 +13,7 @@ Parses XLSX/XLS/CSV files into ElasticSearch using column titles from a specifie
 - support for importing to TLS enabled elasticsearch servers.
 - supports concurrent parsing of excel files and importing for better performance.
 - parses the whole file before starting the import - to make sure your index is not left in an undesired state.
+- specify elasticsearch ingest pipeline for a given spreadsheet.
 
 ## Prerequisites
 The application requires ElasticSearch as its output.
@@ -21,7 +22,7 @@ The application requires ElasticSearch as its output.
 
 2. Grab the latest .jar file from [releases](https://github.com/codingchili/parser-excel-elasticsearch/releases).
 
-Tested with ElasticSearch 5.6.2, 6.4.2 and 7.0.0-alpha1
+Tested with ElasticSearch 5.6.2, 6.4.2, 7.0.0-alpha1 and 7.4.0.
 
 ## Running with docker
 ```console
@@ -35,7 +36,7 @@ connect to it from another machine when binding to all interfaces.
 
 Running the application, filename and index is required, to import from the terminal run:
 ```console
-java -Xmx2g -jar excelastic.jar <fileName> <indexName> --mapping mappingName --clear
+java -Xmx2g -jar excelastic.jar <fileName> <indexName> --mapping mappingName --pipeline pipelineName --clear
 ```
 If running with --clear, then the existing index will be cleared before the import starts.
 

excelastic.png

@@ -10,7 +10,7 @@
 
     <groupId>com.codingchili</groupId>
     <artifactId>excelastic</artifactId>
-    <version>1.3.6</version>
+    <version>1.3.7</version>
     <build>
         <plugins>
             <plugin>

pom.xml

@@ -21,7 +21,7 @@
  */
 public class ApplicationLauncher {
     private final ApplicationLogger logger = new ApplicationLogger(getClass());
-    public static String VERSION = "1.3.6";
+    public static String VERSION = "1.3.7";
     private Vertx vertx;
 
     public static void main(String[] args) {

src/main/java/com/codingchili/ApplicationLauncher.java

@@ -158,10 +158,18 @@ private void updateStatus(HttpClientResponse response, ImportEvent event, int re
     }
 
     private String createImportHeader(ImportEvent event) {
+        JsonObject indexBody = new JsonObject();
+        indexBody.put("_index", event.getIndex())
+                 .put("_type", event.getMapping());
+
+        event.getPipeline()
+        	.filter(value -> !value.isEmpty())
+        	.ifPresent((pipeline) -> indexBody.put("pipeline", pipeline));
+        
         return new JsonObject()
-                .put("index", new JsonObject()
-                        .put("_index", event.getIndex())
-                        .put("_type", event.getMapping())).encode() + "\n";
+                .put("index", indexBody)
+                .encode() + "\n";
+
     }
 
     /**

src/main/java/com/codingchili/Model/ElasticWriter.java

@@ -11,19 +11,22 @@
 /**
  * @author Robin Duda
  * <p>
- * Contains infromation about an import request.
+ * Contains information about an import request.
  */
 public class ImportEvent {
     private static final String ARG_CLEAR = "--clear";
     private static final String ARG_OFFSET = "--offset";
     private static final String ARG_MAPPING = "--mapping";
+    private static final String ARG_PIPELINE = "--pipeline";
     private static final String OFFSET = "offset";
     private static final String MAPPING = "mapping";
+    private static final String PIPELINE = "pipeline";
     private static final String OPTIONS = "options";
     private static final String CLEAR = "clear";
     private FileParser parser;
     private Boolean clearExisting;
     private String mapping;
+    private Optional<String> pipeline;
     private String index;
     private String uploadId;
     private int offset;
@@ -38,6 +41,7 @@ public static ImportEvent fromParams(MultiMap params) {
         return new ImportEvent()
                 .setIndex(params.get(INDEX))
                 .setMapping(getMappingByParams(params))
+                .setPipeline(params.get(PIPELINE))
                 .setClearExisting(params.get(OPTIONS).equals(CLEAR))
                 .setUploadId(params.get(UPLOAD_ID))
                 .setOffset(Integer.parseInt(params.get(OFFSET)));
@@ -53,15 +57,17 @@ public static ImportEvent fromCommandLineArgs(String[] args) {
         return new ImportEvent()
                 .setIndex(args[1])
                 .setOffset(getArgParamValue(args, ARG_OFFSET).map(Integer::parseInt).orElse(1))
-                .setClearExisting(Arrays.asList(args).contains(ARG_CLEAR))
-                .setMapping(getArgParamValue(args, ARG_MAPPING).orElse("default"));
+                .setMapping(getArgParamValue(args, ARG_MAPPING).orElse("default"))
+                .setPipeline(getArgParamValue(args, ARG_PIPELINE).orElse(""))
+                .setClearExisting(Arrays.asList(args).contains(ARG_CLEAR));
+                
     }
 
     private static Optional<String> getArgParamValue(String[] args, String argName) {
         for (int i = 0; i < args.length; i++) {
             if (args[i].equals(argName)) {
                 if (i + 1 < args.length) {
-                    return Optional.of(args[i + i]);
+                    return Optional.of(args[i + 1]);
                 }
             }
         }
@@ -132,4 +138,14 @@ public ImportEvent setIndex(String index) {
         this.index = index;
         return this;
     }
+
+    public Optional<String> getPipeline() {
+        return pipeline;
+    }
+
+    public ImportEvent setPipeline(String pipeline) {
+        this.pipeline = Optional.ofNullable(pipeline);
+        return this;
+    }
+    
 }

src/main/java/com/codingchili/Model/ImportEvent.java

@@ -32,6 +32,10 @@ html(lang='en')
                                 label.col-lg-3.control-label(for='mapping') Mapping
                                 .col-lg-9
                                     input#index.form-control(type='text', name='mapping', placeholder='default')
+                            .form-group
+                                label.col-lg-3.control-label(for='pipeline') Pipeline
+                                .col-lg-9
+                                    input#index.form-control(type='text', name='pipeline', placeholder='none if empty')        
                             a.text-center.clickable#excel-options-show
                                 show excel options
                             .form-group(hidden)#excel-options