enable force-decoding with decode_gzip => force

Author: yaauie

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.5.0
+ - Added ability to force gzip-decoding regardless of filename with `decode_gzip => force`
+
 ## 3.4.1
  - Fixed link formatting for input type (documentation)
 

docs/index.asciidoc

@@ -140,6 +140,18 @@ The name of the S3 bucket.
 
 Whether to delete processed files from the original bucket.
 
+[id="plugins-{type}s-{plugin}-decode_gzip"]
+==== `decode_gzip`
+
+  * Value type is <<string,string>>
+  * Acceptable values are `detect` and `force`
+  * Default value is `detect`
+
+Controls behaviour for performing gzip-decoding on files.
+
+  * `detect` will perform gzip-decoding on files with matching extensions (`.gz` and `.gzip`)
+  * `force` will perform gzip-decoding on all files, regardless of extension or contents.
+
 [id="plugins-{type}s-{plugin}-endpoint"]
 ===== `endpoint`
 

lib/logstash/inputs/s3.rb

@@ -79,6 +79,9 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # be present.
   config :include_object_properties, :validate => :boolean, :default => false
 
+  # Controls which files are handled as gzip.
+  config :decode_gzip, :validate => %w(detect force), :default => 'detect'
+
   public
   def register
     require "fileutils"
@@ -109,6 +112,12 @@ def register
     if !@watch_for_new_files && original_params.include?('interval')
       logger.warn("`watch_for_new_files` has been disabled; `interval` directive will be ignored.")
     end
+
+    @gzip_detector = case @decode_gzip
+                     when "force"  then -> (_) { true }
+                     when "detect" then -> (filename) { filename.end_with?('.gz','.gzip') }
+                     else fail(LogStash::ConfigurationError, "unsupported value `#{@decode_gzip}` for decode_gzip")
+                     end
   end
 
   public
@@ -315,7 +324,7 @@ def read_gzip_file(filename, block)
 
   private
   def gzip?(filename)
-    filename.end_with?('.gz','.gzip')
+    @gzip_detector.call(filename)
   end
 
   private

logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.4.1'
+  s.version         = '3.5.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/inputs/s3_spec.rb

@@ -428,6 +428,13 @@
       include_examples "generated events"
     end
 
+    context 'force gzip decoding' do
+      let(:config) { super().merge('decode_gzip' => 'force') }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
+
+      include_examples "generated events"
+    end
+
     context 'compressed with gzip extension' do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }