From 2c11f3386606abd8254651284fa253d6a6f814c6 Mon Sep 17 00:00:00 2001
From: grmcnair <grmcnair@optusnet.com.au>
Date: Wed, 19 Oct 2016 14:13:08 +1000
Subject: [PATCH] Improve behaviour of lambda function when handling large file

Remove unused variable finalData. This reduces execution memory usage.

Do not add newline to end of each chunk (default is 16K characters) returned by the stream.  Can cause random insertion of newline in the middle of log lines.
---
 s3/s3.js | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/s3/s3.js b/s3/s3.js
index 7fb9ba7..adde9dd 100755
--- a/s3/s3.js
+++ b/s3/s3.js
@@ -26,7 +26,6 @@ function s3LogsToSumo(bucket, objKey,context) {
                 });
             });
     
-    var finalData = '';
     var totalBytes = 0;
     var isCompressed = false;
     if (objKey.match(/\.gz$/)) {
@@ -53,8 +52,7 @@ function s3LogsToSumo(bucket, objKey,context) {
     if (!isCompressed) {
         s3Stream.on('data',function(data) {
                 //console.log("Read bytes:" +data.length);
-                finalData += data;
-                req.write(data+'\n');
+                req.write(data);
                 totalBytes += data.length;
             });
         s3Stream.on('end',finishFnc);
@@ -64,8 +62,7 @@ function s3LogsToSumo(bucket, objKey,context) {
         
         gunzip.on('data',function(data) {
             totalBytes += data.length;
-            req.write(data.toString()+'\n');
-            finalData += data.toString();
+            req.write(data.toString());
         }).on('end',finishFnc)
         .on('error',function(error) {
             context.fail(error);