TAJO-1711: Add basic monitoring page

tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java

@@ -65,6 +65,7 @@
 import org.apache.tajo.util.metrics.TajoSystemMetrics;
 import org.apache.tajo.webapp.QueryExecutorServlet;
 import org.apache.tajo.webapp.StaticHttpServer;
+import org.apache.tajo.webapp.servlet.MasterMetricsServlet;
 import org.apache.tajo.ws.rs.TajoRestService;
 
 import java.io.IOException;
@@ -235,6 +236,7 @@ private void initWebServer() throws Exception {
       webServer = StaticHttpServer.getInstance(this ,"admin", address.getHostName(), address.getPort(),
           true, null, context.getConf(), null);
       webServer.addServlet("queryServlet", "/query_exec", QueryExecutorServlet.class);
+      webServer.addServlet("masterMetricsServlet", "/metrics", MasterMetricsServlet.class);
       webServer.start();
     }
   }

tajo-core/src/main/java/org/apache/tajo/webapp/servlet/AbstractExecutorServlet.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.webapp.servlet;
+
+import org.apache.hadoop.util.StringUtils;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.NotSerializableException;
+import java.io.OutputStream;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class AbstractExecutorServlet extends HttpServlet {
+  protected transient ObjectMapper om = new ObjectMapper();
+
+  protected void writeObject(java.io.ObjectOutputStream stream) throws java.io.IOException {
+    throw new NotSerializableException( getClass().getName() );
+  }
+
+  protected void readObject(java.io.ObjectInputStream stream) throws java.io.IOException, ClassNotFoundException {
+    throw new NotSerializableException( getClass().getName() );
+  }
+
+  protected void errorResponse(HttpServletResponse response, Exception e) throws IOException {
+    errorResponse(response, e, null);
+  }
+
+  protected void errorResponse(HttpServletResponse response, Exception e, String type) throws IOException {
+    errorResponse(response, e.getMessage() + "\n" + StringUtils.stringifyException(e), type);
+  }
+
+  protected void errorResponse(HttpServletResponse response, String message) throws IOException {
+    errorResponse(response, message, null);
+  }
+
+  protected void errorResponse(HttpServletResponse response, String message, String type) throws IOException {
+    Map<String, Object> errorMessage = new HashMap<String, Object>();
+    errorMessage.put("success", "false");
+    errorMessage.put("errorMessage", message);
+    errorMessage.put("timestamp", Calendar.getInstance().getTimeInMillis());
+    writeHttpResponse(response, errorMessage, type);
+  }
+
+  protected void writeHttpResponse(HttpServletResponse response, Map<String, Object> outputMessage) throws IOException {
+    writeHttpResponse(response, outputMessage, null);
+  }
+
+  protected void writeHttpResponse(HttpServletResponse response, Map<String, Object> outputMessage, String type) throws IOException {
+    if(type==null){
+      type = "text/html";
+    }
+    response.setContentType(type);
+
+    OutputStream out = response.getOutputStream();
+    out.write(om.writeValueAsBytes(outputMessage));
+
+    out.flush();
+    out.close();
+  }
+}

tajo-core/src/main/java/org/apache/tajo/webapp/servlet/MasterMetricsServlet.java

@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.webapp.servlet;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.util.metrics.TajoSystemMetrics;
+import org.apache.tajo.webapp.StaticHttpServer;
+import org.codehaus.jackson.map.DeserializationConfig;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.*;
+
+public class MasterMetricsServlet extends AbstractExecutorServlet {
+  private static final Log LOG = LogFactory.getLog(MasterMetricsServlet.class);
+  private static final long serialVersionUID = -1517586415474706579L;
+
+  @Override
+  public void init(ServletConfig config) throws ServletException {
+    om.getDeserializationConfig().disable(
+        DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES);
+  }
+
+  @Override
+  public void service(HttpServletRequest request,
+                      HttpServletResponse response) throws ServletException, IOException {
+    String action = request.getParameter("action");
+    String type = request.getParameter("type");
+    if(type!=null && type.equalsIgnoreCase("json")){
+      type = "application/json";
+    } else {
+      type = "text/html";
+    }
+    Map<String, Object> returnValue = new HashMap<String, Object>();
+    try {
+      if(action == null || action.trim().isEmpty()) {
+        errorResponse(response, "no action parameter.");
+        return;
+      }
+      if("getMetrics".equals(action)) {
+        TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
+        if(master!=null){
+          TajoSystemMetrics systemMetrics = master.getContext().getMetrics();
+          TreeMap<String, Metric> treeMap = new TreeMap<String, Metric>( systemMetrics.getMetrics() );
+          Iterator<String> iteratorKey = treeMap.keySet().iterator( );
+          returnValue.put("metrics", treeMap/*systemMetrics.getMetrics()*/);
+        } else {
+          returnValue.put("metrics", new TreeMap<String, Metric>());
+        }
+      }
+      returnValue.put("success", "true");
+      returnValue.put("timestamp", Calendar.getInstance().getTimeInMillis());
+      writeHttpResponse(response, returnValue, type);
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+      errorResponse(response, e, type);
+    }
+  }
+}

tajo-core/src/main/java/org/apache/tajo/webapp/servlet/WorkerMetricsServlet.java

@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.webapp.servlet;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.util.metrics.TajoSystemMetrics;
+import org.apache.tajo.webapp.StaticHttpServer;
+import org.apache.tajo.worker.TajoWorker;
+import org.codehaus.jackson.map.DeserializationConfig;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.*;
+
+public class WorkerMetricsServlet extends AbstractExecutorServlet {
+  private static final Log LOG = LogFactory.getLog(WorkerMetricsServlet.class);
+  private static final long serialVersionUID = -1517586415428296579L;
+
+  @Override
+  public void init(ServletConfig config) throws ServletException {
+    om.getDeserializationConfig().disable(
+        DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES);
+  }
+
+  @Override
+  public void service(HttpServletRequest request,
+                      HttpServletResponse response) throws ServletException, IOException {
+    String action = request.getParameter("action");
+    String type = request.getParameter("type");
+    if(type!=null && type.equalsIgnoreCase("json")){
+      type = "application/json";
+    } else {
+      type = "text/html";
+    }
+    Map<String, Object> returnValue = new HashMap<String, Object>();
+    try {
+      if(action == null || action.trim().isEmpty()) {
+        errorResponse(response, "no action parameter.");
+        return;
+      }
+      if("getMetrics".equals(action)) {
+        TajoWorker worker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
+        if(worker!=null){
+          TajoSystemMetrics systemMetrics = worker.getWorkerContext().getMetrics();
+          TreeMap<String, Metric> treeMap = new TreeMap<String, Metric>(systemMetrics.getMetrics());
+          Iterator<String> iteratorKey = treeMap.keySet().iterator();
+          returnValue.put("metrics", treeMap/*systemMetrics.getMetrics()*/);
+        } else {
+          returnValue.put("metrics", new TreeMap<String, Metric>());
+        }
+      }
+      returnValue.put("success", "true");
+      returnValue.put("timestamp", Calendar.getInstance().getTimeInMillis());
+      writeHttpResponse(response, returnValue, type);
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+      errorResponse(response, e, type);
+    }
+  }
+}

tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java

@@ -56,6 +56,7 @@
 import org.apache.tajo.util.history.HistoryWriter;
 import org.apache.tajo.util.metrics.TajoSystemMetrics;
 import org.apache.tajo.webapp.StaticHttpServer;
+import org.apache.tajo.webapp.servlet.WorkerMetricsServlet;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -224,6 +225,7 @@ private int initWebServer() {
     try {
       webServer = StaticHttpServer.getInstance(this, "worker", null, httpPort,
           true, null, systemConf, null);
+      webServer.addServlet("workerMetricsServlet", "/metrics", WorkerMetricsServlet.class);
       webServer.start();
       httpPort = webServer.getPort();
       LOG.info("Worker info server started:" + httpPort);

tajo-core/src/main/resources/tajo-metrics.properties

@@ -31,45 +31,53 @@ reporter.ganglia=org.apache.tajo.util.metrics.reporter.GangliaReporter
 ###############################################################################
 
 ###############################################################################
-# tajo master
+# MASTER
 ###############################################################################
-tajomaster.reporters=null
-
-#tajomaster.reporters=file,console
-#tajomaster.console.period=60
-#tajomaster.file.filename=/tmp/tajo/tajomaster-metrics.out
-#tajomaster.file.period=60
-#tajomaster.ganglia.server=my.ganglia.com
-#tajomaster.ganglia.port=8649
-#tajomaster.ganglia.period=60
+MASTER.reporters=null
+#MASTER.reporters=file,console,ganglia
+#MASTER.file.filename=/tmp/tajo/master-metrics.log
+#MASTER.file.period=60
+#MASTER.console.period=60
+#MASTER.ganglia.server=my.ganglia.com
+#MASTER.ganglia.port=8649
+#MASTER.ganglia.period=60
 ###############################################################################
 
 ###############################################################################
-# tajo master-jvm
+# MASTER-JVM
 ###############################################################################
-tajomaster-jvm.reporters=null
-#tajomaster-jvm.reporters=console
-#tajomaster-jvm.console.period=60
-#tajomaster-jvm.file.filename=/tmp/tajo/tajomaster-jvm-metrics.out
-#tajomaster-jvm.file.period=60
+MASTER-JVM.reporters=null
+#MASTER-JVM.reporters=file,console,ganglia
+#MASTER-JVM.file.filename=/tmp/tajo/master-jvm-metrics.log
+#MASTER-JVM.file.period=60
+#MASTER-JVM.console.period=60
+#MASTER-JVM.ganglia.server=my.ganglia.com
+#MASTER-JVM.ganglia.port=8649
+#MASTER-JVM.ganglia.period=60
 ###############################################################################
 
 ###############################################################################
-# worker
+# NODE
 ###############################################################################
-worker.reporters=null
-#worker.reporters=file,console
-#worker.console.period=60
-#worker.file.filename=/tmp/tajo/worker-metrics.out
-#worker.file.period=60
+NODE.reporters=null
+#NODE.reporters=file,console,ganglia
+#NODE.file.filename=/tmp/tajo/node-metrics.log
+#NODE.file.period=60
+#NODE.console.period=60
+#NODE.ganglia.server=my.ganglia.com
+#NODE.ganglia.port=8649
+#NODE.ganglia.period=60
 ###############################################################################
 
 ###############################################################################
-# worker-jvm
+# NODE-JVM
 ###############################################################################
-worker-jvm.reporters=null
-#worker-jvm.reporters=console
-#worker-jvm.console.period=60
-#worker-jvm.file.filename=/tmp/tajo/worker-jvm-metrics.out
-#worker-jvm.file.period=60
+NODE-JVM.reporters=null
+#NODE-JVM.reporters=file,console,ganglia
+#NODE-JVM.file.filename=/tmp/tajo/node-jvm-metrics.log
+#NODE-JVM.file.period=60
+#NODE-JVM.console.period=60
+#NODE-JVM.ganglia.server=my.ganglia.com
+#NODE-JVM.ganglia.port=8649
+#NODE-JVM.ganglia.period=60
 ###############################################################################

tajo-core/src/main/resources/webapps/admin/header.jsp

@@ -26,6 +26,7 @@
     <li><a class='top_menu_item' href='query.jsp'>Query</a></li>
     <li><a class='top_menu_item' href='catalogview.jsp'>Catalog</a></li>
     <li><a class='top_menu_item' href='query_executor.jsp'>Execute Query</a></li>
+    <li><a class='top_menu_item' href='monitoring.jsp'>Monitoring</a></li>
   </ul>
   <br style='clear:left'/>
 </div>