fix: sbin deploy and docs (#3699)

Author: vagetablechicken

docs/zh/deploy/install_deploy.md

@@ -185,7 +185,7 @@ JOB 检查是更灵活的离线任务检查命令，可以按条件筛选job，
 
 ### static-check 静态检查
 
-`static-check`静态检查，根据集群部署配置文件（通过参数`-f,--conf_file`指定），登录各个服务组件的部署地址，可以收集版本信息、配置文件、日志文件，检查版本是否一致，对收集到的配置文件和日志文件做分析。可以在集群未部署前进行检查，避免因程序版本或配置文件错误导致的集群部署失败。或在集群异常时，将分布式的日志文件收集在一起，方便调查问题。
+`static-check`静态检查，根据集群部署配置文件（通过参数`-f,--conf_file`指定），登录各个服务组件的部署地址，可以收集版本信息、配置文件、日志文件，检查版本是否一致，对收集到的配置文件和日志文件做分析。可以在集群*未部署前*进行检查，避免因程序版本或配置文件错误导致的集群部署失败。或在集群异常时，将分布式的日志文件收集在一起，方便调查问题。
 
 ```bash
 openmldb_tool static-check -h
@@ -260,7 +260,7 @@ nameserver:
 
 检查可通过组合FLAG来来指定检查哪些内容，例如，`-V`只检查版本，`-CL`只检查配置文件和日志，`-VCL`检查全部。
 
-- `-V,--version`检查版本，检查各个组件的版本是否一致，如果不一致，会输出不一致的组件和版本信息。
+- `-V,--version`检查版本，检查各个组件的版本是否一致，如果不一致，会输出不一致的组件和版本信息（由于复杂度较高，openmldb-batch包的地址可能查不到，将忽略检查，替换batch包非常容易，可以推后检查）。
 - `-C,--conf`收集配置文件，检查各个组件的配置文件中ZooKeeper地址是否一致等。
 - `-L,--log`收集日志，输出WARNING及以上的日志。
 

docs/zh/maintain/diagnose.md

@@ -38,12 +38,16 @@ docker run -it 4pdosc/openmldb:0.8.4 bash
 
 ```bash
 /work/openmldb/bin/openmldb --zk_cluster=127.0.0.1:2181 --zk_root_path=/openmldb --role=sql_client
+# 或脚本
+/work/openmldb/sbin/openmldb-cli.sh
 ```
 
 成功启动 OpenMLDB CLI 后如下图显示：
 
 ![image](./images/cli_cluster.png)
 
+如果你需要对 OpenMLDB 集群进行配置修改，/work/init.sh使用的是sbin一键部署方式，具体参考[一键部署](../deploy/install_deploy.md#部署方式一一键部署推荐)。
+
 ## 使用流程
 
 参照核心概念，使用 OpenMLDB 的流程一般包含：建立数据库和表、导入离线数据、离线特征计算、SQL 方案上线、导入在线数据、在线实时特征计算六个步骤。

docs/zh/quickstart/openmldb_quickstart.md

@@ -115,7 +115,8 @@ def get_spark_home(server_info: ServerInfo):
                 tm_conf_path = server_info.conf_path_pair("")[0]
                 config_name = "spark.home="
                 log.debug("get %s from %s", config_name, tm_conf_path)
-                grep_str, _ = server_info.cmd_on_host(f"grep {config_name} {tm_conf_path}")
+                # last one option
+                grep_str, _ = server_info.cmd_on_host(f"grep {config_name} {tm_conf_path} | tail -n 1")
 
                 if not grep_str:
                     # TODO(hw):no config in file, get env SPARK_HOME?

python/openmldb_tool/diagnostic_tool/collector.py

@@ -2,15 +2,12 @@
 export OPENMLDB_VERSION=0.8.4
 # openmldb mode: standalone / cluster
 export OPENMLDB_MODE=${OPENMLDB_MODE:=cluster}
-# tablet port
-export OPENMLDB_TABLET_PORT=10921
-# nameserver port
-export OPENMLDB_NAMESERVER_PORT=7527
-# taskmanager port
-export OPENMLDB_TASKMANAGER_PORT=9902
-# apiserver port
-export OPENMLDB_APISERVER_PORT=9080
-
+# openmldb root path
+export OPENMLDB_HOME=
+# the root path of openmldb spark release, default is $OPENMLDB_HOME/spark
+# if not exists, download from online
+export SPARK_HOME=
+export RUNNER_EXISTING_SPARK_HOME=
 # if OPENMLDB_USE_EXISTING_ZK_CLUSTER is set, will use existing zk cluster
 export OPENMLDB_USE_EXISTING_ZK_CLUSTER=false
 # the root path of zookeeper release, default is $OPENMLDB_HOME/zookeeper
@@ -20,17 +17,26 @@ export OPENMLDB_ZK_HOME=
 export OPENMLDB_ZK_CLUSTER=
 # zookeeper root path
 export OPENMLDB_ZK_ROOT_PATH=/openmldb
+
+export OPENMLDB_FORCE_LOCAL=false
+
+export RUNNER_JAVA_HOME=
+
+# if CLEAR_OPENMLDB_INSTALL_DIR is set, all files in the WORKDIR will be deleted when running sbin/clear-all.sh
+export CLEAR_OPENMLDB_INSTALL_DIR=false
+
+# tablet port
+export OPENMLDB_TABLET_PORT=10921
+# nameserver port
+export OPENMLDB_NAMESERVER_PORT=7527
+# taskmanager port
+export OPENMLDB_TASKMANAGER_PORT=9902
+# apiserver port
+export OPENMLDB_APISERVER_PORT=9080
+
 # zookeeper client port, clientPort=2181 in zoo.cfg
 export OPENMLDB_ZK_CLUSTER_CLIENT_PORT=2181
 # zookeeper peer port, which is the first port in this config server.1=zoo1:2888:3888 in zoo.cfg
 export OPENMLDB_ZK_CLUSTER_PEER_PORT=2888
 # zookeeper election port, which is the second port in this config server.1=zoo1:2888:3888 in zoo.cfg
 export OPENMLDB_ZK_CLUSTER_ELECTION_PORT=3888
-
-# openmldb root path
-export OPENMLDB_HOME=
-# the root path of openmldb spark release, default is $OPENMLDB_HOME/spark
-# if not exists, download from online
-export SPARK_HOME=
-# if CLEAR_OPENMLDB_INSTALL_DIR is set, all files in the WORKDIR will be deleted when running sbin/clear-all.sh
-export CLEAR_OPENMLDB_INSTALL_DIR=false

release/conf/openmldb-env.sh

@@ -33,7 +33,7 @@ distribute() {
     type=$4
   fi
   local use_ssh=true
-  if [[ $host = "localhost" || $host = "127.0.0.1" ]]; then
+  if [[ "$OPENMLDB_FORCE_LOCAL" = true || "$host" = "localhost" || "$host" = "127.0.0.1" ]]; then
     use_ssh=false
     if [[ "$dest" = "$src" ]]; then
       echo "skip rsync as dest=src: $dest"
@@ -56,7 +56,10 @@ distribute() {
   else
     if [[ "$type" = "taskmanager" ]]; then
       dir_list=(bin sbin conf taskmanager)
-      if [[ "$use_ssh" = true ]]; then
+      if [[ -n "$RUNNER_EXISTING_SPARK_HOME" ]]; then
+        echo "use existing spark $RUNNER_EXISTING_SPARK_HOME on $host, skip deploy spark"
+      elif [[ "$use_ssh" = true ]]; then
+        run_auto "$host" "mkdir -p ${SPARK_HOME} > /dev/null 2>&1"
         rsync -arz "${SPARK_HOME}/" "$host:${SPARK_HOME}/"
       fi
     else
@@ -146,6 +149,10 @@ function download_spark {
 
 # deploy taskmanagers
 downloaded=false
+if [[ -n "${RUNNER_EXISTING_SPARK_HOME}" ]]; then
+  echo "use $RUNNER_EXISTING_SPARK_HOME, skip download openmldbspark"
+  downloaded=true
+fi
 for line in $(parse_host conf/hosts taskmanager)
 do
   if ! $downloaded; then
@@ -158,7 +165,7 @@ do
 
   echo "deploy taskmanager to $host:$port $dir"
   distribute "$host" "$dir" "$home" taskmanager
-  cmd="cd $dir && OPENMLDB_VERSION=${OPENMLDB_VERSION} SPARK_HOME=${SPARK_HOME} OPENMLDB_HOST=$host OPENMLDB_TASKMANAGER_PORT=$port OPENMLDB_ZK_CLUSTER=${OPENMLDB_ZK_CLUSTER} OPENMLDB_ZK_ROOT_PATH=${OPENMLDB_ZK_ROOT_PATH} sbin/deploy.sh taskmanager"
+  cmd="cd $dir && SPARK_HOME=${SPARK_HOME} OPENMLDB_HOST=$host OPENMLDB_TASKMANAGER_PORT=$port OPENMLDB_ZK_CLUSTER=${OPENMLDB_ZK_CLUSTER} OPENMLDB_ZK_ROOT_PATH=${OPENMLDB_ZK_ROOT_PATH} sbin/deploy.sh taskmanager"
   run_auto "$host" "$cmd"
 done
 

release/sbin/deploy-all.sh

@@ -90,7 +90,7 @@ function parse_host {
 run_auto() {
   local host=$1
   local cmd=$2
-  if [[ $host = "localhost" || $host = "127.0.0.1" ]]; then
+  if [[ "$OPENMLDB_FORCE_LOCAL" = true || "$host" = "localhost" || "$host" = "127.0.0.1" ]]; then
     local cur_dir
     cur_dir=$(pwd)
     bash -c "$cmd"
@@ -105,7 +105,11 @@ if [ -z "${OPENMLDB_HOME}" ]; then
   export OPENMLDB_HOME
 fi
 
-if [ -z "${SPARK_HOME}" ]; then
+if [ -n "$RUNNER_EXISTING_SPARK_HOME" ]; then
+  echo "use existing spark $RUNNER_EXISTING_SPARK_HOME on runner, overwrite SPARK_HOME"
+  SPARK_HOME="$RUNNER_EXISTING_SPARK_HOME"
+  export SPARK_HOME
+elif [ -z "${SPARK_HOME}" ]; then
   SPARK_HOME=${OPENMLDB_HOME}/spark
   export SPARK_HOME
 fi

release/sbin/init.sh

@@ -20,7 +20,7 @@ sbin="$(cd "$(dirname "$0")" || exit 1; pwd)"
 . "$home"/conf/openmldb-env.sh
 . "$sbin"/init.sh
 cd "$home" || exit 1
-
+echo "${OPENMLDB_MODE} ${OPENMLDB_ZK_CLUSTER} ${OPENMLDB_ZK_ROOT_PATH}"
 if [[ -n "$OPENMLDB_MODE" && "$OPENMLDB_MODE" = "cluster" ]]; then
   bin/openmldb --zk_cluster="${OPENMLDB_ZK_CLUSTER}" --zk_root_path="${OPENMLDB_ZK_ROOT_PATH}" --role=sql_client "$@"
 else

release/sbin/openmldb-cli.sh

@@ -38,11 +38,13 @@ else
 
     echo "start taskmanager in $dir with endpoint $host:$port "
     cmd="cd $dir && SPARK_HOME=${SPARK_HOME} bin/start.sh start taskmanager $*"
-    run_auto "$host" "$cmd"
-
-    # Print the log of taskmanager if fail
-    #cmd="cd $dir && cat taskmanager/bin/logs/taskmanager.log"
-    #run_auto "$host" "$cmd"
+    # special for java
+    pre=""
+    if [[ -n $RUNNER_JAVA_HOME ]]; then
+      echo "overwrite java env by RUNNER_JAVA_HOME:$RUNNER_JAVA_HOME"
+      pre="export JAVA_HOME=$RUNNER_JAVA_HOME && export PATH=$JAVA_HOME/bin:$PATH &&"
+    fi
+    run_auto "$host" "$pre $cmd"
   done
   IFS="$old_IFS"
 fi

release/sbin/start-taskmanagers.sh

@@ -33,6 +33,12 @@ do
 
   echo "start zookeeper in $dir with endpoint $host:$port "
   cmd="cd $dir && bin/zkServer.sh start"
-  run_auto "$host" "$cmd"
+  # special for java
+  pre=""
+  if [[ -n $RUNNER_JAVA_HOME ]]; then
+    echo "overwrite java env by RUNNER_JAVA_HOME:$RUNNER_JAVA_HOME"
+    pre="export JAVA_HOME=$RUNNER_JAVA_HOME && export PATH=$JAVA_HOME/bin:$PATH &&"
+  fi
+  run_auto "$host" "$pre $cmd"
 done
-IFS="$old_IFS"
+IFS="$old_IFS"