Merge branch 'master' into 'master'

merge 3.10.0-xinchuang to master

See merge request zstackio/zstack-utility!343

Author: gitlab

baremetalpxeserver/ansible/baremetalpxeserver.py

@@ -27,6 +27,7 @@
 baremetalpxeserver_pushgateway_root="/var/lib/zstack/baremetal/"
 baremetalpxeserver_pushgateway_persistence="/var/lib/zstack/baremetal/persistence.data"
 baremetalpxeserver_pushgateway_port=9093
+update_packages = 'false'
 
 # get parameter from shell
 parser = argparse.ArgumentParser(description='Deploy baremetal pxeserver agent to host')
@@ -109,7 +110,8 @@
     mips64el_ns10 = "dnsmasq nginx vsftpd nmap net-tools"
     dep_pkg = eval("%s_%s" % (host_arch, releasever))
     if zstack_repo != 'false':
-        command = ("pkg_list=`rpm -q %s | grep \"not installed\" | awk '{ print $2 }'` && for pkg in $pkg_list; do yum --disablerepo=* --enablerepo=%s install -y $pkg; done;") % (dep_pkg, zstack_repo)
+        command = ("pkg_list=`rpm -q %s | grep \"not installed\" | awk '{ print $2 }'` && for pkg in %s; do yum --disablerepo=* --enablerepo=%s install -y $pkg; done;") % \
+                  (dep_pkg, dep_pkg if update_packages == 'true' else '$pkg_list', zstack_repo)
         run_remote_command(command, host_post_info)
     else:
         for pkg in ["dnsmasq", "nginx", "vsftpd", "syslinux", "nmap"]:
@@ -129,9 +131,10 @@
 command = """
 basearch=`uname -m`;releasever=`awk '{print $3}' /etc/zstack-release`;
 [ -f /opt/zstack-dvd/$basearch/$releasever/GPL ] || exit 1;
-mkdir -p /var/lib/zstack/baremetal/{dnsmasq,ftp/{ks,zstack-dvd,scripts},tftpboot/{zstack,pxelinux.cfg},vsftpd} /var/log/zstack/baremetal/;
+mkdir -p /var/lib/zstack/baremetal/{dnsmasq,ftp/{ks,zstack-dvd,scripts},tftpboot/{zstack/$basearch,pxelinux.cfg},vsftpd} /var/log/zstack/baremetal/;
 cp -f /usr/share/syslinux/pxelinux.0 /var/lib/zstack/baremetal/tftpboot/;
-cp -f /opt/zstack-dvd/$basearch/$releasever/images/pxeboot/{vmlinuz,initrd.img} /var/lib/zstack/baremetal/tftpboot/zstack/;
+cp -f /opt/zstack-dvd/$basearch/$releasever/EFI/BOOT/grubaa64.efi /var/lib/zstack/baremetal/tftpboot/;
+cp -f /opt/zstack-dvd/$basearch/$releasever/images/pxeboot/{vmlinuz,initrd.img} /var/lib/zstack/baremetal/tftpboot/zstack/$basearch/;
 grep 'zstack-dvd' /etc/fstab || echo "/opt/zstack-dvd/$basearch/$releasever /var/lib/zstack/baremetal/ftp/zstack-dvd none defaults,bind 0 0" >> /etc/fstab;
 mount -a;
 """
@@ -188,11 +191,12 @@
 copy(copy_arg, host_post_info)
 
 # name: copy shellinaboxd
+shellinaboxd_name = "shellinaboxd_{}".format(host_arch)
 VSFTPD_ROOT_PATH = "/var/lib/zstack/baremetal/ftp"
 copy_arg = CopyArg()
-copy_arg.src = "%s/shellinaboxd" % file_root
-copy_arg.dest = VSFTPD_ROOT_PATH
 copy_arg.args = "force=yes"
+copy_arg.src = "%s/%s" % (file_root, shellinaboxd_name)
+copy_arg.dest = os.path.join(VSFTPD_ROOT_PATH, "shellinaboxd")
 copy(copy_arg, host_post_info)
 
 # name: copy noVNC.tar.gz
@@ -202,13 +206,12 @@
 copy_arg.args = "force=yes"
 copy(copy_arg, host_post_info)
 
-if host_arch != "mips64el":
-    # name: copy zwatch-vm-agent
-    zwatch_vm_agent_name = "zwatch-vm-agent{}".format('' if host_arch == 'x86_64' else '_' + host_arch)
-    copy_arg = CopyArg()
-    copy_arg.src = os.path.join(kvm_file_root, zwatch_vm_agent_name)
-    copy_arg.dest = os.path.join(VSFTPD_ROOT_PATH, 'zwatch-vm-agent')
-    copy(copy_arg, host_post_info)
+# name: copy zwatch-vm-agent
+zwatch_vm_agent_name = "zwatch-vm-agent{}".format('' if host_arch == 'x86_64' else '_' + host_arch)
+copy_arg = CopyArg()
+copy_arg.src = os.path.join(kvm_file_root, zwatch_vm_agent_name)
+copy_arg.dest = os.path.join(VSFTPD_ROOT_PATH, 'zwatch-vm-agent')
+copy(copy_arg, host_post_info)
 
 copy_arg = CopyArg()
 copy_arg.src = "%s/agent_version" % file_root
@@ -235,10 +238,11 @@
 copy(copy_arg, host_post_info)
 
 # name: copy pushgateway
+pushgateway_name = "pushgateway_%s" % host_arch
 copy_arg = CopyArg()
-copy_arg.src = "%s/pushgateway" % file_root
-copy_arg.dest = baremetalpxeserver_pushgateway_root
 copy_arg.args = "mode=a+x"
+copy_arg.src = "%s/%s" % (file_root, pushgateway_name)
+copy_arg.dest = "%s/pushgateway" % baremetalpxeserver_pushgateway_root
 copy(copy_arg, host_post_info)
 
 run_remote_command(("systemctl restart pxeServerPushGateway"), host_post_info)

baremetalpxeserver/ansible/pushgateway_aarch64

@@ -0,0 +1 @@
+../../kvmagent/ansible/pushgateway_aarch64

baremetalpxeserver/ansible/pushgateway_mips64el

@@ -0,0 +1 @@
+../../kvmagent/ansible/pushgateway_mips64el

baremetalpxeserver/ansible/pushgateway renamed to baremetalpxeserver/ansible/pushgateway_x86_64

@@ -111,6 +111,7 @@ class PxeServerAgent(object):
     NGINX_TERMINAL_PROXY_CONF_PATH = "/etc/nginx/conf.d/terminal/"
     NOVNC_INSTALL_PATH = BAREMETAL_LIB_PATH + "noVNC/"
     NOVNC_TOKEN_PATH = NOVNC_INSTALL_PATH + "tokens/"
+    ARM_GRUB_CFG = TFTPBOOT_PATH + "grub.cfg"
 
     NMAP_BROADCAST_DHCP_DISCOVER_PATH = "/usr/share/nmap/scripts/broadcast-dhcp-discover.nse"
 
@@ -223,7 +224,10 @@ def init(self, req):
         dhcp_conf = """interface={DHCP_INTERFACE}
 port=0
 bind-interfaces
-dhcp-boot=pxelinux.0
+dhcp-match=x86PC, option:client-arch, 0 #LEGACLY x86
+dhcp-match=AARCH64_EFI, option:client-arch, 11 #EFI AARCH64
+dhcp-boot=tag:x86PC,pxelinux.0
+dhcp-boot=tag:AARCH64_EFI,grubaa64.efi
 enable-tftp
 tftp-root={TFTPBOOT_PATH}
 log-facility={DNSMASQ_LOG_PATH}
@@ -265,7 +269,7 @@ def init(self, req):
 listen_ipv6=YES
 pam_service_name=vsftpd
 userlist_enable=YES
-tcp_wrappers=YES
+#tcp_wrappers=YES
 xferlog_enable=YES
 xferlog_std_format=YES
 xferlog_file={VSFTPD_LOG_PATH}
@@ -278,13 +282,24 @@ def init(self, req):
         pxelinux_cfg = """default zstack_baremetal
 prompt 0
 label zstack_baremetal
-kernel zstack/vmlinuz
+kernel zstack/x86_64/vmlinuz
 ipappend 2
-append initrd=zstack/initrd.img devfs=nomount ksdevice=bootif ks=ftp://{PXESERVER_DHCP_NIC_IP}/ks/inspector_ks.cfg vnc
+append initrd=zstack/x86_64/initrd.img devfs=nomount ksdevice=bootif ks=ftp://{PXESERVER_DHCP_NIC_IP}/ks/inspector_ks.cfg vnc
 """.format(PXESERVER_DHCP_NIC_IP=pxeserver_dhcp_nic_ip)
         with open(self.PXELINUX_DEFAULT_CFG, 'w') as f:
             f.write(pxelinux_cfg)
 
+        #init grub.cfg for arm
+        grub_cfg = """set timeout=10
+menuentry 'ZStack Get Hardware Info' {
+    linux zstack/aarch64/vmlinuz ip=dhcp inst.repo=ftp://%s/zstack-dvd inst.ks=ftp://%s/ks/inspector_ks.cfg \
+        console=tty0 console=ttyS0,115200n8 rd.debug rd.udev.debug systemd.log_level=debug
+    initrd zstack/aarch64/initrd.img
+}
+""" % (pxeserver_dhcp_nic_ip, pxeserver_dhcp_nic_ip)
+        with open(self.ARM_GRUB_CFG, 'w') as f:
+            f.write(grub_cfg)
+
         # init inspector_ks.cfg
         ks_tmpl_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ks_tmpl')
         with open("%s/inspector_ks_tmpl" % ks_tmpl_path, 'r') as fr:
@@ -366,10 +381,10 @@ def ping(self, req):
 
         # DETECT ROGUE DHCP SERVER
         cmd = json_object.loads(req[http.REQUEST_BODY])
-        if platform.machine() == "x86_64":
-            ret, output = bash_ro("nmap -sU -p67 --script broadcast-dhcp-discover -e %s | grep 'Server Identifier'" % cmd.dhcpInterface)
-            if ret == 0:
-                raise PxeServerError("rogue dhcp server[IP:%s] detected" % output.strip().split(' ')[-1])
+        mac = self._get_mac_address(cmd.dhcpInterface)
+        ret, output = bash_ro("nmap -sU -p67 --script broadcast-dhcp-discover --script-args broadcast-dhcp-discover.mac=%s -e %s | grep 'Server Identifier'" % (mac, cmd.dhcpInterface))
+        if ret == 0:
+            raise PxeServerError("rogue dhcp server[IP:%s] detected" % output.strip().split(' ')[-1])
 
         # make sure pxeserver is running if it's Enabled
         if cmd.enabled:
@@ -468,6 +483,23 @@ def _create_pxelinux_cfg(self, cmd):
         with open(pxe_cfg_file, 'w') as f:
             f.write(pxelinux_cfg)
 
+        # init grub.cfg-MAC for arm
+        pxe_cfg_file_arm = os.path.join(self.TFTPBOOT_PATH, "grub.cfg-01-" + cmd.pxeNicMac)
+        pxeserver_dhcp_nic_ip = self._get_ip_address(cmd.dhcpInterface).strip()
+        grub_cfg = """set default="0"
+set timeout=10
+menuentry 'ZStack Baremetal' {{
+    linux {IMAGEUUID}/vmlinuz ip=dhcp inst.repo=ftp://{PXESERVER_DHCP_NIC_IP}/{IMAGEUUID}/ inst.ks=ftp://{PXESERVER_DHCP_NIC_IP}/ks/{KS_CFG_NAME} \
+        console=tty0 console=ttyS0,115200n8 rd.debug rd.udev.debug systemd.log_level=debug
+    initrd {IMAGEUUID}/initrd.img
+}}
+        """.format(IMAGEUUID=cmd.imageUuid,
+                   PXESERVER_DHCP_NIC_IP=pxeserver_dhcp_nic_ip,
+                   KS_CFG_NAME=ks_cfg_name
+                   )
+        with open(pxe_cfg_file_arm, 'w') as f:
+            f.write(grub_cfg)
+
     def _create_preconfiguration_file(self, cmd):
         # in case user didn't seleted a preconfiguration template etc.
         cmd.preconfigurationContent = cmd.preconfigurationContent if cmd.preconfigurationContent != "" else """
@@ -969,12 +1001,18 @@ def delete_bm_configs(self, req):
                 bash_r("rm -f %s/*" % self.NGINX_TERMINAL_PROXY_CONF_PATH)
             if os.path.exists(self.NOVNC_TOKEN_PATH):
                 bash_r("rm -f %s/*" % self.NOVNC_TOKEN_PATH)
+            if os.path.exists(self.TFTPBOOT_PATH):
+                bash_r("rm -f %s/grub.cfg*" % self.TFTPBOOT_PATH)
         else:
             mac_as_name = cmd.pxeNicMac.replace(":", "-")
             pxe_cfg_file = os.path.join(self.PXELINUX_CFG_PATH, "01-" + mac_as_name)
             if os.path.exists(pxe_cfg_file):
                 os.remove(pxe_cfg_file)
 
+            arm_pxe_cfg_file = os.path.join(self.TFTPBOOT_PATH, "grub.cfg-01-" + mac_as_name)
+            if os.path.exists(arm_pxe_cfg_file):
+                os.remove(arm_pxe_cfg_file)
+
             ks_cfg_file = os.path.join(self.KS_CFG_PATH, mac_as_name)
             if os.path.exists(ks_cfg_file):
                 os.remove(ks_cfg_file)
@@ -1098,7 +1136,10 @@ def download_imagestore(self, req):
         # SUSE
         ret5 = bash_r("cp %s %s" % (os.path.join(mount_path, "boot/*/loader/linux"), os.path.join(vmlinuz_path, "vmlinuz")))
         ret6 = bash_r("cp %s %s" % (os.path.join(mount_path, "boot/*/loader/initrd"), os.path.join(vmlinuz_path, "initrd.img")))
-        if (ret1 != 0 or ret2 != 0) and (ret3 != 0 or ret4 != 0) and (ret5 != 0 or ret6 != 0):
+        # ns10
+        ret7 = bash_r("cp %s %s" % (os.path.join(mount_path, "images/pxeboot/vmlinuz"), os.path.join(vmlinuz_path, "vmlinuz")))
+        ret8 = bash_r("cp %s %s" % (os.path.join(mount_path, "images/pxeboot/initrd.img"), os.path.join(vmlinuz_path, "initrd.img")))
+        if (ret1 != 0 or ret2 != 0) and (ret3 != 0 or ret4 != 0) and (ret5 != 0 or ret6 != 0) and (ret7 != 0 or ret8 != 0):
             raise PxeServerError("failed to copy vmlinuz and initrd.img from image[uuid:%s] to baremetal tftp server" % cmd.imageUuid)
 
         logger.info("successfully downloaded image[uuid:%s] and mounted it" % cmd.imageUuid)

baremetalpxeserver/ansible/shellinaboxd_aarch64

@@ -74,7 +74,11 @@
 else:
     src_pkg_imagestorebackupstorage = "zstack-store.bin"
     src_pkg_exporter = "collectd_exporter"
-dst_pkg_imagestorebackupstorage = "zstack-store.bin"
+
+if client != "true":
+    dst_pkg_imagestorebackupstorage = "zstack-store.bin"
+else:
+    dst_pkg_imagestorebackupstorage = "zstack-store-client.bin"
 dst_pkg_exporter = "collectd_exporter"
 
 # include zstacklib.py
@@ -109,6 +113,7 @@
     aarch64_ns10 = "qemu-img fuse-sshfs nmap collectd"
     mips64el_ns10 = "qemu-img-ev fuse-sshfs nmap collectd"
     x86_64_ns10 = "qemu-img fuse-sshfs nmap collectd"
+
     qemu_pkg = eval("%s_%s" % (host_arch, releasever))
     # skip these packages
     _skip_list = re.split(r'[|;,\s]\s*', skip_packages)
@@ -191,8 +196,7 @@
     run_remote_command("sudo chown -R -H --dereference %s: %s" % (remote_user, fs_rootpath), host_post_info)
 
 # name: restart image store server
-server = client != "true" or run_remote_command("pkill -0 zstore", host_post_info, return_status=True, return_output=False)
-if server:
+if client != "true":
     # integrate zstack-store with init.d
     run_remote_command("/bin/cp -f /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage /etc/init.d/", host_post_info)
     if distro in RPM_BASED_OS:

baremetalpxeserver/ansible/shellinaboxd renamed to baremetalpxeserver/ansible/shellinaboxd_x86_64

@@ -122,6 +122,7 @@ DEFAULT_MN_PORT='8080'
 MN_PORT="$DEFAULT_MN_PORT"
 
 DEFAULT_UI_PORT='5000'
+RESOLV_CONF='/etc/resolv.conf'
 
 BASEARCH=`uname -m`
 ZSTACK_RELEASE=''
@@ -859,6 +860,11 @@ zstack  hard  nproc  $nr_open
 EOF
 }
 
+do_check_resolv_conf(){
+    [ ! -f $RESOLV_CONF ] && touch $RESOLV_CONF
+    chmod a+r $RESOLV_CONF
+}
+
 do_config_ansible(){
     mkdir -p /etc/ansible
     mkdir -p /var/log/ansible
@@ -953,6 +959,7 @@ You can also add '-q' to installer, then Installer will help you to remove it.
     fi
     do_enable_sudo
     do_config_limits
+    do_check_resolv_conf
     pass
 }
 

baremetalpxeserver/baremetalpxeserver/pxeserveragent.py

@@ -1,2 +1,4 @@
 zwatch-vm-agent=4.0.0
 md5-zwatch-vm-agent=43a93d10db8b4661bab173ea788bc502
+md5-zwatch-vm-agent_aarch64=aa19a4f6ef9929ac64505117ac0277aa
+md5-zwatch-vm-agent_mips64el=e580f8982e140ceb76a915771178f522