From 32ec11bf10db4e58a090e005f1a4ad0da8ec849f Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 15 Mar 2018 18:33:20 +0500 Subject: [PATCH 01/19] work in progress commit Signed-off-by: Slach --- Vagrantfile | 3 ++- go/agent/agent.go | 2 +- go/app/http.go | 2 +- go/cmd/orchestrator-agent/main.go | 2 +- go/http/api.go | 4 ++-- go/inst/binlog.go | 4 ++-- go/osagent/osagent.go | 26 ++++++++++++++------------ go/ssl/ssl.go | 2 +- 8 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 409e7ae..33bc989 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = BOX config.vm.box_download_insecure = true config.vm.box_check_update = false - config.vm.synced_folder '.', '/orchestrator-agent', type: 'rsync', rsync__auto: true + config.vm.synced_folder '.', '/orchestrator-agent' + #, type: 'rsync', rsync__auto: true config.vm.hostname = "orchestrator-agent" config.vm.network "private_network", ip: "192.168.57.211", virtualbox__inet: true diff --git a/go/agent/agent.go b/go/agent/agent.go index 4c8e46f..ac6948a 100644 --- a/go/agent/agent.go +++ b/go/agent/agent.go @@ -23,10 +23,10 @@ import ( "net/http" "time" - "github.com/outbrain/golib/log" "github.com/github/orchestrator-agent/go/config" "github.com/github/orchestrator-agent/go/osagent" "github.com/github/orchestrator-agent/go/ssl" + "github.com/outbrain/golib/log" ) var httpTimeout = time.Duration(time.Duration(config.Config.HttpTimeoutSeconds) * time.Second) diff --git a/go/app/http.go b/go/app/http.go index c6a6c9d..3849464 100644 --- a/go/app/http.go +++ b/go/app/http.go @@ -28,11 +28,11 @@ import ( nethttp "net/http" - "github.com/outbrain/golib/log" "github.com/github/orchestrator-agent/go/agent" "github.com/github/orchestrator-agent/go/config" "github.com/github/orchestrator-agent/go/http" "github.com/github/orchestrator-agent/go/ssl" + "github.com/outbrain/golib/log" ) // Http starts serving HTTP (api/web) requests diff --git a/go/cmd/orchestrator-agent/main.go b/go/cmd/orchestrator-agent/main.go index e35af58..6e9d5e3 100644 --- a/go/cmd/orchestrator-agent/main.go +++ b/go/cmd/orchestrator-agent/main.go @@ -23,10 +23,10 @@ import ( "os/signal" "syscall" - "github.com/outbrain/golib/log" "github.com/github/orchestrator-agent/go/agent" "github.com/github/orchestrator-agent/go/app" "github.com/github/orchestrator-agent/go/config" + "github.com/outbrain/golib/log" ) var AppVersion string diff --git a/go/http/api.go b/go/http/api.go index c52a9ee..b8050e6 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -28,11 +28,11 @@ import ( "strconv" "time" - "github.com/go-martini/martini" - "github.com/martini-contrib/render" "github.com/github/orchestrator-agent/go/agent" "github.com/github/orchestrator-agent/go/config" "github.com/github/orchestrator-agent/go/osagent" + "github.com/go-martini/martini" + "github.com/martini-contrib/render" ) type HttpAPI struct{} diff --git a/go/inst/binlog.go b/go/inst/binlog.go index 151c178..63f5186 100644 --- a/go/inst/binlog.go +++ b/go/inst/binlog.go @@ -135,7 +135,7 @@ func (this *BinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoor if fileNum == 0 { return result, errors.New("Log file number is zero, cannot detect previous file") } - newNumStr := fmt.Sprintf("%d", fileNum - offset) + newNumStr := fmt.Sprintf("%d", fileNum-offset) newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr tokens := strings.Split(this.LogFile, ".") @@ -154,7 +154,7 @@ func (this *BinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) result := BinlogCoordinates{LogPos: 0, Type: this.Type} fileNum, numLen := this.FileNumber() - newNumStr := fmt.Sprintf("%d", fileNum + 1) + newNumStr := fmt.Sprintf("%d", fileNum+1) newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr tokens := strings.Split(this.LogFile, ".") diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 4967bda..2058688 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -27,9 +27,9 @@ import ( "strconv" "strings" - "github.com/outbrain/golib/log" "github.com/github/orchestrator-agent/go/config" "github.com/github/orchestrator-agent/go/inst" + "github.com/outbrain/golib/log" ) const ( @@ -181,14 +181,14 @@ func MySQLBinlogBinaryContents(binlogFiles []string, startPosition int64, stopPo cmd = fmt.Sprintf("%s | head -c %d", cmd, stopPosition) } if i == 0 && startPosition != 0 { - cmd = fmt.Sprintf("%s | tail -c+%d", cmd, startPosition + 1) + cmd = fmt.Sprintf("%s | tail -c+%d", cmd, startPosition+1) } if i > 0 { // At any case, we drop out binlog header (magic + format_description) for next relay logs if headerSize, err = MySQLBinlogContentHeaderSize(binlogFile); err != nil { return "", log.Errore(err) } - cmd = fmt.Sprintf("%s | tail -c+%d", cmd, headerSize + 1) + cmd = fmt.Sprintf("%s | tail -c+%d", cmd, headerSize+1) } cmd = fmt.Sprintf("%s >> %s", cmd, tmpFile.Name()) if _, err := commandOutput(sudoCmd(cmd)); err != nil { @@ -351,15 +351,17 @@ func LogicalVolumes(volumeName string, filterPattern string) ([]LogicalVolume, e logicalVolumes := []LogicalVolume{} for _, lineTokens := range tokens { - logicalVolume := LogicalVolume{ - Name: lineTokens[1], - GroupName: lineTokens[2], - Path: lineTokens[3], - } - logicalVolume.SnapshotPercent, err = strconv.ParseFloat(lineTokens[4], 32) - logicalVolume.IsSnapshot = (err == nil) - if strings.Contains(logicalVolume.Name, filterPattern) { - logicalVolumes = append(logicalVolumes, logicalVolume) + if len(lineTokens) >= 5 { + logicalVolume := LogicalVolume{ + Name: lineTokens[1], + GroupName: lineTokens[2], + Path: lineTokens[3], + } + logicalVolume.SnapshotPercent, err = strconv.ParseFloat(lineTokens[4], 32) + logicalVolume.IsSnapshot = (err == nil) + if strings.Contains(logicalVolume.Name, filterPattern) { + logicalVolumes = append(logicalVolumes, logicalVolume) + } } } return logicalVolumes, nil diff --git a/go/ssl/ssl.go b/go/ssl/ssl.go index 87b2623..8c7f740 100644 --- a/go/ssl/ssl.go +++ b/go/ssl/ssl.go @@ -8,9 +8,9 @@ import ( nethttp "net/http" "strings" + "github.com/github/orchestrator-agent/go/config" "github.com/go-martini/martini" "github.com/outbrain/golib/log" - "github.com/github/orchestrator-agent/go/config" ) var cipherSuites = []uint16{ From 0d08d34065bf02c29e0bc169fde01544b8b1bd23 Mon Sep 17 00:00:00 2001 From: Slach Date: Sat, 17 Mar 2018 20:49:19 +0500 Subject: [PATCH 02/19] extract some LVM related commands into customizable config parameters add sourceHost parameter into /api/post-copy/ Signed-off-by: Slach --- README.md | 7 +++++++ conf/orchestrator-agent.conf.json | 2 ++ docker/entrypoint.sh | 34 +++++++++++++++++++------------ go/config/config.go | 10 +++++++++ go/http/api.go | 4 ++-- go/osagent/osagent.go | 32 +++++++++++++++-------------- 6 files changed, 59 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f7a4609..5928e86 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,11 @@ The following is a complete list of configuration parameters: * `SnapshotMountPoint` (string), a known mountpoint onto which a `mount` command will mount snapshot volumes * `ContinuousPollSeconds` (uint), internal clocking interval (default 60 seconds) * `ResubmitAgentIntervalMinutes` (uint), interval at which the agent re-submits itself to *orchestrator* daemon +* `LogicalVolumesCommand` (string), command which list logical volumes, default implementation used lvs +* `GetMountCommand` (string), command which get mount point parameters by mount point name, default implementation over cat /etc/mtab +* `UnmountCommand` (string), command which unmount current mount point, default implementation just execute umount +* `MountLVCommand` (string), command which mount selected snapshot, default implementation execute mount selected LVM snapshot +* `RemoveLVCommand` (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot * `CreateSnapshotCommand` (string), command which creates new LVM snapshot of MySQL data * `AvailableLocalSnapshotHostsCommand` (string), command which returns list of hosts in local DC on which recent snapshots are available * `AvailableSnapshotHostsCommand` (string), command which returns list of hosts in all DCs on which recent snapshots are available @@ -127,6 +132,8 @@ An example configuration file may be: "AgentsServer": "https://my.orchestrator.daemon:3001", "ContinuousPollSeconds" : 60, "ResubmitAgentIntervalMinutes": 60, + "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent", + "GetMountCommand": "grep %s /etc/mtab", "CreateSnapshotCommand": "/path/to/snapshot-command.bash", "AvailableLocalSnapshotHostsCommand": "/path/to/snapshot-local-availability-command.bash", "AvailableSnapshotHostsCommand": "/path/to/snapshot-availability-command.bash", diff --git a/conf/orchestrator-agent.conf.json b/conf/orchestrator-agent.conf.json index d90ea6b..beca568 100644 --- a/conf/orchestrator-agent.conf.json +++ b/conf/orchestrator-agent.conf.json @@ -4,6 +4,8 @@ "AgentsServerPort": ":3001", "ContinuousPollSeconds" : 60, "ResubmitAgentIntervalMinutes": 60, + "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent", + "GetMountCommand": "grep %s /etc/mtab", "CreateSnapshotCommand": "echo 'no action'", "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1", "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1", diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6ce0c35..abe6e2f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,25 +1,33 @@ #!/bin/sh if [ ! -e /etc/orchestrator-agent.conf.json ] ; then +# @TODO think about full customization of orchestrator-agent.conf.json via ENV variables +# @TODO think about mysql running not inside this docker cat < /etc/orchestrator-agent.conf.json { - "SnapshotMountPoint": "/tmp", - "AgentsServer": "http://localhost", - "AgentsServerPort": ":3001", - "ContinuousPollSeconds" : 60, - "ResubmitAgentIntervalMinutes": 60, - "CreateSnapshotCommand": "echo 'no action'", - "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1", - "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1", - "SnapshotVolumesFilter": "-my-snapshot-", - "MySQLDatadirCommand": "echo '~/tmp'", + "SnapshotMountPoint": "${SnapshotMountPoint:-/mysql-data}", + "AgentsServer": "${AgentsServer:-http://localhost}", + "AgentsServerPort": "${AgentsServerPort:-:3001}", + "ContinuousPollSeconds" : ${ContinuousPollSeconds:-60}, + "ResubmitAgentIntervalMinutes": ${ResubmitAgentIntervalMinutes:-10}, + "LogicalVolumesCommand": "${LogicalVolumesCommand:-lvs --noheading -o lv_name,vg_name,lv_path,snap_percent}", + "GetMountCommand": "${GetMountCommand:-grep %s /etc/mtab}", + "UnmountCommand": "${UnmountCommand:-umount %s}", + "MountLVCommand": "${MountLVCommand:-mount %s %s %s}", + "RemoveLVCommand": "${RemoveLVCommand:-lvremove --force %s}", + + "CreateSnapshotCommand": "${CreateSnapshotCommand:-echo \'no action\'}", + "AvailableLocalSnapshotHostsCommand": "${AvailableLocalSnapshotHostsCommand:-echo 127.0.0.1}", + "AvailableSnapshotHostsCommand": "${AvailableSnapshotHostsCommand:-printf \'localhost\n127.0.0.1\'}", + "SnapshotVolumesFilter": "${SnapshotVolumesFilter:--mysql-snapshot-}", + "MySQLDatadirCommand": "mysql -B --skip-column-names -e 'SELECT @@datadir'", "MySQLPortCommand": "echo '3306'", "MySQLDeleteDatadirContentCommand": "echo 'will not do'", "MySQLServiceStopCommand": "/etc/init.d/mysqld stop", "MySQLServiceStartCommand": "/etc/init.d/mysqld start", "MySQLServiceStatusCommand": "/etc/init.d/mysqld status", - "ReceiveSeedDataCommand": "echo 'not implemented here'", - "SendSeedDataCommand": "echo 'not implemented here'", - "PostCopyCommand": "echo 'post copy'", + "ReceiveSeedDataCommand": "${ReceiveSeedDataCommand:-echo \'not implemented here\'}", + "SendSeedDataCommand": "${SendSeedDataCommand:-echo \'not implemented here\'}", + "PostCopyCommand": "${PostCopyCommand:-echo \'post copy\'}", "HTTPPort": 3002, "HTTPAuthUser": "", "HTTPAuthPassword": "", diff --git a/go/config/config.go b/go/config/config.go index d270242..cb0130d 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -28,6 +28,11 @@ type Configuration struct { SnapshotMountPoint string // The single, agreed-upon mountpoint for logical volume snapshots ContinuousPollSeconds uint // Poll interval for continuous operation ResubmitAgentIntervalMinutes uint // Poll interval for resubmitting this agent on orchestrator agents API + LogicalVolumesCommand string // Command which list logical volumes, default implementation used lvs + GetMountCommand string // Command which get mount point parameters by mount point name, default implementation over cat %s/etc/mtab + UnmountCommand string // Command which unmount current mount point, default implementation just execute `umount %s` + MountLVCommand string // Command which mount selected snapshot into mount point + RemoveLVCommand string // Command which remove selected snapshot from disk CreateSnapshotCommand string // Command which creates a snapshot logical volume. It's a "do it yourself" implementation AvailableLocalSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in local datacenter AvailableSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in any datacenter @@ -72,6 +77,11 @@ func NewConfiguration() *Configuration { ContinuousPollSeconds: 60, ResubmitAgentIntervalMinutes: 60, CreateSnapshotCommand: "", + LogicalVolumesCommand: "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent", + GetMountCommand: "grep %s /etc/mtab", + UnmountCommand: "umount %s", + MountLVCommand: "mount %s %s %s", + RemoveLVCommand: "lvremove --force %s", AvailableLocalSnapshotHostsCommand: "", AvailableSnapshotHostsCommand: "", SnapshotVolumesFilter: "", diff --git a/go/http/api.go b/go/http/api.go index b8050e6..4984df9 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -363,7 +363,7 @@ func (this *HttpAPI) PostCopy(params martini.Params, r render.Render, req *http. if err := this.validateToken(r, req); err != nil { return } - err := osagent.PostCopy() + err := osagent.PostCopy(params["sourceHost"]) if err != nil { r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()}) return @@ -616,7 +616,7 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) { m.Get("/api/mysql-start", this.MySQLStart) m.Get("/api/delete-mysql-datadir", this.DeleteMySQLDataDir) m.Get("/api/mysql-datadir-available-space", this.GetMySQLDataDirAvailableDiskSpace) - m.Get("/api/post-copy", this.PostCopy) + m.Get("/api/post-copy/:sourceHost", this.PostCopy) m.Get("/api/receive-mysql-seed-data/:seedId", this.ReceiveMySQLSeedData) m.Get("/api/send-mysql-seed-data/:targetHost/:seedId", this.SendMySQLSeedData) m.Get("/api/abort-seed/:seedId", this.AbortSeed) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 2058688..993de5f 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -343,7 +343,7 @@ func Hostname() (string, error) { } func LogicalVolumes(volumeName string, filterPattern string) ([]LogicalVolume, error) { - output, err := commandOutput(sudoCmd(fmt.Sprintf("lvs --noheading -o lv_name,vg_name,lv_path,snap_percent %s", volumeName))) + output, err := commandOutput(sudoCmd(config.Config.LogicalVolumesCommand + " " + volumeName)) tokens, err := outputTokens(`[ \t]+`, output, err) if err != nil { return nil, err @@ -392,7 +392,7 @@ func GetMount(mountPoint string) (Mount, error) { IsMounted: false, } - output, err := commandOutput(fmt.Sprintf("grep %s /etc/mtab", mountPoint)) + output, err := commandOutput(sudoCmd(fmt.Sprintf(config.Config.GetMountCommand, mountPoint))) tokens, err := outputTokens(`[ \t]+`, output, err) if err != nil { // when grep does not find rows, it returns an error. So this is actually OK @@ -400,14 +400,16 @@ func GetMount(mountPoint string) (Mount, error) { } for _, lineTokens := range tokens { - mount.IsMounted = true - mount.Device = lineTokens[0] - mount.Path = lineTokens[1] - mount.FileSystem = lineTokens[2] - mount.LVPath, _ = GetLogicalVolumePath(mount.Device) - mount.DiskUsage, _ = DiskUsage(mountPoint) - mount.MySQLDataPath, _ = HeuristicMySQLDataPath(mountPoint) - mount.MySQLDiskUsage, _ = DiskUsage(mount.MySQLDataPath) + if len(lineTokens) >= 3 { + mount.IsMounted = true + mount.Device = lineTokens[0] + mount.Path = lineTokens[1] + mount.FileSystem = lineTokens[2] + mount.LVPath, _ = GetLogicalVolumePath(mount.Device) + mount.DiskUsage, _ = DiskUsage(mountPoint) + mount.MySQLDataPath, _ = HeuristicMySQLDataPath(mountPoint) + mount.MySQLDiskUsage, _ = DiskUsage(mount.MySQLDataPath) + } } return mount, nil } @@ -429,7 +431,7 @@ func MountLV(mountPoint string, volumeName string) (Mount, error) { if fsType == "xfs" { mountOptions = "-o nouuid" } - _, err = commandOutput(sudoCmd(fmt.Sprintf("mount %s %s %s", mountOptions, volumeName, mountPoint))) + _, err = commandOutput(sudoCmd(fmt.Sprintf(config.Config.MountLVCommand, mountOptions, volumeName, mountPoint))) if err != nil { return mount, err } @@ -438,7 +440,7 @@ func MountLV(mountPoint string, volumeName string) (Mount, error) { } func RemoveLV(volumeName string) error { - _, err := commandOutput(sudoCmd(fmt.Sprintf("lvremove --force %s", volumeName))) + _, err := commandOutput(sudoCmd(fmt.Sprintf(config.Config.RemoveLVCommand, volumeName))) return err } @@ -452,7 +454,7 @@ func Unmount(mountPoint string) (Mount, error) { Path: mountPoint, IsMounted: false, } - _, err := commandOutput(sudoCmd(fmt.Sprintf("umount %s", mountPoint))) + _, err := commandOutput(sudoCmd(fmt.Sprintf(config.Config.UnmountCommand, mountPoint))) if err != nil { return mount, err } @@ -520,8 +522,8 @@ func GetMySQLDataDirAvailableDiskSpace() (int64, error) { } // PostCopy executes a post-copy command -- after LVM copy is done, before service starts. Some cleanup may go here. -func PostCopy() error { - _, err := commandOutput(config.Config.PostCopyCommand) +func PostCopy(sourceHost string) error { + _, err := commandOutput(sudoCmd(config.Config.PostCopyCommand + " " + sourceHost)) return err } From 56e4362e00e403cf0fc9fc77f0e5026b8325bd3f Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 18 Mar 2018 16:25:09 +0500 Subject: [PATCH 03/19] add customizable MySQLTailErrorLogCommand Signed-off-by: Slach --- README.md | 3 ++- docker/entrypoint.sh | 1 + go/config/config.go | 4 +++- go/osagent/osagent.go | 7 ++++++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5928e86..9646fa9 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,8 @@ The following is a complete list of configuration parameters: * `GetMountCommand` (string), command which get mount point parameters by mount point name, default implementation over cat /etc/mtab * `UnmountCommand` (string), command which unmount current mount point, default implementation just execute umount * `MountLVCommand` (string), command which mount selected snapshot, default implementation execute mount selected LVM snapshot -* `RemoveLVCommand` (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot +* `RemoveLVCommand` (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot +* `MySQLTailErrorLogCommand` (string), command which return last 20 lines from MySQL @@log_error file * `CreateSnapshotCommand` (string), command which creates new LVM snapshot of MySQL data * `AvailableLocalSnapshotHostsCommand` (string), command which returns list of hosts in local DC on which recent snapshots are available * `AvailableSnapshotHostsCommand` (string), command which returns list of hosts in all DCs on which recent snapshots are available diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index abe6e2f..245d8e2 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -14,6 +14,7 @@ cat < /etc/orchestrator-agent.conf.json "UnmountCommand": "${UnmountCommand:-umount %s}", "MountLVCommand": "${MountLVCommand:-mount %s %s %s}", "RemoveLVCommand": "${RemoveLVCommand:-lvremove --force %s}", + "MySQLTailErrorLogCommand": "${MySQLTailErrorLogCommand:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}", "CreateSnapshotCommand": "${CreateSnapshotCommand:-echo \'no action\'}", "AvailableLocalSnapshotHostsCommand": "${AvailableLocalSnapshotHostsCommand:-echo 127.0.0.1}", diff --git a/go/config/config.go b/go/config/config.go index cb0130d..db5dbc0 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -33,6 +33,7 @@ type Configuration struct { UnmountCommand string // Command which unmount current mount point, default implementation just execute `umount %s` MountLVCommand string // Command which mount selected snapshot into mount point RemoveLVCommand string // Command which remove selected snapshot from disk + MySQLTailErrorLogCommand string // Command which return last 20 lines from @@log_error file CreateSnapshotCommand string // Command which creates a snapshot logical volume. It's a "do it yourself" implementation AvailableLocalSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in local datacenter AvailableSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in any datacenter @@ -79,9 +80,10 @@ func NewConfiguration() *Configuration { CreateSnapshotCommand: "", LogicalVolumesCommand: "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent", GetMountCommand: "grep %s /etc/mtab", - UnmountCommand: "umount %s", + UnmountCommand: "umount %s", MountLVCommand: "mount %s %s %s", RemoveLVCommand: "lvremove --force %s", + MySQLTailErrorLogCommand: `tail -n 20 $(egrep "log[-_]error" /etc/my.cnf | cut -d "=" -f 2)`, AvailableLocalSnapshotHostsCommand: "", AvailableSnapshotHostsCommand: "", SnapshotVolumesFilter: "", diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 993de5f..f01a713 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -545,7 +545,12 @@ func HeuristicMySQLDataPath(mountPoint string) (string, error) { if datadir == "" { return "", errors.New("Cannot detect MySQL datadir") } - datadir = re.FindStringSubmatch(datadir)[1] + matches := re.FindStringSubmatch(datadir) + if len(matches) > 1 { + datadir = re.FindStringSubmatch(datadir)[1] + } else { + return "", errors.New("Cannot detect MySQL datadir") + } } } From caa8db669283d6a30bd47e898661758e86cd6b65 Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 18 Mar 2018 17:13:48 +0500 Subject: [PATCH 04/19] add backward compatibility for old orchestrator post-copy call Signed-off-by: Slach --- go/http/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/http/api.go b/go/http/api.go index 4984df9..70deaf6 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -616,6 +616,7 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) { m.Get("/api/mysql-start", this.MySQLStart) m.Get("/api/delete-mysql-datadir", this.DeleteMySQLDataDir) m.Get("/api/mysql-datadir-available-space", this.GetMySQLDataDirAvailableDiskSpace) + m.Get("/api/post-copy", this.PostCopy) m.Get("/api/post-copy/:sourceHost", this.PostCopy) m.Get("/api/receive-mysql-seed-data/:seedId", this.ReceiveMySQLSeedData) m.Get("/api/send-mysql-seed-data/:targetHost/:seedId", this.SendMySQLSeedData) From ac7cf938cbdc7be97c6ae2961cf605cd49b3a32c Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 18 Mar 2018 19:05:31 +0500 Subject: [PATCH 05/19] add customizable GetLogicalVolumeFSTypeCommand Signed-off-by: Slach --- README.md | 3 ++- docker/entrypoint.sh | 2 +- go/config/config.go | 2 ++ go/osagent/osagent.go | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9646fa9..793ae24 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,8 @@ The following is a complete list of configuration parameters: * `UnmountCommand` (string), command which unmount current mount point, default implementation just execute umount * `MountLVCommand` (string), command which mount selected snapshot, default implementation execute mount selected LVM snapshot * `RemoveLVCommand` (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot -* `MySQLTailErrorLogCommand` (string), command which return last 20 lines from MySQL @@log_error file +* `MySQLTailErrorLogCommand` (string), command which return last 20 lines from MySQL @@log_error file +* `GetLogicalVolumeFSTypeCommand` (string), command which return logical volume filesystem type * `CreateSnapshotCommand` (string), command which creates new LVM snapshot of MySQL data * `AvailableLocalSnapshotHostsCommand` (string), command which returns list of hosts in local DC on which recent snapshots are available * `AvailableSnapshotHostsCommand` (string), command which returns list of hosts in all DCs on which recent snapshots are available diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 245d8e2..634e53a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,7 +15,7 @@ cat < /etc/orchestrator-agent.conf.json "MountLVCommand": "${MountLVCommand:-mount %s %s %s}", "RemoveLVCommand": "${RemoveLVCommand:-lvremove --force %s}", "MySQLTailErrorLogCommand": "${MySQLTailErrorLogCommand:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}", - + "GetLogicalVolumeFSTypeCommand": "${GetLogicalVolumeFSTypeCommand:-blkid %s}", "CreateSnapshotCommand": "${CreateSnapshotCommand:-echo \'no action\'}", "AvailableLocalSnapshotHostsCommand": "${AvailableLocalSnapshotHostsCommand:-echo 127.0.0.1}", "AvailableSnapshotHostsCommand": "${AvailableSnapshotHostsCommand:-printf \'localhost\n127.0.0.1\'}", diff --git a/go/config/config.go b/go/config/config.go index db5dbc0..62bd980 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -34,6 +34,7 @@ type Configuration struct { MountLVCommand string // Command which mount selected snapshot into mount point RemoveLVCommand string // Command which remove selected snapshot from disk MySQLTailErrorLogCommand string // Command which return last 20 lines from @@log_error file + GetLogicalVolumeFSTypeCommand string // Command which return logical volume filesystem type CreateSnapshotCommand string // Command which creates a snapshot logical volume. It's a "do it yourself" implementation AvailableLocalSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in local datacenter AvailableSnapshotHostsCommand string // Command which returns list of hosts (one host per line) with available snapshots in any datacenter @@ -84,6 +85,7 @@ func NewConfiguration() *Configuration { MountLVCommand: "mount %s %s %s", RemoveLVCommand: "lvremove --force %s", MySQLTailErrorLogCommand: `tail -n 20 $(egrep "log[-_]error" /etc/my.cnf | cut -d "=" -f 2)`, + GetLogicalVolumeFSTypeCommand: "blkid %s", AvailableLocalSnapshotHostsCommand: "", AvailableSnapshotHostsCommand: "", SnapshotVolumesFilter: "", diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index f01a713..b58978d 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -375,7 +375,7 @@ func GetLogicalVolumePath(volumeName string) (string, error) { } func GetLogicalVolumeFSType(volumeName string) (string, error) { - command := fmt.Sprintf("blkid %s", volumeName) + command := fmt.Sprintf(config.Config.GetLogicalVolumeFSTypeCommand, volumeName) output, err := commandOutput(sudoCmd(command)) lines, err := outputLines(output, err) re := regexp.MustCompile(`TYPE="(.*?)"`) From 0d74d97671f0ec53cc6d996a790f10458e9cbe26 Mon Sep 17 00:00:00 2001 From: Slach Date: Mon, 19 Mar 2018 08:34:53 +0500 Subject: [PATCH 06/19] add include /etc/ files for iject DAEMONOPTS or other settings before daemon running Signed-off-by: Slach --- etc/init.d/orchestrator-agent.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/init.d/orchestrator-agent.bash b/etc/init.d/orchestrator-agent.bash index 01620fe..c97e182 100644 --- a/etc/init.d/orchestrator-agent.bash +++ b/etc/init.d/orchestrator-agent.bash @@ -26,6 +26,11 @@ DESC="orchestrator-agent: MySQL management agent" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME +# This files can be used to inject pre-service execution +# scripts, such as exporting variables or whatever. It's yours! +[ -f /etc/default/orchestrator-agent ] && . /etc/default/orchestrator-agent +[ -f /etc/orchestrator-agent_profile ] && . /etc/orchestrator-agent_profile + case "$1" in start) printf "%-50s" "Starting $NAME..." From 3055395481740b8572e586c39951ede200fefb01 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 10:26:04 +0500 Subject: [PATCH 07/19] add config reload by SIGHUP and commandOutput to DEBUG log Signed-off-by: Slach --- go/cmd/orchestrator-agent/main.go | 17 ++++++++++++++--- go/config/config.go | 7 +++++++ go/osagent/osagent.go | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/go/cmd/orchestrator-agent/main.go b/go/cmd/orchestrator-agent/main.go index 6e9d5e3..57fca7f 100644 --- a/go/cmd/orchestrator-agent/main.go +++ b/go/cmd/orchestrator-agent/main.go @@ -33,11 +33,22 @@ var AppVersion string func acceptSignal() { c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGHUP) // Block until a signal is received. - sig := <-c - log.Fatalf("Got signal: %+v", sig) + signal.Notify(c, syscall.SIGHUP, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGTERM) + go func() { + for sig := range c { + switch sig { + case syscall.SIGHUP: + log.Infof("Received SIGHUP. Reloading configuration") + config.Reload() + case syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT: + log.Infof("Received %s. Shutting down orchestrator-agent", sig.String()) + // probably should poke other go routines to stop cleanly here ... + os.Exit(0) + } + } + }() } // main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces. diff --git a/go/config/config.go b/go/config/config.go index 62bd980..d2da786 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -72,6 +72,7 @@ type Configuration struct { } var Config = NewConfiguration() +var configFileNames []string func NewConfiguration() *Configuration { return &Configuration{ @@ -144,9 +145,15 @@ func Read(file_names ...string) *Configuration { for _, file_name := range file_names { read(file_name) } + configFileNames = file_names return Config } +// Reload +func Reload() *Configuration { + return Read(configFileNames...) +} + // ForceRead reads configuration from given file name or bails out if it fails func ForceRead(file_name string) *Configuration { _, err := read(file_name) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index b58978d..6f01c0a 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -296,7 +296,7 @@ func commandOutput(commandText string) ([]byte, error) { if err != nil { return nil, log.Errore(err) } - + log.Debugf("commandOutput: %s", outputBytes) return outputBytes, nil } From 5d1b823251576e7d9e257a24fcacf13104a594f4 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 11:29:43 +0500 Subject: [PATCH 08/19] fix MySQLTailErrorLogCommand Signed-off-by: Slach --- go/osagent/osagent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 6f01c0a..894eedd 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -568,7 +568,7 @@ func AvailableSnapshots(requireLocal bool) ([]string, error) { } func MySQLErrorLogTail() ([]string, error) { - output, err := commandOutput(sudoCmd(`tail -n 20 $(egrep "log[-_]error" /etc/my.cnf | cut -d "=" -f 2)`)) + output, err := commandOutput(sudoCmd(config.Config.MySQLTailErrorLogCommand)) tail, err := outputLines(output, err) return tail, err } From fb2e214ace5a7035ee86b5b071b20c6d10ece03b Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 11:49:33 +0500 Subject: [PATCH 09/19] add reload and kill TERM without HUP ;) Signed-off-by: Slach --- etc/init.d/orchestrator-agent.bash | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/etc/init.d/orchestrator-agent.bash b/etc/init.d/orchestrator-agent.bash index c97e182..b0dfc7a 100644 --- a/etc/init.d/orchestrator-agent.bash +++ b/etc/init.d/orchestrator-agent.bash @@ -65,21 +65,39 @@ case "$1" in PID=$(cat $PIDFILE) cd $DAEMON_PATH if [ -f $PIDFILE ]; then - kill -HUP $PID - printf "%s\n" "Ok" + kill -TERM $PID rm -f $PIDFILE + # Wait for orchestrator to stop otherwise restart may fail. + # (The newly restarted process may be unable to bind to the + # currently bound socket.) + while ps -p $PID >/dev/null 2>&1; do + printf "." + sleep 1 + done + printf "\n" + printf "Ok\n" else printf "%s\n" "pidfile not found" exit 1 fi ;; - restart) $0 stop $0 start ;; - + reload) + printf "%-50s" "Reloading $NAME" + PID=$(cat $PIDFILE) + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + kill -HUP $PID + printf "%s\n" "Ok" + else + printf "%s\n" "pidfile not found" + exit 1 + fi + ;; *) - echo "Usage: $0 {status|start|stop|restart}" + echo "Usage: $0 {status|start|stop|restart|reload}" exit 1 esac From 16f920c1195e592f3c455b543af4d7e2e8ac58f1 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 12:59:06 +0500 Subject: [PATCH 10/19] extract const SeedTransferPort to configurable parameter Signed-off-by: Slach --- README.md | 1 + go/config/config.go | 2 ++ go/osagent/osagent.go | 33 +++++++++++++++++++++++---------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 793ae24..2597a70 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ The following is a complete list of configuration parameters: * `PostCopyCommand` (string), command to be executed after the seed is complete (cleanup) * `AgentsServer` (string), **Required** URL of your **orchestrator** daemon, You must add the port the orchestrator server expects to talk to agents to (see below, e.g. `https://my.orchestrator.daemon:3001`) * `HTTPPort` (uint), Port to listen on +* `SeedTransferPort` (uint), TCP Port to seed data transfer * `HTTPAuthUser` (string), Basic auth user (default empty, meaning no auth) * `HTTPAuthPassword` (string), Basic auth password * `UseSSL` (bool), If `true` then serving via `https` protocol diff --git a/go/config/config.go b/go/config/config.go index d2da786..fc90821 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -52,6 +52,7 @@ type Configuration struct { AgentsServer string // HTTP address of the orchestrator agents server AgentsServerPort string // HTTP port of the orchestrator agents server HTTPPort uint // HTTP port on which this service listens + SeedTransferPort uint // TCP port for data seed transfer HTTPAuthUser string // Username for HTTP Basic authentication (blank disables authentication) HTTPAuthPassword string // Password for HTTP Basic authentication UseSSL bool // If true, service will serve HTTPS only @@ -103,6 +104,7 @@ func NewConfiguration() *Configuration { AgentsServer: "", AgentsServerPort: "", HTTPPort: 3002, + SeedTransferPort: 21234, HTTPAuthUser: "", HTTPAuthPassword: "", UseSSL: false, diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 894eedd..fcba449 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -32,9 +32,6 @@ import ( "github.com/outbrain/golib/log" ) -const ( - SeedTransferPort = 21234 -) var activeCommands = make(map[string]*exec.Cmd) @@ -259,11 +256,6 @@ func init() { os.Setenv("PATH", fmt.Sprintf("%s:/usr/sbin:/usr/bin:/sbin:/bin", osPath)) } -func commandSplit(commandText string) (string, []string) { - tokens := regexp.MustCompile(`[ ]+`).Split(strings.TrimSpace(commandText), -1) - return tokens[0], tokens[1:] -} - func execCmd(commandText string) (*exec.Cmd, string, error) { commandBytes := []byte(commandText) tmpFile, err := ioutil.TempFile("", "orchestrator-agent-cmd-") @@ -594,9 +586,17 @@ func ReceiveMySQLSeedData(seedId string) error { if err != nil { return log.Errore(err) } + var receiveCmd string + if strings.Contains(config.Config.ReceiveSeedDataCommand,"%s") { + receiveCmd = fmt.Sprintf(config.Config.ReceiveSeedDataCommand, directory, config.Config.SeedTransferPort) + } else { + //old behavior backwards + receiveCmd = fmt.Sprintf("%s %s %d", config.Config.ReceiveSeedDataCommand, directory, config.Config.SeedTransferPort) + } + err = commandRun( - fmt.Sprintf("%s %s %d", config.Config.ReceiveSeedDataCommand, directory, SeedTransferPort), + receiveCmd, func(cmd *exec.Cmd) { activeCommands[seedId] = cmd log.Debug("ReceiveMySQLSeedData command completed") @@ -612,7 +612,20 @@ func SendMySQLSeedData(targetHostname string, directory string, seedId string) e if directory == "" { return log.Error("Empty directory in SendMySQLSeedData") } - err := commandRun(fmt.Sprintf("%s %s %s %d", config.Config.SendSeedDataCommand, directory, targetHostname, SeedTransferPort), + var sendCmd string + if strings.Contains(config.Config.SendSeedDataCommand,"%s") { + sendCmd = fmt.Sprintf( + config.Config.SendSeedDataCommand, + directory, targetHostname, config.Config.SeedTransferPort, + ) + } else { + sendCmd = fmt.Sprintf( + "%s %s %s %d", + config.Config.SendSeedDataCommand, directory, targetHostname, config.Config.SeedTransferPort, + ) + } + err := commandRun( + sendCmd, func(cmd *exec.Cmd) { activeCommands[seedId] = cmd log.Debug("SendMySQLSeedData command completed") From 0772999cffe058613d9528d8606cbc28d868ea22 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 13:35:43 +0500 Subject: [PATCH 11/19] add sudoCmd to send \ receive seed data commands Signed-off-by: Slach --- go/osagent/osagent.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index fcba449..166e538 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -32,7 +32,6 @@ import ( "github.com/outbrain/golib/log" ) - var activeCommands = make(map[string]*exec.Cmd) // LogicalVolume describes an LVM volume @@ -587,16 +586,15 @@ func ReceiveMySQLSeedData(seedId string) error { return log.Errore(err) } var receiveCmd string - if strings.Contains(config.Config.ReceiveSeedDataCommand,"%s") { + if strings.Contains(config.Config.ReceiveSeedDataCommand, "%s") { receiveCmd = fmt.Sprintf(config.Config.ReceiveSeedDataCommand, directory, config.Config.SeedTransferPort) } else { //old behavior backwards receiveCmd = fmt.Sprintf("%s %s %d", config.Config.ReceiveSeedDataCommand, directory, config.Config.SeedTransferPort) } - err = commandRun( - receiveCmd, + sudoCmd(receiveCmd), func(cmd *exec.Cmd) { activeCommands[seedId] = cmd log.Debug("ReceiveMySQLSeedData command completed") @@ -613,7 +611,7 @@ func SendMySQLSeedData(targetHostname string, directory string, seedId string) e return log.Error("Empty directory in SendMySQLSeedData") } var sendCmd string - if strings.Contains(config.Config.SendSeedDataCommand,"%s") { + if strings.Contains(config.Config.SendSeedDataCommand, "%s") { sendCmd = fmt.Sprintf( config.Config.SendSeedDataCommand, directory, targetHostname, config.Config.SeedTransferPort, @@ -625,7 +623,7 @@ func SendMySQLSeedData(targetHostname string, directory string, seedId string) e ) } err := commandRun( - sendCmd, + sudoCmd(sendCmd), func(cmd *exec.Cmd) { activeCommands[seedId] = cmd log.Debug("SendMySQLSeedData command completed") From b2d10eddd0339c39e127ca0682cb6bae4f72bb09 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 17:07:26 +0500 Subject: [PATCH 12/19] add backward compatibility to postCopy Signed-off-by: Slach --- go/osagent/osagent.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index 166e538..f6719d0 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -514,7 +514,13 @@ func GetMySQLDataDirAvailableDiskSpace() (int64, error) { // PostCopy executes a post-copy command -- after LVM copy is done, before service starts. Some cleanup may go here. func PostCopy(sourceHost string) error { - _, err := commandOutput(sudoCmd(config.Config.PostCopyCommand + " " + sourceHost)) + var postCopyCmd string + if strings.Contains(config.Config.PostCopyCommand,"%s") { + postCopyCmd = fmt.Sprintf(config.Config.PostCopyCommand, sourceHost) + } else { + postCopyCmd = config.Config.PostCopyCommand+" "+sourceHost + } + _, err := commandOutput(sudoCmd(postCopyCmd)) return err } From 46468cd55c02954e18217b8314f435198cb45310 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 18:05:24 +0500 Subject: [PATCH 13/19] add backward compatibility to postCopy Signed-off-by: Slach --- go/osagent/osagent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/osagent/osagent.go b/go/osagent/osagent.go index f6719d0..14f2527 100644 --- a/go/osagent/osagent.go +++ b/go/osagent/osagent.go @@ -515,10 +515,10 @@ func GetMySQLDataDirAvailableDiskSpace() (int64, error) { // PostCopy executes a post-copy command -- after LVM copy is done, before service starts. Some cleanup may go here. func PostCopy(sourceHost string) error { var postCopyCmd string - if strings.Contains(config.Config.PostCopyCommand,"%s") { + if strings.Contains(config.Config.PostCopyCommand, "%s") { postCopyCmd = fmt.Sprintf(config.Config.PostCopyCommand, sourceHost) } else { - postCopyCmd = config.Config.PostCopyCommand+" "+sourceHost + postCopyCmd = config.Config.PostCopyCommand + " " + sourceHost } _, err := commandOutput(sudoCmd(postCopyCmd)) return err From 9450b7da2bbd2274a5774478e72057d4ca7cb2ff Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 19:54:06 +0500 Subject: [PATCH 14/19] add /etc/profile.d/orchestrator-agent usage Signed-off-by: Slach --- etc/init.d/orchestrator-agent.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/init.d/orchestrator-agent.bash b/etc/init.d/orchestrator-agent.bash index b0dfc7a..c40ea3e 100644 --- a/etc/init.d/orchestrator-agent.bash +++ b/etc/init.d/orchestrator-agent.bash @@ -30,6 +30,7 @@ SCRIPTNAME=/etc/init.d/$NAME # scripts, such as exporting variables or whatever. It's yours! [ -f /etc/default/orchestrator-agent ] && . /etc/default/orchestrator-agent [ -f /etc/orchestrator-agent_profile ] && . /etc/orchestrator-agent_profile +[ -f /etc/profile.d/orchestrator-agent ] && . /etc/profile.d/orchestrator-agent case "$1" in start) From 66cb2a2b1a10f48ac54d009be9104aae27330dfa Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Mar 2018 14:49:52 +0500 Subject: [PATCH 15/19] fixed Docker entrypoint after code review and also fixed some mistypes in init.d script Signed-off-by: Slach --- .gitignore | 2 + docker/entrypoint.sh | 70 +++++++++++++++--------------- etc/init.d/orchestrator-agent.bash | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index bda99c7..7eb68a7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .vendor/ .gopath/ .idea/ +*.deb +*.pcap *.log diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 634e53a..aadbb87 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,46 +1,44 @@ #!/bin/sh if [ ! -e /etc/orchestrator-agent.conf.json ] ; then -# @TODO think about full customization of orchestrator-agent.conf.json via ENV variables -# @TODO think about mysql running not inside this docker cat < /etc/orchestrator-agent.conf.json { - "SnapshotMountPoint": "${SnapshotMountPoint:-/mysql-data}", - "AgentsServer": "${AgentsServer:-http://localhost}", - "AgentsServerPort": "${AgentsServerPort:-:3001}", - "ContinuousPollSeconds" : ${ContinuousPollSeconds:-60}, - "ResubmitAgentIntervalMinutes": ${ResubmitAgentIntervalMinutes:-10}, - "LogicalVolumesCommand": "${LogicalVolumesCommand:-lvs --noheading -o lv_name,vg_name,lv_path,snap_percent}", - "GetMountCommand": "${GetMountCommand:-grep %s /etc/mtab}", - "UnmountCommand": "${UnmountCommand:-umount %s}", - "MountLVCommand": "${MountLVCommand:-mount %s %s %s}", - "RemoveLVCommand": "${RemoveLVCommand:-lvremove --force %s}", - "MySQLTailErrorLogCommand": "${MySQLTailErrorLogCommand:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}", - "GetLogicalVolumeFSTypeCommand": "${GetLogicalVolumeFSTypeCommand:-blkid %s}", - "CreateSnapshotCommand": "${CreateSnapshotCommand:-echo \'no action\'}", - "AvailableLocalSnapshotHostsCommand": "${AvailableLocalSnapshotHostsCommand:-echo 127.0.0.1}", - "AvailableSnapshotHostsCommand": "${AvailableSnapshotHostsCommand:-printf \'localhost\n127.0.0.1\'}", - "SnapshotVolumesFilter": "${SnapshotVolumesFilter:--mysql-snapshot-}", - "MySQLDatadirCommand": "mysql -B --skip-column-names -e 'SELECT @@datadir'", - "MySQLPortCommand": "echo '3306'", - "MySQLDeleteDatadirContentCommand": "echo 'will not do'", - "MySQLServiceStopCommand": "/etc/init.d/mysqld stop", - "MySQLServiceStartCommand": "/etc/init.d/mysqld start", - "MySQLServiceStatusCommand": "/etc/init.d/mysqld status", - "ReceiveSeedDataCommand": "${ReceiveSeedDataCommand:-echo \'not implemented here\'}", - "SendSeedDataCommand": "${SendSeedDataCommand:-echo \'not implemented here\'}", - "PostCopyCommand": "${PostCopyCommand:-echo \'post copy\'}", - "HTTPPort": 3002, - "HTTPAuthUser": "", - "HTTPAuthPassword": "", - "UseSSL": false, - "SSLCertFile": "", - "SSLPrivateKeyFile": "", - "HttpTimeoutSeconds": 10, - "ExecWithSudo": false, + "SnapshotMountPoint": "${SNAPSHOT_MOUNT_POINT:-/mysql-data}", + "AgentsServer": "${AGENTS_SERVER:-http://localhost}", + "AgentsServerPort": "${AGENTS_SERVER_PORT:-:3001}", + "ContinuousPollSeconds" : ${CONTINUOUS_POLL_SECONDS:-60}, + "ResubmitAgentIntervalMinutes": ${RESUBMIT_AGENTINTERVAL_MINUTES:-10}, + "LogicalVolumesCommand": "${LOGICAL_VOLUMES_COMMAND:-lvs --noheading -o lv_name,vg_name,lv_path,snap_percent}", + "GetMountCommand": "${GET_MOUNT_COMMAND:-grep %s /etc/mtab}", + "UnmountCommand": "${UNMOUNT_COMMAND:-umount %s}", + "MountLVCommand": "${MOUNT_LV_COMMAND:-mount %s %s %s}", + "RemoveLVCommand": "${REMOVE_LV_COMMAND:-lvremove --force %s}", + "MySQLTailErrorLogCommand": "${MYSQL_TAIL_ERROR_LOG_COMMAND:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}", + "GetLogicalVolumeFSTypeCommand": "${GET_LOGICAL_VOLUME_FS_TYPE_COMMAND:-blkid %s}", + "CreateSnapshotCommand": "${CREATE_SNAPSHOT_COMMAND:-echo \'no action\'}", + "AvailableLocalSnapshotHostsCommand": "${AVAILABLE_LOCAL_SNAPSHOT_HOSTS_COMMAND:-echo 127.0.0.1}", + "AvailableSnapshotHostsCommand": "${AVAILABLE_SNAPSHOT_HOSTS_COMMAND:-printf \'localhost\n127.0.0.1\'}", + "SnapshotVolumesFilter": "${SNAPSHOT_VOLUMES_FILTER:--mysql-snapshot-}", + "MySQLDatadirCommand": "${MYSQL_DATADIR_COMMAND:-mysql -B --skip-column-names -e 'SELECT @@datadir'}", + "MySQLPortCommand": "${MYSQL_PORT_COMMAND:-echo '3306'}", + "MySQLDeleteDatadirContentCommand": "${MYSQL_DELETE_DATADIR_CONTENT_COMMAND:-echo 'will not do'}", + "MySQLServiceStopCommand": "${MYSQL_SERVICE_STOP_COMMAND:-/etc/init.d/mysqld stop}", + "MySQLServiceStartCommand": "${MYSQL_SERVICE_START_COMMAND:-/etc/init.d/mysqld start}", + "MySQLServiceStatusCommand": "${MYSQL_SERVICE_STATUS_COMMAND:-/etc/init.d/mysqld status}", + "ReceiveSeedDataCommand": "${RECEIVE_SEED_DATA_COMMAND:-echo \'not implemented here\'}", + "SendSeedDataCommand": "${SEND_SEED_DATA_COMMAND:-echo \'not implemented here\'}", + "PostCopyCommand": "${POST_COPY_COMMAND:-echo \'post copy\'}", + "HTTPPort": ${HTTP_PORT:-3002}, + "HTTPAuthUser": "${HTTP_AUTH_USER}", + "HTTPAuthPassword": "${HTTP_AUTH_PASSWORD}", + "UseSSL": ${USE_SSL:-false}, + "SSLCertFile": "${SSL_CERT_FILE}", + "SSLPrivateKeyFile": "${SSL_PRIVATE_KEY_FILE}", + "HttpTimeoutSeconds": ${HTTP_TIMEOUT_SECONDS:-10}, + "ExecWithSudo": ${EXEC_WITH_SUDO:-false}, "CustomCommands": { "true": "/bin/true" }, - "TokenHintFile": "" + "TokenHintFile": "${TOKEN_HINT_FILE}" } EOF fi diff --git a/etc/init.d/orchestrator-agent.bash b/etc/init.d/orchestrator-agent.bash index c40ea3e..c22e257 100644 --- a/etc/init.d/orchestrator-agent.bash +++ b/etc/init.d/orchestrator-agent.bash @@ -68,7 +68,7 @@ case "$1" in if [ -f $PIDFILE ]; then kill -TERM $PID rm -f $PIDFILE - # Wait for orchestrator to stop otherwise restart may fail. + # Wait for orchestrator-agent to stop otherwise restart may fail. # (The newly restarted process may be unable to bind to the # currently bound socket.) while ps -p $PID >/dev/null 2>&1; do From 5c0dddb635723e236d2b43b32322eb26bd238e8d Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Mar 2018 15:33:18 +0500 Subject: [PATCH 16/19] return rsync folder mounting mode back Signed-off-by: Slach --- Vagrantfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 33bc989..409e7ae 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,8 +15,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = BOX config.vm.box_download_insecure = true config.vm.box_check_update = false - config.vm.synced_folder '.', '/orchestrator-agent' - #, type: 'rsync', rsync__auto: true + config.vm.synced_folder '.', '/orchestrator-agent', type: 'rsync', rsync__auto: true config.vm.hostname = "orchestrator-agent" config.vm.network "private_network", ip: "192.168.57.211", virtualbox__inet: true From 8febb275937fcce1754e45b02c3a92a19d39ce02 Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Mar 2018 16:43:32 +0500 Subject: [PATCH 17/19] return blocking behavior Signed-off-by: Slach --- go/cmd/orchestrator-agent/main.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/go/cmd/orchestrator-agent/main.go b/go/cmd/orchestrator-agent/main.go index 57fca7f..b76ff39 100644 --- a/go/cmd/orchestrator-agent/main.go +++ b/go/cmd/orchestrator-agent/main.go @@ -36,19 +36,17 @@ func acceptSignal() { // Block until a signal is received. signal.Notify(c, syscall.SIGHUP, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGTERM) - go func() { - for sig := range c { - switch sig { - case syscall.SIGHUP: - log.Infof("Received SIGHUP. Reloading configuration") - config.Reload() - case syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT: - log.Infof("Received %s. Shutting down orchestrator-agent", sig.String()) - // probably should poke other go routines to stop cleanly here ... - os.Exit(0) - } + for sig := range c { + switch sig { + case syscall.SIGHUP: + log.Infof("Received SIGHUP. Reloading configuration") + config.Reload() + case syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT: + log.Infof("Received %s. Shutting down orchestrator-agent", sig.String()) + // probably should poke other go routines to stop cleanly here ... + os.Exit(0) } - }() + } } // main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces. From 17700a79420686876fa17fe4adb9300aecf21bb3 Mon Sep 17 00:00:00 2001 From: Slach Date: Sat, 31 Mar 2018 13:54:24 +0500 Subject: [PATCH 18/19] get sourceHost via query string Signed-off-by: Slach --- go/http/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/http/api.go b/go/http/api.go index 70deaf6..383a6d9 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -363,6 +363,11 @@ func (this *HttpAPI) PostCopy(params martini.Params, r render.Render, req *http. if err := this.validateToken(r, req); err != nil { return } + + if params["sourceHost"]=="" { + params["sourceHost"] = req.Form.Get("sourceHost") + } + err := osagent.PostCopy(params["sourceHost"]) if err != nil { r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()}) From 4056d5880c17c23fd9ac56402aa3696353d98d91 Mon Sep 17 00:00:00 2001 From: Slach Date: Sat, 31 Mar 2018 20:22:41 +0500 Subject: [PATCH 19/19] parse sourceHost over http.Request.URL.Query Signed-off-by: Slach --- go/http/api.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/go/http/api.go b/go/http/api.go index 383a6d9..ec4167d 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -363,11 +363,10 @@ func (this *HttpAPI) PostCopy(params martini.Params, r render.Render, req *http. if err := this.validateToken(r, req); err != nil { return } - - if params["sourceHost"]=="" { - params["sourceHost"] = req.Form.Get("sourceHost") + qs := req.URL.Query() + if sourceHost, exists := params["sourceHost"]; !exists || sourceHost=="" { + params["sourceHost"] = qs.Get("sourceHost") } - err := osagent.PostCopy(params["sourceHost"]) if err != nil { r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()})