From a4ae1df15a8ae02518b8d6266349cd4696c1e2b9 Mon Sep 17 00:00:00 2001 From: zhiyuan Date: Mon, 2 Dec 2024 18:37:45 +0800 Subject: [PATCH 1/3] feat: add process metrics --- go.mod | 18 ++- go.sum | 43 ++++++- pkg/metrics/process.go | 249 ++++++++++++++++++++++++++++++++++++ pkg/metrics/process_test.go | 104 +++++++++++++++ 4 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 pkg/metrics/process.go create mode 100644 pkg/metrics/process_test.go diff --git a/go.mod b/go.mod index d1fb6e3..7a3cadc 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.1 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.20.5 + github.com/shirou/gopsutil/v4 v4.24.11 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.65.0 @@ -20,10 +22,14 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/ebitengine/purego v0.8.1 // indirect github.com/fatih/color v1.16.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -31,20 +37,30 @@ require ( github.com/hashicorp/yamux v0.1.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect github.com/smartystreets/goconvey v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.4.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 289cf2b..19e1252 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,17 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/bytedance/mockey v1.2.10 h1:4JlMpkm7HMXmTUtItid+iCu2tm61wvq+ca1X2u7ymzE= github.com/bytedance/mockey v1.2.10/go.mod h1:bNrUnI1u7+pAc0TYDgPATM+wF2yzHxmNH+iDXg4AOCU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE= +github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -13,10 +19,13 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -39,10 +48,16 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -59,6 +74,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -66,8 +83,20 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/shirou/gopsutil/v4 v4.24.11 h1:WaU9xqGFKvFfsUv94SXcUPD7rCkU0vr/asVdQOBZNj8= +github.com/shirou/gopsutil/v4 v4.24.11/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -79,10 +108,16 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -108,16 +143,20 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= diff --git a/pkg/metrics/process.go b/pkg/metrics/process.go new file mode 100644 index 0000000..79f30b7 --- /dev/null +++ b/pkg/metrics/process.go @@ -0,0 +1,249 @@ +package metrics + +/* +@ Package Description: +This package current collect process cpu and memory usage rate, memory rss usage, requests already processed by the process and can easily to expand other stats. +At the same time, by integrating Prometheus, process related metrics were exposed to the outside. + +@ How to use this package ？ +You can follow below step to record process metrics: +1. initialize process metrics server. for example: + @Parameter: listenAddress -- (string) server listen address. + errChan := metrics.InitProcessMetricsServer(":9090") + select { + case err := <-errChan: + log.Fatalf("Failed to init process metrics server: %v", err) + case <-os.Interrupt: + log.Println("Received interrupt signal, shutting down.") + } + +2.You can invoke RecordProcessResourceUsageMetrics to record process resource usage metrics, for example: +2.1 Record current process metrics. + @Parameter: pid -- (int32) process pid. + err := metrics.RecordProcessResourceUsageMetrics(43123) + if err != nil { + log.Warn(err) + } +2.2 Record multiple process metrics. + for _, pid := range pids { + err := metrics.RecordProcessResourceUsageMetrics(pid) + if err != nil { + log.Warn(err) + } + } + +3. You can invoke RecordProcessRequestMetrics to record requests metrics processed by specific process, for example: + @Parameter: pid -- (int32) process pid. + @Parameter: traceID -- (string) request trace id. + @Parameter: duration -- (float64) duration second for request is processed. + start := time.Now() + process(requests) + duration := time.Since(start).Seconds() + err := metrics.RecordProcessRequestMetrics(43123, "1111-2222-3333", duration) + if err != nil { + log.Warn(err) + } + +@ How kusion integrate with this package to record module process metrics ? +Kusion should manage the module subprocess information and regularly collect subprocess resource usage metrics. +At the same time, Kusion should record the request metrics processed by specific module process. + +@ How to fetch the process metrics ? +You can test it locally by using curl command. for example: +curl -v http://127.0.0.1:9090/process/metrics + +# HELP process_cpu_percent How many percent of the CPU time process uses. +# TYPE process_cpu_percent gauge +process_cpu_percent{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585"} 0.5315134891429808 +# HELP process_memory_percent How many percent of the memory process uses. +# TYPE process_memory_percent gauge +process_memory_percent{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585"} 0.037479400634765625 +# HELP process_memory_rss How many rss of the memory process uses. +# TYPE process_memory_rss gauge +process_memory_rss{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585"} 6.438912e+06 +# HELP process_per_request_duration_seconds Process per request duration in seconds. +# TYPE process_per_request_duration_seconds gauge +process_per_request_duration_seconds{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",trace_id="11111"} 5.000896791 +# HELP process_request_duration_seconds Process request duration in seconds. +# TYPE process_request_duration_seconds histogram +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.01"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.025"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.05"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.1"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.25"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="0.5"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="1"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="2.5"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="5"} 0 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="10"} 1 +process_request_duration_seconds_bucket{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585",le="+Inf"} 1 +process_request_duration_seconds_sum{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585"} 5.000896791 +process_request_duration_seconds_count{hostname="xxxMacBook-Pro.local",pid="43638",pid_name="metrics",ppid="43585"} 1 +*/ + +import ( + "fmt" + "net/http" + "os" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/shirou/gopsutil/v4/process" +) + +var ( + ProcessCPUPercent = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "process_cpu_percent", + Help: "How many percent of the CPU time process uses.", + }, + []string{"hostname", "pid", "pid_name", "ppid"}, + ) + + ProcessMemoryPercent = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "process_memory_percent", + Help: "How many percent of the memory process uses.", + }, + []string{"hostname", "pid", "pid_name", "ppid"}, + ) + + ProcessMemoryRSS = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "process_memory_rss", + Help: "How many rss of the memory process uses.", + }, + []string{"hostname", "pid", "pid_name", "ppid"}, + ) + + ProcessRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "process_request_duration_seconds", + Help: "Process request duration in seconds.", + Buckets: []float64{.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"hostname", "pid", "pid_name", "ppid"}, + ) + + ProcessPerRequestDuration = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "process_per_request_duration_seconds", + Help: "Process per request duration in seconds.", + }, + []string{"hostname", "pid", "pid_name", "ppid", "trace_id"}, + ) +) + +func InitProcessMetricsServer(listenAddress string) <-chan error { + if listenAddress == "" { + listenAddress = os.Getenv("PROCESS_METRICS_LISTEN_ADDRESS") + if listenAddress == "" { + listenAddress = ":9090" + } + } + customRegistry := prometheus.NewRegistry() + customRegistry.MustRegister(ProcessCPUPercent) + customRegistry.MustRegister(ProcessMemoryPercent) + customRegistry.MustRegister(ProcessMemoryRSS) + customRegistry.MustRegister(ProcessRequestDuration) + customRegistry.MustRegister(ProcessPerRequestDuration) + + var errCH chan error + go func() { + http.Handle("/process/metrics", promhttp.HandlerFor(customRegistry, promhttp.HandlerOpts{})) + errCH <- http.ListenAndServe(listenAddress, nil) + }() + return errCH +} + +func RecordProcessResourceUsageMetrics(pid int32) error { + pi, err := CollectProcessInfo(pid) + if err != nil { + return err + } + hostName, _ := os.Hostname() + labels := prometheus.Labels{ + "hostname": hostName, + "pid": fmt.Sprintf("%v", pi.Pid), + "pid_name": pi.ProcessName, + "ppid": fmt.Sprintf("%v", pi.ParentPid), + } + ProcessCPUPercent.With(labels).Set(pi.CPUInfo.CPUPercent) + ProcessMemoryPercent.With(labels).Set(float64(pi.MemoryInfo.MemoryPercent)) + ProcessMemoryRSS.With(labels).Set(float64(pi.MemoryInfo.RSS)) + return nil +} + +func RecordProcessRequestMetrics(pid int32, traceID string, duration float64) error { + pi, err := CollectProcessInfo(pid) + if err != nil { + return err + } + hostName, _ := os.Hostname() + labels := prometheus.Labels{ + "hostname": hostName, + "pid": fmt.Sprintf("%v", pi.Pid), + "pid_name": pi.ProcessName, + "ppid": fmt.Sprintf("%v", pi.ParentPid), + } + ProcessRequestDuration.With(labels).Observe(duration) + labels["trace_id"] = traceID + ProcessPerRequestDuration.With(labels).Set(duration) + return nil +} + +type ProcessInfo struct { + Pid int32 + ParentPid int32 + ProcessName string + CPUInfo *CPUInfo + MemoryInfo *MemoryInfo +} + +type CPUInfo struct { + CPUPercent float64 +} + +type MemoryInfo struct { + MemoryPercent float32 + *process.MemoryInfoStat +} + +func CollectProcessInfo(pid int32) (*ProcessInfo, error) { + p, err := process.NewProcess(pid) + if err != nil { + return nil, fmt.Errorf("failed to create process[%v] instance: %v\n", pid, err) + } + name, err := p.Name() + if err != nil { + return nil, fmt.Errorf("failed to get process[%v] name: %v", pid, err) + } + parentPid, err := p.Ppid() + if err != nil { + return nil, fmt.Errorf("failed to get process[%v] parent process pid: %v", pid, err) + } + cpuPercent, err := p.CPUPercent() + if err != nil { + return nil, fmt.Errorf("failed to get process[%v] cpu percent: %v", pid, err) + } + memoryPercent, err := p.MemoryPercent() + if err != nil { + fmt.Errorf("failed to get process[%v] memory percent: %v", pid, err) + } + memInfo, err := p.MemoryInfo() + if err != nil { + fmt.Errorf("failed to get process[%v] memory info: %v", pid, err) + } + + pi := &ProcessInfo{ + Pid: pid, + ParentPid: parentPid, + ProcessName: name, + CPUInfo: &CPUInfo{CPUPercent: cpuPercent}, + MemoryInfo: &MemoryInfo{ + MemoryPercent: memoryPercent, + MemoryInfoStat: memInfo, + }, + } + return pi, nil +} diff --git a/pkg/metrics/process_test.go b/pkg/metrics/process_test.go new file mode 100644 index 0000000..45762a1 --- /dev/null +++ b/pkg/metrics/process_test.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCollectProcessInfo(t *testing.T) { + tests := []struct { + name string + pid func() int32 + wantErr bool + }{ + { + name: "pid is invalid", + pid: func() int32 { + return -1 + }, + wantErr: true, + }, + { + name: "pid is valid", + pid: func() int32 { + return int32(os.Getpid()) + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := CollectProcessInfo(tt.pid()) + assert.Equal(t, err != nil, tt.wantErr) + }) + } +} + +func TestRecordProcessRequestMetrics(t *testing.T) { + tests := []struct { + name string + pid func() int32 + traceID string + duration float64 + wantErr bool + }{ + { + name: "pid is invalid", + traceID: "111-222-333", + duration: float64(1), + pid: func() int32 { + return -1 + }, + wantErr: true, + }, + { + name: "pid is valid", + traceID: "111-222-333", + duration: float64(1), + pid: func() int32 { + return int32(os.Getpid()) + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RecordProcessRequestMetrics(tt.pid(), tt.traceID, tt.duration) + assert.Equal(t, err != nil, tt.wantErr) + }) + } +} + +func TestRecordProcessResourceUsageMetrics(t *testing.T) { + tests := []struct { + name string + pid func() int32 + wantErr bool + }{ + { + name: "pid is invalid", + pid: func() int32 { + return -1 + }, + wantErr: true, + }, + { + name: "pid is valid", + pid: func() int32 { + return int32(os.Getpid()) + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RecordProcessResourceUsageMetrics(tt.pid()) + assert.Equal(t, err != nil, tt.wantErr) + }) + } +} From e70f4d13b1c5bad9a76084f583eb196a5b17b3a5 Mon Sep 17 00:00:00 2001 From: zhiyuan Date: Mon, 2 Dec 2024 19:02:00 +0800 Subject: [PATCH 2/3] chore: update metrics notes --- pkg/metrics/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/process.go b/pkg/metrics/process.go index 79f30b7..8982359 100644 --- a/pkg/metrics/process.go +++ b/pkg/metrics/process.go @@ -95,7 +95,7 @@ var ( ProcessCPUPercent = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "process_cpu_percent", - Help: "How many percent of the CPU time process uses.", + Help: "How many percent of the cpu time process uses.", }, []string{"hostname", "pid", "pid_name", "ppid"}, ) From 2063067a0ef601a3e6346ddba38fa879719ac8f3 Mon Sep 17 00:00:00 2001 From: zhiyuan Date: Wed, 4 Dec 2024 10:08:29 +0800 Subject: [PATCH 3/3] chore: update variable name and return result --- pkg/metrics/process.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/metrics/process.go b/pkg/metrics/process.go index 8982359..682a6c9 100644 --- a/pkg/metrics/process.go +++ b/pkg/metrics/process.go @@ -148,12 +148,12 @@ func InitProcessMetricsServer(listenAddress string) <-chan error { customRegistry.MustRegister(ProcessRequestDuration) customRegistry.MustRegister(ProcessPerRequestDuration) - var errCH chan error + var errChan chan error go func() { http.Handle("/process/metrics", promhttp.HandlerFor(customRegistry, promhttp.HandlerOpts{})) - errCH <- http.ListenAndServe(listenAddress, nil) + errChan <- http.ListenAndServe(listenAddress, nil) }() - return errCH + return errChan } func RecordProcessResourceUsageMetrics(pid int32) error { @@ -228,11 +228,11 @@ func CollectProcessInfo(pid int32) (*ProcessInfo, error) { } memoryPercent, err := p.MemoryPercent() if err != nil { - fmt.Errorf("failed to get process[%v] memory percent: %v", pid, err) + return nil, fmt.Errorf("failed to get process[%v] memory percent: %v", pid, err) } memInfo, err := p.MemoryInfo() if err != nil { - fmt.Errorf("failed to get process[%v] memory info: %v", pid, err) + return nil, fmt.Errorf("failed to get process[%v] memory info: %v", pid, err) } pi := &ProcessInfo{