diff --git a/go.mod b/go.mod index 88e565e..cd0aead 100644 --- a/go.mod +++ b/go.mod @@ -3,25 +3,56 @@ module github.com/databendcloud/db-archiver go 1.21.0 require ( + github.com/avast/retry-go v3.0.0+incompatible github.com/codesuki/go-time-series v0.0.0-20210430055340-c4c8d8fa61d4 github.com/datafuselabs/databend-go v0.6.6 + github.com/gin-gonic/gin v1.10.0 github.com/go-sql-driver/mysql v1.8.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 + gorm.io/driver/sqlite v1.5.6 + gorm.io/gorm v1.25.10 ) require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/BurntSushi/toml v1.2.1 // indirect - github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/benbjohnson/clock v1.3.5 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 12e436a..16c0b27 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,14 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/codesuki/go-time-series v0.0.0-20210430055340-c4c8d8fa61d4 h1:xKzsxCG6QVIh31ZIXuRR/eCvEflCFwpeET6cvTFYhVo= github.com/codesuki/go-time-series v0.0.0-20210430055340-c4c8d8fa61d4/go.mod h1:Rm6RJZPJg9b/vwne8fiAcfh0X5QFNszEhijK6d6qW9k= github.com/datafuselabs/databend-go v0.6.6 h1:xSXjvs1OSY3+gt/qAhejqX73Fw0Wy5eGE2CcIfIxapk= @@ -15,17 +23,57 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -33,11 +81,23 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= @@ -46,9 +106,30 @@ go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGX go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= +gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/app.go b/pkg/app.go new file mode 100644 index 0000000..3c925d5 --- /dev/null +++ b/pkg/app.go @@ -0,0 +1,19 @@ +package pkg + +import ( + "github.com/databendcloud/db-archiver/config" + "github.com/databendcloud/db-archiver/pkg/models" +) + +type App struct { + cfg *config.Config + dao *models.DAO +} + +func NewApp(conf *config.Config) (*App, error) { + var err error + app := &App{cfg: conf} + app.dao = models.NewDao() + + return app, err +} diff --git a/pkg/controller/synctask_controller.go b/pkg/controller/synctask_controller.go new file mode 100644 index 0000000..063f42f --- /dev/null +++ b/pkg/controller/synctask_controller.go @@ -0,0 +1,20 @@ +package controller + +import ( + "github.com/gin-gonic/gin" + + "github.com/databendcloud/db-archiver/pkg/logic/sync_logic" +) + +type SyncTaskController struct { + l *sync_logic.SyncLogic +} + +func NewSyncTaskController(l *sync_logic.SyncLogic) *SyncTaskController { + return &SyncTaskController{l: l} +} + +func (ctl *SyncTaskController) InjectRouters(r *gin.Engine) { + //g := r.Group("/api/v1") + //g.POST() +} diff --git a/pkg/dto/synctask_dto.go b/pkg/dto/synctask_dto.go new file mode 100644 index 0000000..76d3a17 --- /dev/null +++ b/pkg/dto/synctask_dto.go @@ -0,0 +1 @@ +package dto diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 0000000..c770926 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,34 @@ +package errors + +import "github.com/pkg/errors" + +type SyncTaskError struct { + Kind string + StatusCode int + Message string +} + +func (err *SyncTaskError) Error() string { + return err.Message +} + +var ( + TaskNotFound = SyncTaskError{"TaskNotFound", 404, "Sync task not found"} + TaskRunning = SyncTaskError{"TaskRunning", 404, "Sync task is running"} +) + +func Is(err error, target error) bool { + return errors.Is(err, target) +} + +func As(err error, target interface{}) bool { + return errors.As(err, target) +} + +func AsCloudControlError(err error, gwErr **SyncTaskError) bool { + return errors.As(err, gwErr) +} + +func Wrapf(err error, format string, args ...interface{}) error { + return errors.Wrapf(err, format, args...) +} diff --git a/pkg/logic/sync_logic/sync_logic.go b/pkg/logic/sync_logic/sync_logic.go new file mode 100644 index 0000000..5c884a1 --- /dev/null +++ b/pkg/logic/sync_logic/sync_logic.go @@ -0,0 +1,14 @@ +package sync_logic + +import ( + "github.com/databendcloud/db-archiver/config" + "github.com/databendcloud/db-archiver/pkg/models" +) + +type SyncLogicImpl struct { + dao *models.DAO + cfg *config.Config +} + +type SyncLogic interface { +} \ No newline at end of file diff --git a/pkg/models/dao.go b/pkg/models/dao.go new file mode 100644 index 0000000..4df2bad --- /dev/null +++ b/pkg/models/dao.go @@ -0,0 +1,19 @@ +package models + +import ( + "gorm.io/driver/sqlite" + _ "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type DAO struct { + db *gorm.DB +} + +func NewDao() *DAO { + db, err := gorm.Open(sqlite.Open("test.db"), &gorm.Config{}) + if err != nil { + panic("failed to connect database") + } + return &DAO{db: db} +} diff --git a/pkg/models/sync.go b/pkg/models/sync.go new file mode 100644 index 0000000..3904101 --- /dev/null +++ b/pkg/models/sync.go @@ -0,0 +1,132 @@ +package models + +import ( + "time" + + "gorm.io/gorm" + + terr "github.com/databendcloud/db-archiver/pkg/errors" +) + +type SyncTask struct { + SyncTaskID int64 `json:"syncTaskID" gorm:"column:sync_task_id;type:bigint;primary_key"` + SourceHost string `json:"sourceHost" gorm:"column:source_host;type:varchar(255)"` + SourcePort int `json:"sourcePort" gorm:"column:source_port;type:int"` + SourceUser string `json:"sourceUser" gorm:"column:source_user;type:varchar(255)"` + SourcePass string `json:"sourcePass" gorm:"column:source_pass;type:varchar(255)"` + SourceDB string `json:"sourceDB" gorm:"column:source_db;type:varchar(255)"` + SourceTable string `json:"sourceTable" gorm:"column:source_table;type:varchar(255)"` + SourceQuery string `json:"sourceQuery" gorm:"column:source_query;type:varchar(255)"` // select * from table where condition + SourceWhereCondition string `json:"sourceWhereCondition" gorm:"column:source_where_condition;type:varchar(255)"` //example: where id > 100 and id < 200 and time > '2023-01-01' + SourceSplitKey string `json:"sourceSplitKey" gorm:"column:source_split_key;type:varchar(255)"` // primary split key for split table, only for int type + // the format of time field must be: 2006-01-02 15:04:05 + SourceSplitTimeKey string `json:"SourceSplitTimeKey" gorm:"column:source_split_time_key;type:varchar(255)"` // time field for split table + TimeSplitUnit string `json:"TimeSplitUnit" default:"hour" gorm:"column:time_split_unit;type:varchar(255)"` // time split unit, default is hour, option is: minute, hour, day + + // Databend configuration + DatabendDSN string `json:"databendDSN" default:"localhost:8000" gorm:"column:databend_dsn;type:varchar(255)"` + DatabendDatabase string `json:"databendDatabase" gorm:"column:databend_database;type:varchar(255)"` + DatabendTable string `json:"databendTable" gorm:"column:databend_table;type:varchar(255)"` + BatchSize int `json:"batchSize" default:"1000" gorm:"column:batch_size;type:int"` + //BatchMaxInterval int `json:"batchMaxInterval" default:"3"` // for rate limit control + + // related docs: https://docs.databend.com/sql/sql-commands/dml/dml-copy-into-table + CopyPurge bool `json:"copyPurge" default:"false" gorm:"column:copy_purge;type:bool"` + CopyForce bool `json:"copyForce" default:"false" gorm:"column:copy_force;type:bool"` + UserStage string `json:"userStage" default:"~" gorm:"column:user_stage;type:varchar(255)"` + DeleteAfterSync bool `json:"deleteAfterSync" default:"false" gorm:"column:delete_after_sync;type:bool"` + MaxThread int `json:"maxThread" default:"1" gorm:"column:max_thread;type:int"` + CreatedAt time.Time `json:"createdAt" gorm:"column:created_at;type:timestamp with time zone;default:now()"` + UpdatedAt time.Time `json:"updatedAt" gorm:"column:updated_at;type:timestamp with time zone;default:now()"` + DeletedAt time.Time `json:"deletedAt" gorm:"column:deleted_at;type:timestamp with time zone;default:now()"` + ShouldRunning bool `json:"shouldRunning" default:"true" gorm:"column:should_running;type:bool"` + SyncRate string `json:"syncRate" gorm:"column:sync_rate;type:varchar(255)"` // 100 rows/s + SyncPercentage int `json:"syncPercentage" default:"0" gorm:"column:sync_percentage;type:int"` // 0-100 + Status string `json:"status" gorm:"column:status;type:varchar(255)"` +} + +func (SyncTask) TableName() string { + return "sync_config" +} + +func (s *SyncTask) DeepCopy() *SyncTask { + st := *s + return &st +} + +func (dao *DAO) CreateSync(syncConfig *SyncTask) error { + return dao.db.Create(syncConfig).Error +} + +func (dao *DAO) GetTask(syncTaskId int64) (*SyncTask, error) { + t := &SyncTask{} + if err := dao.db.Where(&SyncTask{SyncTaskID: syncTaskId}).First(&t).Error; err != nil { + switch err { + case gorm.ErrRecordNotFound: + return nil, terr.Wrapf(&terr.TaskNotFound, "syncTaskId: %d", syncTaskId) + default: + return nil, err + } + } + return t, nil +} + +func (dao *DAO) UpdateTaskStatus(syncTaskId int64, status string) error { + t, err := dao.GetTask(syncTaskId) + if err != nil { + return err + } + t.Status = status + t.UpdatedAt = time.Now() + if err := dao.db.Model(&SyncTask{}).Where("sync_task_id = ?", t.SyncTaskID).Select("status").Updates(t).Error; err != nil { + return err + } + return nil +} + +func (dao *DAO) UpdateTaskRate(syncTaskId int64, rate string, syncPercentage int) error { + t, err := dao.GetTask(syncTaskId) + if err != nil { + return err + } + t.SyncRate = rate + t.SyncPercentage = syncPercentage + t.UpdatedAt = time.Now() + if err := dao.db.Model(&SyncTask{}).Where("sync_task_id = ?", t.SyncTaskID).Select("sync_rate, sync_percentage").Updates(t).Error; err != nil { + return err + } + return nil +} + +func (dao *DAO) ListSyncTasks() ([]SyncTask, error) { + var tasks []SyncTask + if err := dao.db.Find(&tasks).Error; err != nil { + return nil, err + } + return tasks, nil +} + +func (dao *DAO) DeleteSync(syncTaskID int64) error { + t, err := dao.GetTask(syncTaskID) + if err != nil { + return nil + } + // can't delete a running task + if t.ShouldRunning { + return terr.Wrapf(&terr.TaskRunning, "syncTaskId: %d", syncTaskID) + } + return dao.db.Where("sync_task_id = ?", syncTaskID).Delete(&SyncTask{}).Error +} + +func (dao *DAO) StartStopTask(syncTaskID int64, start bool) error { + t, err := dao.GetTask(syncTaskID) + if err != nil { + return err + } + t.ShouldRunning = start + t.UpdatedAt = time.Now() + if err := dao.db.Model(&SyncTask{}).Where("sync_task_id = ?", t.SyncTaskID).Select("should_running").Updates(t).Error; err != nil { + return err + } + return nil +} diff --git a/pkg/server/server.go b/pkg/server/server.go new file mode 100644 index 0000000..abb4e43 --- /dev/null +++ b/pkg/server/server.go @@ -0,0 +1 @@ +package server