gorm supports master-slave config

.github/RELEASE.md

@@ -1,5 +1,5 @@
 ## Change log
 
-- Add request_id to gorm print sql log.
-- Add a new arbitrary condition query api interface in web and microservices.
-- Update documentation.
+- The make proto command supports code generation for a given proto file.
+- Gorm supports master-slave configuration and custom plugins.
+- Logger library add sync function.

configs/serverNameExample.yml

@@ -84,6 +84,11 @@ mysql:
   maxIdleConns: 3                  # set the maximum number of connections in the idle connection pool
   maxOpenConns: 100            # set the maximum number of open database connections
   connMaxLifetime: 30            # sets the maximum time for which the connection can be reused, in minutes
+  #slavesDsn:                          # sets slaves mysql dsn, array type
+  #  - "your slave dsn 1"
+  #  - "your slave dsn 2"
+  #mastersDsn:                        # sets masters mysql dsn, array type, non-required field, if there is only one master, there is no need to set the mastersDsn field, the default dsn field is mysql master.
+  #  - "your master dsn"
 
 
 # redis settings

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/go-playground/validator/v10 v10.14.0
 	github.com/go-redis/redis/extra/redisotel v0.3.0
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-sql-driver/mysql v1.7.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/snappy v0.0.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
@@ -51,8 +51,8 @@ require (
 	google.golang.org/grpc v1.53.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/yaml.v3 v3.0.1
-	gorm.io/driver/mysql v1.3.5
-	gorm.io/gorm v1.23.8
+	gorm.io/driver/mysql v1.5.1
+	gorm.io/gorm v1.25.4
 )
 
 require (
@@ -172,4 +172,5 @@ require (
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gorm.io/plugin/dbresolver v1.4.7 // indirect
 )

go.sum

@@ -236,6 +236,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
@@ -1156,8 +1158,19 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
 gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
+gorm.io/driver/mysql v1.4.3 h1:/JhWJhO2v17d8hjApTltKNADm7K7YI2ogkR7avJUL3k=
+gorm.io/driver/mysql v1.4.3/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
+gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
+gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
 gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
+gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
+gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw=
+gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+gorm.io/plugin/dbresolver v1.4.7 h1:ZwtwmJQxTx9us7o6zEHFvH1q4OeEo1pooU7efmnunJA=
+gorm.io/plugin/dbresolver v1.4.7/go.mod h1:l4Cn87EHLEYuqUncpEeTC2tTJQkjngPSD+lo8hIvcT0=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config/serverNameExample.go

@@ -103,12 +103,14 @@ type GrpcClient struct {
 }
 
 type Mysql struct {
-	ConnMaxLifetime int    `yaml:"connMaxLifetime" json:"connMaxLifetime"`
-	Dsn             string `yaml:"dsn" json:"dsn"`
-	EnableLog       bool   `yaml:"enableLog" json:"enableLog"`
-	MaxIdleConns    int    `yaml:"maxIdleConns" json:"maxIdleConns"`
-	MaxOpenConns    int    `yaml:"maxOpenConns" json:"maxOpenConns"`
-	SlowThreshold   int    `yaml:"slowThreshold" json:"slowThreshold"`
+	ConnMaxLifetime int      `yaml:"connMaxLifetime" json:"connMaxLifetime"`
+	Dsn             string   `yaml:"dsn" json:"dsn"`
+	EnableLog       bool     `yaml:"enableLog" json:"enableLog"`
+	MastersDsn      []string `yaml:"mastersDsn" json:"mastersDsn"`
+	MaxIdleConns    int      `yaml:"maxIdleConns" json:"maxIdleConns"`
+	MaxOpenConns    int      `yaml:"maxOpenConns" json:"maxOpenConns"`
+	SlavesDsn       []string `yaml:"slavesDsn" json:"slavesDsn"`
+	SlowThreshold   int      `yaml:"slowThreshold" json:"slowThreshold"`
 }
 
 type Redis struct {

internal/config/serverNameExample_cc.go

@@ -1,4 +1,4 @@
-// code generated by sponge.
+// code generated by https://github.com/zhufuyi/sponge
 
 package config
 

internal/model/init.go

@@ -53,6 +53,15 @@ func InitMysql() {
 		opts = append(opts, mysql.WithEnableTrace())
 	}
 
+	// setting mysql slave and master dsn addresses
+	//opts = append(opts, mysql.WithRWSeparation(
+	//	config.Get().Mysql.SlavesDsn,
+	//	config.Get().Mysql.MastersDsn...,
+	//))
+
+	// add custom gorm plugin
+	//opts = append(opts, mysql.WithGormPlugin(yourPlugin))
+
 	var err error
 	db, err = mysql.Init(config.Get().Mysql.Dsn, opts...)
 	if err != nil {

pkg/conf/parse.go

@@ -95,6 +95,11 @@ func hideSensitiveFields(line string, fields ...string) string {
 		}
 	}
 
+	// replace dsn
+	if strings.Contains(line, "@") && strings.Contains(line, ":") {
+		return replaceDSN(line)
+	}
+
 	return line
 }
 

pkg/conf/parse_test.go

@@ -40,6 +40,9 @@ func Test_hideSensitiveFields(t *testing.T) {
 	str := Show(c, keywords...)
 
 	fmt.Printf(hideSensitiveFields(str))
+
+	str = "\ndefault:123456@192.168.3.37:6379/0\n"
+	fmt.Printf(hideSensitiveFields(str))
 }
 
 // test listening for profile updates

pkg/logger/method.go

@@ -63,7 +63,8 @@ func Errorf(format string, a ...interface{}) {
 
 // Sync flushing any buffered log entries, applications should take care to call Sync before exiting.
 func Sync() error {
-	return getLogger().Sync()
+	//return getLogger().Sync()
+	return nil
 }
 
 // WithFields carrying field information

pkg/mysql/gorm.go

@@ -12,6 +12,7 @@ import (
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
+	"gorm.io/plugin/dbresolver"
 )
 
 // Init mysql
@@ -33,6 +34,7 @@ func Init(dns string, opts ...Option) (*gorm.DB, error) {
 	}
 	db.Set("gorm:table_options", "CHARSET=utf8mb4") // automatic appending of table suffixes when creating tables
 
+	// register trace plugin
 	if o.enableTrace {
 		err = db.Use(otelgorm.NewPlugin())
 		if err != nil {
@@ -40,6 +42,22 @@ func Init(dns string, opts ...Option) (*gorm.DB, error) {
 		}
 	}
 
+	// register read-write separation plugin
+	if len(o.slavesDsn) > 0 {
+		err = db.Use(rwSeparationPlugin(o))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// register plugins
+	for _, plugin := range o.plugins {
+		err = db.Use(plugin)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return db, nil
 }
 
@@ -77,3 +95,25 @@ func gormConfig(o *options) *gorm.Config {
 
 	return config
 }
+
+func rwSeparationPlugin(o *options) gorm.Plugin {
+	slaves := []gorm.Dialector{}
+	for _, dsn := range o.slavesDsn {
+		slaves = append(slaves, mysqlDriver.New(mysqlDriver.Config{
+			DSN: dsn,
+		}))
+	}
+
+	masters := []gorm.Dialector{}
+	for _, dsn := range o.mastersDsn {
+		masters = append(masters, mysqlDriver.New(mysqlDriver.Config{
+			DSN: dsn,
+		}))
+	}
+
+	return dbresolver.Register(dbresolver.Config{
+		Sources:  masters,
+		Replicas: slaves,
+		Policy:   dbresolver.RandomPolicy{},
+	})
+}

pkg/mysql/gorm_test.go

@@ -33,10 +33,19 @@ func Test_gormConfig(t *testing.T) {
 		WithMaxOpenConns(50),
 		WithConnMaxLifetime(time.Minute*3),
 		WithEnableForeignKey(),
+		WithLogRequestIDKey("request_id"),
+		WithRWSeparation([]string{
+			"root:123456@(192.168.3.37:3306)/slave1",
+			"root:123456@(192.168.3.37:3306)/slave2"},
+			"root:123456@(192.168.3.37:3306)/master"),
+		WithGormPlugin(nil),
 	)
 
 	c := gormConfig(o)
 	assert.NotNil(t, c)
+
+	err := rwSeparationPlugin(o)
+	assert.NotNil(t, err)
 }
 
 func TestGetTableName(t *testing.T) {

pkg/mysql/option.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"go.uber.org/zap"
+	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 )
 
@@ -24,6 +25,11 @@ type options struct {
 	requestIDKey string
 	gLog         *zap.Logger
 	logLevel     logger.LogLevel
+
+	slavesDsn  []string
+	mastersDsn []string
+
+	plugins []gorm.Plugin
 }
 
 func (o *options) apply(opts ...Option) {
@@ -122,3 +128,18 @@ func WithLogRequestIDKey(key string) Option {
 		o.requestIDKey = key
 	}
 }
+
+// WithRWSeparation setting read-write separation
+func WithRWSeparation(slavesDsn []string, mastersDsn ...string) Option {
+	return func(o *options) {
+		o.slavesDsn = slavesDsn
+		o.mastersDsn = mastersDsn
+	}
+}
+
+// WithGormPlugin setting gorm plugin
+func WithGormPlugin(plugins ...gorm.Plugin) Option {
+	return func(o *options) {
+		o.plugins = plugins
+	}
+}