feat: 同步订阅完成

This commit is contained in:
周杰 2025-08-19 13:52:01 +08:00
parent ea0bc7be8c
commit 6e8fdc6b7c
9 changed files with 479 additions and 0 deletions

2
.gitignore vendored
View File

@ -17,6 +17,8 @@
# Dependency directories (remove the comment below to include it)
# vendor/
.idea
*.log
# Go workspace file
go.work

1
client_api.go Normal file
View File

@ -0,0 +1 @@
package user_core_client

19
client_sync.go Normal file
View File

@ -0,0 +1,19 @@
package user_core_client
import (
"context"
)
type EventTyped interface {
GetEventType() string
}
type UserCentSync interface {
Start(ctx context.Context) error
Stop(ctx context.Context) error
// SetCallUserListener 设置用户事件回调函数
SetCallUserListener(func(context.Context, *UserEvent) error)
// SetErrorListener 设置错误回调函数
SetErrorListener(func(context.Context, error) error)
}

215
client_sync_kafka_impl.go Normal file
View File

@ -0,0 +1,215 @@
package user_core_client
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/segmentio/kafka-go"
"github.com/segmentio/kafka-go/sasl"
"github.com/segmentio/kafka-go/sasl/plain"
"github.com/segmentio/kafka-go/sasl/scram"
"strings"
"time"
)
// Kafka 相关常量
const (
// UserSyncTopic 用户同步的Kafka主题
UserSyncTopic = "user-sync"
)
type EventType string
// 用户事件类型常量
const (
UserCreateEvent EventType = "user-create-event" // 用户创建事件
UserUpdateEvent EventType = "user-update-event" // 用户更新事件
UserDeleteEvent EventType = "user-delete-event" // 用户删除事件
)
type OptionKafkaSync = func(s *kafkaSync)
func OptKafkaLogger(l Logger) func(s *kafkaSync) {
return func(s *kafkaSync) {
s.logger = l
}
}
type KafkaConfig struct {
Brokers []string `json:"brokers"` // kafka broker 地址
GroupID string `json:"group_id"` // kafka group id
SaslUser string `json:"sasluser"` // kafka 用户名
SaslPassword string `json:"saslpassword"` // kafka 密码
SaslMechanism string `json:"saslmechanism"` // 可选值: plain / scram-sha256 / scram-sha512
}
func NewUserCentKafkaSync(platformCode string, kafkaConfig *KafkaConfig, opts ...OptionKafkaSync) UserCentSync {
k := &kafkaSync{
platformCode: platformCode,
kafkaConfig: kafkaConfig,
}
for _, opt := range opts {
opt(k)
}
if k.logger == nil {
mLog := NewLogger()
k.logger = mLog
}
return k
}
type kafkaSync struct {
platformCode string
kafkaConfig *KafkaConfig
kafkaReader *kafka.Reader
errHandler func(ctx context.Context, err error) error
eventHandler func(ctx context.Context, event *UserEvent) error
cancelFunc context.CancelFunc
logger Logger
}
func (k *kafkaSync) Stop(ctx context.Context) error {
// 取消上下文
if k.cancelFunc != nil {
k.cancelFunc()
}
// 关闭 kafka 消费者
if k.kafkaReader != nil {
if err := k.kafkaReader.Close(); err != nil {
return fmt.Errorf("close kafka reader failed, err: %v", err)
}
}
return nil
}
func (k *kafkaSync) Start(ctx context.Context) error {
// 初始化 SASL 机制
var mechanism sasl.Mechanism
switch strings.ToLower(k.kafkaConfig.SaslMechanism) {
case "scram-sha256":
mechanism, _ = scram.Mechanism(scram.SHA256, k.kafkaConfig.SaslUser, k.kafkaConfig.SaslPassword)
case "scram-sha512":
mechanism, _ = scram.Mechanism(scram.SHA512, k.kafkaConfig.SaslUser, k.kafkaConfig.SaslPassword)
case "plain", "": // 默认 plain
mechanism = plain.Mechanism{
Username: k.kafkaConfig.SaslUser,
Password: k.kafkaConfig.SaslPassword,
}
default:
return fmt.Errorf("unsupported sasl mechanism: %s", k.kafkaConfig.SaslMechanism)
}
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
SASLMechanism: mechanism,
}
// 初始化 kafka 消费者
k.kafkaReader = kafka.NewReader(kafka.ReaderConfig{
Brokers: k.kafkaConfig.Brokers,
GroupID: k.kafkaConfig.GroupID,
Topic: UserSyncTopic, // 订阅主题
MinBytes: 10, // 最小字节数
MaxBytes: 10e6, // 最大字节数
MaxWait: 10 * time.Second, // 最大等待时间
CommitInterval: 100 * time.Millisecond, // 提交偏移量的时间间隔
StartOffset: kafka.LastOffset, // 从最后一个偏移量开始消费
WatchPartitionChanges: true, // 监听分区变化
Dialer: dialer, // 自定义拨号器
})
sCtx, cancelFunc := context.WithCancel(ctx)
k.cancelFunc = cancelFunc
// 启动消费循环
go k.startLoop(sCtx)
return nil
}
func (k *kafkaSync) SetCallUserListener(f func(context.Context, *UserEvent) error) {
k.eventHandler = f
}
func (k *kafkaSync) callEventHandler(ctx context.Context, message *kafka.Message) error {
if k.eventHandler == nil {
return nil
}
var userEvent UserEvent
err := json.Unmarshal(message.Value, &userEvent)
if err != nil {
return fmt.Errorf("unmarshal user event failed topic: %s message: %s err: %v", message.Topic, message.Value, err)
}
// 过滤非当前平台的事件
if userEvent.PlatformCode != k.platformCode {
return nil
}
k.logger.Printf("received event topic: %s message: %s", message.Topic, message.Value)
return k.eventHandler(ctx, &userEvent)
}
func (k *kafkaSync) SetErrorListener(f func(context.Context, error) error) {
k.errHandler = f
}
func (k *kafkaSync) callErrorListener(ctx context.Context, err error) error {
if k.errHandler == nil {
return nil
}
return k.errHandler(ctx, err)
}
func (k *kafkaSync) startLoop(ctx context.Context) {
k.logger.Printf("kafka reader start loop")
defer k.logger.Printf("kafka reader stop")
for {
// 从 kafka 读取消息, 该消息会阻塞,直到有消息可读
// 并且里面也判断了 ctx 的取消,所以不再判断
message, err := k.kafkaReader.FetchMessage(ctx)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
// 其他错误, 直接返回
k.logger.Printf("kafka reader fetch message failed, err: %v", err)
continue
}
// 解析消息
var event BaseEvent
if err := json.Unmarshal(message.Value, &event); err != nil {
if err := k.callErrorListener(ctx, err); err != nil {
k.logger.Printf("call error listener failed, err: %v", err)
continue
}
}
// 是用户事件 就处理
if strings.HasPrefix(event.EventType, "user-") {
if err := k.callEventHandler(ctx, &message); err != nil {
k.logger.Printf("call event handler failed, err: %v", err)
continue
}
}
// 提交偏移量
if err := k.kafkaReader.CommitMessages(ctx, message); err != nil {
k.logger.Printf("commit message failed, err: %v", err)
continue
}
}
}

51
client_sync_test.go Normal file
View File

@ -0,0 +1,51 @@
package user_core_client
import (
"context"
"encoding/json"
"log"
"os"
"os/signal"
"syscall"
"testing"
)
func init() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
}
func TestUserSyncClient(t *testing.T) {
client := NewUserCentKafkaSync("user-core", &KafkaConfig{
Brokers: []string{"101.133.147.225:9092"},
GroupID: "test-group",
SaslUser: "admin",
SaslPassword: "admin123",
SaslMechanism: "plain",
}, OptKafkaLogger(log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)))
ctx := context.Background()
if err := client.Start(ctx); err != nil {
t.Fatalf("start user sync client failed,err:%v", err)
}
defer func() {
log.Printf("stop user sync client ...")
if err := client.Stop(ctx); err != nil {
log.Printf("stop user sync client failed,err:%v", err)
} else {
log.Printf("stop user sync client success")
}
}()
client.SetCallUserListener(func(ctx context.Context, event *UserEvent) error {
marshal, _ := json.Marshal(event)
log.Printf("user event: %s", marshal)
return nil
})
// 优雅的关闭 监听 ctrl + c
ch := make(chan os.Signal, 1)
signal.Notify(ch, os.Interrupt, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGUSR1)
<-ch
}

55
event_types.go Normal file
View File

@ -0,0 +1,55 @@
package user_core_client
import "time"
// Event 接口定义了所有事件应具备的基本元数据
type Event interface {
GetEventID() string // 获取事件唯一ID
GetEventType() string // 获取事件类型
GetTimestamp() time.Time // 获取事件发生时间
}
// User 用户数据模型
type User struct {
ID int64 `json:"id" ` // 主键ID
CreatedAt time.Time `json:"created_at"` // 创建时间
UpdatedAt time.Time `json:"updated_at"` // 更新时间
DeletedAt uint `json:"deleted_at"` // 软删除时间 毫秒 0 表示未删除
EncryptedPhone string `json:"encrypted_phone" ` // 加密后的手机号
Username string `json:"username" ` // 用户名(可重复)
LoginName string `json:"login_name" ` // 登录名
Gender string `json:"gender" ` // 性别 (male 男性, female 女性, unknown 未知)
PhoneMask string `json:"phone_mask" ` // 手机号掩码(前三后四位)
Status string `json:"status" ` // 用户状态enabled/disabled
}
type BaseEvent struct {
EventID string `json:"event_id"` // 事件ID
EventType string `json:"event_type"` // 事件类型
Timestamp time.Time `json:"timestamp"` // 事件时间戳
}
func (b *BaseEvent) GetEventID() string {
return b.EventID
}
func (b *BaseEvent) GetEventType() string {
return b.EventType
}
func (b *BaseEvent) GetTimestamp() time.Time {
return b.Timestamp
}
// UserEvent 用户事件结构
// 用于在Kafka消息中传输用户相关事件
type UserEvent struct {
BaseEvent
PlatformCode string `json:"platform_code"` // 平台代码
UserID int64 `json:"user_id"` // 用户ID
Username string `json:"username"` // 用户名
UserData *User `json:"user_data,omitempty"` // 用户数据,改为具体类型
}

14
go.mod Normal file
View File

@ -0,0 +1,14 @@
module user_core/client
go 1.23.0
require github.com/segmentio/kafka-go v0.4.48
require (
github.com/klauspost/compress v1.18.0 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
golang.org/x/text v0.28.0 // indirect
)

71
go.sum Normal file
View File

@ -0,0 +1,71 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

51
logger.go Normal file
View File

@ -0,0 +1,51 @@
package user_core_client
import (
"fmt"
"io"
"log"
"os"
"sync/atomic"
)
type Logger interface {
Printf(format string, v ...any)
}
func NewLogger() *SdkLogger {
return &SdkLogger{
logEnabled: atomic.Bool{},
logger: log.New(io.Discard, "user-core-sdk", log.LstdFlags|log.Lshortfile),
}
}
type SdkLogger struct {
logEnabled atomic.Bool
logger *log.Logger
}
func (l *SdkLogger) Debug(debug bool) *SdkLogger {
if debug {
l.logger.SetOutput(os.Stdout)
} else {
l.logger.SetOutput(io.Discard)
}
l.logEnabled.Store(debug)
return l
}
func (l *SdkLogger) SetOutput(w io.Writer) *SdkLogger {
if w == nil {
l.logger.SetOutput(io.Discard)
} else {
l.logger.SetOutput(w)
}
return l
}
func (l *SdkLogger) Printf(format string, v ...any) {
if !l.logEnabled.Load() {
return
}
_ = l.logger.Output(2, fmt.Sprintf(format, v...))
}