diff --git a/cmd/main.go b/cmd/main.go index 8cd5e17..b5e54e8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,36 +1,679 @@ package main import ( + "bufio" "context" + "encoding/base64" "fmt" - "strings" + "os" - "github.com/Leufolium/test/es" - "github.com/mozillazg/go-pinyin" + myCrypto "github.com/Leufolium/test/crypto" + "github.com/Leufolium/test/dbstruct" + "github.com/Leufolium/test/mongo" ) func main() { fmt.Println("Start importing...") - client, err := es.NewElasticSearch() + client, err := mongo.NewMongo() if err != nil { - fmt.Printf("es client init fail : %v", err) + fmt.Printf("mongo client init fail : %v", err) return } ctx := context.Background() - list, err := client.GetStreamerAcctList(ctx) - if err != nil { - fmt.Printf("GetStreamerAcctList err :%v", err) + mids := []int64{ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 14, + 15, + 16, + 17, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 34, + 35, + 36, + 37, + 38, + 44, + 48, + 50, + 58, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 81, + 82, + 83, + 85, + 87, + 88, + 89, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 109, + 110, + 111, + 112, + 113, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 127, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 159, + 245, + 283, + 447, + 493, + 501, + 507, + 750, + 779, + 876, + 897, + 898, + 996, + 1071, + 1188, + 1258, + 1259, + 1336, + 1365, + 1471, + 1624, + 1649, + 1740, + 1832, + 1850, + 1958, + 1982, + 2391, + 2432, + 2508, + 2608, + 2634, + 2686, + 2895, + 2953, + 2997, + 3169, + 3343, + 3364, + 3367, + 3489, + 3536, + 3940, + 4474, + 4497, + 4865, + 5051, + 5161, + 5545, + 5620, + 5681, + 6069, + 6145, + 6520, + 6856, + 7520, + 7875, + 8184, + 9429, + 10351, + 11146, + 11583, + 12771, + 12955, + 13208, + 13324, + 13820, + 13862, + 15851, + 15949, + 15966, + 16311, + 18370, + 18488, + 19230, + 19316, + 20475, + 21951, + 22358, + 23152, + 24457, + 24582, + 25058, + 25114, + 25315, + 26283, + 29980, + 31476, + 31551, + 33487, + 34365, + 35042, + 35373, + 36032, + 36540, + 37567, + 37634, + 37939, + 38971, + 39190, + 40441, + 42720, + 46302, + 48900, + 49436, + 51444, + 51609, + 52499, + 54525, + 55108, + 57245, + 57272, + 60553, + 60978, + 61060, + 65028, + 66771, + 66930, + 66961, + 67378, + 67379, + 67383, + 67420, + 68283, + 68559, + 69379, + 70822, + 71287, + 76278, + 76723, + 77377, + 79390, + 79843, + 80077, + 82391, + 82414, + 84663, + 85910, + 87404, + 87907, + 89818, + 90281, + 90356, + 91236, + 91856, + 93963, + 97359, + 97857, + 99796, + 100601, + 103920, + 105604, + 143961, + 144384, + 145309, + 145740, + 145806, + 146166, + 146340, + 146460, + 146545, + 147322, + 148509, + 152001, + 153632, + 158395, + 159566, + 161609, + 162710, + 162816, + 163374, + 163608, + 163660, + 164533, + 164630, + 166037, + 167340, + 169095, + 172477, + 172731, + 173051, + 173304, + 174440, + 175834, + 176342, + 177637, + 178210, + 179782, + 180889, + 182308, + 184293, + 185893, + 187760, + 190618, + 191324, + 192434, + 192549, + 193781, + 194448, + 195763, + 197811, + 199188, + 202826, + 204372, + 205940, + 206164, + 206580, + 208778, + 209473, + 210852, + 212561, + 213014, + 213554, + 214911, + 215008, + 215135, + 215240, + 217606, + 217860, + 218851, + 220411, + 226924, + 227196, + 228265, + 228359, + 229743, + 230965, + 239368, + 240225, + 242263, + 243225, + 243875, + 249042, + 252780, + 254325, + 254587, + 258846, + 259111, + 259476, + 263758, + 263921, + 264877, + 267713, + 267884, + 268493, + 268954, + 269339, + 269583, + 269632, + 270017, + 270232, + 270602, + 270888, + 273575, + 274697, + 275013, + 275732, + 276039, + 276071, + 276120, + 276466, + 279590, + 280318, + 280544, + 293390, + 293753, + 296457, + 296564, + 300771, + 301759, + 302200, + 308732, + 308873, + 310356, + 311294, + 312807, + 317098, + 317333, + 320610, + 322645, + 323517, + 326483, + 327391, + 328717, + 329947, + 330130, + 330205, + 330649, + 335318, + 335373, + 335577, + 335736, + 337587, + 337626, + 337829, + 338334, + 338605, + 341103, + 341351, + 341834, + 342076, + 342468, + 342969, + 344012, + 344664, + 344844, + 345300, + 345521, + 345859, + 346732, + 347900, + 348187, + 348500, + 348616, + 348991, + 349805, + 351513, + 352007, + 352353, + 352787, + 352921, + 352924, + 353051, + 353654, + 353681, + 353921, + 354058, + 354125, + 354236, + 355229, + 356071, + 356408, + 356641, + 356737, + 357096, + 357146, + 357181, + 357669, + 358886, + 359048, + 359406, + 360080, + 360323, + 360412, + 361340, + 365051, + 366194, + 367375, + 367849, + 370070, + 370793, + 371145, + 371600, + 373789, + 374749, + 376093, + 376243, + 377767, + 378027, + 378518, + 378891, + 379005, + 379253, + 380316, + 380647, + 380765, + 381577, + 382101, + 382874, + 383225, + 386910, + 387243, + 388957, + 389643, + 389928, + 390102, + 391886, + 392278, + 392896, + 393152, + 393298, + 393529, + 396777, + 397554, + 399206, + 399212, + 399876, + 401922, + 402096, + 404583, + 404820, + 406241, + 406445, + 408607, + 408708, + 408893, + 409130, + 409145, + 409201, + 410700, + 412300, + 412533, + 412764, + 412769, + 414419, + 414568, + 414570, + 414996, + 416251, + 418307, + 420413, + 421224, + 421229, + 421408, + 421498, + 421672, + 421876, + 421951, + 422423, + 424452, + 425002, + 425180, + 425534, + 425674, + 426274, + 426738, + 427317, + 429542, + 429897, + 430335, + 430734, + 431977, + 432362, + 432447, + 432893, + 432959, + 435296, + 436833, + 436883, + 437166, + 437347, + 437450, + 437526, + 437934, + 437955, + 438312, + 438643, + 439926, + 440514, + 440977, + 441008, + 441744, + 441793, + 441883, + 443406, + 445007, + 445068, + 445162, + 445215, + 445553, + 445716, + 445860, + 446143, + 446204, + 446887, + 447665, + 447925, + 449525, + 449933, + 450124, + 450203, + 450388, + 450488, + 450739, + 450832, + 450834, + 451329, + 451333, + 451444, + 451513, + 451771, + 454716, + 454721, + 455290, + 455709, + 456325, + 456385, + 456449, + 456473, + 456921, + 457540, + 458707, + 459369, + 459482, + 459716, + 459920, + 460252, + 460716, } - fmt.Printf("总共%v个", len(list)) - for _, acct := range list { - acct.PinYin = strings.Join(pinyin.LazyConvert(acct.Name, nil), "") - err := client.UpdateStreamerAcct(ctx, acct) - if err != nil { - fmt.Printf("UpdateStreamerAcct err :%v", err) - } + accounts, err := client.GetAccountListByMids(ctx, mids) + if err != nil { + fmt.Printf("GetAccountListByMids fail, err :%v\n", err) + } + + zones, err := client.GetZoneListByMids(ctx, mids) + if err != nil { + fmt.Printf("GetZoneListByMids fail, err :%v\n", err) + } + + outfilePath := "/app/dataprep/outfile.txt" + outfile, err := os.OpenFile(outfilePath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + fmt.Printf("Open File Err : %v", err) + } + defer outfile.Close() + writer := bufio.NewWriter(outfile) + + cryptoService := &myCrypto.CryptoService{} + cryptoService.Init() + + zoneMp := make(map[int64]*dbstruct.Zone, 0) + for _, zone := range zones { + zoneMp[zone.GetMid()] = zone + } + + for _, account := range accounts { + base64DecryptedBytes, _ := base64.StdEncoding.DecodeString(*account.MobilePhone) + phone, _ := cryptoService.DecryptByAES(base64DecryptedBytes) + phoneStr := string(phone) + mid := *account.Mid + name := *account.Name + userId := *account.UserId + zoneMomentCount, _ := client.GetZoneMomentCountByMid(ctx, mid) + str := fmt.Sprintf("%v %v %v %v %v\n", mid, name, userId, phoneStr, zoneMomentCount) + writer.WriteString(str) } fmt.Println("End importing...") diff --git a/crypto/cryptoService.go b/crypto/cryptoService.go new file mode 100644 index 0000000..4252738 --- /dev/null +++ b/crypto/cryptoService.go @@ -0,0 +1,162 @@ +package myCrypto + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "os" + "sync" +) + +var ( + instance *CryptoService + once sync.Once +) + +func CryptoServiceInstance() *CryptoService { + once.Do(func() { + instance = &CryptoService{} + }) + return instance +} + +type CryptoService struct { + rsaPriKey *rsa.PrivateKey + rsaPubKey *rsa.PublicKey + aesPriKey []byte +} + +func (cryptoService *CryptoService) Init() { + cryptoService.readAESPriKey() +} + +func (cryptoService *CryptoService) EncryptByRSA(msg []byte) (encryptedBytes []byte, err error) { + //公钥加密 + encryptedBytes, err = rsa.EncryptPKCS1v15(rand.Reader, cryptoService.rsaPubKey, msg) + if err != nil { + fmt.Printf("EncryptByRSA failed!, err: %v", err) + return + } + return +} + +func (cryptoService *CryptoService) DecryptByRSA(encryptedBytes []byte) (decryptedBytes []byte, err error) { + //私钥解密 + decryptedBytes, err = rsa.DecryptPKCS1v15(rand.Reader, cryptoService.rsaPriKey, encryptedBytes) + if err != nil { + fmt.Printf("DecryptByRSA failed!, err: %v", err) + return + } + + return +} + +func (cryptoService *CryptoService) EncryptByAES(msg []byte) (encryptedBytes []byte, err error) { + //CBC加密 + block, err := aes.NewCipher(cryptoService.aesPriKey) //block + if err != nil { + fmt.Printf("AES NewCipher failed!, err: %v", err) + return nil, err + } + blockSize := block.BlockSize() //密钥块长度 + paddedData := cryptoService.pkcs5Padding(msg, blockSize) //填充 + blockMode := cipher.NewCBCEncrypter(block, cryptoService.aesPriKey[:blockSize]) //加密模式 + encryptedBytes = make([]byte, len(paddedData)) //创建数组 + blockMode.CryptBlocks(encryptedBytes, paddedData) //加密 + return +} + +func (cryptoService *CryptoService) DecryptByAES(encryptedBytes []byte) (decryptedBytes []byte, err error) { + //CBC解密 + block, _ := aes.NewCipher(cryptoService.aesPriKey) // 分组秘钥 + blockSize := block.BlockSize() // 获取秘钥块的长度 + blockMode := cipher.NewCBCDecrypter(block, cryptoService.aesPriKey[:blockSize]) // 加密模式 + decryptedBytes = make([]byte, len(encryptedBytes)) // 创建数组 + blockMode.CryptBlocks(decryptedBytes, encryptedBytes) // 解密 + decryptedBytes = pkcs5UnPadding(decryptedBytes) // 去除补全码 + return +} + +func (cryptoService *CryptoService) readRSAPubKey() (err error) { + //读取公钥 + file, err := os.Open("public.pem") + if err != nil { + fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err) + return + } + fileInfo, err := file.Stat() + if err != nil { + fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err) + return + } + rsaPubKeyBytes := make([]byte, fileInfo.Size()) + file.Read(rsaPubKeyBytes) // 读取公钥文件内容 + file.Close() + + // 2 pem解码 + block, _ := pem.Decode(rsaPubKeyBytes) // 解码 + + // 3 x509解析得到公钥 + pubInterface, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err) + return + } + // 断言类型转换 + cryptoService.rsaPubKey = pubInterface.(*rsa.PublicKey) + return +} + +func (cryptoService *CryptoService) readRSAPriKey() (err error) { + //读取私钥 + file, err := os.Open("private.pem") + if err != nil { + fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err) + return + } + fileInfo, err := file.Stat() + if err != nil { + fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err) + return + } + rsaPriKeyBytes := make([]byte, fileInfo.Size()) + file.Read(rsaPriKeyBytes) + file.Close() + + block, _ := pem.Decode(rsaPriKeyBytes) + + fmt.Printf("私钥:\n") + fmt.Printf("%v", base64.StdEncoding.EncodeToString(block.Bytes)) + + rsaPriKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + fmt.Printf("Decoding rsa primary key failed, check your config, please!, err: %v", err) + return + } + cryptoService.rsaPriKey = rsaPriKey + return +} + +func (cryptoService *CryptoService) readAESPriKey() (aesPriKey []byte) { + //读取私钥 + aesPriKey = []byte("Xbz1145141919810") + return +} + +func (cryptoService *CryptoService) pkcs5Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize //判断缺少几位长度。最少1,最多 blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) //补足位数。把切片[]byte{byte(padding)}复制padding个 + return append(ciphertext, padtext...) +} + +func pkcs5UnPadding(origData []byte) []byte { + length := len(origData) + unpadding := int(origData[length-1]) + return origData[:(length - unpadding)] +} diff --git a/dbstruct/zone.go b/dbstruct/zone.go new file mode 100644 index 0000000..1b89ec0 --- /dev/null +++ b/dbstruct/zone.go @@ -0,0 +1,43 @@ +package dbstruct + +type Zone struct { + Id *int64 `json:"id" bson:"_id"` // 空间表id + Mid *int64 `json:"mid" bson:"mid"` // 用户表id + Profile *string `json:"profile" bson:"profile"` // 空间简介 + LastZoneMomentCt *int64 `json:"last_zone_moment_ct" bson:"last_zone_moment_ct"` // 最后空间动态创建时间 + ZoneMomentCount *int64 `json:"zone_moment_count" bson:"zone_moment_count"` // 空间内动态总数 + ImageCount *int64 `json:"image_count" bson:"image_count"` // 空间内图片总数 + VideoCount *int64 `json:"video_count" bson:"video_count"` // 空间内视频总数 + IsZoneThirdPartnerHided *int64 `json:"is_zone_third_partner_hided" bson:"is_zone_third_partner_hided"` // 是否隐藏空间代运营 + Ct *int64 `json:"ct" bson:"ct"` // 创建时间 + Ut *int64 `json:"ut" bson:"ut"` // 更新时间 + DelFlag *int64 `json:"del_flag" bson:"del_flag"` // 删除标记 +} + +func (p *Zone) GetId() int64 { + if p != nil && p.Id != nil { + return *p.Id + } + return 0 +} + +func (p *Zone) GetMid() int64 { + if p != nil && p.Mid != nil { + return *p.Mid + } + return 0 +} + +func (p *Zone) GetLastZoneMomentCt() int64 { + if p != nil && p.LastZoneMomentCt != nil { + return *p.LastZoneMomentCt + } + return 0 +} + +func (p *Zone) GetIsZoneThirdPartnerHided() int64 { + if p != nil && p.IsZoneThirdPartnerHided != nil { + return *p.IsZoneThirdPartnerHided + } + return 0 +} diff --git a/mongo/client.go b/mongo/client.go index 8eff837..f868f83 100644 --- a/mongo/client.go +++ b/mongo/client.go @@ -11,7 +11,7 @@ import ( func NewMongoClient() (*qmgo.Client, error) { clientCfg := &qmgo.Config{ - Uri: "mongodb://root:Wishpal2024@dds-bp1a72ede37c55d4242490.mongodb.rds.aliyuncs.com:3717/admin", + Uri: "mongodb://root:Wishpal2024@dds-bp1da1ddd62bede41.mongodb.rds.aliyuncs.com:3717,dds-bp1da1ddd62bede42.mongodb.rds.aliyuncs.com:3717/admin?replicaSet=mgset-77304659", ConnectTimeoutMS: goproto.Int64(30000), MaxPoolSize: goproto.Uint64(16), MinPoolSize: goproto.Uint64(0), diff --git a/mongo/mongo.go b/mongo/mongo.go index 675c5da..b4f42d7 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -43,6 +43,12 @@ const ( DBLogin = "login" COLLogin = "login" + + DBZone = "zone" + COLZone = "zone" + + DBZoneMoment = "zone_moment" + COLZoneMoment = "zone_moment" ) func (m *Mongo) getColUserIdSeq() *qmgo.Collection { @@ -68,6 +74,16 @@ func (m *Mongo) getColAccountIdSeq() *qmgo.Collection { return m.clientMix.Database(DBAccountIdSeq).Collection(COLAccountIdSeq) } +// 空间表 +func (m *Mongo) getColZone() *qmgo.Collection { + return m.clientMix.Database(DBZone).Collection(COLZone) +} + +// 私密圈动态表 +func (m *Mongo) getColZoneMoment() *qmgo.Collection { + return m.clientMix.Database(DBZoneMoment).Collection(COLZoneMoment) +} + func (m *Mongo) CreateBatch(ctx context.Context, userIdSeqs []*UserIdMap) error { col := m.getColUserIdMap() _, err := col.InsertMany(ctx, userIdSeqs) @@ -169,6 +185,28 @@ func (m *Mongo) GetAccountList(ctx context.Context) ([]*dbstruct.Account, error) return list, err } +func (m *Mongo) GetAccountListByMids(ctx context.Context, mids []int64) ([]*dbstruct.Account, error) { + col := m.getColAccount() + list := make([]*dbstruct.Account, 0) + + query := qmgo.M{ + "_id": qmgo.M{ + "$in": mids, + }, + "del_flag": 0, + } + + err := col.Find(ctx, query).All(&list) + if err == qmgo.ErrNoSuchDocuments { + err = nil + return nil, err + } + if err != nil { + return nil, err + } + return list, err +} + func (m *Mongo) UpdateAccount(ctx context.Context, account *dbstruct.Account) error { col := m.getColAccount() set := qmgo.M{ @@ -181,3 +219,29 @@ func (m *Mongo) UpdateAccount(ctx context.Context, account *dbstruct.Account) er err := col.UpdateId(ctx, *account.Mid, up) return err } + +func (m *Mongo) GetZoneListByMids(ctx context.Context, mids []int64) ([]*dbstruct.Zone, error) { + col := m.getColZone() + list := make([]*dbstruct.Zone, 0) + query := qmgo.M{ + "del_flag": 0, + "mid": qmgo.M{ + "$in": mids, + }, + } + err := col.Find(ctx, query).All(&list) + if err == qmgo.ErrNoSuchDocuments { + err = nil + return list, err + } + return list, err +} + +func (m *Mongo) GetZoneMomentCountByMid(ctx context.Context, mid int64) (int64, error) { + col := m.getColZoneMoment() + query := qmgo.M{ + "mid": mid, + "del_flag": 0, + } + return col.Find(ctx, query).Count() +}