This commit is contained in:
Leufolium 2024-07-26 22:47:38 +08:00
parent 7f7005a762
commit e3ca244313
5 changed files with 928 additions and 16 deletions

View File

@ -1,36 +1,679 @@
package main
import (
"bufio"
"context"
"encoding/base64"
"fmt"
"strings"
"os"
"github.com/Leufolium/test/es"
"github.com/mozillazg/go-pinyin"
myCrypto "github.com/Leufolium/test/crypto"
"github.com/Leufolium/test/dbstruct"
"github.com/Leufolium/test/mongo"
)
func main() {
fmt.Println("Start importing...")
client, err := es.NewElasticSearch()
client, err := mongo.NewMongo()
if err != nil {
fmt.Printf("es client init fail : %v", err)
fmt.Printf("mongo client init fail : %v", err)
return
}
ctx := context.Background()
list, err := client.GetStreamerAcctList(ctx)
if err != nil {
fmt.Printf("GetStreamerAcctList err :%v", err)
mids := []int64{
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
14,
15,
16,
17,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
34,
35,
36,
37,
38,
44,
48,
50,
58,
61,
62,
63,
64,
65,
66,
67,
68,
69,
71,
72,
73,
74,
75,
76,
77,
78,
79,
81,
82,
83,
85,
87,
88,
89,
91,
92,
93,
94,
95,
96,
97,
98,
99,
100,
101,
102,
103,
104,
105,
106,
107,
109,
110,
111,
112,
113,
115,
116,
117,
118,
119,
120,
121,
122,
123,
124,
125,
127,
131,
132,
133,
134,
135,
136,
137,
138,
139,
140,
141,
142,
144,
145,
146,
147,
148,
149,
150,
151,
159,
245,
283,
447,
493,
501,
507,
750,
779,
876,
897,
898,
996,
1071,
1188,
1258,
1259,
1336,
1365,
1471,
1624,
1649,
1740,
1832,
1850,
1958,
1982,
2391,
2432,
2508,
2608,
2634,
2686,
2895,
2953,
2997,
3169,
3343,
3364,
3367,
3489,
3536,
3940,
4474,
4497,
4865,
5051,
5161,
5545,
5620,
5681,
6069,
6145,
6520,
6856,
7520,
7875,
8184,
9429,
10351,
11146,
11583,
12771,
12955,
13208,
13324,
13820,
13862,
15851,
15949,
15966,
16311,
18370,
18488,
19230,
19316,
20475,
21951,
22358,
23152,
24457,
24582,
25058,
25114,
25315,
26283,
29980,
31476,
31551,
33487,
34365,
35042,
35373,
36032,
36540,
37567,
37634,
37939,
38971,
39190,
40441,
42720,
46302,
48900,
49436,
51444,
51609,
52499,
54525,
55108,
57245,
57272,
60553,
60978,
61060,
65028,
66771,
66930,
66961,
67378,
67379,
67383,
67420,
68283,
68559,
69379,
70822,
71287,
76278,
76723,
77377,
79390,
79843,
80077,
82391,
82414,
84663,
85910,
87404,
87907,
89818,
90281,
90356,
91236,
91856,
93963,
97359,
97857,
99796,
100601,
103920,
105604,
143961,
144384,
145309,
145740,
145806,
146166,
146340,
146460,
146545,
147322,
148509,
152001,
153632,
158395,
159566,
161609,
162710,
162816,
163374,
163608,
163660,
164533,
164630,
166037,
167340,
169095,
172477,
172731,
173051,
173304,
174440,
175834,
176342,
177637,
178210,
179782,
180889,
182308,
184293,
185893,
187760,
190618,
191324,
192434,
192549,
193781,
194448,
195763,
197811,
199188,
202826,
204372,
205940,
206164,
206580,
208778,
209473,
210852,
212561,
213014,
213554,
214911,
215008,
215135,
215240,
217606,
217860,
218851,
220411,
226924,
227196,
228265,
228359,
229743,
230965,
239368,
240225,
242263,
243225,
243875,
249042,
252780,
254325,
254587,
258846,
259111,
259476,
263758,
263921,
264877,
267713,
267884,
268493,
268954,
269339,
269583,
269632,
270017,
270232,
270602,
270888,
273575,
274697,
275013,
275732,
276039,
276071,
276120,
276466,
279590,
280318,
280544,
293390,
293753,
296457,
296564,
300771,
301759,
302200,
308732,
308873,
310356,
311294,
312807,
317098,
317333,
320610,
322645,
323517,
326483,
327391,
328717,
329947,
330130,
330205,
330649,
335318,
335373,
335577,
335736,
337587,
337626,
337829,
338334,
338605,
341103,
341351,
341834,
342076,
342468,
342969,
344012,
344664,
344844,
345300,
345521,
345859,
346732,
347900,
348187,
348500,
348616,
348991,
349805,
351513,
352007,
352353,
352787,
352921,
352924,
353051,
353654,
353681,
353921,
354058,
354125,
354236,
355229,
356071,
356408,
356641,
356737,
357096,
357146,
357181,
357669,
358886,
359048,
359406,
360080,
360323,
360412,
361340,
365051,
366194,
367375,
367849,
370070,
370793,
371145,
371600,
373789,
374749,
376093,
376243,
377767,
378027,
378518,
378891,
379005,
379253,
380316,
380647,
380765,
381577,
382101,
382874,
383225,
386910,
387243,
388957,
389643,
389928,
390102,
391886,
392278,
392896,
393152,
393298,
393529,
396777,
397554,
399206,
399212,
399876,
401922,
402096,
404583,
404820,
406241,
406445,
408607,
408708,
408893,
409130,
409145,
409201,
410700,
412300,
412533,
412764,
412769,
414419,
414568,
414570,
414996,
416251,
418307,
420413,
421224,
421229,
421408,
421498,
421672,
421876,
421951,
422423,
424452,
425002,
425180,
425534,
425674,
426274,
426738,
427317,
429542,
429897,
430335,
430734,
431977,
432362,
432447,
432893,
432959,
435296,
436833,
436883,
437166,
437347,
437450,
437526,
437934,
437955,
438312,
438643,
439926,
440514,
440977,
441008,
441744,
441793,
441883,
443406,
445007,
445068,
445162,
445215,
445553,
445716,
445860,
446143,
446204,
446887,
447665,
447925,
449525,
449933,
450124,
450203,
450388,
450488,
450739,
450832,
450834,
451329,
451333,
451444,
451513,
451771,
454716,
454721,
455290,
455709,
456325,
456385,
456449,
456473,
456921,
457540,
458707,
459369,
459482,
459716,
459920,
460252,
460716,
}
fmt.Printf("总共%v个", len(list))
for _, acct := range list {
acct.PinYin = strings.Join(pinyin.LazyConvert(acct.Name, nil), "")
err := client.UpdateStreamerAcct(ctx, acct)
if err != nil {
fmt.Printf("UpdateStreamerAcct err :%v", err)
}
accounts, err := client.GetAccountListByMids(ctx, mids)
if err != nil {
fmt.Printf("GetAccountListByMids fail, err :%v\n", err)
}
zones, err := client.GetZoneListByMids(ctx, mids)
if err != nil {
fmt.Printf("GetZoneListByMids fail, err :%v\n", err)
}
outfilePath := "/app/dataprep/outfile.txt"
outfile, err := os.OpenFile(outfilePath, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
fmt.Printf("Open File Err : %v", err)
}
defer outfile.Close()
writer := bufio.NewWriter(outfile)
cryptoService := &myCrypto.CryptoService{}
cryptoService.Init()
zoneMp := make(map[int64]*dbstruct.Zone, 0)
for _, zone := range zones {
zoneMp[zone.GetMid()] = zone
}
for _, account := range accounts {
base64DecryptedBytes, _ := base64.StdEncoding.DecodeString(*account.MobilePhone)
phone, _ := cryptoService.DecryptByAES(base64DecryptedBytes)
phoneStr := string(phone)
mid := *account.Mid
name := *account.Name
userId := *account.UserId
zoneMomentCount, _ := client.GetZoneMomentCountByMid(ctx, mid)
str := fmt.Sprintf("%v %v %v %v %v\n", mid, name, userId, phoneStr, zoneMomentCount)
writer.WriteString(str)
}
fmt.Println("End importing...")

162
crypto/cryptoService.go Normal file
View File

@ -0,0 +1,162 @@
package myCrypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"os"
"sync"
)
var (
instance *CryptoService
once sync.Once
)
func CryptoServiceInstance() *CryptoService {
once.Do(func() {
instance = &CryptoService{}
})
return instance
}
type CryptoService struct {
rsaPriKey *rsa.PrivateKey
rsaPubKey *rsa.PublicKey
aesPriKey []byte
}
func (cryptoService *CryptoService) Init() {
cryptoService.readAESPriKey()
}
func (cryptoService *CryptoService) EncryptByRSA(msg []byte) (encryptedBytes []byte, err error) {
//公钥加密
encryptedBytes, err = rsa.EncryptPKCS1v15(rand.Reader, cryptoService.rsaPubKey, msg)
if err != nil {
fmt.Printf("EncryptByRSA failed!, err: %v", err)
return
}
return
}
func (cryptoService *CryptoService) DecryptByRSA(encryptedBytes []byte) (decryptedBytes []byte, err error) {
//私钥解密
decryptedBytes, err = rsa.DecryptPKCS1v15(rand.Reader, cryptoService.rsaPriKey, encryptedBytes)
if err != nil {
fmt.Printf("DecryptByRSA failed!, err: %v", err)
return
}
return
}
func (cryptoService *CryptoService) EncryptByAES(msg []byte) (encryptedBytes []byte, err error) {
//CBC加密
block, err := aes.NewCipher(cryptoService.aesPriKey) //block
if err != nil {
fmt.Printf("AES NewCipher failed!, err: %v", err)
return nil, err
}
blockSize := block.BlockSize() //密钥块长度
paddedData := cryptoService.pkcs5Padding(msg, blockSize) //填充
blockMode := cipher.NewCBCEncrypter(block, cryptoService.aesPriKey[:blockSize]) //加密模式
encryptedBytes = make([]byte, len(paddedData)) //创建数组
blockMode.CryptBlocks(encryptedBytes, paddedData) //加密
return
}
func (cryptoService *CryptoService) DecryptByAES(encryptedBytes []byte) (decryptedBytes []byte, err error) {
//CBC解密
block, _ := aes.NewCipher(cryptoService.aesPriKey) // 分组秘钥
blockSize := block.BlockSize() // 获取秘钥块的长度
blockMode := cipher.NewCBCDecrypter(block, cryptoService.aesPriKey[:blockSize]) // 加密模式
decryptedBytes = make([]byte, len(encryptedBytes)) // 创建数组
blockMode.CryptBlocks(decryptedBytes, encryptedBytes) // 解密
decryptedBytes = pkcs5UnPadding(decryptedBytes) // 去除补全码
return
}
func (cryptoService *CryptoService) readRSAPubKey() (err error) {
//读取公钥
file, err := os.Open("public.pem")
if err != nil {
fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err)
return
}
fileInfo, err := file.Stat()
if err != nil {
fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err)
return
}
rsaPubKeyBytes := make([]byte, fileInfo.Size())
file.Read(rsaPubKeyBytes) // 读取公钥文件内容
file.Close()
// 2 pem解码
block, _ := pem.Decode(rsaPubKeyBytes) // 解码
// 3 x509解析得到公钥
pubInterface, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err)
return
}
// 断言类型转换
cryptoService.rsaPubKey = pubInterface.(*rsa.PublicKey)
return
}
func (cryptoService *CryptoService) readRSAPriKey() (err error) {
//读取私钥
file, err := os.Open("private.pem")
if err != nil {
fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err)
return
}
fileInfo, err := file.Stat()
if err != nil {
fmt.Printf("Decoding rsa public key failed, check your config, please!, err: %v", err)
return
}
rsaPriKeyBytes := make([]byte, fileInfo.Size())
file.Read(rsaPriKeyBytes)
file.Close()
block, _ := pem.Decode(rsaPriKeyBytes)
fmt.Printf("私钥:\n")
fmt.Printf("%v", base64.StdEncoding.EncodeToString(block.Bytes))
rsaPriKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
fmt.Printf("Decoding rsa primary key failed, check your config, please!, err: %v", err)
return
}
cryptoService.rsaPriKey = rsaPriKey
return
}
func (cryptoService *CryptoService) readAESPriKey() (aesPriKey []byte) {
//读取私钥
aesPriKey = []byte("Xbz1145141919810")
return
}
func (cryptoService *CryptoService) pkcs5Padding(ciphertext []byte, blockSize int) []byte {
padding := blockSize - len(ciphertext)%blockSize //判断缺少几位长度。最少1最多 blockSize
padtext := bytes.Repeat([]byte{byte(padding)}, padding) //补足位数。把切片[]byte{byte(padding)}复制padding个
return append(ciphertext, padtext...)
}
func pkcs5UnPadding(origData []byte) []byte {
length := len(origData)
unpadding := int(origData[length-1])
return origData[:(length - unpadding)]
}

43
dbstruct/zone.go Normal file
View File

@ -0,0 +1,43 @@
package dbstruct
type Zone struct {
Id *int64 `json:"id" bson:"_id"` // 空间表id
Mid *int64 `json:"mid" bson:"mid"` // 用户表id
Profile *string `json:"profile" bson:"profile"` // 空间简介
LastZoneMomentCt *int64 `json:"last_zone_moment_ct" bson:"last_zone_moment_ct"` // 最后空间动态创建时间
ZoneMomentCount *int64 `json:"zone_moment_count" bson:"zone_moment_count"` // 空间内动态总数
ImageCount *int64 `json:"image_count" bson:"image_count"` // 空间内图片总数
VideoCount *int64 `json:"video_count" bson:"video_count"` // 空间内视频总数
IsZoneThirdPartnerHided *int64 `json:"is_zone_third_partner_hided" bson:"is_zone_third_partner_hided"` // 是否隐藏空间代运营
Ct *int64 `json:"ct" bson:"ct"` // 创建时间
Ut *int64 `json:"ut" bson:"ut"` // 更新时间
DelFlag *int64 `json:"del_flag" bson:"del_flag"` // 删除标记
}
func (p *Zone) GetId() int64 {
if p != nil && p.Id != nil {
return *p.Id
}
return 0
}
func (p *Zone) GetMid() int64 {
if p != nil && p.Mid != nil {
return *p.Mid
}
return 0
}
func (p *Zone) GetLastZoneMomentCt() int64 {
if p != nil && p.LastZoneMomentCt != nil {
return *p.LastZoneMomentCt
}
return 0
}
func (p *Zone) GetIsZoneThirdPartnerHided() int64 {
if p != nil && p.IsZoneThirdPartnerHided != nil {
return *p.IsZoneThirdPartnerHided
}
return 0
}

View File

@ -11,7 +11,7 @@ import (
func NewMongoClient() (*qmgo.Client, error) {
clientCfg := &qmgo.Config{
Uri: "mongodb://root:Wishpal2024@dds-bp1a72ede37c55d4242490.mongodb.rds.aliyuncs.com:3717/admin",
Uri: "mongodb://root:Wishpal2024@dds-bp1da1ddd62bede41.mongodb.rds.aliyuncs.com:3717,dds-bp1da1ddd62bede42.mongodb.rds.aliyuncs.com:3717/admin?replicaSet=mgset-77304659",
ConnectTimeoutMS: goproto.Int64(30000),
MaxPoolSize: goproto.Uint64(16),
MinPoolSize: goproto.Uint64(0),

View File

@ -43,6 +43,12 @@ const (
DBLogin = "login"
COLLogin = "login"
DBZone = "zone"
COLZone = "zone"
DBZoneMoment = "zone_moment"
COLZoneMoment = "zone_moment"
)
func (m *Mongo) getColUserIdSeq() *qmgo.Collection {
@ -68,6 +74,16 @@ func (m *Mongo) getColAccountIdSeq() *qmgo.Collection {
return m.clientMix.Database(DBAccountIdSeq).Collection(COLAccountIdSeq)
}
// 空间表
func (m *Mongo) getColZone() *qmgo.Collection {
return m.clientMix.Database(DBZone).Collection(COLZone)
}
// 私密圈动态表
func (m *Mongo) getColZoneMoment() *qmgo.Collection {
return m.clientMix.Database(DBZoneMoment).Collection(COLZoneMoment)
}
func (m *Mongo) CreateBatch(ctx context.Context, userIdSeqs []*UserIdMap) error {
col := m.getColUserIdMap()
_, err := col.InsertMany(ctx, userIdSeqs)
@ -169,6 +185,28 @@ func (m *Mongo) GetAccountList(ctx context.Context) ([]*dbstruct.Account, error)
return list, err
}
func (m *Mongo) GetAccountListByMids(ctx context.Context, mids []int64) ([]*dbstruct.Account, error) {
col := m.getColAccount()
list := make([]*dbstruct.Account, 0)
query := qmgo.M{
"_id": qmgo.M{
"$in": mids,
},
"del_flag": 0,
}
err := col.Find(ctx, query).All(&list)
if err == qmgo.ErrNoSuchDocuments {
err = nil
return nil, err
}
if err != nil {
return nil, err
}
return list, err
}
func (m *Mongo) UpdateAccount(ctx context.Context, account *dbstruct.Account) error {
col := m.getColAccount()
set := qmgo.M{
@ -181,3 +219,29 @@ func (m *Mongo) UpdateAccount(ctx context.Context, account *dbstruct.Account) er
err := col.UpdateId(ctx, *account.Mid, up)
return err
}
func (m *Mongo) GetZoneListByMids(ctx context.Context, mids []int64) ([]*dbstruct.Zone, error) {
col := m.getColZone()
list := make([]*dbstruct.Zone, 0)
query := qmgo.M{
"del_flag": 0,
"mid": qmgo.M{
"$in": mids,
},
}
err := col.Find(ctx, query).All(&list)
if err == qmgo.ErrNoSuchDocuments {
err = nil
return list, err
}
return list, err
}
func (m *Mongo) GetZoneMomentCountByMid(ctx context.Context, mid int64) (int64, error) {
col := m.getColZoneMoment()
query := qmgo.M{
"mid": mid,
"del_flag": 0,
}
return col.Find(ctx, query).Count()
}