95 lines
2.1 KiB
Go
95 lines
2.1 KiB
Go
package importfunc
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"data_prep/dbstruct"
|
|
"data_prep/mongo"
|
|
)
|
|
|
|
func ImportUserIdMap() {
|
|
|
|
mcli, err := mongo.NewMongo()
|
|
if err != nil {
|
|
fmt.Printf("mongo client init fail : %v", err)
|
|
return
|
|
}
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 100; i++ {
|
|
pathurl := fmt.Sprintf("/app/data_prep/file/user_id_map%v.txt", i)
|
|
go func(i int) {
|
|
fmt.Printf("Importing %dth", i)
|
|
infile, err := os.Open(pathurl)
|
|
if err != nil {
|
|
fmt.Printf("Open File Err : %v", err)
|
|
}
|
|
reader := bufio.NewReader(infile)
|
|
userIdMaps := make([]*dbstruct.UserIdMap, 0)
|
|
for {
|
|
str, err := reader.ReadString('\n')
|
|
if len(str) == 0 {
|
|
continue
|
|
}
|
|
strs := strings.Split(str, ":")
|
|
seq, _ := strconv.Atoi(strs[0])
|
|
userId, _ := strconv.Atoi(strs[1])
|
|
userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{
|
|
Seq: int64(seq),
|
|
UserId: int64(userId),
|
|
})
|
|
if len(userIdMaps) == 1000 {
|
|
err := mcli.CreateMappedUserIds(ctx, userIdMaps)
|
|
if err != nil {
|
|
fmt.Printf("CreateMappedUserIds err :%v", err)
|
|
}
|
|
}
|
|
if err == io.EOF {
|
|
err := mcli.CreateMappedUserIds(ctx, userIdMaps)
|
|
if err != nil {
|
|
fmt.Printf("CreateMappedUserIds err :%v", err)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
infile.Close()
|
|
fmt.Printf("%dth imported", i)
|
|
}(i)
|
|
}
|
|
|
|
fmt.Printf("Import into test success")
|
|
}
|
|
|
|
func ImportAsList(mcli *mongo.Mongo, ctx context.Context, list []int64, s int) {
|
|
fmt.Printf("Importing %dth", s)
|
|
userIdMaps := make([]*dbstruct.UserIdMap, 0)
|
|
for i := range list {
|
|
seq := int64(1000000) + int64(i)
|
|
userId := list[i]
|
|
userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{
|
|
Seq: int64(seq),
|
|
UserId: int64(userId),
|
|
})
|
|
if len(userIdMaps) == 10000 {
|
|
err := mcli.CreateMappedUserIds(ctx, userIdMaps)
|
|
if err != nil {
|
|
fmt.Printf("CreateMappedUserIds err :%v", err)
|
|
}
|
|
userIdMaps = make([]*dbstruct.UserIdMap, 0)
|
|
}
|
|
}
|
|
if len(userIdMaps) > 0 {
|
|
err := mcli.CreateMappedUserIds(ctx, userIdMaps)
|
|
if err != nil {
|
|
fmt.Printf("CreateMappedUserIds err :%v", err)
|
|
}
|
|
}
|
|
fmt.Printf("%dth imported", s)
|
|
}
|