92 lines
1.7 KiB
Go
92 lines
1.7 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/Leufolium/test/mongo"
|
|
)
|
|
|
|
func main() {
|
|
fmt.Println("Start importing...")
|
|
|
|
client, err := mongo.NewMongo()
|
|
if err != nil {
|
|
fmt.Printf("mongo client init fail : %v", err)
|
|
return
|
|
}
|
|
|
|
infilePath := "/app/dataprep/test/file/outfile.txt"
|
|
//infilePath := "/Users/PC/test/file/outfile.txt"
|
|
|
|
infile, err := os.Open(infilePath)
|
|
if err != nil {
|
|
fmt.Printf("Open File Err : %v", err)
|
|
}
|
|
|
|
defer infile.Close()
|
|
|
|
reader := bufio.NewReader(infile)
|
|
|
|
list := make([]*mongo.UserIdMap, 0)
|
|
|
|
for {
|
|
str, err := reader.ReadString('\n')
|
|
if err == io.EOF {
|
|
err := client.CreateBatch(context.Background(), list)
|
|
if err != nil {
|
|
fmt.Printf("CreateBatch fail : %v", err)
|
|
return
|
|
}
|
|
if len(list) > 0 {
|
|
fmt.Printf("%v\n", list[len(list)-1])
|
|
}
|
|
|
|
break
|
|
}
|
|
if err != nil {
|
|
fmt.Printf("ReadString fail : %v", err)
|
|
return
|
|
}
|
|
|
|
strs := strings.Split(str, ":")
|
|
seqStr := strs[0]
|
|
userIdStr := strs[1]
|
|
userIdStr = strings.ReplaceAll(userIdStr, "\n", "")
|
|
userIdStr = strings.ReplaceAll(userIdStr, "\r", "")
|
|
|
|
seq, err := strconv.Atoi(seqStr)
|
|
if err != nil {
|
|
fmt.Printf("atoi fail : %v", err)
|
|
return
|
|
}
|
|
userId, err := strconv.Atoi(userIdStr)
|
|
if err != nil {
|
|
fmt.Printf("atoi fail : %v", err)
|
|
return
|
|
}
|
|
|
|
list = append(list, &mongo.UserIdMap{
|
|
Seq: int64(seq),
|
|
UserId: int64(userId),
|
|
})
|
|
|
|
if len(list) == 10000 {
|
|
err := client.CreateBatch(context.Background(), list)
|
|
if err != nil {
|
|
fmt.Printf("CreateBatch fail : %v", err)
|
|
return
|
|
}
|
|
fmt.Printf("%v\n", list[9999])
|
|
list = make([]*mongo.UserIdMap, 0)
|
|
}
|
|
}
|
|
|
|
fmt.Println("End importing...")
|
|
}
|