data_prep/importfunc/import_user_id_map.go

package importfunc

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"data_prep/dbstruct"
	"data_prep/mongo"
)

func ImportUserIdMap() {

	mcli, err := mongo.NewMongo()
	if err != nil {
		fmt.Printf("mongo client init fail : %v", err)
		return
	}
	ctx := context.Background()

	for i := 1; i < 100; i++ {
		go func(i int) {
			fmt.Printf("Importing %dth", i)
			pathurl := fmt.Sprintf("/app/data_prep/file/user_id_map%v.txt", i)
			infile, err := os.Open(pathurl)
			if err != nil {
				fmt.Printf("Open File Err : %v", err)
				return
			}
			reader := bufio.NewReader(infile)
			userIdMaps := make([]*dbstruct.UserIdMap, 0)
			for {
				str, err := reader.ReadString('\n')
				if len(str) == 0 {
					continue
				}
				strs := strings.Split(str, ":")
				seq, _ := strconv.Atoi(strs[0])
				userId, _ := strconv.Atoi(strs[1])
				userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{
					Seq:    int64(seq),
					UserId: int64(userId),
				})
				if len(userIdMaps) == 1000 {
					err := mcli.CreateMappedUserIds(ctx, userIdMaps)
					if err != nil {
						fmt.Printf("CreateMappedUserIds err :%v", err)
					}
				}
				if err == io.EOF {
					err := mcli.CreateMappedUserIds(ctx, userIdMaps)
					if err != nil {
						fmt.Printf("CreateMappedUserIds err :%v", err)
					}
					break
				}
			}
			infile.Close()
			fmt.Printf("%dth imported", i)
		}(i)
	}

	fmt.Printf("Import into test success")
}

func ImportAsList(mcli *mongo.Mongo, ctx context.Context, list []int64, s int) {
	fmt.Printf("Importing %dth", s)
	userIdMaps := make([]*dbstruct.UserIdMap, 0)
	for i := range list {
		seq := int64(1000000) + int64(i)
		userId := list[i]
		userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{
			Seq:    int64(seq),
			UserId: int64(userId),
		})
		if len(userIdMaps) == 10000 {
			err := mcli.CreateMappedUserIds(ctx, userIdMaps)
			if err != nil {
				fmt.Printf("CreateMappedUserIds err :%v", err)
			}
			userIdMaps = make([]*dbstruct.UserIdMap, 0)
		}
	}
	if len(userIdMaps) > 0 {
		err := mcli.CreateMappedUserIds(ctx, userIdMaps)
		if err != nil {
			fmt.Printf("CreateMappedUserIds err :%v", err)
		}
	}
	fmt.Printf("%dth imported", s)
}