dtp/cmd/main.go

package main

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/Leufolium/test/mongo"
)

func main() {
	fmt.Println("Start importing...")

	client, err := mongo.NewMongo()
	if err != nil {
		fmt.Printf("mongo client init fail : %v", err)
		return
	}

	infilePath := "/app/dataprep/file/outfile.txt"

	infile, err := os.Open(infilePath)
	if err != nil {
		fmt.Printf("Open File Err : %v", err)
	}

	defer infile.Close()

	reader := bufio.NewReader(infile)

	list := make([]*mongo.UserIdMap, 0)

	for {
		str, err := reader.ReadString('\n')
		if err == io.EOF {
			err := client.CreateBatch(context.Background(), list)
			if err != nil {
				fmt.Printf("CreateBatch fail : %v", err)
				return
			}
			if len(list) > 0 {
				fmt.Printf("%v\n", list[len(list)-1])
			}

			break
		}
		if err != nil {
			fmt.Printf("ReadString fail : %v", err)
			return
		}

		strs := strings.Split(str, ":")
		seqStr := strs[0]
		userIdStr := strs[1]
		userIdStr = strings.ReplaceAll(userIdStr, "\n", "")
		userIdStr = strings.ReplaceAll(userIdStr, "\r", "")

		seq, err := strconv.Atoi(seqStr)
		if err != nil {
			fmt.Printf("atoi fail : %v", err)
			return
		}
		userId, err := strconv.Atoi(userIdStr)
		if err != nil {
			fmt.Printf("atoi fail : %v", err)
			return
		}

		list = append(list, &mongo.UserIdMap{
			Seq:    int64(seq),
			UserId: int64(userId),
		})

		if len(list) == 10000 {
			err := client.CreateBatch(context.Background(), list)
			if err != nil {
				fmt.Printf("CreateBatch fail : %v", err)
				return
			}
			fmt.Printf("%v\n", list[9999])
			list = make([]*mongo.UserIdMap, 0)
		}
	}

	fmt.Println("End importing...")
}