1

importfunc/generate_user_id_map.go

@@ -10,28 +10,14 @@ import (
 func GenerateUserIdMap() {
 
 	list := make([]int64, 0)
-	for i := int64(1000000); i < 100000000; i++ {
+	for i := int64(1000000); i < 10000000; i++ {
 		list = append(list, i)
 	}
-	rand.Shuffle(99000000, func(i, j int) {
+	rand.Shuffle(9000000, func(i, j int) {
 		list[i], list[j] = list[j], list[i]
 	})
 
-	j := 1
-	for i := 0; i < len(list); i += 1000000 {
-		lower := i
-		upper := i + 999999
-		if upper >= len(list) {
-			upper = len(list) - 1
-		}
-		WriteAsList(list[lower:upper], j)
-		j++
-	}
-
-}
-
-func WriteAsList(list []int64, seq int) {
-	pathurl := fmt.Sprintf("/Users/Caeru/Desktop/file/user_id_map%v.txt", seq)
+	pathurl := "/Users/Caeru/Desktop/file/user_id_map.txt"
 	outfile, err := os.OpenFile(pathurl, os.O_WRONLY|os.O_CREATE, 0666)
 	if err != nil {
 		fmt.Printf("Open File Err : %v", err)
@@ -42,8 +28,7 @@ func WriteAsList(list []int64, seq int) {
 	writer := bufio.NewWriter(outfile)
 
 	for i := range list {
-		outfile.WriteString(fmt.Sprintf("%v:%v\n", 1000000*seq+i, list[i]))
+		outfile.WriteString(fmt.Sprintf("%v:%v\n", 1000000+i, list[i]))
 	}
 	writer.Flush()
-	fmt.Printf("%dth file has generated\n", seq)
 }

importfunc/import_user_id_map.go

@@ -1,10 +1,13 @@
 package importfunc
 
 import (
+	"bufio"
 	"context"
 	"fmt"
-	"math/rand"
-	"time"
+	"io"
+	"os"
+	"strconv"
+	"strings"
 
 	"data_prep/dbstruct"
 	"data_prep/mongo"
@@ -21,18 +24,37 @@ func ImportUserIdMap() {
 
 	userIdMaps := make([]*dbstruct.UserIdMap, 0)
 
-	// 打乱
-	list := make([]int64, 0)
-	for i := int64(1000000); i < 10000000; i++ {
-		list = append(list, i)
+	pathurl := "/app/file/user_id_map.txt"
+	infile, err := os.Open(pathurl)
+	if err != nil {
+		fmt.Printf("Open File Err : %v", err)
+		return
 	}
-	rand.Shuffle(9900000, func(i, j int) {
-		list[i], list[j] = list[j], list[i]
-	})
+	reader := bufio.NewReader(infile)
 
-	for i := range list {
-		seq := int64(1000000) + int64(i)
-		userId := list[i]
+	for {
+		str, err := reader.ReadString('\n')
+		if len(str) == 0 {
+			continue
+		}
+		if err == io.EOF {
+			err := mcli.CreateMappedUserIds(ctx, userIdMaps)
+			if err != nil {
+				fmt.Printf("CreateMappedUserIds err :%v", err)
+			}
+			break
+		}
+		str = strings.ReplaceAll(str, "\n", "")
+		str = strings.ReplaceAll(str, "\r", "")
+		strs := strings.Split(str, ":")
+		seq, err := strconv.Atoi(strs[0])
+		if err != nil {
+			fmt.Printf("Atoi err :%v", err)
+		}
+		userId, err := strconv.Atoi(strs[1])
+		if err != nil {
+			fmt.Printf("Atoi err :%v", err)
+		}
 		userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{
 			Seq:    int64(seq),
 			UserId: int64(userId),
@@ -43,16 +65,9 @@ func ImportUserIdMap() {
 				fmt.Printf("CreateMappedUserIds err :%v", err)
 			}
 			userIdMaps = make([]*dbstruct.UserIdMap, 0)
-			time.Sleep(time.Second)
-		}
-	}
-
-	if len(userIdMaps) > 0 {
-		err := mcli.CreateMappedUserIds(ctx, userIdMaps)
-		if err != nil {
-			fmt.Printf("CreateMappedUserIds err :%v", err)
 		}
 	}
+	infile.Close()
 
 	fmt.Printf("Import into test success")
 }