diff --git a/importfunc/generate_user_id_map.go b/importfunc/generate_user_id_map.go index 6256efc..811c145 100644 --- a/importfunc/generate_user_id_map.go +++ b/importfunc/generate_user_id_map.go @@ -10,28 +10,14 @@ import ( func GenerateUserIdMap() { list := make([]int64, 0) - for i := int64(1000000); i < 100000000; i++ { + for i := int64(1000000); i < 10000000; i++ { list = append(list, i) } - rand.Shuffle(99000000, func(i, j int) { + rand.Shuffle(9000000, func(i, j int) { list[i], list[j] = list[j], list[i] }) - j := 1 - for i := 0; i < len(list); i += 1000000 { - lower := i - upper := i + 999999 - if upper >= len(list) { - upper = len(list) - 1 - } - WriteAsList(list[lower:upper], j) - j++ - } - -} - -func WriteAsList(list []int64, seq int) { - pathurl := fmt.Sprintf("/Users/Caeru/Desktop/file/user_id_map%v.txt", seq) + pathurl := "/Users/Caeru/Desktop/file/user_id_map.txt" outfile, err := os.OpenFile(pathurl, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { fmt.Printf("Open File Err : %v", err) @@ -42,8 +28,7 @@ func WriteAsList(list []int64, seq int) { writer := bufio.NewWriter(outfile) for i := range list { - outfile.WriteString(fmt.Sprintf("%v:%v\n", 1000000*seq+i, list[i])) + outfile.WriteString(fmt.Sprintf("%v:%v\n", 1000000+i, list[i])) } writer.Flush() - fmt.Printf("%dth file has generated\n", seq) } diff --git a/importfunc/import_user_id_map.go b/importfunc/import_user_id_map.go index b2e0a2d..db4bd3d 100644 --- a/importfunc/import_user_id_map.go +++ b/importfunc/import_user_id_map.go @@ -1,10 +1,13 @@ package importfunc import ( + "bufio" "context" "fmt" - "math/rand" - "time" + "io" + "os" + "strconv" + "strings" "data_prep/dbstruct" "data_prep/mongo" @@ -21,18 +24,37 @@ func ImportUserIdMap() { userIdMaps := make([]*dbstruct.UserIdMap, 0) - // 打乱 - list := make([]int64, 0) - for i := int64(1000000); i < 10000000; i++ { - list = append(list, i) + pathurl := "/app/file/user_id_map.txt" + infile, err := os.Open(pathurl) + if err != nil { + fmt.Printf("Open File Err : %v", err) + return } - rand.Shuffle(9900000, func(i, j int) { - list[i], list[j] = list[j], list[i] - }) + reader := bufio.NewReader(infile) - for i := range list { - seq := int64(1000000) + int64(i) - userId := list[i] + for { + str, err := reader.ReadString('\n') + if len(str) == 0 { + continue + } + if err == io.EOF { + err := mcli.CreateMappedUserIds(ctx, userIdMaps) + if err != nil { + fmt.Printf("CreateMappedUserIds err :%v", err) + } + break + } + str = strings.ReplaceAll(str, "\n", "") + str = strings.ReplaceAll(str, "\r", "") + strs := strings.Split(str, ":") + seq, err := strconv.Atoi(strs[0]) + if err != nil { + fmt.Printf("Atoi err :%v", err) + } + userId, err := strconv.Atoi(strs[1]) + if err != nil { + fmt.Printf("Atoi err :%v", err) + } userIdMaps = append(userIdMaps, &dbstruct.UserIdMap{ Seq: int64(seq), UserId: int64(userId), @@ -43,16 +65,9 @@ func ImportUserIdMap() { fmt.Printf("CreateMappedUserIds err :%v", err) } userIdMaps = make([]*dbstruct.UserIdMap, 0) - time.Sleep(time.Second) - } - } - - if len(userIdMaps) > 0 { - err := mcli.CreateMappedUserIds(ctx, userIdMaps) - if err != nil { - fmt.Printf("CreateMappedUserIds err :%v", err) } } + infile.Close() fmt.Printf("Import into test success") }