From e8649e1111c2d1b3ac5e654b12b013793c3f3e14 Mon Sep 17 00:00:00 2001 From: Leufolium Date: Sun, 28 Apr 2024 21:16:18 +0800 Subject: [PATCH 1/5] BY ROBIN AT 20240428 --- library/contentaudit/imageaudit/imageaudit.go | 53 ++++++++++++------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/library/contentaudit/imageaudit/imageaudit.go b/library/contentaudit/imageaudit/imageaudit.go index 5ea73d74..20a08668 100644 --- a/library/contentaudit/imageaudit/imageaudit.go +++ b/library/contentaudit/imageaudit/imageaudit.go @@ -48,7 +48,7 @@ func Run(batchId string) (successNum int, failNum int, err error) { // 2.创建请求 // oss不在上海的服务器，需要调用Advance接口 - req, err := createScanImageAdvanceRequest(ctrlBlock) + reqs, err := createScanImageAdvanceRequest(ctrlBlock) if err != nil { logger.Info("Create Scan ImageRequest fail: %v", err) handleBatchError(ctrlBlock, err) @@ -60,25 +60,29 @@ func Run(batchId string) (successNum int, failNum int, err error) { runtime := &teautils.RuntimeOptions{ ConnectTimeout: tea.Int(30000), } - _result, err := defaultImageAuditClient.ScanImageAdvance(req, runtime) - if err != nil { - if _t, ok := err.(*tea.SDKError); ok { - logger.Error("ScanImageAdvance fail, errinfo : %v", util.DerefString(_t.Data)) - } - handleBatchError(ctrlBlock, err) - failNum = len(imageaudittasks) - return - } - logger.Info("Receive the response from ScanImageAdvance: %v", _result.String()) - // 4.处理应答 - err = handleScanImageResponse(ctrlBlock, _result) + for _, req := range reqs { + var _result *imageaudit.ScanImageResponse + _result, err = defaultImageAuditClient.ScanImageAdvance(req, runtime) + if err != nil { + if _t, ok := err.(*tea.SDKError); ok { + logger.Error("ScanImageAdvance fail, errinfo : %v", util.DerefString(_t.Data)) + } + handleBatchError(ctrlBlock, err) + failNum = len(imageaudittasks) + return + } + logger.Info("Receive the response from ScanImageAdvance: %v", _result.String()) + + // 4.处理应答 + err = handleScanImageResponse(ctrlBlock, _result) + } successNum = len(imageaudittasks) return } -func createScanImageAdvanceRequest(ctrlBlock *ImageAuditTaskBatchControlBlock) (request *imageaudit.ScanImageAdvanceRequest, err error) { +func createScanImageAdvanceRequest(ctrlBlock *ImageAuditTaskBatchControlBlock) (requests []*imageaudit.ScanImageAdvanceRequest, err error) { httpClient := http.Client{} imageauditIds := ctrlBlock.ImageAuditIds reqTasks := make([]*imageaudit.ScanImageAdvanceRequestTask, 0) @@ -90,7 +94,6 @@ func createScanImageAdvanceRequest(ctrlBlock *ImageAuditTaskBatchControlBlock) ( logger.Error("httpClient Get fail : %v", err) return } - reqTasks = append(reqTasks, &imageaudit.ScanImageAdvanceRequestTask{ DataId: goproto.String(imageauditIds[i]), ImageURLObject: file.Body, @@ -101,10 +104,24 @@ func createScanImageAdvanceRequest(ctrlBlock *ImageAuditTaskBatchControlBlock) ( }) } - request = &imageaudit.ScanImageAdvanceRequest{ - Scene: scenes, - Task: reqTasks, + // 阿里云一次最多审10条，将待审图片按10条拆分 + requests = make([]*imageaudit.ScanImageAdvanceRequest, 0) + i := 0 + for ; i < len(reqTasks)/10; i++ { + request := &imageaudit.ScanImageAdvanceRequest{ + Scene: scenes, + Task: reqTasks[i*10 : (i+1)*10], + } + requests = append(requests, request) } + if i*10 < len(reqTasks) { + request := &imageaudit.ScanImageAdvanceRequest{ + Scene: scenes, + Task: reqTasks[i*10:], + } + requests = append(requests, request) + } + logger.Info("本次打包:%v", reqs) return } From 442b84a0cdec0d9dbb1ba4ea36b622cebf68022f Mon Sep 17 00:00:00 2001 From: Leufolium Date: Sun, 28 Apr 2024 21:26:59 +0800 Subject: [PATCH 2/5] by Robin at 20240428 --- app/mix/service/xxljob_tasks.go | 8 ++++++++ dbstruct/daily_statement.go | 1 + 2 files changed, 9 insertions(+) diff --git a/app/mix/service/xxljob_tasks.go b/app/mix/service/xxljob_tasks.go index 09b00274..9f61a6b7 100644 --- a/app/mix/service/xxljob_tasks.go +++ b/app/mix/service/xxljob_tasks.go @@ -94,6 +94,13 @@ func (s *CronService) CreateDailyStatement(ctx context.Context, param *xxl.RunRe return fmt.Sprintf("query h5 call count fail : %v", err) } + //获取站外user_id访问zone访问量 + zoneListFromOusideCount, err := DefaultScriptsService.QueryCallCount(consts.ZoneListFromOutsideCountCallUrl, logpath) + if err != nil { + logger.Error("query list_by_user_id_from_outside call count fail : %v", err) + return fmt.Sprintf("query list_by_user_id_from_outside call count fail : %v", err) + } + //获取用户总量 accountCount, err := _DefaultAccount.OpCount(&gin.Context{}, &accountproto.OpCountReq{ CtLowerBound: goproto.Int64(int64(consts.AppEnterProductionTime)), @@ -140,6 +147,7 @@ func (s *CronService) CreateDailyStatement(ctx context.Context, param *xxl.RunRe LastHourNewUserCountFromAndroid: goproto.Int64(androidCount), LastHourNewUserCountFromIos: goproto.Int64(iosCount), LastHourNewUserCountFromH5: goproto.Int64(h5Count), + ZoneListFromOutsideCount: goproto.Int64(int64(zoneListFromOusideCount)), StartTime: goproto.Int64(startTimeStamp), EndTime: goproto.Int64(endTimeStamp), } diff --git a/dbstruct/daily_statement.go b/dbstruct/daily_statement.go index bcba203e..a8a16a49 100644 --- a/dbstruct/daily_statement.go +++ b/dbstruct/daily_statement.go @@ -9,6 +9,7 @@ type DailyStatement struct { LastHourNewUserCountFromAndroid *int64 `json:"last_hour_new_user_count_from_android" bson:"last_hour_new_user_count_from_android"` // 安卓注册总量 LastHourNewUserCountFromIos *int64 `json:"last_hour_new_user_count_from_ios" bson:"last_hour_new_user_count_from_ios"` // ios注册总量 LastHourNewUserCountFromH5 *int64 `json:"last_hour_new_user_count_from_h5" bson:"last_hour_new_user_count_from_h5"` // h5注册总量 + ZoneListFromOutsideCount *int64 `json:"zone_list_from_outside_count" bson:"zone_list_from_outside_count"` // 站外user_id查询空间总量 StartTime *int64 `json:"start_time" bson:"start_time"` // 起始时间 EndTime *int64 `json:"end_time" bson:"end_time"` // 结束时间 Ct *int64 `json:"ct" bson:"ct"` // 创建时间 From 24735fa03e519c1977f9650e92f3e2adf93e9ee0 Mon Sep 17 00:00:00 2001 From: Leufolium Date: Sun, 28 Apr 2024 21:32:52 +0800 Subject: [PATCH 3/5] by Robin at 20240428 --- etc/mix/mix-prod.yaml | 2 +- etc/mix/mix-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/mix/mix-prod.yaml b/etc/mix/mix-prod.yaml index ee927ec4..fa2852ce 100644 --- a/etc/mix/mix-prod.yaml +++ b/etc/mix/mix-prod.yaml @@ -75,7 +75,7 @@ apollo: dysmsapi: access_key_id: "LTAI5tMZEAgdr7ZqHQfBF314" access_key_secret: "Qd8Sf94841y22vUjAhXCpLTdo2r0ys" - sign_name: "成都心意到了科技" + sign_name: "铁粉空间" template_code: "SMS_290196756" imageaudit: diff --git a/etc/mix/mix-test.yaml b/etc/mix/mix-test.yaml index cd30bb7f..8b596ce6 100644 --- a/etc/mix/mix-test.yaml +++ b/etc/mix/mix-test.yaml @@ -74,7 +74,7 @@ apollo: dysmsapi: access_key_id: "LTAI5tMZEAgdr7ZqHQfBF314" access_key_secret: "Qd8Sf94841y22vUjAhXCpLTdo2r0ys" - sign_name: "成都心意到了科技" + sign_name: "铁粉空间" template_code: "SMS_290196756" imageaudit: From 217a53d6b216cfaf4229e2683e986fab82876648 Mon Sep 17 00:00:00 2001 From: Leufolium Date: Sun, 28 Apr 2024 21:33:24 +0800 Subject: [PATCH 4/5] by Robin at 20240428 --- api/consts/consts.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/consts/consts.go b/api/consts/consts.go index 8ebf7a24..a371b1b7 100644 --- a/api/consts/consts.go +++ b/api/consts/consts.go @@ -95,5 +95,8 @@ const ReservedUserIdRegexesConfig = PackageRootPath + "/etc/mix/resource/reg_res // H5调用路径 const H5CallUrl = "/api/streamer/list_ext_by_user_id" +// 站外user_id访问zone调用路径 +const ZoneListFromOutsideCountCallUrl = "/api/zone/list_by_user_id_from_outside" + // 应用上线时间，2024-01-02 14:00:00 const AppEnterProductionTime = 1704175200 From 58a238e9c94b1bc24038726b4dd2978450a94a96 Mon Sep 17 00:00:00 2001 From: Leufolium Date: Sun, 28 Apr 2024 21:51:16 +0800 Subject: [PATCH 5/5] by Robin at 20240428 --- library/contentaudit/imageaudit/imageaudit.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/library/contentaudit/imageaudit/imageaudit.go b/library/contentaudit/imageaudit/imageaudit.go index 20a08668..1c9aaf7a 100644 --- a/library/contentaudit/imageaudit/imageaudit.go +++ b/library/contentaudit/imageaudit/imageaudit.go @@ -61,6 +61,7 @@ func Run(batchId string) (successNum int, failNum int, err error) { ConnectTimeout: tea.Int(30000), } + results := make([]*imageaudit.ScanImageResponseBodyDataResults, 0) for _, req := range reqs { var _result *imageaudit.ScanImageResponse _result, err = defaultImageAuditClient.ScanImageAdvance(req, runtime) @@ -72,12 +73,13 @@ func Run(batchId string) (successNum int, failNum int, err error) { failNum = len(imageaudittasks) return } + results = append(results, _result.Body.Data.Results...) logger.Info("Receive the response from ScanImageAdvance: %v", _result.String()) - - // 4.处理应答 - err = handleScanImageResponse(ctrlBlock, _result) } + // 4.处理应答 + err = handleScanImageResponseBodyDataResults(ctrlBlock, results) + successNum = len(imageaudittasks) return } @@ -126,8 +128,7 @@ func createScanImageAdvanceRequest(ctrlBlock *ImageAuditTaskBatchControlBlock) ( return } -func handleScanImageResponse(ctrlBlock *ImageAuditTaskBatchControlBlock, resp *imageaudit.ScanImageResponse) (err error) { - results := resp.Body.Data.Results +func handleScanImageResponseBodyDataResults(ctrlBlock *ImageAuditTaskBatchControlBlock, results []*imageaudit.ScanImageResponseBodyDataResults) (err error) { taskCtrlBlocks := ctrlBlock.TaskCtrlBlocks img2taskIndexMap := ctrlBlock.Img2taskIndexMap actionMap := ctrlBlock.ActionMap