diff --git a/common/create_bug.go b/common/create_bug.go index 2d5c137..7c53d39 100644 --- a/common/create_bug.go +++ b/common/create_bug.go @@ -12,13 +12,15 @@ import ( const API_tbapi = "http://qa.flatincbr.work:5354" const BugCategory_Anomaly = "monkey异常" +const BugCategory_Anomaly_adr = "monkey异常-安卓端" +const BugCategory_Anomaly_ios = "monkey异常-iOS端" type BugInfo struct { ProjectName string `json:"project_name"` // tb项目名称 Category string `json:"category"` // 提单配置中的缺陷类型 Title string `json:"title"` // tb缺陷的标题 Content string `json:"content"` // tb缺陷的备注 - Pf string `json:"pf"` // 所属平台，除了adr/ios其它不处理 + Platform string `json:"platform"` // 所属平台，除了adr/ios其它不处理 } func NewBug(project_name string) BugInfo { @@ -47,7 +49,7 @@ func (b *BugInfo) SetContent(content string) *BugInfo { // 设置所属平台：adr/ios func (b *BugInfo) SetPlatform(pf string) *BugInfo { - b.Pf = pf + b.Platform = pf return b } diff --git a/common/push.go b/common/push.go index 606c76e..8a02f23 100644 --- a/common/push.go +++ b/common/push.go @@ -27,6 +27,11 @@ func PushDingTalk() { // push.SendActionCardMessage() } +func PushLog(title, str string) { + cli := dingtalk.InitDingTalkWithSecret(Bot_Test_Token, Bot_Test_Secret) + cli.SendMarkDownMessage(title, str) +} + func PushCorntaskLog(str string) { cli := dingtalk.InitDingTalkWithSecret(Bot_Test_Token, Bot_Test_Secret) cli.SendMarkDownMessage("定时任务", str) diff --git a/crontask/monkey.go b/crontask/monkey.go index 2acc7ba..96f2a2e 100644 --- a/crontask/monkey.go +++ b/crontask/monkey.go @@ -47,14 +47,22 @@ func CheckMonkeyTasks() { if strings.Contains(task.Product, "-") { product_name = strings.Split(task.Product, "-")[0] } - var d models.Device - db.Table("device").Model(models.Device{}).Where("project = ? AND product_name = ? AND platform = ? AND status = ?", task.Project, product_name, task.Platform, "online").First(&d) + // 获取product.id + var _p models.QaProduct + db_oms.Table(_p.TableName()).Model(models.QaProduct{}).Where("label = ?", product_name).Last(&_p) + if _p.ID < 1 { + fmt.Println("找不到产品,", product_name) + // 找不到对应的产品 + continue + } + var d models.DeviceV2 + db_oms.Table("qa_devices").Model(models.DeviceV2{}).Where("product_id = ? AND platform = ? AND status = ? AND is_enabled = 1", _p.ID, task.Platform, "online").First(&d) if d.ID < 1 { // 没有空闲设备 continue } // 占用设备 - db.Table("device").Model(models.Device{}).Where("udid = ?", d.Udid).Update("status", "busy") + db_oms.Table("qa_devices").Model(models.DeviceV2{}).Where("udid = ?", d.Udid).Update("status", "busy") go monkey.RunAndroidMonkeyCmd(task, d.Udid) db.Table(task.TableName()).Model(models.MonkeyTask{}).Where("id = ?", task.Id).Update("start_time", time.Now().Unix()) common.PushCorntaskLog("执行Monkey任务：" + task.Project + "-" + d.Udid) @@ -109,6 +117,8 @@ func CheckMonkeyTasks() { list[i].Status = "FINISH" db.Table(v.TableName()).Model(models.MonkeyTask{}).Where("id = ?", v.Id).Update("status", list[i].Status) db.Table(v.TableName()).Model(models.MonkeyTask{}).Where("id = ?", v.Id).Update("end_time", time.Now().Unix()) + // 释放设备 + db_oms.Table("qa_devices").Model(models.DeviceV2{}).Where("udid = ?", v.Devices).Update("status", "online") common.PushCorntaskLog("[autogo] 任务标记为FINISH, 但容器未退出, task_id=" + cast.ToString(v.Id)) } diff --git a/docs/docs.go b/docs/docs.go index 3a232c1..b2f0445 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1353,6 +1353,48 @@ const docTemplate = `{ } } }, + "/api/tool/v1/bigdata/query": { + "get": { + "description": "通过指定一些参数查询打点数据", + "consumes": [ + "application/x-www-form-urlencoded" + ], + "tags": [ + "工具相关 /api/tool/v1/" + ], + "summary": "埋点查询", + "parameters": [ + { + "type": "string", + "description": "产品key", + "name": "project_key", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔", + "name": "keyword", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "请求数量，默认为10", + "name": "count", + "in": "query" + } + ], + "responses": { + "200": { + "description": "返回打点信息", + "schema": { + "$ref": "#/definitions/models.Response" + } + } + } + } + }, "/api/tool/v1/voice/list": { "get": { "description": "根据表单id获取声音评测内容，返回分组和分组文件信息", diff --git a/docs/swagger.json b/docs/swagger.json index 7a55be3..dfa7e4f 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1341,6 +1341,48 @@ } } }, + "/api/tool/v1/bigdata/query": { + "get": { + "description": "通过指定一些参数查询打点数据", + "consumes": [ + "application/x-www-form-urlencoded" + ], + "tags": [ + "工具相关 /api/tool/v1/" + ], + "summary": "埋点查询", + "parameters": [ + { + "type": "string", + "description": "产品key", + "name": "project_key", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔", + "name": "keyword", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "请求数量，默认为10", + "name": "count", + "in": "query" + } + ], + "responses": { + "200": { + "description": "返回打点信息", + "schema": { + "$ref": "#/definitions/models.Response" + } + } + } + } + }, "/api/tool/v1/voice/list": { "get": { "description": "根据表单id获取声音评测内容，返回分组和分组文件信息", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 630dacd..a5f881b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -929,6 +929,34 @@ paths: summary: 更新Bugly凭据 tags: - 设置相关 /api/setting/v2/ + /api/tool/v1/bigdata/query: + get: + consumes: + - application/x-www-form-urlencoded + description: 通过指定一些参数查询打点数据 + parameters: + - description: 产品key + in: query + name: project_key + required: true + type: string + - description: 过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔 + in: query + name: keyword + required: true + type: string + - description: 请求数量，默认为10 + in: query + name: count + type: integer + responses: + "200": + description: 返回打点信息 + schema: + $ref: '#/definitions/models.Response' + summary: 埋点查询 + tags: + - 工具相关 /api/tool/v1/ /api/tool/v1/voice/list: get: consumes: diff --git a/monkey/anomaly.go b/monkey/anomaly.go index 1280f57..6e89f99 100644 --- a/monkey/anomaly.go +++ b/monkey/anomaly.go @@ -126,6 +126,12 @@ func createTbBug(ano models.MonkeyAnomaly) { } defer dbsql.Close(db) + // 先查询当前monkey任务已有的异常 + var list []models.MonkeyAnomaly + db.Table(ano.TableName()).Model(models.MonkeyAnomaly{}).Where("task_id = ?", ano.TaskId).Find(&list) + // 标记当前异常是否已上报过 + is_exist := false + var task models.MonkeyTask db.Model(models.MonkeyTask{}).Where("id = ?", ano.TaskId).Last(&task) if task.Id < 1 { @@ -152,7 +158,7 @@ func createTbBug(ano models.MonkeyAnomaly) { content += "> " + vv + "\n" switch bug_type { case "EXCEPTION": - // 取第一行和第三行内容拼接 + // 取第一行（异常发生未知）和第三行（异常类型）内容拼接 if i == 0 && ii == 0 { _strs := strings.Split(vv, "): ") if len(_strs) > 1 { @@ -162,6 +168,13 @@ func createTbBug(ano models.MonkeyAnomaly) { _strs := strings.Split(vv, "): ") if len(_strs) > 1 { title += _strs[1] + // 比较历史异常中是否已经出现 + for _, v := range list { + if strings.Contains(v.AnomalyInfos, _strs[1]) { + // 相似，不上报了，但还是会存库里 + is_exist = true + } + } } } default: @@ -178,12 +191,24 @@ func createTbBug(ano models.MonkeyAnomaly) { content += "> " + "(...更多请查看Monkey报告)" + "\n" content += "\n" - bug := common.NewBug(task.Project) //task.Project + // 如果已标记当前异常已经上报过，则跳过提单逻辑 + if is_exist { + // 这里可以再额外处理 + common.PushLog("Monkey", "相似异常已存在，不再提单，task_id="+cast.ToString(ano.TaskId)+"\n"+title) + return + } + + // 初始化提单bug对象 + bug := common.NewBug(task.Project) // task.Project bug.SetTitle(title). SetPlatform(task.Platform). - SetContent(content). - SetCategory(common.BugCategory_Anomaly) + SetContent(content) + if task.Platform == "adr" { + bug.SetCategory(common.BugCategory_Anomaly_adr) + } else { + bug.SetCategory(common.BugCategory_Anomaly_ios) + } id, err := bug.Create() if err != nil { diff --git a/monkey/device.go b/monkey/device.go index 112a6c5..3f5a08f 100644 --- a/monkey/device.go +++ b/monkey/device.go @@ -57,6 +57,13 @@ func GetDevicesByTaskId(c *gin.Context) { return } defer dbsql.Close(db) + db_oms, err := dbsql.GetConn(dbsql.DSN_qaoms()) + if err != nil { + rsp.Error(err.Error()) + c.JSON(http.StatusOK, rsp) + return + } + defer dbsql.Close(db_oms) var task models.MonkeyTask db.Model(models.MonkeyTask{}).Where("id = ?", task_id).Last(&task) @@ -67,19 +74,19 @@ func GetDevicesByTaskId(c *gin.Context) { return } - var devices []models.Device + var devices []models.DeviceV2 if strings.Contains(task.Devices, ",") { list := strings.Split(task.Devices, ",") for _, v := range list { - var d models.Device - db.Model(models.Device{}).Where("udid = ?", v).Last(&d) + var d models.DeviceV2 + db_oms.Model(models.DeviceV2{}).Where("udid = ?", v).Last(&d) if d.ID > 0 { devices = append(devices, d) } } } else { - var d models.Device - db.Model(models.Device{}).Where("udid = ?", task.Devices).Last(&d) + var d models.DeviceV2 + db_oms.Model(models.DeviceV2{}).Where("udid = ?", task.Devices).Last(&d) if d.ID > 0 { devices = append(devices, d) } diff --git a/qatool/bigdata.go b/qatool/bigdata.go new file mode 100644 index 0000000..eb2cca4 --- /dev/null +++ b/qatool/bigdata.go @@ -0,0 +1,166 @@ +package qatool + +import ( + "autogo/controllers" + "crypto/md5" + "encoding/hex" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/imroc/req/v3" + "github.com/spf13/cast" +) + +type BigData struct { + Keyword string `json:"keyword"` + Action string `json:"action"` + Did string `json:"did"` + PkgName string `json:"pkg_name"` + RawData string `json:"raw_data"` + SLogTime string `json:"slogtime"` + LogTime int64 `json:"logtime"` + MD5 string `json:"md5"` +} + +// 实现 sort.Interface 接口的 Len、Less 和 Swap 方法 +type BigDataList []BigData + +func (a BigDataList) Len() int { return len(a) } +func (a BigDataList) Less(i, j int) bool { return a[i].LogTime < a[j].LogTime } +func (a BigDataList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// @Tags 工具相关 /api/tool/v1/ +// @Summary 埋点查询 +// @Description 通过指定一些参数查询打点数据 +// @accept x-www-form-urlencoded +// @Param project_key query string true "产品key" +// @Param action query string false "过滤action事件" +// @Param keyword query string false "过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔" +// @Param count query int false "请求数量，默认为10" +// @Param did query string false "可选-did过滤" +// @Param pkg query string false "可选-包名过滤" +// @Success 200 {object} models.Response "返回打点信息" +// @Router /api/tool/v1/bigdata/query [get] +func GetProjectBigData(c *gin.Context) { + rsp := controllers.NewResponse() + + var actions []string + + // 访问日志路径需要的产品参数 + project_key := c.Query("project_key") + if project_key == "" { + c.JSON(http.StatusOK, rsp.Error("project_key needed")) + return + } + + // filter-传入did以过滤 + did := c.DefaultQuery("did", "") + + // 获取事件名称 + action := c.Query("action") + if action == "" { + c.JSON(http.StatusOK, rsp.Error("action or keyword needed")) + return + } + if strings.Contains(action, ",") { + actions = strings.Split(action, ",") + } else { + actions = append(actions, action) + } + + count := cast.ToInt(c.DefaultQuery("count", "10")) + + pkg_name := c.Query("pkg") + + // 开始获取数据处理... + var list []BigData + for _, v := range actions { + _list := GetBigData(project_key, did, "action="+v, count) + for _, vv := range _list { + var bd BigData + bd.Keyword = v + bd.RawData = vv + for _, vvv := range strings.Split(vv, "`") { + if strings.Split(vvv, "=")[0] == "action" { + bd.Action = strings.Split(vvv, "=")[1] + } + if strings.Split(vvv, "=")[0] == "did" { + bd.Did = strings.Split(vvv, "=")[1] + } + if strings.Split(vvv, "=")[0] == "pkg" { + bd.PkgName = strings.Split(vvv, "=")[1] + } + if strings.Split(vvv, "=")[0] == "logtime" { + bd.LogTime = cast.ToInt64(strings.Split(vvv, "=")[1]) + } + if strings.Split(vvv, "=")[0] == "slogtime" { + bd.SLogTime = strings.Split(vvv, "=")[1] + } + } + md5Hash := md5.Sum([]byte(vv)) + bd.MD5 = hex.EncodeToString(md5Hash[:]) + + // 为空 或者 包名与数据匹配 + if pkg_name == "" || pkg_name == bd.PkgName { + list = append(list, bd) + } + } + } + + // 遍历获取到的数据，初步解析 + + // 使用 sort.Sort 函数进行排序 + sort.Sort(BigDataList(list)) + + rsp.Success() + rsp.Data = list + c.JSON(http.StatusOK, rsp) +} + +func GetBigData(project_key, did, kw string, count int) []string { + var strs []string + m := make(map[string]string) + m["filepath"] = "apps/odps_app_log/action_log/app_" + project_key + "_action_" + getTodayStr() + ".log" + m["filter"] = did // "ad82204d-b395-4000-9a6c-c693bb859dec" + m["keywords"] = kw //"on_time" + m["count"] = cast.ToString(count) + resp, err := req. + SetQueryParams(m). + Get("http://47.74.249.92:7912/log/grep") + if err != nil { + fmt.Println(err) + return strs + } + if resp.IsErrorState() { + fmt.Println(resp.Err) + return strs + } + + strs = strings.Split(resp.String(), "\n") + // 去除头一行说明 + if strings.Contains(strs[0], "===") { + strs = strs[1:] + } + // 去除尾部可能存在的空行 + if len(strs[len(strs)-1]) < 32 { + strs = strs[:len(strs)-1] + } + + return strs +} + +func getTodayStr() string { + // 设置上海时区 + loc, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + fmt.Println("无法加载时区信息:", err) + return "" + } + now := time.Now().In(loc) + // 格式化日期 + return now.Format("20060102") +} diff --git a/router/v1.go b/router/v1.go index 0b43b3a..5ae29ad 100644 --- a/router/v1.go +++ b/router/v1.go @@ -39,4 +39,5 @@ func setRoute(r *gin.Engine) { // 工具接口 r.GET("/api/tool/v1/voice/list", qatool.GetVoiceTestData) + r.GET("/api/tool/v1/bigdata/query", qatool.GetProjectBigData) }