package qatool import ( "autogo/controllers" "crypto/md5" "encoding/hex" "fmt" "net/http" "sort" "strings" "time" "github.com/gin-gonic/gin" "github.com/imroc/req/v3" "github.com/spf13/cast" ) type BigData struct { Keyword string `json:"keyword"` Action string `json:"action"` Did string `json:"did"` PkgName string `json:"pkg_name"` RawData string `json:"raw_data"` SLogTime string `json:"slogtime"` LogTime int64 `json:"logtime"` MD5 string `json:"md5"` } // 实现 sort.Interface 接口的 Len、Less 和 Swap 方法 type BigDataList []BigData func (a BigDataList) Len() int { return len(a) } func (a BigDataList) Less(i, j int) bool { return a[i].LogTime < a[j].LogTime } func (a BigDataList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } // @Tags 工具相关 /api/tool/v1/ // @Summary 埋点查询 // @Description 通过指定一些参数查询打点数据 // @accept x-www-form-urlencoded // @Param project_key query string true "产品key" // @Param action query string false "过滤action事件" // @Param keyword query string false "过滤关键字,可以是action,也可以是打点内容,模糊匹配,多个用逗号分隔" // @Param count query int false "请求数量,默认为10" // @Param did query string false "可选-did过滤" // @Param pkg query string false "可选-包名过滤" // @Success 200 {object} models.Response "返回打点信息" // @Router /api/tool/v1/bigdata/query [get] func GetProjectBigData(c *gin.Context) { rsp := controllers.NewResponse() var actions []string // 访问日志路径需要的产品参数 project_key := c.Query("project_key") if project_key == "" { c.JSON(http.StatusOK, rsp.Error("project_key needed")) return } // filter-传入did以过滤 did := c.DefaultQuery("did", "") // 获取事件名称 action := c.Query("action") if action == "" { c.JSON(http.StatusOK, rsp.Error("action or keyword needed")) return } if strings.Contains(action, ",") { actions = strings.Split(action, ",") } else { actions = append(actions, action) } count := cast.ToInt(c.DefaultQuery("count", "10")) pkg_name := c.Query("pkg") // 开始获取数据处理... var list []BigData for _, v := range actions { _list := GetBigData(project_key, did, "action="+v, count) for _, vv := range _list { var bd BigData bd.Keyword = v bd.RawData = vv for _, vvv := range strings.Split(vv, "`") { if strings.Split(vvv, "=")[0] == "action" { bd.Action = strings.Split(vvv, "=")[1] } if strings.Split(vvv, "=")[0] == "did" { bd.Did = strings.Split(vvv, "=")[1] } if strings.Split(vvv, "=")[0] == "pkg" { bd.PkgName = strings.Split(vvv, "=")[1] } if strings.Split(vvv, "=")[0] == "logtime" { bd.LogTime = cast.ToInt64(strings.Split(vvv, "=")[1]) } if strings.Split(vvv, "=")[0] == "slogtime" { bd.SLogTime = strings.Split(vvv, "=")[1] } } md5Hash := md5.Sum([]byte(vv)) bd.MD5 = hex.EncodeToString(md5Hash[:]) // 为空 或者 包名与数据匹配 if pkg_name == "" || pkg_name == bd.PkgName { list = append(list, bd) } } } // 遍历获取到的数据,初步解析 // 使用 sort.Sort 函数进行排序 sort.Sort(BigDataList(list)) rsp.Success() rsp.Data = list c.JSON(http.StatusOK, rsp) } func GetBigData(project_key, did, kw string, count int) []string { var strs []string m := make(map[string]string) m["filepath"] = "apps/odps_app_log/action_log/app_" + project_key + "_action_" + getTodayStr() + ".log" m["filter"] = did // "ad82204d-b395-4000-9a6c-c693bb859dec" m["keywords"] = kw //"on_time" m["count"] = cast.ToString(count) resp, err := req. SetQueryParams(m). Get("http://47.74.249.92:7912/log/grep") if err != nil { fmt.Println(err) return strs } if resp.IsErrorState() { fmt.Println(resp.Err) return strs } strs = strings.Split(resp.String(), "\n") // 去除头一行说明 if strings.Contains(strs[0], "===") { strs = strs[1:] } // 去除尾部可能存在的空行 if len(strs[len(strs)-1]) < 32 { strs = strs[:len(strs)-1] } return strs } func getTodayStr() string { // 设置上海时区 loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { fmt.Println("无法加载时区信息:", err) return "" } now := time.Now().In(loc) // 格式化日期 return now.Format("20060102") }