|
|
|
|
@ -0,0 +1,166 @@
|
|
|
|
|
package qatool
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"autogo/controllers"
|
|
|
|
|
"crypto/md5"
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
"fmt"
|
|
|
|
|
"net/http"
|
|
|
|
|
"sort"
|
|
|
|
|
"strings"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"github.com/gin-gonic/gin"
|
|
|
|
|
"github.com/imroc/req/v3"
|
|
|
|
|
"github.com/spf13/cast"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type BigData struct {
|
|
|
|
|
Keyword string `json:"keyword"`
|
|
|
|
|
Action string `json:"action"`
|
|
|
|
|
Did string `json:"did"`
|
|
|
|
|
PkgName string `json:"pkg_name"`
|
|
|
|
|
RawData string `json:"raw_data"`
|
|
|
|
|
SLogTime string `json:"slogtime"`
|
|
|
|
|
LogTime int64 `json:"logtime"`
|
|
|
|
|
MD5 string `json:"md5"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 实现 sort.Interface 接口的 Len、Less 和 Swap 方法
|
|
|
|
|
type BigDataList []BigData
|
|
|
|
|
|
|
|
|
|
func (a BigDataList) Len() int { return len(a) }
|
|
|
|
|
func (a BigDataList) Less(i, j int) bool { return a[i].LogTime < a[j].LogTime }
|
|
|
|
|
func (a BigDataList) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
|
|
|
|
|
|
// @Tags 工具相关 /api/tool/v1/
|
|
|
|
|
// @Summary 埋点查询
|
|
|
|
|
// @Description 通过指定一些参数查询打点数据
|
|
|
|
|
// @accept x-www-form-urlencoded
|
|
|
|
|
// @Param project_key query string true "产品key"
|
|
|
|
|
// @Param action query string false "过滤action事件"
|
|
|
|
|
// @Param keyword query string false "过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔"
|
|
|
|
|
// @Param count query int false "请求数量，默认为10"
|
|
|
|
|
// @Param did query string false "可选-did过滤"
|
|
|
|
|
// @Param pkg query string false "可选-包名过滤"
|
|
|
|
|
// @Success 200 {object} models.Response "返回打点信息"
|
|
|
|
|
// @Router /api/tool/v1/bigdata/query [get]
|
|
|
|
|
func GetProjectBigData(c *gin.Context) {
|
|
|
|
|
rsp := controllers.NewResponse()
|
|
|
|
|
|
|
|
|
|
var actions []string
|
|
|
|
|
|
|
|
|
|
// 访问日志路径需要的产品参数
|
|
|
|
|
project_key := c.Query("project_key")
|
|
|
|
|
if project_key == "" {
|
|
|
|
|
c.JSON(http.StatusOK, rsp.Error("project_key needed"))
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// filter-传入did以过滤
|
|
|
|
|
did := c.DefaultQuery("did", "")
|
|
|
|
|
|
|
|
|
|
// 获取事件名称
|
|
|
|
|
action := c.Query("action")
|
|
|
|
|
if action == "" {
|
|
|
|
|
c.JSON(http.StatusOK, rsp.Error("action or keyword needed"))
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if strings.Contains(action, ",") {
|
|
|
|
|
actions = strings.Split(action, ",")
|
|
|
|
|
} else {
|
|
|
|
|
actions = append(actions, action)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
count := cast.ToInt(c.DefaultQuery("count", "10"))
|
|
|
|
|
|
|
|
|
|
pkg_name := c.Query("pkg")
|
|
|
|
|
|
|
|
|
|
// 开始获取数据处理...
|
|
|
|
|
var list []BigData
|
|
|
|
|
for _, v := range actions {
|
|
|
|
|
_list := GetBigData(project_key, did, "action="+v, count)
|
|
|
|
|
for _, vv := range _list {
|
|
|
|
|
var bd BigData
|
|
|
|
|
bd.Keyword = v
|
|
|
|
|
bd.RawData = vv
|
|
|
|
|
for _, vvv := range strings.Split(vv, "`") {
|
|
|
|
|
if strings.Split(vvv, "=")[0] == "action" {
|
|
|
|
|
bd.Action = strings.Split(vvv, "=")[1]
|
|
|
|
|
}
|
|
|
|
|
if strings.Split(vvv, "=")[0] == "did" {
|
|
|
|
|
bd.Did = strings.Split(vvv, "=")[1]
|
|
|
|
|
}
|
|
|
|
|
if strings.Split(vvv, "=")[0] == "pkg" {
|
|
|
|
|
bd.PkgName = strings.Split(vvv, "=")[1]
|
|
|
|
|
}
|
|
|
|
|
if strings.Split(vvv, "=")[0] == "logtime" {
|
|
|
|
|
bd.LogTime = cast.ToInt64(strings.Split(vvv, "=")[1])
|
|
|
|
|
}
|
|
|
|
|
if strings.Split(vvv, "=")[0] == "slogtime" {
|
|
|
|
|
bd.SLogTime = strings.Split(vvv, "=")[1]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
md5Hash := md5.Sum([]byte(vv))
|
|
|
|
|
bd.MD5 = hex.EncodeToString(md5Hash[:])
|
|
|
|
|
|
|
|
|
|
// 为空 或者 包名与数据匹配
|
|
|
|
|
if pkg_name == "" || pkg_name == bd.PkgName {
|
|
|
|
|
list = append(list, bd)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 遍历获取到的数据，初步解析
|
|
|
|
|
|
|
|
|
|
// 使用 sort.Sort 函数进行排序
|
|
|
|
|
sort.Sort(BigDataList(list))
|
|
|
|
|
|
|
|
|
|
rsp.Success()
|
|
|
|
|
rsp.Data = list
|
|
|
|
|
c.JSON(http.StatusOK, rsp)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func GetBigData(project_key, did, kw string, count int) []string {
|
|
|
|
|
var strs []string
|
|
|
|
|
m := make(map[string]string)
|
|
|
|
|
m["filepath"] = "apps/odps_app_log/action_log/app_" + project_key + "_action_" + getTodayStr() + ".log"
|
|
|
|
|
m["filter"] = did // "ad82204d-b395-4000-9a6c-c693bb859dec"
|
|
|
|
|
m["keywords"] = kw //"on_time"
|
|
|
|
|
m["count"] = cast.ToString(count)
|
|
|
|
|
resp, err := req.
|
|
|
|
|
SetQueryParams(m).
|
|
|
|
|
Get("http://47.74.249.92:7912/log/grep")
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Println(err)
|
|
|
|
|
return strs
|
|
|
|
|
}
|
|
|
|
|
if resp.IsErrorState() {
|
|
|
|
|
fmt.Println(resp.Err)
|
|
|
|
|
return strs
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
strs = strings.Split(resp.String(), "\n")
|
|
|
|
|
// 去除头一行说明
|
|
|
|
|
if strings.Contains(strs[0], "===") {
|
|
|
|
|
strs = strs[1:]
|
|
|
|
|
}
|
|
|
|
|
// 去除尾部可能存在的空行
|
|
|
|
|
if len(strs[len(strs)-1]) < 32 {
|
|
|
|
|
strs = strs[:len(strs)-1]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return strs
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func getTodayStr() string {
|
|
|
|
|
// 设置上海时区
|
|
|
|
|
loc, err := time.LoadLocation("Asia/Shanghai")
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Println("无法加载时区信息:", err)
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
now := time.Now().In(loc)
|
|
|
|
|
// 格式化日期
|
|
|
|
|
return now.Format("20060102")
|
|
|
|
|
}
|
