autogo/qatool/bigdata.go

package qatool

import (
	"autogo/controllers"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/imroc/req/v3"
	"github.com/spf13/cast"
)

type BigData struct {
	Keyword  string `json:"keyword"`
	Action   string `json:"action"`
	Did      string `json:"did"`
	PkgName  string `json:"pkg_name"`
	RawData  string `json:"raw_data"`
	SLogTime string `json:"slogtime"`
	LogTime  int64  `json:"logtime"`
	MD5      string `json:"md5"`
}

// 实现 sort.Interface 接口的 Len、Less 和 Swap 方法
type BigDataList []BigData

func (a BigDataList) Len() int           { return len(a) }
func (a BigDataList) Less(i, j int) bool { return a[i].LogTime < a[j].LogTime }
func (a BigDataList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

// @Tags 工具相关 /api/tool/v1/
// @Summary 埋点查询
// @Description 通过指定一些参数查询打点数据
// @accept x-www-form-urlencoded
// @Param project_key query string true "产品key"
// @Param action query string false "过滤action事件"
// @Param keyword query string false "过滤关键字，可以是action，也可以是打点内容，模糊匹配,多个用逗号分隔"
// @Param count query int false "请求数量，默认为10"
// @Param did query string false "可选-did过滤"
// @Param pkg query string false "可选-包名过滤"
// @Success 200 {object} models.Response "返回打点信息"
// @Router /api/tool/v1/bigdata/query [get]
func GetProjectBigData(c *gin.Context) {
	rsp := controllers.NewResponse()

	var actions []string

	// 访问日志路径需要的产品参数
	project_key := c.Query("project_key")
	if project_key == "" {
		c.JSON(http.StatusOK, rsp.Error("project_key needed"))
		return
	}

	// filter-传入did以过滤
	did := c.DefaultQuery("did", "")

	// 获取事件名称
	action := c.Query("action")
	if action == "" {
		c.JSON(http.StatusOK, rsp.Error("action or keyword needed"))
		return
	}
	if strings.Contains(action, ",") {
		actions = strings.Split(action, ",")
	} else {
		actions = append(actions, action)
	}

	count := cast.ToInt(c.DefaultQuery("count", "10"))

	pkg_name := c.Query("pkg")

	// 开始获取数据处理...
	var list []BigData
	for _, v := range actions {
		_list := GetBigData(project_key, did, "action="+v, count)
		for _, vv := range _list {
			var bd BigData
			bd.Keyword = v
			bd.RawData = vv
			for _, vvv := range strings.Split(vv, "`") {
				if strings.Split(vvv, "=")[0] == "action" {
					bd.Action = strings.Split(vvv, "=")[1]
				}
				if strings.Split(vvv, "=")[0] == "did" {
					bd.Did = strings.Split(vvv, "=")[1]
				}
				if strings.Split(vvv, "=")[0] == "pkg" {
					bd.PkgName = strings.Split(vvv, "=")[1]
				}
				if strings.Split(vvv, "=")[0] == "logtime" {
					bd.LogTime = cast.ToInt64(strings.Split(vvv, "=")[1])
				}
				if strings.Split(vvv, "=")[0] == "slogtime" {
					bd.SLogTime = strings.Split(vvv, "=")[1]
				}
			}
			md5Hash := md5.Sum([]byte(vv))
			bd.MD5 = hex.EncodeToString(md5Hash[:])

			// 为空 或者 包名与数据匹配
			if pkg_name == "" || pkg_name == bd.PkgName {
				list = append(list, bd)
			}
		}
	}

	// 遍历获取到的数据，初步解析

	// 使用 sort.Sort 函数进行排序
	sort.Sort(BigDataList(list))

	rsp.Success()
	rsp.Data = list
	c.JSON(http.StatusOK, rsp)
}

func GetBigData(project_key, did, kw string, count int) []string {
	var strs []string
	m := make(map[string]string)
	m["filepath"] = "apps/odps_app_log/action_log/app_" + project_key + "_action_" + getTodayStr() + ".log"
	m["filter"] = did  // "ad82204d-b395-4000-9a6c-c693bb859dec"
	m["keywords"] = kw //"on_time"
	m["count"] = cast.ToString(count)
	resp, err := req.
		SetQueryParams(m).
		Get("http://47.74.249.92:7912/log/grep")
	if err != nil {
		fmt.Println(err)
		return strs
	}
	if resp.IsErrorState() {
		fmt.Println(resp.Err)
		return strs
	}

	strs = strings.Split(resp.String(), "\n")
	// 去除头一行说明
	if strings.Contains(strs[0], "===") {
		strs = strs[1:]
	}
	// 去除尾部可能存在的空行
	if len(strs[len(strs)-1]) < 32 {
		strs = strs[:len(strs)-1]
	}

	return strs
}

func getTodayStr() string {
	// 设置上海时区
	loc, err := time.LoadLocation("Asia/Shanghai")
	if err != nil {
		fmt.Println("无法加载时区信息:", err)
		return ""
	}
	now := time.Now().In(loc)
	// 格式化日期
	return now.Format("20060102")
}