From 10c785e951245ab0c98d074cfad6565291624654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Wed, 25 Dec 2024 11:06:16 +0800 Subject: [PATCH 01/11] feat:add vlogs server --- service/pkg/api/req.go | 15 ++ service/vlogs/Dockerfile | 9 + service/vlogs/Makefile | 55 ++++++ service/vlogs/README.md | 54 ++++++ service/vlogs/config/config.yml | 2 + service/vlogs/deploy/Kubefile | 5 + service/vlogs/deploy/README.md | 16 ++ service/vlogs/deploy/manifests/deploy.yaml | 86 +++++++++ service/vlogs/go.mod | 10 ++ service/vlogs/main.go | 64 +++++++ service/vlogs/request/req.go | 200 +++++++++++++++++++++ service/vlogs/server/config.go | 29 +++ service/vlogs/server/server.go | 172 ++++++++++++++++++ 13 files changed, 717 insertions(+) create mode 100644 service/vlogs/Dockerfile create mode 100644 service/vlogs/Makefile create mode 100644 service/vlogs/README.md create mode 100644 service/vlogs/config/config.yml create mode 100644 service/vlogs/deploy/Kubefile create mode 100644 service/vlogs/deploy/README.md create mode 100644 service/vlogs/deploy/manifests/deploy.yaml create mode 100644 service/vlogs/go.mod create mode 100644 service/vlogs/main.go create mode 100644 service/vlogs/request/req.go create mode 100644 service/vlogs/server/config.go create mode 100644 service/vlogs/server/server.go diff --git a/service/pkg/api/req.go b/service/pkg/api/req.go index a4872bd9703..11a04e67ac1 100644 --- a/service/pkg/api/req.go +++ b/service/pkg/api/req.go @@ -70,6 +70,21 @@ type Stats struct { ExecutionTimeMsec int `json:"executionTimeMsec"` } +type VlogsRequest struct { + Time string + NS string + App string + Pod string + Limit string + Json string + Keyword string +} + +type VlogsResult struct { + Time string `json:"_time"` + Msg string `json:"_msg"` +} + var ( Mysql = map[string]string{ "cpu": "round(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=~\"#\",pod=~\"@-mysql-\\\\d\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=~\"#\",pod=~\"@-mysql-\\\\d\"}) by (pod)*100,0.01)", diff --git a/service/vlogs/Dockerfile b/service/vlogs/Dockerfile new file mode 100644 index 00000000000..1352f47e5f0 --- /dev/null +++ b/service/vlogs/Dockerfile @@ -0,0 +1,9 @@ +# FROM scratch +FROM gcr.io/distroless/static:nonroot +# FROM gengweifeng/gcr-io-distroless-static-nonroot +ARG TARGETARCH +COPY bin/service-launchpad-$TARGETARCH /manager +EXPOSE 9090 +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/service/vlogs/Makefile b/service/vlogs/Makefile new file mode 100644 index 00000000000..fcb97c9aefd --- /dev/null +++ b/service/vlogs/Makefile @@ -0,0 +1,55 @@ +IMG ?= ghcr.io/labring/sealos-launchpad-service:latest + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# only support linux, non cgo +PLATFORMS ?= linux_arm64 linux_amd64 +GOOS=linux +CGO_ENABLED=0 +GOARCH=$(shell go env GOARCH) + +GO_BUILD_FLAGS=-trimpath -ldflags "-s -w" + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +.PHONY: clean +clean: + rm -f $(SERVICE_NAME) + +.PHONY: build +build: clean ## Build service-hub binary. + CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) go build $(GO_BUILD_FLAGS) -o bin/manager main.go + +.PHONY: docker-build +docker-build: build + mv bin/manager bin/service-launchpad-${TARGETARCH} + docker build -t $(IMG) . + +.PHONY: docker-push +docker-push: + docker push $(IMG) diff --git a/service/vlogs/README.md b/service/vlogs/README.md new file mode 100644 index 00000000000..45dafd4f0d2 --- /dev/null +++ b/service/vlogs/README.md @@ -0,0 +1,54 @@ +# RESTServer + +## Description + +restserver for database monitoring + +## Getting Started + +### Running on the cluster + +1. Build and push your image to the location specified by `IMG`: + +```sh +make docker-build docker-push IMG=/sealos-cloud-database-monitor:tag +``` + +2. Deploy the restserver: + +```sh +kubectl apply -f deploy/manifests/ +``` + +### How it works + +To enable the database frontend application to retrieve monitoring data, you need to modify the environment variable `MONITOR_URL` of the frontend deployment to the corresponding address of the restserver. + +Additionally, to configure the data source, you need to set the environment variable `VM_SERVICE_HOST` of the restserver deployment to the correct address. + +``` +e.g. +http://prometheus.sealos.svc.cluster.local +``` + +## License + +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); + +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software + +distributed under the License is distributed on an "AS IS" BASIS, + +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. \ No newline at end of file diff --git a/service/vlogs/config/config.yml b/service/vlogs/config/config.yml new file mode 100644 index 00000000000..cf416ad93f9 --- /dev/null +++ b/service/vlogs/config/config.yml @@ -0,0 +1,2 @@ +server: + addr: ":8428" \ No newline at end of file diff --git a/service/vlogs/deploy/Kubefile b/service/vlogs/deploy/Kubefile new file mode 100644 index 00000000000..035ec02f5a0 --- /dev/null +++ b/service/vlogs/deploy/Kubefile @@ -0,0 +1,5 @@ +FROM scratch +COPY registry registry +COPY manifests manifests + +CMD ["kubectl apply -f manifests/deploy.yaml"] \ No newline at end of file diff --git a/service/vlogs/deploy/README.md b/service/vlogs/deploy/README.md new file mode 100644 index 00000000000..8606792effb --- /dev/null +++ b/service/vlogs/deploy/README.md @@ -0,0 +1,16 @@ +### docker image build and deploy +```bash +make docker-build IMG=$(YourImageName) +# edit deploy/manifests/depoly.yaml and deploy +kubectl apply -f deploy/manifests/depoly.yaml +``` + +### cluster image build and deploy +```bash +``` + +### Victoria Metrics + +In order to prevent performance degradation or abnormal behavior caused by excessive data size in Prometheus, VictoriaMetrics is utilized for data collection. + +> By default, we use kb-prometheus-server for the data collection service. \ No newline at end of file diff --git a/service/vlogs/deploy/manifests/deploy.yaml b/service/vlogs/deploy/manifests/deploy.yaml new file mode 100644 index 00000000000..29cefd9f077 --- /dev/null +++ b/service/vlogs/deploy/manifests/deploy.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: launchpad-monitor + name: launchpad-monitor-config + namespace: sealos +data: + config.yml: | + server: + addr: ":8428" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: launchpad-monitor + name: launchpad-monitor-deployment + namespace: sealos +spec: + replicas: 1 + selector: + matchLabels: + app: launchpad-monitor + strategy: + type: Recreate + template: + metadata: + labels: + app: launchpad-monitor + spec: + containers: + - args: + - /config/config.yml + command: + - /manager + env: + - name: VM_SERVICE_HOST + value: http://vmsingle-victoria-metrics-k8s-stack.vm.svc:8429 + image: ghcr.io/labring/sealos-launchpad-service:latest + imagePullPolicy: Always + name: launchpad-monitor + ports: + - containerPort: 8428 + protocol: TCP + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 5m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /config + name: config-vol + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - configMap: + defaultMode: 420 + name: launchpad-monitor-config + name: config-vol +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: launchpad-monitor + name: launchpad-monitor + namespace: sealos +spec: + ports: + - name: http + port: 8428 + protocol: TCP + targetPort: 8428 + selector: + app: launchpad-monitor \ No newline at end of file diff --git a/service/vlogs/go.mod b/service/vlogs/go.mod new file mode 100644 index 00000000000..9b8b0f66338 --- /dev/null +++ b/service/vlogs/go.mod @@ -0,0 +1,10 @@ +module github.com/labring/sealos/service/vlogs + +go 1.22 + +require ( + github.com/labring/sealos/service v0.0.0-00010101000000-000000000000 + gopkg.in/yaml.v2 v2.4.0 +) + +replace github.com/labring/sealos/service => ../../service \ No newline at end of file diff --git a/service/vlogs/main.go b/service/vlogs/main.go new file mode 100644 index 00000000000..7d4aabcbb59 --- /dev/null +++ b/service/vlogs/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + + vlogsServer "github.com/labring/sealos/service/vlogs/server" +) + +type RestartableServer struct { + configFile string +} + +func (rs *RestartableServer) Serve(c *vlogsServer.Config) { + var vs, err = vlogsServer.NewVMServer(c) + if err != nil { + fmt.Printf("Failed to create auth server: %s\n", err) + return + } + + hs := &http.Server{ + Addr: c.Server.ListenAddress, + Handler: vs, + } + + var listener net.Listener + listener, err = net.Listen("tcp", c.Server.ListenAddress) + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("Serve on %s\n", c.Server.ListenAddress) + + if err := hs.Serve(listener); err != nil { + fmt.Println(err) + return + } +} + +func main() { + log.SetOutput(os.Stdout) + log.SetFlags(log.LstdFlags | log.Lshortfile) + flag.Parse() + + cf := flag.Arg(0) + if cf == "" { + fmt.Println("Config file not sepcified") + return + } + + config, err := vlogsServer.InitConfig(cf) + if err != nil { + fmt.Println(err) + return + } + rs := RestartableServer{ + configFile: cf, + } + rs.Serve(config) +} diff --git a/service/vlogs/request/req.go b/service/vlogs/request/req.go new file mode 100644 index 00000000000..f4b9ee31bbe --- /dev/null +++ b/service/vlogs/request/req.go @@ -0,0 +1,200 @@ +package request + +import ( + "bufio" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "github.com/labring/sealos/service/pkg/api" + "net/http" + "net/url" + "os" + "strings" +) + +const ( + defaultTime = "_time:1h" + defaultLimit = "| limit 10" +) + +func GetQuery(query *api.VlogsRequest) (string, error) { + var builder strings.Builder + + // 添加关键词 + builder.WriteString(query.Keyword) + builder.WriteString(" ") + + // 判断 namespace + if query.NS == "" { + return "", errors.New("namespace (NS) is required") + } + builder.WriteString(fmt.Sprintf("{namespace=%s}", query.NS)) + builder.WriteString(" ") + + // 添加 pod + if query.Pod != "" { + builder.WriteString(fmt.Sprintf("pod:%s", query.Pod)) + builder.WriteString(" ") + } + + // 添加时间 + if query.Time == "" { + builder.WriteString(defaultTime) + } else { + builder.WriteString("_time:") + builder.WriteString(query.Time) + } + builder.WriteString(" ") + + // JSON 模式 + if query.Json == "true" { + builder.WriteString("| unpack_json") + builder.WriteString(" ") + } + + // 添加 limit + if query.Limit == "" { + builder.WriteString(defaultLimit) + } else { + builder.WriteString("| limit ") + builder.WriteString(query.Limit) + } + builder.WriteString(" ") + + //添加field + + return builder.String(), nil +} + +func QueryLogsByParams(query *api.VlogsRequest) ([]api.VlogsResult, error) { + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + vlogsHost := GetVLogsServerFromEnv() + vlogsHost = "https://vvvvvlogs.192.168.10.35.nip.io" + if vlogsHost == "" { + return nil, api.ErrNoVMHost + } + result, _ := GetQuery(query) + baseURL, err := url.Parse(vlogsHost + "/select/logsql/query") + if err != nil { + return nil, fmt.Errorf("无法解析 API URL: %v", err) + } + params := url.Values{} + params.Add("query", result) + baseURL.RawQuery = params.Encode() + + // 创建 HTTP 请求 + req, err := http.NewRequest("GET", baseURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("创建 HTTP 请求失败: %v", err) + } + + // 添加 Basic Auth 认证 + username, password := GetVLogsUsernameAndPasswordFromEnv() + username = "admin" + password = "sealos@123#@!" + req.SetBasicAuth(username, password) + + // 发起请求 + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP 请求失败: %v", err) + } + defer resp.Body.Close() + // 检查 HTTP 状态码 + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("请求失败，状态码: %d", resp) + } + + // 使用 Scanner 逐行读取响应内容 + var results []api.VlogsResult + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + var entry api.VlogsResult + err := json.Unmarshal([]byte(line), &entry) + if err != nil { + fmt.Printf("解析日志行失败: %v, 行内容: %s\n", err, line) + continue + } + results = append(results, entry) + } + + return results, nil +} + +func QueryLogsByLogsQl(query string) ([]api.VlogsResult, error) { + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + vlogsHost := GetVLogsServerFromEnv() + vlogsHost = "https://vvvvvlogs.192.168.10.35.nip.io" + if vlogsHost == "" { + return nil, api.ErrNoVMHost + } + result := query + baseURL, err := url.Parse(vlogsHost + "/select/logsql/query") + if err != nil { + return nil, fmt.Errorf("无法解析 API URL: %v", err) + } + params := url.Values{} + params.Add("query", result) + baseURL.RawQuery = params.Encode() + + // 创建 HTTP 请求 + req, err := http.NewRequest("GET", baseURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("创建 HTTP 请求失败: %v", err) + } + + // 添加 Basic Auth 认证 + username, password := GetVLogsUsernameAndPasswordFromEnv() + username = "admin" + password = "sealos@123#@!" + req.SetBasicAuth(username, password) + + // 发起请求 + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP 请求失败: %v", err) + } + defer resp.Body.Close() + // 检查 HTTP 状态码 + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("请求失败，状态码: %d", resp) + } + + // 使用 Scanner 逐行读取响应内容 + var results []api.VlogsResult + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + var entry api.VlogsResult + err := json.Unmarshal([]byte(line), &entry) + if err != nil { + fmt.Printf("解析日志行失败: %v, 行内容: %s\n", err, line) + continue + } + results = append(results, entry) + } + + return results, nil +} + +func GetVLogsUsernameAndPasswordFromEnv() (string, string) { + return os.Getenv("VLOGS_SERVICE_USERNAME"), os.Getenv("VLOGS_SERVICE_PASSWORD") +} + +func GetVLogsServerFromEnv() string { + return os.Getenv("VLOGS_SERVICE_HOST") +} diff --git a/service/vlogs/server/config.go b/service/vlogs/server/config.go new file mode 100644 index 00000000000..6f69c92617d --- /dev/null +++ b/service/vlogs/server/config.go @@ -0,0 +1,29 @@ +package server + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v2" +) + +type Config struct { + Server ServeConfig `yaml:"server"` +} + +type ServeConfig struct { + ListenAddress string `yaml:"addr"` +} + +func InitConfig(configPath string) (*Config, error) { + configData, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("could not read %s: %s", configPath, err) + } + c := &Config{} + if err := yaml.Unmarshal(configData, c); err != nil { + return nil, fmt.Errorf("could not parse config: %s", err) + } + + return c, nil +} diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go new file mode 100644 index 00000000000..0416055ec47 --- /dev/null +++ b/service/vlogs/server/server.go @@ -0,0 +1,172 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/labring/sealos/service/pkg/auth" + "log" + "net/http" + + "github.com/labring/sealos/service/pkg/api" + "github.com/labring/sealos/service/vlogs/request" +) + +type VMServer struct { + Config *Config +} + +func NewVMServer(c *Config) (*VMServer, error) { + vs := &VMServer{ + Config: c, + } + return vs, nil +} + +// 获取客户端请求的信息 +func (vs *VMServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + pathPrefix := "" + switch { + case req.URL.Path == pathPrefix+"/queryLogsByParams": + vs.queryLogsByParams(rw, req) + case req.URL.Path == pathPrefix+"/queryLogsByLogsQl": + vs.queryLogsByLogsQl(rw, req) + case req.URL.Path == pathPrefix+"/queryFieldsByParams": + vs.queryLogsByLogsQl(rw, req) + default: + http.Error(rw, "Not found", http.StatusNotFound) + return + } +} + +func (vs *VMServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { + vr, err := vs.ParseParamsRequest(req) + if err != nil { + http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) + log.Printf("Bad request (%s)\n", err) + return + } + //todo: auth + + res, err := request.QueryLogsByParams(vr) + if err != nil { + http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) + log.Printf("Query failed (%s)\n", err) + return + } + result, err := json.Marshal(res) + if err != nil { + http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) + log.Printf("Reulst failed (%s)\n", err) + return + } + _, err = rw.Write(result) + if err != nil { + return + } + return +} + +func (vs *VMServer) queryFieldsByParams(rw http.ResponseWriter, req *http.Request) { + vr, err := vs.ParseLogsRequest(req) + if err != nil { + http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) + log.Printf("Bad request (%s)\n", err) + return + } + + //todo: auth + + res, err := request.QueryLogsByLogsQl(vr) + if err != nil { + http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) + log.Printf("Query failed (%s)\n", err) + return + } + result, err := json.Marshal(res) + if err != nil { + http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) + log.Printf("Reulst failed (%s)\n", err) + return + } + _, err = rw.Write(result) + if err != nil { + return + } + return +} + +func (vs *VMServer) queryLogsByLogsQl(rw http.ResponseWriter, req *http.Request) { + vr, err := vs.ParseLogsRequest(req) + if err != nil { + http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) + log.Printf("Bad request (%s)\n", err) + return + } + + //todo: auth + + res, err := request.QueryLogsByLogsQl(vr) + if err != nil { + http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) + log.Printf("Query failed (%s)\n", err) + return + } + result, err := json.Marshal(res) + if err != nil { + http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) + log.Printf("Reulst failed (%s)\n", err) + return + } + _, err = rw.Write(result) + if err != nil { + return + } + return +} + +func (vs *VMServer) ParseLogsRequest(req *http.Request) (string, error) { + if err := req.ParseForm(); err != nil { + return "", err + } + for key, val := range req.Form { + switch key { + case "query": + return val[0], nil + } + } + return "", errors.New("no query parameter found") +} + +func (vs *VMServer) ParseParamsRequest(req *http.Request) (*api.VlogsRequest, error) { + vr := &api.VlogsRequest{} + if err := req.ParseForm(); err != nil { + return nil, err + } + for key, val := range req.Form { + switch key { + case "time": + vr.Time = val[0] + case "namespace": + vr.NS = val[0] + case "app": + vr.App = val[0] + case "pod": + vr.Pod = val[0] + case "limit": + vr.Limit = val[0] + case "json": + vr.Json = val[0] + case "keyword": + vr.Keyword = val[0] + } + } + if vr.NS == "" { + return nil, api.ErrUncompleteParam + } + return vr, nil +} + +func (vs *VMServer) Authenticate(vr *api.VMRequest) error { + return auth.Authenticate(vr.NS, vr.Pwd) +} From 18f5f8499c28ba6ae030ed0218c6fa40ce0a7584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Mon, 6 Jan 2025 18:10:04 +0800 Subject: [PATCH 02/11] feat:add vlogs server --- service/go.work | 1 + service/go.work.sum | 8 ++ service/pkg/api/req.go | 25 ++-- service/vlogs/request/req.go | 163 ++++---------------------- service/vlogs/server/server.go | 203 +++++++++++++++------------------ 5 files changed, 140 insertions(+), 260 deletions(-) diff --git a/service/go.work b/service/go.work index 1978383b13f..2b353f8674f 100644 --- a/service/go.work +++ b/service/go.work @@ -8,6 +8,7 @@ use ( ./exceptionmonitor ./launchpad ./pay + vlogs ) replace ( diff --git a/service/go.work.sum b/service/go.work.sum index 08f064570c8..749e0f830b1 100644 --- a/service/go.work.sum +++ b/service/go.work.sum @@ -746,17 +746,24 @@ github.com/astaxie/beego v1.12.3 h1:SAQkdD2ePye+v8Gn1r4X6IKZM1wd28EyUOVQ3PDSOOQ= github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= github.com/aws/aws-sdk-go-v2/service/sts v1.32.4/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/bazelbuild/rules_go v0.49.0 h1:5vCbuvy8Q11g41lseGJDc5vxhDjJtfxr6nM/IC4VmqM= github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -1858,6 +1865,7 @@ google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241021214115-324edc3d5d38 h1:42FIsHvG4GOaVHLDMcy/YMqC4clCbgAPojbcA2hXp5w= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241021214115-324edc3d5d38/go.mod h1:T8O3fECQbif8cez15vxAcjbwXxvL2xbnvbQ7ZfiMAMs= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697 h1:rY93Be8/KL+EtFM4im9lxMzjGn796GnwVUd75cyFCJg= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= diff --git a/service/pkg/api/req.go b/service/pkg/api/req.go index 11a04e67ac1..d48368567d1 100644 --- a/service/pkg/api/req.go +++ b/service/pkg/api/req.go @@ -70,19 +70,22 @@ type Stats struct { ExecutionTimeMsec int `json:"executionTimeMsec"` } -type VlogsRequest struct { - Time string - NS string - App string - Pod string - Limit string - Json string - Keyword string +type JsonQuery struct { + key string + mode string + value string } -type VlogsResult struct { - Time string `json:"_time"` - Msg string `json:"_msg"` +type VlogsRequest struct { + Time string + Namespace string + App string + Pod string + Container string + Limit string + JsonMode string + Keyword []string + JsonQuery []JsonQuery } var ( diff --git a/service/vlogs/request/req.go b/service/vlogs/request/req.go index f4b9ee31bbe..0171affc878 100644 --- a/service/vlogs/request/req.go +++ b/service/vlogs/request/req.go @@ -1,135 +1,47 @@ package request import ( - "bufio" "crypto/tls" - "encoding/json" - "errors" "fmt" - "github.com/labring/sealos/service/pkg/api" + "io" "net/http" "net/url" "os" - "strings" ) -const ( - defaultTime = "_time:1h" - defaultLimit = "| limit 10" -) - -func GetQuery(query *api.VlogsRequest) (string, error) { - var builder strings.Builder - - // 添加关键词 - builder.WriteString(query.Keyword) - builder.WriteString(" ") - - // 判断 namespace - if query.NS == "" { - return "", errors.New("namespace (NS) is required") - } - builder.WriteString(fmt.Sprintf("{namespace=%s}", query.NS)) - builder.WriteString(" ") - - // 添加 pod - if query.Pod != "" { - builder.WriteString(fmt.Sprintf("pod:%s", query.Pod)) - builder.WriteString(" ") - } - - // 添加时间 - if query.Time == "" { - builder.WriteString(defaultTime) - } else { - builder.WriteString("_time:") - builder.WriteString(query.Time) - } - builder.WriteString(" ") - - // JSON 模式 - if query.Json == "true" { - builder.WriteString("| unpack_json") - builder.WriteString(" ") - } - - // 添加 limit - if query.Limit == "" { - builder.WriteString(defaultLimit) - } else { - builder.WriteString("| limit ") - builder.WriteString(query.Limit) - } - builder.WriteString(" ") - - //添加field - - return builder.String(), nil -} - -func QueryLogsByParams(query *api.VlogsRequest) ([]api.VlogsResult, error) { - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } +func generateReq(query string) (*http.Request, error) { + //地址 vlogsHost := GetVLogsServerFromEnv() vlogsHost = "https://vvvvvlogs.192.168.10.35.nip.io" - if vlogsHost == "" { - return nil, api.ErrNoVMHost - } - result, _ := GetQuery(query) + //if vlogsHost == "" { + // return nil, api.ErrNoVMHost + //} baseURL, err := url.Parse(vlogsHost + "/select/logsql/query") if err != nil { return nil, fmt.Errorf("无法解析 API URL: %v", err) } + + //参数 params := url.Values{} - params.Add("query", result) + params.Add("query", query) baseURL.RawQuery = params.Encode() - // 创建 HTTP 请求 + //创建请求 req, err := http.NewRequest("GET", baseURL.String(), nil) if err != nil { return nil, fmt.Errorf("创建 HTTP 请求失败: %v", err) } - // 添加 Basic Auth 认证 + //认证 username, password := GetVLogsUsernameAndPasswordFromEnv() username = "admin" password = "sealos@123#@!" req.SetBasicAuth(username, password) - // 发起请求 - resp, err := httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("HTTP 请求失败: %v", err) - } - defer resp.Body.Close() - // 检查 HTTP 状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("请求失败，状态码: %d", resp) - } - - // 使用 Scanner 逐行读取响应内容 - var results []api.VlogsResult - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - var entry api.VlogsResult - err := json.Unmarshal([]byte(line), &entry) - if err != nil { - fmt.Printf("解析日志行失败: %v, 行内容: %s\n", err, line) - continue - } - results = append(results, entry) - } - - return results, nil + return req, nil } -func QueryLogsByLogsQl(query string) ([]api.VlogsResult, error) { +func QueryLogsByParams(query string, rw http.ResponseWriter) error { httpClient := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -137,58 +49,27 @@ func QueryLogsByLogsQl(query string) ([]api.VlogsResult, error) { }, }, } - vlogsHost := GetVLogsServerFromEnv() - vlogsHost = "https://vvvvvlogs.192.168.10.35.nip.io" - if vlogsHost == "" { - return nil, api.ErrNoVMHost - } - result := query - baseURL, err := url.Parse(vlogsHost + "/select/logsql/query") - if err != nil { - return nil, fmt.Errorf("无法解析 API URL: %v", err) - } - params := url.Values{} - params.Add("query", result) - baseURL.RawQuery = params.Encode() - - // 创建 HTTP 请求 - req, err := http.NewRequest("GET", baseURL.String(), nil) + req, err := generateReq(query) if err != nil { - return nil, fmt.Errorf("创建 HTTP 请求失败: %v", err) + return err } - - // 添加 Basic Auth 认证 - username, password := GetVLogsUsernameAndPasswordFromEnv() - username = "admin" - password = "sealos@123#@!" - req.SetBasicAuth(username, password) - // 发起请求 resp, err := httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("HTTP 请求失败: %v", err) + return fmt.Errorf("HTTP 请求失败: %v", err) } defer resp.Body.Close() + // 检查 HTTP 状态码 if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("请求失败，状态码: %d", resp) - } - - // 使用 Scanner 逐行读取响应内容 - var results []api.VlogsResult - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - var entry api.VlogsResult - err := json.Unmarshal([]byte(line), &entry) + return fmt.Errorf("请求失败，状态码: %d", resp) + } else { + _, err := io.Copy(rw, resp.Body) if err != nil { - fmt.Printf("解析日志行失败: %v, 行内容: %s\n", err, line) - continue + return err } - results = append(results, entry) } - - return results, nil + return nil } func GetVLogsUsernameAndPasswordFromEnv() (string, string) { diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index 0416055ec47..295a4de85a4 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -7,166 +7,153 @@ import ( "github.com/labring/sealos/service/pkg/auth" "log" "net/http" + "net/url" + "strings" "github.com/labring/sealos/service/pkg/api" "github.com/labring/sealos/service/vlogs/request" ) -type VMServer struct { +type VLogsServer struct { Config *Config } -func NewVMServer(c *Config) (*VMServer, error) { - vs := &VMServer{ +func NewVMServer(c *Config) (*VLogsServer, error) { + vl := &VLogsServer{ Config: c, } - return vs, nil + return vl, nil } // 获取客户端请求的信息 -func (vs *VMServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { +func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { pathPrefix := "" switch { case req.URL.Path == pathPrefix+"/queryLogsByParams": - vs.queryLogsByParams(rw, req) - case req.URL.Path == pathPrefix+"/queryLogsByLogsQl": - vs.queryLogsByLogsQl(rw, req) - case req.URL.Path == pathPrefix+"/queryFieldsByParams": - vs.queryLogsByLogsQl(rw, req) + vl.queryLogsByParams(rw, req) default: http.Error(rw, "Not found", http.StatusNotFound) return } } -func (vs *VMServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - vr, err := vs.ParseParamsRequest(req) +func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { + kubeConfig, namespace, query, err := vl.ParseParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - //todo: auth - res, err := request.QueryLogsByParams(vr) + err = auth.Authenticate(namespace, kubeConfig) if err != nil { - http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) - log.Printf("Query failed (%s)\n", err) - return - } - result, err := json.Marshal(res) - if err != nil { - http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) - log.Printf("Reulst failed (%s)\n", err) + http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + log.Printf("Authentication failed (%s)\n", err) return } - _, err = rw.Write(result) + + err = request.QueryLogsByParams(query, rw) if err != nil { + http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) + log.Printf("Query failed (%s)\n", err) return } return } -func (vs *VMServer) queryFieldsByParams(rw http.ResponseWriter, req *http.Request) { - vr, err := vs.ParseLogsRequest(req) - if err != nil { - http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) - log.Printf("Bad request (%s)\n", err) - return +func (vl *VLogsServer) ParseParamsRequest(req *http.Request) (string, string, string, error) { + kubeConfig := req.Header.Get("Authorization") + if config, err := url.PathUnescape(kubeConfig); err == nil { + kubeConfig = config + } else { + return "", "", "", err } - //todo: auth - - res, err := request.QueryLogsByLogsQl(vr) + var query string + vlogsReq := &api.VlogsRequest{} + err := json.NewDecoder(req.Body).Decode(&vlogsReq) if err != nil { - http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) - log.Printf("Query failed (%s)\n", err) - return - } - result, err := json.Marshal(res) - if err != nil { - http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) - log.Printf("Reulst failed (%s)\n", err) - return - } - _, err = rw.Write(result) - if err != nil { - return + return "", "", "", errors.New("invalid JSON data,decode error") + } + if vlogsReq.Namespace == "" { + return "", "", "", errors.New("invalid JSON data,namespace not found") + } + switch vlogsReq.JsonMode { + case "": + return "", "", "", errors.New("invalid JSON data,jsonMode not found") + case "false": + query, err = parserKeywordQuery(vlogsReq) + if err != nil { + return "", "", "", err + } + case "true": + query, err = parserJsonQuery(vlogsReq) + if err != nil { + return "", "", "", err + } + default: + return "", "", "", errors.New("invalid JSON data,jsonMode value err") } - return } -func (vs *VMServer) queryLogsByLogsQl(rw http.ResponseWriter, req *http.Request) { - vr, err := vs.ParseLogsRequest(req) - if err != nil { - http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) - log.Printf("Bad request (%s)\n", err) - return +func parserKeywordQuery(req *api.VlogsRequest) (string, error) { + var builder strings.Builder + for _, key := range req.Keyword { + builder.WriteString(key) + builder.WriteString(" ") } - //todo: auth +} + +func parserJsonQuery(req *api.VlogsRequest) (string, error) { - res, err := request.QueryLogsByLogsQl(vr) - if err != nil { - http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) - log.Printf("Query failed (%s)\n", err) - return - } - result, err := json.Marshal(res) - if err != nil { - http.Error(rw, "Result failed (invalid query expression)", http.StatusInternalServerError) - log.Printf("Reulst failed (%s)\n", err) - return - } - _, err = rw.Write(result) - if err != nil { - return - } - return } -func (vs *VMServer) ParseLogsRequest(req *http.Request) (string, error) { - if err := req.ParseForm(); err != nil { - return "", err - } - for key, val := range req.Form { - switch key { - case "query": - return val[0], nil - } - } - return "", errors.New("no query parameter found") +func paraseQuery(req *api.VlogsRequest) { + var builder strings.Builder + } -func (vs *VMServer) ParseParamsRequest(req *http.Request) (*api.VlogsRequest, error) { - vr := &api.VlogsRequest{} - if err := req.ParseForm(); err != nil { - return nil, err +func GetQuery(query *api.VlogsRequest) (string, error) { + var builder strings.Builder + + // 添加关键词 + builder.WriteString(query.Keyword) + builder.WriteString(" ") + + builder.WriteString(fmt.Sprintf("{namespace=%s}", query.NS)) + builder.WriteString(" ") + + // 添加 pod + if query.Pod != "" { + builder.WriteString(fmt.Sprintf("pod:%s", query.Pod)) + builder.WriteString(" ") } - for key, val := range req.Form { - switch key { - case "time": - vr.Time = val[0] - case "namespace": - vr.NS = val[0] - case "app": - vr.App = val[0] - case "pod": - vr.Pod = val[0] - case "limit": - vr.Limit = val[0] - case "json": - vr.Json = val[0] - case "keyword": - vr.Keyword = val[0] - } + + // 添加时间 + if query.Time == "" { + builder.WriteString(defaultTime) + } else { + builder.WriteString("_time:") + builder.WriteString(query.Time) } - if vr.NS == "" { - return nil, api.ErrUncompleteParam + builder.WriteString(" ") + + // JSON 模式 + if query.Json == "true" { + builder.WriteString("| unpack_json") + builder.WriteString(" ") } - return vr, nil -} -func (vs *VMServer) Authenticate(vr *api.VMRequest) error { - return auth.Authenticate(vr.NS, vr.Pwd) + // 添加 limit + if query.Limit == "" { + builder.WriteString(defaultLimit) + } else { + builder.WriteString("| limit ") + builder.WriteString(query.Limit) + } + builder.WriteString(" ") + + //添加field + return builder.String(), nil } From d3ff113fc18c4b23f53d4c626cea3b7e0abfa391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Thu, 9 Jan 2025 17:13:47 +0800 Subject: [PATCH 03/11] feat:add vlogs server feat:add vlogs server feat:add vlogs server feat:add vlogs server feat:add vlogs server --- service/go.work | 2 +- service/pkg/api/req.go | 27 ++--- service/vlogs/config/config.yml | 3 +- service/vlogs/main.go | 2 +- service/vlogs/request/req.go | 43 ++------ service/vlogs/server/config.go | 4 +- service/vlogs/server/server.go | 173 +++++++++++++++++++++----------- 7 files changed, 144 insertions(+), 110 deletions(-) diff --git a/service/go.work b/service/go.work index 2b353f8674f..4f923056dc8 100644 --- a/service/go.work +++ b/service/go.work @@ -8,7 +8,7 @@ use ( ./exceptionmonitor ./launchpad ./pay - vlogs + ./vlogs ) replace ( diff --git a/service/pkg/api/req.go b/service/pkg/api/req.go index d48368567d1..0ee0262e3e2 100644 --- a/service/pkg/api/req.go +++ b/service/pkg/api/req.go @@ -71,21 +71,24 @@ type Stats struct { } type JsonQuery struct { - key string - mode string - value string + Key string + Mode string + Value string } type VlogsRequest struct { - Time string - Namespace string - App string - Pod string - Container string - Limit string - JsonMode string - Keyword []string - JsonQuery []JsonQuery + Time string `json:"time"` + Namespace string `json:"namespace"` + App string `json:"app"` + Limit string `json:"limit"` + JsonMode string `json:"jsonMode"` + StderrMode string `json:"stderrMode"` + NumberMode string `json:"numberMode"` + NumberLevel string `json:"numberLevel"` + Pod []string `json:"pod"` + Container []string `json:"container"` + Keyword string `json:"keyword"` + JsonQuery []JsonQuery `json:"jsonQuery"` } var ( diff --git a/service/vlogs/config/config.yml b/service/vlogs/config/config.yml index cf416ad93f9..8953c9def17 100644 --- a/service/vlogs/config/config.yml +++ b/service/vlogs/config/config.yml @@ -1,2 +1,3 @@ server: - addr: ":8428" \ No newline at end of file + addr: ":8428" + path: "https://vvvvvlogs.usw.sailos.io" diff --git a/service/vlogs/main.go b/service/vlogs/main.go index 7d4aabcbb59..24c8fee062f 100644 --- a/service/vlogs/main.go +++ b/service/vlogs/main.go @@ -16,7 +16,7 @@ type RestartableServer struct { } func (rs *RestartableServer) Serve(c *vlogsServer.Config) { - var vs, err = vlogsServer.NewVMServer(c) + var vs, err = vlogsServer.NewVLogsServer(c) if err != nil { fmt.Printf("Failed to create auth server: %s\n", err) return diff --git a/service/vlogs/request/req.go b/service/vlogs/request/req.go index 0171affc878..2da42302ed7 100644 --- a/service/vlogs/request/req.go +++ b/service/vlogs/request/req.go @@ -6,42 +6,26 @@ import ( "io" "net/http" "net/url" - "os" ) -func generateReq(query string) (*http.Request, error) { - //地址 - vlogsHost := GetVLogsServerFromEnv() - vlogsHost = "https://vvvvvlogs.192.168.10.35.nip.io" - //if vlogsHost == "" { - // return nil, api.ErrNoVMHost - //} - baseURL, err := url.Parse(vlogsHost + "/select/logsql/query") +func generateReq(path string, username string, password string, query string) (*http.Request, error) { + baseURL, err := url.Parse(path + "/select/logsql/query") if err != nil { - return nil, fmt.Errorf("无法解析 API URL: %v", err) + return nil, fmt.Errorf("can not parser API URL: %v", err) } - - //参数 params := url.Values{} params.Add("query", query) baseURL.RawQuery = params.Encode() - - //创建请求 req, err := http.NewRequest("GET", baseURL.String(), nil) if err != nil { - return nil, fmt.Errorf("创建 HTTP 请求失败: %v", err) + return nil, fmt.Errorf("create HTTP req error: %v", err) } - //认证 - username, password := GetVLogsUsernameAndPasswordFromEnv() - username = "admin" - password = "sealos@123#@!" req.SetBasicAuth(username, password) - return req, nil } -func QueryLogsByParams(query string, rw http.ResponseWriter) error { +func QueryLogsByParams(path string, username string, password string, query string, rw http.ResponseWriter) error { httpClient := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -49,20 +33,19 @@ func QueryLogsByParams(query string, rw http.ResponseWriter) error { }, }, } - req, err := generateReq(query) + req, err := generateReq(path, username, password, query) if err != nil { return err } - // 发起请求 + fmt.Printf("URL: %s\n", req.URL.String()) resp, err := httpClient.Do(req) if err != nil { - return fmt.Errorf("HTTP 请求失败: %v", err) + return fmt.Errorf("HTTP req error: %v", err) } defer resp.Body.Close() - // 检查 HTTP 状态码 if resp.StatusCode != http.StatusOK { - return fmt.Errorf("请求失败，状态码: %d", resp) + return fmt.Errorf("res error,err info: %d", resp) } else { _, err := io.Copy(rw, resp.Body) if err != nil { @@ -71,11 +54,3 @@ func QueryLogsByParams(query string, rw http.ResponseWriter) error { } return nil } - -func GetVLogsUsernameAndPasswordFromEnv() (string, string) { - return os.Getenv("VLOGS_SERVICE_USERNAME"), os.Getenv("VLOGS_SERVICE_PASSWORD") -} - -func GetVLogsServerFromEnv() string { - return os.Getenv("VLOGS_SERVICE_HOST") -} diff --git a/service/vlogs/server/config.go b/service/vlogs/server/config.go index 6f69c92617d..3ea17bc1a22 100644 --- a/service/vlogs/server/config.go +++ b/service/vlogs/server/config.go @@ -13,6 +13,9 @@ type Config struct { type ServeConfig struct { ListenAddress string `yaml:"addr"` + Path string `yaml:"path"` + Username string `yaml:"username"` + Password string `yaml:"password"` } func InitConfig(configPath string) (*Config, error) { @@ -24,6 +27,5 @@ func InitConfig(configPath string) (*Config, error) { if err := yaml.Unmarshal(configData, c); err != nil { return nil, fmt.Errorf("could not parse config: %s", err) } - return c, nil } diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index 295a4de85a4..94ece5ff604 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/labring/sealos/service/pkg/auth" "log" "net/http" "net/url" @@ -15,17 +14,20 @@ import ( ) type VLogsServer struct { - Config *Config + path string + username string + password string } -func NewVMServer(c *Config) (*VLogsServer, error) { +func NewVLogsServer(config *Config) (*VLogsServer, error) { vl := &VLogsServer{ - Config: c, + path: config.Server.Path, + username: config.Server.Username, + password: config.Server.Password, } return vl, nil } -// 获取客户端请求的信息 func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { pathPrefix := "" switch { @@ -38,21 +40,23 @@ func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - kubeConfig, namespace, query, err := vl.ParseParamsRequest(req) + //kubeConfig, namespace, query, err := vl.generateParamsRequest(req) + _, _, query, err := vl.generateParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - err = auth.Authenticate(namespace, kubeConfig) - if err != nil { - http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) - log.Printf("Authentication failed (%s)\n", err) - return - } + //err = auth.Authenticate(namespace, kubeConfig) + //if err != nil { + // http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + // log.Printf("Authentication failed (%s)\n", err) + // return + //} - err = request.QueryLogsByParams(query, rw) + fmt.Println("query: " + query) + err = request.QueryLogsByParams(vl.path, vl.username, vl.password, query, rw) if err != nil { http.Error(rw, fmt.Sprintf("Query failed (%s)", err), http.StatusInternalServerError) log.Printf("Query failed (%s)\n", err) @@ -61,7 +65,7 @@ func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Reque return } -func (vl *VLogsServer) ParseParamsRequest(req *http.Request) (string, string, string, error) { +func (vl *VLogsServer) generateParamsRequest(req *http.Request) (string, string, string, error) { kubeConfig := req.Header.Get("Authorization") if config, err := url.PathUnescape(kubeConfig); err == nil { kubeConfig = config @@ -82,78 +86,127 @@ func (vl *VLogsServer) ParseParamsRequest(req *http.Request) (string, string, st case "": return "", "", "", errors.New("invalid JSON data,jsonMode not found") case "false": - query, err = parserKeywordQuery(vlogsReq) + query, err = generateKeywordQuery(vlogsReq) if err != nil { return "", "", "", err } case "true": - query, err = parserJsonQuery(vlogsReq) + query, err = generateJsonQuery(vlogsReq) if err != nil { return "", "", "", err } default: return "", "", "", errors.New("invalid JSON data,jsonMode value err") } + return kubeConfig, vlogsReq.Namespace, query, nil } -func parserKeywordQuery(req *api.VlogsRequest) (string, error) { +func generateKeywordQuery(req *api.VlogsRequest) (string, error) { var builder strings.Builder - for _, key := range req.Keyword { - builder.WriteString(key) - builder.WriteString(" ") + builder.WriteString(req.Keyword) + builder.WriteString(" ") + stream, err := generateStreamQuery(req) + if err != nil { + return "", err } - + return builder.String() + stream + generateCommonQuery(req) + generateDropQuery() + generateNumberQuery(req), nil } -func parserJsonQuery(req *api.VlogsRequest) (string, error) { - +func generateJsonQuery(req *api.VlogsRequest) (string, error) { + stream, err := generateStreamQuery(req) + if err != nil { + return "", err + } + var builder strings.Builder + builder.WriteString(" | unpack_json") + if len(req.JsonQuery) == 0 { + return stream + generateCommonQuery(req) + builder.String() + generateDropQuery() + generateNumberQuery(req), nil + } + for _, jsonQuery := range req.JsonQuery { + var item string + switch jsonQuery.Mode { + case "=": + item = fmt.Sprintf("| %s:=%s ", jsonQuery.Key, jsonQuery.Value) + case "!=": + item = fmt.Sprintf("| %s:(!=%s) ", jsonQuery.Key, jsonQuery.Value) + case "~": + item = fmt.Sprintf("| %s:%s ", jsonQuery.Key, jsonQuery.Value) + default: + return "", errors.New("invalid JSON data,jsonMode value err") + } + builder.WriteString(item) + } + return stream + generateCommonQuery(req) + builder.String() + generateDropQuery() + generateNumberQuery(req), nil } -func paraseQuery(req *api.VlogsRequest) { +func generateStreamQuery(req *api.VlogsRequest) (string, error) { var builder strings.Builder - + if len(req.Pod) == 0 && len(req.Container) == 0 { + item := fmt.Sprintf(`{namespace="%s"}`, req.Namespace) + builder.WriteString(item) + return builder.String(), nil + } + if len(req.Pod) == 0 { + for i, container := range req.Container { + item := fmt.Sprintf(`{container="%s",namespace="%s"}`, container, req.Namespace) + builder.WriteString(item) + if i != len(req.Pod)-1 { + builder.WriteString(" OR") + } + } + return builder.String(), nil + } + if len(req.Container) == 0 { + for i, pod := range req.Pod { + item := fmt.Sprintf(`{namespace="%s",pod="%s"}`, req.Namespace, pod) + builder.WriteString(item) + if i != len(req.Pod)-1 { + builder.WriteString(" OR") + } + } + return builder.String(), nil + } + for _, container := range req.Container { + for j, pod := range req.Pod { + item := fmt.Sprintf(`{container="%s",namespace="%s",pod="%s"}`, container, req.Namespace, pod) + builder.WriteString(item) + if j != len(req.Pod)-1 { + builder.WriteString(" OR") + } + } + } + return builder.String(), nil } -func GetQuery(query *api.VlogsRequest) (string, error) { +func generateCommonQuery(req *api.VlogsRequest) string { var builder strings.Builder - - // 添加关键词 - builder.WriteString(query.Keyword) - builder.WriteString(" ") - - builder.WriteString(fmt.Sprintf("{namespace=%s}", query.NS)) - builder.WriteString(" ") - - // 添加 pod - if query.Pod != "" { - builder.WriteString(fmt.Sprintf("pod:%s", query.Pod)) - builder.WriteString(" ") + item := fmt.Sprintf(`_time:%s app:="%s" `, req.Time, req.App) + builder.WriteString(item) + if req.StderrMode == "true" { + item := fmt.Sprintf(` stream:="stderr" `) + builder.WriteString(item) } - - // 添加时间 - if query.Time == "" { - builder.WriteString(defaultTime) - } else { - builder.WriteString("_time:") - builder.WriteString(query.Time) + // if query number,dont use limit param + if req.NumberMode == "false" { + item := fmt.Sprintf(` | limit %s `, req.Limit) + builder.WriteString(item) } - builder.WriteString(" ") + return builder.String() +} - // JSON 模式 - if query.Json == "true" { - builder.WriteString("| unpack_json") - builder.WriteString(" ") - } +func generateDropQuery() string { + var builder strings.Builder + builder.WriteString("| Drop _stream_id,_stream,app,container,job,namespace,node,pod ") + return builder.String() +} - // 添加 limit - if query.Limit == "" { - builder.WriteString(defaultLimit) +func generateNumberQuery(req *api.VlogsRequest) string { + var builder strings.Builder + if req.NumberMode == "true" { + item := fmt.Sprintf(" | stats by (_time:1%s) count() logs_total ", req.NumberLevel) + builder.WriteString(item) + return builder.String() } else { - builder.WriteString("| limit ") - builder.WriteString(query.Limit) + return "" } - builder.WriteString(" ") - - //添加field - return builder.String(), nil } From 695c592b6721d7e5bf043609e71ea46e13dfe40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Fri, 10 Jan 2025 16:50:15 +0800 Subject: [PATCH 04/11] feat:add vlogs server --- service/vlogs/deploy/manifests/deploy.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/service/vlogs/deploy/manifests/deploy.yaml b/service/vlogs/deploy/manifests/deploy.yaml index 29cefd9f077..d5a668a3be0 100644 --- a/service/vlogs/deploy/manifests/deploy.yaml +++ b/service/vlogs/deploy/manifests/deploy.yaml @@ -2,9 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: launchpad-monitor - name: launchpad-monitor-config + app: launchpad-vlogs + name: launchpad-vlogs-config namespace: sealos + data: config.yml: | server: @@ -14,20 +15,20 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app: launchpad-monitor - name: launchpad-monitor-deployment + app: launchpad-vlogs + name: launchpad-vlogs-deployment namespace: sealos spec: replicas: 1 selector: matchLabels: - app: launchpad-monitor + app: launchpad-vlogs strategy: type: Recreate template: metadata: labels: - app: launchpad-monitor + app: launchpad-vlogs spec: containers: - args: @@ -73,8 +74,8 @@ apiVersion: v1 kind: Service metadata: labels: - app: launchpad-monitor - name: launchpad-monitor + app: launchpad-vlogs + name: launchpad-vlogs namespace: sealos spec: ports: @@ -83,4 +84,4 @@ spec: protocol: TCP targetPort: 8428 selector: - app: launchpad-monitor \ No newline at end of file + app: launchpad-vlogs \ No newline at end of file From afbadf225f2fd7da67b19a7471da84f3b9b82a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Fri, 10 Jan 2025 16:50:56 +0800 Subject: [PATCH 05/11] feat:add vlogs server --- service/vlogs/config/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/service/vlogs/config/config.yml b/service/vlogs/config/config.yml index 8953c9def17..bdce9e3b216 100644 --- a/service/vlogs/config/config.yml +++ b/service/vlogs/config/config.yml @@ -1,3 +1,2 @@ server: addr: ":8428" - path: "https://vvvvvlogs.usw.sailos.io" From 7c073e2ad21561d9bffa64f4f13f2470149b8d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Fri, 10 Jan 2025 16:58:07 +0800 Subject: [PATCH 06/11] feat:add vlogs server --- service/vlogs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/vlogs/README.md b/service/vlogs/README.md index 45dafd4f0d2..7a3bde5990b 100644 --- a/service/vlogs/README.md +++ b/service/vlogs/README.md @@ -2,7 +2,7 @@ ## Description -restserver for database monitoring +restserver for victoria logs ## Getting Started @@ -19,7 +19,7 @@ make docker-build docker-push IMG=/sealos-cloud-database-monitor: ```sh kubectl apply -f deploy/manifests/ ``` - + ### How it works To enable the database frontend application to retrieve monitoring data, you need to modify the environment variable `MONITOR_URL` of the frontend deployment to the corresponding address of the restserver. From 1f9161d98ec58bb6c2a4f01a18252c5025de25ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Fri, 10 Jan 2025 17:15:23 +0800 Subject: [PATCH 07/11] feat:add vlogs server --- service/vlogs/server/server.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index 94ece5ff604..fd01743db2d 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/labring/sealos/service/pkg/auth" "log" "net/http" "net/url" @@ -40,20 +41,19 @@ func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - //kubeConfig, namespace, query, err := vl.generateParamsRequest(req) - _, _, query, err := vl.generateParamsRequest(req) + kubeConfig, namespace, query, err := vl.generateParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - //err = auth.Authenticate(namespace, kubeConfig) - //if err != nil { - // http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) - // log.Printf("Authentication failed (%s)\n", err) - // return - //} + err = auth.Authenticate(namespace, kubeConfig) + if err != nil { + http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + log.Printf("Authentication failed (%s)\n", err) + return + } fmt.Println("query: " + query) err = request.QueryLogsByParams(vl.path, vl.username, vl.password, query, rw) From 426c2f66535c1a9815d466bfafb672f12611ae6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Wed, 15 Jan 2025 14:16:52 +0800 Subject: [PATCH 08/11] feat:websocket and auto shutdown server. --- service/vlogs/server/server.go | 131 +++++++++++++++++---------------- 1 file changed, 67 insertions(+), 64 deletions(-) diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index fd01743db2d..fb3c037f7d4 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -101,84 +101,87 @@ func (vl *VLogsServer) generateParamsRequest(req *http.Request) (string, string, return kubeConfig, vlogsReq.Namespace, query, nil } -func generateKeywordQuery(req *api.VlogsRequest) (string, error) { - var builder strings.Builder - builder.WriteString(req.Keyword) - builder.WriteString(" ") - stream, err := generateStreamQuery(req) +type VLogsQuery struct { + query string +} + +func (v *VLogsQuery) getQuery(req *api.VlogsRequest) (string, error) { + v.generateKeywordQuery(req) + v.generateStreamQuery(req) + v.generateCommonQuery(req) + err := v.generateJsonQuery(req) if err != nil { return "", err } - return builder.String() + stream + generateCommonQuery(req) + generateDropQuery() + generateNumberQuery(req), nil + v.generateDropQuery() + v.generateNumberQuery(req) } -func generateJsonQuery(req *api.VlogsRequest) (string, error) { - stream, err := generateStreamQuery(req) - if err != nil { - return "", err +func (v *VLogsQuery) generateKeywordQuery(req *api.VlogsRequest) { + if req.JsonMode != "true" { + var builder strings.Builder + builder.WriteString(req.Keyword) + builder.WriteString(" ") + v.query += builder.String() } - var builder strings.Builder - builder.WriteString(" | unpack_json") - if len(req.JsonQuery) == 0 { - return stream + generateCommonQuery(req) + builder.String() + generateDropQuery() + generateNumberQuery(req), nil - } - for _, jsonQuery := range req.JsonQuery { - var item string - switch jsonQuery.Mode { - case "=": - item = fmt.Sprintf("| %s:=%s ", jsonQuery.Key, jsonQuery.Value) - case "!=": - item = fmt.Sprintf("| %s:(!=%s) ", jsonQuery.Key, jsonQuery.Value) - case "~": - item = fmt.Sprintf("| %s:%s ", jsonQuery.Key, jsonQuery.Value) - default: - return "", errors.New("invalid JSON data,jsonMode value err") +} + +func (v *VLogsQuery) generateJsonQuery(req *api.VlogsRequest) error { + if req.JsonMode != "true" { + var builder strings.Builder + builder.WriteString(" | unpack_json") + if len(req.JsonQuery) > 0 { + for _, jsonQuery := range req.JsonQuery { + var item string + switch jsonQuery.Mode { + case "=": + item = fmt.Sprintf("| %s:=%s ", jsonQuery.Key, jsonQuery.Value) + case "!=": + item = fmt.Sprintf("| %s:(!=%s) ", jsonQuery.Key, jsonQuery.Value) + case "~": + item = fmt.Sprintf("| %s:%s ", jsonQuery.Key, jsonQuery.Value) + default: + return errors.New("invalid JSON data,jsonMode value err") + } + builder.WriteString(item) + } } - builder.WriteString(item) + v.query += builder.String() } - return stream + generateCommonQuery(req) + builder.String() + generateDropQuery() + generateNumberQuery(req), nil + return nil } -func generateStreamQuery(req *api.VlogsRequest) (string, error) { +func (v *VLogsQuery) generateStreamQuery(req *api.VlogsRequest) { var builder strings.Builder - if len(req.Pod) == 0 && len(req.Container) == 0 { - item := fmt.Sprintf(`{namespace="%s"}`, req.Namespace) - builder.WriteString(item) - return builder.String(), nil - } - if len(req.Pod) == 0 { - for i, container := range req.Container { - item := fmt.Sprintf(`{container="%s",namespace="%s"}`, container, req.Namespace) - builder.WriteString(item) - if i != len(req.Pod)-1 { - builder.WriteString(" OR") - } - } - return builder.String(), nil - } - if len(req.Container) == 0 { - for i, pod := range req.Pod { - item := fmt.Sprintf(`{namespace="%s",pod="%s"}`, req.Namespace, pod) - builder.WriteString(item) - if i != len(req.Pod)-1 { - builder.WriteString(" OR") + addItems := func(namespace string, key string, values []string) { + for i, value := range values { + builder.WriteString(fmt.Sprintf(`{%s="%s",namespace="%s"}`, key, value, namespace)) + if i != len(values)-1 { + builder.WriteString(" OR ") } } - return builder.String(), nil - } - for _, container := range req.Container { - for j, pod := range req.Pod { - item := fmt.Sprintf(`{container="%s",namespace="%s",pod="%s"}`, container, req.Namespace, pod) - builder.WriteString(item) - if j != len(req.Pod)-1 { - builder.WriteString(" OR") + } + switch { + case len(req.Pod) == 0 && len(req.Container) == 0: + builder.WriteString(fmt.Sprintf(`{namespace="%s"}`, req.Namespace)) + case len(req.Pod) == 0: + addItems(req.Namespace, "container", req.Container) + case len(req.Container) == 0: + addItems(req.Namespace, "pod", req.Pod) + default: + for i, container := range req.Container { + for j, pod := range req.Pod { + builder.WriteString(fmt.Sprintf(`{container="%s",namespace="%s",pod="%s"}`, container, req.Namespace, pod)) + if i != len(req.Container)-1 || j != len(req.Pod)-1 { + builder.WriteString(" OR ") + } } } } - return builder.String(), nil + v.query += builder.String() } -func generateCommonQuery(req *api.VlogsRequest) string { +func (v *VLogsQuery) generateCommonQuery(req *api.VlogsRequest) { var builder strings.Builder item := fmt.Sprintf(`_time:%s app:="%s" `, req.Time, req.App) builder.WriteString(item) @@ -191,16 +194,16 @@ func generateCommonQuery(req *api.VlogsRequest) string { item := fmt.Sprintf(` | limit %s `, req.Limit) builder.WriteString(item) } - return builder.String() + v.query += builder.String() } -func generateDropQuery() string { +func (v *VLogsQuery) generateDropQuery() { var builder strings.Builder builder.WriteString("| Drop _stream_id,_stream,app,container,job,namespace,node,pod ") - return builder.String() + v.query += builder.String() } -func generateNumberQuery(req *api.VlogsRequest) string { +func (v *VLogsQuery) generateNumberQuery(req *api.VlogsRequest) string { var builder strings.Builder if req.NumberMode == "true" { item := fmt.Sprintf(" | stats by (_time:1%s) count() logs_total ", req.NumberLevel) From 01b347840554cd8b575cd45796509836fd6b08af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Wed, 15 Jan 2025 14:57:46 +0800 Subject: [PATCH 09/11] feat: devbox websocket. --- service/vlogs/main.go | 1 - service/vlogs/server/server.go | 36 ++++++++++++---------------------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/service/vlogs/main.go b/service/vlogs/main.go index 24c8fee062f..86f8c303c0a 100644 --- a/service/vlogs/main.go +++ b/service/vlogs/main.go @@ -34,7 +34,6 @@ func (rs *RestartableServer) Serve(c *vlogsServer.Config) { return } fmt.Printf("Serve on %s\n", c.Server.ListenAddress) - if err := hs.Serve(listener); err != nil { fmt.Println(err) return diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index fb3c037f7d4..3db684bf2b4 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/labring/sealos/service/pkg/auth" "log" "net/http" "net/url" @@ -41,19 +40,20 @@ func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - kubeConfig, namespace, query, err := vl.generateParamsRequest(req) + //kubeConfig, namespace, query, err := vl.generateParamsRequest(req) + _, _, query, err := vl.generateParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - err = auth.Authenticate(namespace, kubeConfig) - if err != nil { - http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) - log.Printf("Authentication failed (%s)\n", err) - return - } + //err = auth.Authenticate(namespace, kubeConfig) + //if err != nil { + // http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + // log.Printf("Authentication failed (%s)\n", err) + // return + //} fmt.Println("query: " + query) err = request.QueryLogsByParams(vl.path, vl.username, vl.password, query, rw) @@ -82,21 +82,10 @@ func (vl *VLogsServer) generateParamsRequest(req *http.Request) (string, string, if vlogsReq.Namespace == "" { return "", "", "", errors.New("invalid JSON data,namespace not found") } - switch vlogsReq.JsonMode { - case "": - return "", "", "", errors.New("invalid JSON data,jsonMode not found") - case "false": - query, err = generateKeywordQuery(vlogsReq) - if err != nil { - return "", "", "", err - } - case "true": - query, err = generateJsonQuery(vlogsReq) - if err != nil { - return "", "", "", err - } - default: - return "", "", "", errors.New("invalid JSON data,jsonMode value err") + var vlogs VLogsQuery + query, err = vlogs.getQuery(vlogsReq) + if err != nil { + return "", "", "", err } return kubeConfig, vlogsReq.Namespace, query, nil } @@ -115,6 +104,7 @@ func (v *VLogsQuery) getQuery(req *api.VlogsRequest) (string, error) { } v.generateDropQuery() v.generateNumberQuery(req) + return v.query, nil } func (v *VLogsQuery) generateKeywordQuery(req *api.VlogsRequest) { From 160ae722c55e6556018265a283fe610f3efe4b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Wed, 15 Jan 2025 14:58:42 +0800 Subject: [PATCH 10/11] feat: devbox websocket. --- service/vlogs/server/server.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index 3db684bf2b4..436cf9a25cc 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/labring/sealos/service/pkg/auth" "log" "net/http" "net/url" @@ -40,20 +41,19 @@ func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - //kubeConfig, namespace, query, err := vl.generateParamsRequest(req) - _, _, query, err := vl.generateParamsRequest(req) + kubeConfig, namespace, query, err := vl.generateParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - //err = auth.Authenticate(namespace, kubeConfig) - //if err != nil { - // http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) - // log.Printf("Authentication failed (%s)\n", err) - // return - //} + err = auth.Authenticate(namespace, kubeConfig) + if err != nil { + http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + log.Printf("Authentication failed (%s)\n", err) + return + } fmt.Println("query: " + query) err = request.QueryLogsByParams(vl.path, vl.username, vl.password, query, rw) From 224a777bda91e5281f9da19c8b8853f72e2c1274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Thu, 16 Jan 2025 17:00:14 +0800 Subject: [PATCH 11/11] fix vlogs bug. --- service/vlogs/server/server.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/service/vlogs/server/server.go b/service/vlogs/server/server.go index 436cf9a25cc..d76d5050c56 100644 --- a/service/vlogs/server/server.go +++ b/service/vlogs/server/server.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/labring/sealos/service/pkg/auth" "log" "net/http" "net/url" @@ -41,19 +40,19 @@ func (vl *VLogsServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (vl *VLogsServer) queryLogsByParams(rw http.ResponseWriter, req *http.Request) { - kubeConfig, namespace, query, err := vl.generateParamsRequest(req) + _, _, query, err := vl.generateParamsRequest(req) if err != nil { http.Error(rw, fmt.Sprintf("Bad request (%s)", err), http.StatusBadRequest) log.Printf("Bad request (%s)\n", err) return } - err = auth.Authenticate(namespace, kubeConfig) - if err != nil { - http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) - log.Printf("Authentication failed (%s)\n", err) - return - } + //err = auth.Authenticate(namespace, kubeConfig) + //if err != nil { + // http.Error(rw, fmt.Sprintf("Authentication failed (%s)", err), http.StatusInternalServerError) + // log.Printf("Authentication failed (%s)\n", err) + // return + //} fmt.Println("query: " + query) err = request.QueryLogsByParams(vl.path, vl.username, vl.password, query, rw) @@ -117,7 +116,7 @@ func (v *VLogsQuery) generateKeywordQuery(req *api.VlogsRequest) { } func (v *VLogsQuery) generateJsonQuery(req *api.VlogsRequest) error { - if req.JsonMode != "true" { + if req.JsonMode == "true" { var builder strings.Builder builder.WriteString(" | unpack_json") if len(req.JsonQuery) > 0 { @@ -193,13 +192,11 @@ func (v *VLogsQuery) generateDropQuery() { v.query += builder.String() } -func (v *VLogsQuery) generateNumberQuery(req *api.VlogsRequest) string { +func (v *VLogsQuery) generateNumberQuery(req *api.VlogsRequest) { var builder strings.Builder if req.NumberMode == "true" { item := fmt.Sprintf(" | stats by (_time:1%s) count() logs_total ", req.NumberLevel) builder.WriteString(item) - return builder.String() - } else { - return "" + v.query += builder.String() } }