diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 297324e..4881132 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,10 +1,10 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-10-10T17:36:54Z by kres latest. +# Generated on 2024-01-12T12:34:10Z by kres latest. name: default concurrency: - group: ${{ github.event.label == null && github.head_ref || github.run_id }} + group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true "on": push: @@ -20,15 +20,18 @@ concurrency: jobs: default: permissions: + actions: read contents: write + issues: read packages: write + pull-requests: read runs-on: - self-hosted - generic if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/')) services: buildkitd: - image: moby/buildkit:v0.12.2 + image: moby/buildkit:v0.12.4 options: --privileged ports: - 1234:1234 @@ -89,8 +92,8 @@ jobs: - name: Generate Checksums if: startsWith(github.ref, 'refs/tags/') run: | - sha256sum _out/* > _out/sha256sum.txt - sha512sum _out/* > _out/sha512sum.txt + sha256sum _out/discovery-service-* > _out/sha256sum.txt + sha512sum _out/discovery-service-* > _out/sha512sum.txt - name: release-notes if: startsWith(github.ref, 'refs/tags/') run: | @@ -102,5 +105,5 @@ jobs: body_path: _out/RELEASE_NOTES.md draft: "true" files: |- - _out/* + _out/discovery-service-* _out/sha*.txt diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index 6a8e4a2..3b5d7bb 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-10-10T17:36:54Z by kres latest. +# Generated on 2024-01-12T12:34:10Z by kres latest. name: slack-notify "on": @@ -14,14 +14,15 @@ jobs: runs-on: - self-hosted - generic - if: ${{ github.event.workflow_run.conclusion != 'skipped' }} + if: github.event.workflow_run.conclusion != 'skipped' steps: - - name: Retrieve Workflow Run Info - id: retrieve-workflow-run-info - uses: potiuk/get-workflow-origin@v1_5 - with: - sourceRunId: ${{ github.event.workflow_run.id }} - token: ${{ secrets.GITHUB_TOKEN }} + - name: Get PR number + id: get-pr-number + if: github.event.workflow_run.event == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: | + echo pull_request_number=$(gh pr view -R ${{ github.repository }} ${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }} --json number --jq .number) >> $GITHUB_OUTPUT - name: Slack Notify uses: slackapi/slack-github-action@v1 with: @@ -38,7 +39,7 @@ jobs: "fields": [ { "type": "mrkdwn", - "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.retrieve-workflow-run-info.outputs.pullRequestNumber, github.event.workflow_run.display_title) || format('*Build:* {0}#{1} (`{2}`)', github.repository, github.sha, github.ref_name) }}" + "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}" }, { "type": "mrkdwn", diff --git a/.golangci.yml b/.golangci.yml index 9426b94..67cc272 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-10-10T17:36:54Z by kres latest. +# Generated on 2024-01-12T12:34:10Z by kres latest. # options for analysis running run: @@ -150,6 +150,9 @@ linters: - wrapcheck - depguard # Disabled because starting with golangci-lint 1.53.0 it doesn't allow denylist alone anymore - tagalign + - inamedparam + - testifylint # complains about our assert recorder and has a number of false positives for assert.Greater(t, thing, 1) + - protogetter # complains about us using Value field on typed spec, instead of GetValue which has a different signature # abandoned linters for which golangci shows the warning that the repo is archived by the owner - interfacer - maligned diff --git a/Dockerfile b/Dockerfile index be957ad..ef0f888 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,19 +2,19 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-10-10T17:36:54Z by kres latest. +# Generated on 2024-01-12T12:34:10Z by kres latest. ARG TOOLCHAIN # cleaned up specs and compiled versions FROM scratch AS generate -FROM ghcr.io/siderolabs/ca-certificates:v1.6.0-alpha.0-10-gd3d7d29 AS image-ca-certificates +FROM ghcr.io/siderolabs/ca-certificates:v1.6.0 AS image-ca-certificates -FROM ghcr.io/siderolabs/fhs:v1.6.0-alpha.0-10-gd3d7d29 AS image-fhs +FROM ghcr.io/siderolabs/fhs:v1.6.0 AS image-fhs # runs markdownlint -FROM docker.io/node:20.8.0-alpine3.18 AS lint-markdown +FROM docker.io/node:21.4.0-alpine3.18 AS lint-markdown WORKDIR /src RUN npm i -g markdownlint-cli@0.37.0 RUN npm i sentences-per-line@0.2.1 diff --git a/Makefile b/Makefile index 331a060..0195ad9 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-10-10T17:36:54Z by kres latest. +# Generated on 2024-01-12T14:05:03Z by kres 0e666ea-dirty. # common variables @@ -14,15 +14,15 @@ WITH_RACE ?= false REGISTRY ?= ghcr.io USERNAME ?= siderolabs REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME) -PROTOBUF_GO_VERSION ?= 1.31.0 +PROTOBUF_GO_VERSION ?= 1.32.0 GRPC_GO_VERSION ?= 1.3.0 -GRPC_GATEWAY_VERSION ?= 2.18.0 +GRPC_GATEWAY_VERSION ?= 2.19.0 VTPROTOBUF_VERSION ?= 0.5.0 DEEPCOPY_VERSION ?= v0.5.5 -GOLANGCILINT_VERSION ?= v1.54.2 +GOLANGCILINT_VERSION ?= v1.55.2 GOFUMPT_VERSION ?= v0.5.0 -GO_VERSION ?= 1.21.1 -GOIMPORTS_VERSION ?= v0.13.0 +GO_VERSION ?= 1.21.6 +GOIMPORTS_VERSION ?= v0.17.0 GO_BUILDFLAGS ?= GO_LDFLAGS ?= CGO_ENABLED ?= 0 @@ -184,7 +184,7 @@ image-discovery-service: ## Builds image for discovery-service. .PHONY: rekres rekres: @docker pull $(KRES_IMAGE) - @docker run --rm --net=host -v $(PWD):/src -w /src -e GITHUB_TOKEN $(KRES_IMAGE) + @docker run --rm --net=host --user $(shell id -u):$(shell id -g) -v $(PWD):/src -w /src -e GITHUB_TOKEN $(KRES_IMAGE) .PHONY: help help: ## This help menu. diff --git a/cmd/discovery-service/main.go b/cmd/discovery-service/main.go index 9e35c3b..ee9c99e 100644 --- a/cmd/discovery-service/main.go +++ b/cmd/discovery-service/main.go @@ -34,6 +34,7 @@ import ( "google.golang.org/grpc/status" "github.com/siderolabs/discovery-service/internal/landing" + "github.com/siderolabs/discovery-service/internal/limiter" _ "github.com/siderolabs/discovery-service/internal/proto" "github.com/siderolabs/discovery-service/internal/state" "github.com/siderolabs/discovery-service/pkg/limits" @@ -124,7 +125,7 @@ func run(ctx context.Context, logger *zap.Logger) error { recoveryOpt := grpc_recovery.WithRecoveryHandler(recoveryHandler(logger)) - limiter := limits.NewIPRateLimiter(limits.RequestsPerSecondMax, limits.BurstSizeMax) + limiter := limiter.NewIPRateLimiter(limits.IPRateRequestsPerSecondMax, limits.IPRateBurstSizeMax) //nolint:contextcheck serverOptions := []grpc.ServerOption{ diff --git a/go.mod b/go.mod index e08b1f8..1a7bdb6 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,22 @@ module github.com/siderolabs/discovery-service -go 1.21.3 +go 1.21.6 require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/siderolabs/discovery-api v0.1.3 github.com/siderolabs/discovery-client v0.1.5 + github.com/siderolabs/gen v0.4.7 github.com/siderolabs/go-debug v0.2.3 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 - go4.org/netipx v0.0.0-20230824141953-6213f710f925 + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/sync v0.4.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.58.3 - google.golang.org/protobuf v1.31.0 + google.golang.org/grpc v1.60.1 + google.golang.org/protobuf v1.32.0 ) require ( @@ -24,16 +25,15 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/siderolabs/gen v0.4.5 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7a682d3..949c707 100644 --- a/go.sum +++ b/go.sum @@ -48,29 +48,29 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/siderolabs/discovery-api v0.1.3 h1:37ue+0w2A7Q2FrhyuDbfdhL4VPvDTpCzUYGvibhMwv0= github.com/siderolabs/discovery-api v0.1.3/go.mod h1:fC6DOJwYQy2QsMCLLTvoScKmBCMNza+VwK2/RHLsoHU= github.com/siderolabs/discovery-client v0.1.5 h1:CyaOOynanZdB29v46lyEOaNfPoBnKjjEBwdYbyCZEh4= github.com/siderolabs/discovery-client v0.1.5/go.mod h1:XFSNX7ADu+4r3j/m299V6pP7f4vEDnSJJhgc5yZE73g= -github.com/siderolabs/gen v0.4.5 h1:rwXUVJlL7hYza1LrSVXfT905ZC9Rgei37jMKKs/+eP0= -github.com/siderolabs/gen v0.4.5/go.mod h1:wS8tFq7sn5vqKAuyS30vJUig3tX5v6q79VG4KfUnILM= +github.com/siderolabs/gen v0.4.7 h1:lM69UYggT7yzpubf7hEFaNujPdY55Y9zvQf/NC18GvA= +github.com/siderolabs/gen v0.4.7/go.mod h1:4PBYMdXxTg292IDRq4CGn5AymyDxJVEDvobVKDqFBEA= github.com/siderolabs/go-debug v0.2.3 h1:O9luHW4P++gQqOKzMnUgGIiQGsg9QaQmi6qI0JfQHXI= github.com/siderolabs/go-debug v0.2.3/go.mod h1:45N6ALRTyrbtKAzbehGwz3VoE5MZqEKd/xrv7dawR30= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -94,8 +94,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ= -go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -119,7 +119,6 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,8 +130,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= @@ -157,19 +156,19 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/limiter/iprate.go b/internal/limiter/iprate.go new file mode 100644 index 0000000..2624776 --- /dev/null +++ b/internal/limiter/iprate.go @@ -0,0 +1,85 @@ +// Copyright (c) 2021 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +// Package limiter provides service resource limiters. +package limiter + +import ( + "context" + "net/netip" + "sync" + "time" + + "golang.org/x/time/rate" + + "github.com/siderolabs/discovery-service/pkg/limits" +) + +// IPRateLimiter applies the same limits to a group of IP addresses. +type IPRateLimiter struct { + ips map[netip.Addr]*rate.Limiter + mu sync.Mutex + rateLimit rate.Limit + bucketSize int +} + +// NewIPRateLimiter returns a new IPRateLimiter. +func NewIPRateLimiter(rateLimit rate.Limit, bucketSize int) *IPRateLimiter { + return &IPRateLimiter{ + ips: make(map[netip.Addr]*rate.Limiter), + rateLimit: rateLimit, + bucketSize: bucketSize, + } +} + +// Get returns the rate limiter for the provided IP address if it exists or creates a new one. +func (iPRL *IPRateLimiter) Get(ip netip.Addr) *rate.Limiter { + iPRL.mu.Lock() + defer iPRL.mu.Unlock() + + limiter, exists := iPRL.ips[ip] + + if !exists { + limiter = rate.NewLimiter(iPRL.rateLimit, iPRL.bucketSize) + iPRL.ips[ip] = limiter + } + + return limiter +} + +// Len returns the number of limiters. +func (iPRL *IPRateLimiter) Len() int { + iPRL.mu.Lock() + defer iPRL.mu.Unlock() + + return len(iPRL.ips) +} + +// RunGC periodically clears IPs. +func (iPRL *IPRateLimiter) RunGC(ctx context.Context) { + ticker := time.NewTicker(limits.IPRateGarbageCollectionPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + iPRL.DoGC(time.Now()) + case <-ctx.Done(): + return + } + } +} + +// DoGC runs a single round of garbage collection. +func (iPRL *IPRateLimiter) DoGC(now time.Time) { + iPRL.mu.Lock() + for key, val := range iPRL.ips { + // AllowN on success consumes the tokens, but as the limiter is going to be dropped, it doesn't matter + if val.AllowN(now, iPRL.bucketSize) { + delete(iPRL.ips, key) + } + } + iPRL.mu.Unlock() +} diff --git a/pkg/limits/limits_test.go b/internal/limiter/iprate_test.go similarity index 72% rename from pkg/limits/limits_test.go rename to internal/limiter/iprate_test.go index 9cbbdef..85d8c06 100644 --- a/pkg/limits/limits_test.go +++ b/internal/limiter/iprate_test.go @@ -3,27 +3,28 @@ // Use of this software is governed by the Business Source License // included in the LICENSE file. -package limits_test +package limiter_test import ( + "net/netip" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/siderolabs/discovery-service/pkg/limits" + "github.com/siderolabs/discovery-service/internal/limiter" ) func TestDoGC(t *testing.T) { t.Parallel() - const ( - testIP1 = "1.1.1.1" - testIP2 = "2.2.2.2" + var ( + testIP1 = netip.MustParseAddr("1.1.1.1") + testIP2 = netip.MustParseAddr("2.2.2.2") ) - limiter := limits.NewIPRateLimiter(1, 1) + limiter := limiter.NewIPRateLimiter(1, 1) // hit the bucket size lim1 := limiter.Get(testIP1) @@ -49,12 +50,12 @@ func TestDoGC(t *testing.T) { func TestIndependentLimiters(t *testing.T) { t.Parallel() - const ( - testIP1 = "1.1.1.1" - testIP2 = "2.2.2.2" + var ( + testIP1 = netip.MustParseAddr("1.1.1.1") + testIP2 = netip.MustParseAddr("2.2.2.2") ) - limiter := limits.NewIPRateLimiter(1, 1) + limiter := limiter.NewIPRateLimiter(1, 1) require.True(t, limiter.Get(testIP1).Allow()) require.False(t, limiter.Get(testIP1).Allow()) diff --git a/internal/state/affiliate.go b/internal/state/affiliate.go index 08d91db..235a773 100644 --- a/internal/state/affiliate.go +++ b/internal/state/affiliate.go @@ -7,6 +7,7 @@ package state import ( "bytes" + "slices" "time" "github.com/siderolabs/discovery-service/pkg/limits" @@ -114,5 +115,10 @@ func (affiliate *Affiliate) GarbageCollect(now time.Time) (remove, changed bool) affiliate.endpoints = affiliate.endpoints[:n] + // garbage collect extra capacity + if cap(affiliate.endpoints) > 8 && cap(affiliate.endpoints) > 2*len(affiliate.endpoints) { + affiliate.endpoints = slices.Clip(affiliate.endpoints) + } + return } diff --git a/internal/state/cluster.go b/internal/state/cluster.go index e2dc509..e969a5f 100644 --- a/internal/state/cluster.go +++ b/internal/state/cluster.go @@ -6,9 +6,13 @@ package state import ( + "slices" "sync" "time" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/discovery-service/pkg/limits" ) @@ -90,13 +94,7 @@ func (cluster *Cluster) List() []*AffiliateExport { cluster.affiliatesMu.Lock() defer cluster.affiliatesMu.Unlock() - result := make([]*AffiliateExport, 0, len(cluster.affiliates)) - - for _, affiliate := range cluster.affiliates { - result = append(result, affiliate.Export()) - } - - return result + return xslices.Map(maps.Values(cluster.affiliates), func(affiliate *Affiliate) *AffiliateExport { return affiliate.Export() }) } // Subscribe to the affiliate updates. @@ -108,11 +106,7 @@ func (cluster *Cluster) Subscribe(ch chan<- *Notification) ([]*AffiliateExport, cluster.subscriptionsMu.Lock() defer cluster.subscriptionsMu.Unlock() - snapshot := make([]*AffiliateExport, 0, len(cluster.affiliates)) - - for _, affiliate := range cluster.affiliates { - snapshot = append(snapshot, affiliate.Export()) - } + snapshot := xslices.Map(maps.Values(cluster.affiliates), func(affiliate *Affiliate) *AffiliateExport { return affiliate.Export() }) subscription := &Subscription{ cluster: cluster, @@ -129,14 +123,12 @@ func (cluster *Cluster) unsubscribe(subscription *Subscription) { cluster.subscriptionsMu.Lock() defer cluster.subscriptionsMu.Unlock() - for i := range cluster.subscriptions { - if cluster.subscriptions[i] == subscription { - cluster.subscriptions[i] = cluster.subscriptions[len(cluster.subscriptions)-1] - cluster.subscriptions[len(cluster.subscriptions)-1] = nil - cluster.subscriptions = cluster.subscriptions[:len(cluster.subscriptions)-1] + idx := slices.Index(cluster.subscriptions, subscription) - return - } + if idx != -1 { + cluster.subscriptions[idx] = cluster.subscriptions[len(cluster.subscriptions)-1] + cluster.subscriptions[len(cluster.subscriptions)-1] = nil + cluster.subscriptions = cluster.subscriptions[:len(cluster.subscriptions)-1] } } @@ -171,7 +163,7 @@ func (cluster *Cluster) GarbageCollect(now time.Time) (removedAffiliates int, em func (cluster *Cluster) notify(notifications ...*Notification) { cluster.subscriptionsMu.Lock() - subscriptions := append([]*Subscription(nil), cluster.subscriptions...) + subscriptions := slices.Clone(cluster.subscriptions) cluster.subscriptionsMu.Unlock() for _, notification := range notifications { diff --git a/pkg/limits/limits.go b/pkg/limits/limits.go index aee885a..cb6dd1b 100644 --- a/pkg/limits/limits.go +++ b/pkg/limits/limits.go @@ -7,90 +7,23 @@ package limits import ( - "context" - "sync" "time" - - "golang.org/x/time/rate" ) // Service limits. const ( - ClusterIDMax = 256 - AffiliateIDMax = 256 - AffiliateDataMax = 2048 - AffiliateEndpointMax = 32 - TTLMax = 30 * time.Minute - ClusterAffiliatesMax = 1024 - AffiliateEndpointsMax = 64 - RequestsPerSecondMax = 5 - BurstSizeMax = 30 - GarbageCollectionPeriod = time.Minute + ClusterIDMax = 256 + AffiliateIDMax = 256 + AffiliateDataMax = 2048 + AffiliateEndpointMax = 32 + TTLMax = 30 * time.Minute + ClusterAffiliatesMax = 1024 + AffiliateEndpointsMax = 64 ) -// IPRateLimiter applies the same limits to a group of IP addresses. -type IPRateLimiter struct { - ips map[string]*rate.Limiter - mu sync.Mutex - rateLimit rate.Limit - bucketSize int -} - -// NewIPRateLimiter returns a new IPRateLimiter. -func NewIPRateLimiter(rateLimit rate.Limit, bucketSize int) *IPRateLimiter { - return &IPRateLimiter{ - ips: make(map[string]*rate.Limiter), - rateLimit: rateLimit, - bucketSize: bucketSize, - } -} - -// Get returns the rate limiter for the provided IP address if it exists or creates a new one. -func (iPRL *IPRateLimiter) Get(ip string) *rate.Limiter { - iPRL.mu.Lock() - defer iPRL.mu.Unlock() - - limiter, exists := iPRL.ips[ip] - - if !exists { - limiter = rate.NewLimiter(iPRL.rateLimit, iPRL.bucketSize) - iPRL.ips[ip] = limiter - } - - return limiter -} - -// Len returns the number of limiters. -func (iPRL *IPRateLimiter) Len() int { - iPRL.mu.Lock() - defer iPRL.mu.Unlock() - - return len(iPRL.ips) -} - -// RunGC periodically clears IPs. -func (iPRL *IPRateLimiter) RunGC(ctx context.Context) { - ticker := time.NewTicker(GarbageCollectionPeriod) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - iPRL.DoGC(time.Now()) - case <-ctx.Done(): - return - } - } -} - -// DoGC runs a single round of garbage collection. -func (iPRL *IPRateLimiter) DoGC(now time.Time) { - iPRL.mu.Lock() - for key, val := range iPRL.ips { - // AllowN on success consumes the tokens, but as the limiter is going to be dropped, it doesn't matter - if val.AllowN(now, iPRL.bucketSize) { - delete(iPRL.ips, key) - } - } - iPRL.mu.Unlock() -} +// IP Rate Limiter. +const ( + IPRateRequestsPerSecondMax = 5 + IPRateBurstSizeMax = 30 + IPRateGarbageCollectionPeriod = time.Minute +) diff --git a/pkg/server/client_test.go b/pkg/server/client_test.go index 63133dd..4888d4b 100644 --- a/pkg/server/client_test.go +++ b/pkg/server/client_test.go @@ -418,7 +418,7 @@ func clusterSimulator(t *testing.T, endpoint string, logger *zap.Logger, numAffi require.NoError(t, affiliates[i].SetLocalData(&client.Affiliate{ Affiliate: &clientpb.Affiliate{ NodeId: fmt.Sprintf("affiliate-%d", i), - Hostname: fmt.Sprintf("%d", i), + Hostname: strconv.Itoa(i), }, Endpoints: []*clientpb.Endpoint{ { diff --git a/pkg/server/limiting.go b/pkg/server/limiting.go index 907027b..172b40c 100644 --- a/pkg/server/limiting.go +++ b/pkg/server/limiting.go @@ -7,18 +7,19 @@ package server import ( "context" + "net/netip" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/siderolabs/discovery-service/pkg/limits" + "github.com/siderolabs/discovery-service/internal/limiter" ) -func pause(ctx context.Context, limiter *limits.IPRateLimiter) error { +func pause(ctx context.Context, limiter *limiter.IPRateLimiter) error { iPAddr := extractIPAddressFromTags(ctx) - if iPAddr != "" { + if !IsZero(iPAddr) { limit := limiter.Get(iPAddr) err := limit.Wait(ctx) @@ -31,7 +32,7 @@ func pause(ctx context.Context, limiter *limits.IPRateLimiter) error { } // RateLimitUnaryServerInterceptor limits Unary PRCs from an IPAdress. -func RateLimitUnaryServerInterceptor(limiter *limits.IPRateLimiter) grpc.UnaryServerInterceptor { +func RateLimitUnaryServerInterceptor(limiter *limiter.IPRateLimiter) grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { err = pause(ctx, limiter) if err != nil { @@ -43,7 +44,7 @@ func RateLimitUnaryServerInterceptor(limiter *limits.IPRateLimiter) grpc.UnarySe } // RateLimitStreamServerInterceptor limits Stream PRCs from an IPAdress. -func RateLimitStreamServerInterceptor(limiter *limits.IPRateLimiter) grpc.StreamServerInterceptor { +func RateLimitStreamServerInterceptor(limiter *limiter.IPRateLimiter) grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() @@ -56,15 +57,15 @@ func RateLimitStreamServerInterceptor(limiter *limits.IPRateLimiter) grpc.Stream } } -func extractIPAddressFromTags(ctx context.Context) string { +func extractIPAddressFromTags(ctx context.Context) netip.Addr { if tags := grpc_ctxtags.Extract(ctx); tags != nil { values := tags.Values() - if stringValue, ok := values["peer.address"]; ok { - if iPString, ok := stringValue.(string); ok { - return iPString + if addrV, ok := values["peer.address"]; ok { + if addr, ok := addrV.(netip.Addr); ok { + return addr } } } - return "" + return netip.Addr{} } diff --git a/pkg/server/logging.go b/pkg/server/logging.go index f2c76e8..2df50fa 100644 --- a/pkg/server/logging.go +++ b/pkg/server/logging.go @@ -59,7 +59,7 @@ func AddPeerAddressStreamServerInterceptor() grpc.StreamServerInterceptor { func extractPeerAddress(ctx context.Context) { if peerAddress := PeerAddress(ctx); !IsZero(peerAddress) { if tags := grpc_ctxtags.Extract(ctx); tags != nil { - tags.Set("peer.address", peerAddress.String()) + tags.Set("peer.address", peerAddress) } } } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index bb885db..fd4e180 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -31,6 +31,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" + "github.com/siderolabs/discovery-service/internal/limiter" _ "github.com/siderolabs/discovery-service/internal/proto" "github.com/siderolabs/discovery-service/internal/state" "github.com/siderolabs/discovery-service/pkg/limits" @@ -87,7 +88,7 @@ func setupServer(t *testing.T, rateLimit rate.Limit, redirectEndpoint string) *t testServer.address = testServer.lis.Addr().String() - limiter := limits.NewIPRateLimiter(rateLimit, limits.BurstSizeMax) + limiter := limiter.NewIPRateLimiter(rateLimit, limits.IPRateBurstSizeMax) testServer.serverOptions = []grpc.ServerOption{ grpc.ChainUnaryInterceptor( @@ -516,7 +517,7 @@ func testHitRateLimit(client pb.ClusterClient, ip string) func(t *testing.T) { ctx = metadata.AppendToOutgoingContext(ctx, "X-Real-IP", ip) - for i := 0; i < limits.BurstSizeMax; i++ { + for i := 0; i < limits.IPRateBurstSizeMax; i++ { _, err := client.Hello(ctx, &pb.HelloRequest{ ClusterId: "fake", ClientVersion: "v0.12.0",