[TT-13723] [release-5.3] Update to Go 1.23

.github/workflows/ci-tests.yml

@@ -17,36 +17,84 @@ on:
       - master
       - release-**
 
-# Only have one runner per PR, and per merged commit.
-#
-# - As a PR gets new commits, any old run jobs get cancelled (PR number)
-# - As a commit gets merged, it doesn't cancel previous running PR's (github.sha)
 concurrency:
-  group: ${{ github.event.pull_request.number || github.sha }}-ci-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   PYTHON_VERSION: "3.11"
   PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
 
 jobs:
+  golangci-lint:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    steps:
+      - name: "Checkout PR"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: "Get base ref"
+        run: |
+          git fetch origin ${{ github.base_ref }}
+          git rev-parse origin/${{ github.base_ref }}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/golangci-lint
+          key: 'golangci-lint-${{ runner.os }}-${{ hashFiles(''**/go.sum'') }}'
+          restore-keys: |
+            golangci-lint-${{ runner.os }}-
+
+      - name: Setup CI Tooling
+        uses: shrink/actions-docker-extract@v3
+        with:
+          image: tykio/ci-tools:latest
+          path: /usr/local/bin/golangci-lint
+          destination: /usr/local/bin
+
+      - run: golangci-lint version && golangci-lint cache status
+
+      - name: golangci-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          golangci-lint run --out-format colored-line-number,checkstyle:golangci-lint-report.json --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} -v ./...
+
+      - name: golangci-lint-on-push
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          golangci-lint run --out-format checkstyle:golangci-lint-report.json --issues-exit-code=0 -v ./...
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: golangcilint
+          retention-days: 1
+          path: |
+            golangci-lint-report.json
+
   test:
     name: Go ${{ matrix.go-version }} Redis ${{ matrix.redis-version }}
+    if: ${{ !github.event.pull_request.draft }}
+    needs: golangci-lint
+    # Runs on is pinned to a version that provides python 3.10.
+    # See: https://github.com/actions/runner-images?tab=readme-ov-file#available-images
+    # Avoid using ubuntu-latest as it would upgrade python unattended.
     runs-on: ubuntu-22.04
     strategy:
-      fail-fast: true
-      # This workflow isn't designed to be run as a pipeline, several issues:
-      #
-      # - contains golangci-lint jobs, sonarcloud (would duplicate)
-      # - cache config not suitable for multiple pipelines
-      # - python tests should be separate job, or no job
-      #
-      # Keep it to a single job run from the matrix as configured
-      # until we get a chance to redesign the pipeline properly.
+      fail-fast: false
       matrix:
         redis-version: [7]
-        python-version: ["3.11"]
-        go-version: [1.22.x]
+        go-version: [1.23.x]
 
     env:
       REDIS_IMAGE: redis:${{ matrix.redis-version }}
@@ -57,15 +105,18 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Setup Golang
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-
+      # Regardless that the base image provides a python release, we need
+      # setup-python so it properly configures the python3-venv.
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Print runtime python version
+        run: python3 -c 'import sys; print("%d.%d" % (sys.version_info[0], sys.version_info[1]))'
+
+      - name: Print runtime pip version
+        run: pip -V && pip3 -V
 
       - name: Setup CI Tooling
         uses: shrink/actions-docker-extract@v3
@@ -74,15 +125,11 @@ jobs:
           path: /usr/local/bin/.
           destination: /usr/local/bin
 
-      - name: Cache
-        uses: actions/cache@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('**/go.sum') }}
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
 
       - name: Install Dependencies and basic hygiene test
         id: hygiene
@@ -113,33 +160,64 @@ jobs:
             exit 1
           fi
 
-      - name: Fetch base branch
-        if: ${{ github.event_name == 'pull_request' }}
-        run: git fetch origin ${{ github.base_ref }}
-
-      - name: Print CPU info
-        run: grep '^model name' /proc/cpuinfo
+      - name: Bring up test services
+        run: task services:up
 
-      - name: Print Go env
-        run: go env
+      - name: Preflight Python tests
+        if: runner.debug == '1'
+        run: TYK_LOGLEVEL=debug go test -p 1 -parallel 1 -race -v ./dlpython ./coprocess/...
 
       - name: Run Gateway Tests
         id: ci-tests
         run: |
           task test:e2e-combined args="-race -timeout=15m"
           task test:coverage
 
-      # golangci-lint actions *require* issues-exit-code=0 to pass data along to sonarcloud
-      # rather than erroring out on github issues directly with out-format github.
-      - name: golangci-lint
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: coverage
+          retention-days: 1
+          path: coverage/gateway-all.cov
 
-      - name: golangci-lint-on-push
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: testjson
+          retention-days: 1
+          path: coverage/gateway-all.json
+
+  sonar-cloud-analysis:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    needs: [test, golangci-lint]
+    steps:
+      - name: "Checkout repository"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage
+
+      - name: Download golangcilint artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: golangcilint
+
+      - name: Check reports existence
+        id: check_files
+        uses: andstor/file-existence-action@v3
+        with:
+          files: 'gateway-all.cov, golangci-lint-report.json'
+          fail: true
+
+      - name: Install Dependencies
+        env:
+          TOKEN: '${{ secrets.ORG_GH_TOKEN }}'
+        run: git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com"
 
       - name: SonarCloud Scan
         if: always()
@@ -149,12 +227,12 @@ jobs:
             -Dsonar.organization=tyktechnologies
             -Dsonar.projectKey=TykTechnologies_tyk
             -Dsonar.sources=.
-            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go
+            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go,templates/**
             -Dsonar.coverage.exclusions=**/*_test.go,**/mock/*
             -Dsonar.test.inclusions=**/*_test.go
             -Dsonar.tests=.
-            -Dsonar.go.coverage.reportPaths=coverage/gateway-all.cov
-            -Dsonar.go.golangci-lint.reportPaths=golanglint.xml
+            -Dsonar.go.coverage.reportPaths=gateway-all.cov
+            -Dsonar.go.golangci-lint.reportPaths=golangci-lint-report.json
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/plugin-compiler-build.yml

@@ -8,19 +8,29 @@ on:
       - master
       - release-**
     tags:
-      - 'v*'
+      - "v*"
 
 env:
-  GOLANG_CROSS: 1.22-bullseye
+  GOLANG_CROSS: 1.23-bullseye
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   docker-build:
     runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
     permissions:
       id-token: write
     steps:
+      - name: "Reclaim some runner space"
+        run: sudo rm -rf /usr/local/bin/* /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
 
       - name: Configure AWS Credentials
         id: configure-aws
@@ -58,7 +68,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push to dockerhub/ECR
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ci/images/plugin-compiler/Dockerfile
@@ -67,6 +77,6 @@ jobs:
           labels: ${{ steps.set-metadata.outputs.labels }}
           tags: ${{ steps.set-metadata.outputs.tags }}
           build-args: |
-            BASE-IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }}
+            BASE_IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }}
             GITHUB_SHA=${{ github.sha }}
             GITHUB_TAG=${{ github.ref_name }}

.github/workflows/release.yml

@@ -21,7 +21,7 @@
       - master
       - release-**
     tags:
       - 'v*'
 env:
   GOPRIVATE: github.com/TykTechnologies
   VARIATION: inverted
@@ -41,9 +41,9 @@
       fail-fast: false
       matrix:
         golang_cross:
-          - 1.22-bullseye
+          - 1.23-bullseye
         include:
-          - golang_cross: 1.22-bullseye
+          - golang_cross: 1.23-bullseye
             goreleaser: 'ci/goreleaser/goreleaser.yml'
             cgo: 1
             rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
@@ -127,12 +127,12 @@
           mask-aws-account-id: false
       - uses: aws-actions/amazon-ecr-login@v2
         id: ecr
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         with:
           mask-password: 'true'
       - name: Docker metadata for CI
         id: ci_metadata
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         uses: docker/metadata-action@v5
         with:
           images: ${{ steps.ecr.outputs.registry }}/tyk
@@ -146,7 +146,7 @@
             type=semver,pattern={{major}}.{{minor}},prefix=v
             type=semver,pattern={{version}},prefix=v
       - name: push image to CI
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         uses: docker/build-push-action@v6
         with:
           context: "dist"
@@ -174,7 +174,7 @@
             type=semver,pattern={{version}}
           labels: "org.opencontainers.image.title=tyk-gateway  (distroless) \norg.opencontainers.image.description=Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
       - name: push image to prod
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         uses: docker/build-push-action@v6
         with:
           context: "dist"
@@ -189,7 +189,7 @@
           labels: ${{ steps.tag_metadata.outputs.labels }}
       - name: save deb
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         with:
           name: deb
           retention-days: 1
@@ -199,7 +199,7 @@
             !dist/*fips*.deb
       - name: save rpm
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
+        if: ${{ matrix.golang_cross == '1.23-bullseye' }}
         with:
           name: rpm
           retention-days: 1

.taskfiles/hooks.yml

@@ -0,0 +1,23 @@
+version: "3"
+
+includes:
+  lint_dep: lint.yml
+
+tasks:
+  pre-commit:
+    desc: "Run pre-commit hooks"
+    cmds:
+      - task: :lint:tidy
+      - task: lint_dep:compile-tests
+
+  pre-push:
+    desc: "Run pre-push hooks"
+    cmds:
+      - task: :lint:tidy
+      - task: lint_dep:vet
+      - task: lint_dep:golangci-lint
+        vars:
+          args: "{{.golangci}} --new-from-rev=origin/master"
+      - task: :lint:x-tyk-gateway
+      - task: lint_dep:compile-tests
+      - task: lint_dep:simple-build

.taskfiles/test.yml

@@ -2,7 +2,7 @@
 version: "3"
 
 includes:
-  deps: ./deps/Taskfile.yml
+  deps: ./deps.yml
   services:
     taskfile: ../docker/services/Taskfile.yml
     dir: ../docker/services