Generate protobuf code for Go and Python services

Part of open-telemetry#1787
to generate profobuf files for all services.

This should help unblock
open-telemetry#1754 to allow
dependabot to manage dependecy upgrades for the Go services.

There are new Makefile recipes for managing the protobuf files.

1. docker-generate-protobuf - to generate the protobuf files with docker
   so that the only dependency on the machine needed is docker.
2. clean - to remove the protobuf files generated
3. check-clean-work-tree - to check that the working tree is clean and
   to help with verifying that the protobuf files are updated for all of
   the services when there are changes to the protobuf definition.

There's a new check in the GitHub Actions workflow to verify that the
protobuf code is generated. It is only verifying that the protobuf code
is generated for Go and Python, but other services can apply the same
workflow by updating the docker-gen-proto.sh script to uncomment the
function call for the service.

The dockerfiles for the Go and Python services have been updated to not
generate the protobuf files during the build process. Instead, this
function has been refactored into the genproto directory of each
service.

Signed-off-by: Charlie Le <charlie_le@apple.com>

Author: CharlieTLe

.github/workflows/gen-check.yaml

@@ -0,0 +1,19 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+name: Clean Generation
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  protobufcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Generate
+        run: make clean docker-generate-protobuf
+      - name: Check Clean Work Tree
+        run: make check-clean-work-tree

.gitignore

@@ -52,9 +52,6 @@ test/tracetesting/tracetesting-vars.yaml
 /src/frontend/pb/
 /src/frontend/protos/
 /src/paymentservice/demo.proto
-/src/recommendationservice/demo_pb2*.py
 /src/shippingservice/proto/
-/src/productcatalogservice/genproto
 /src/currencyservice/proto
-/src/checkoutservice/genproto
 /src/accountingservice/genproto

.licenserc.json

@@ -45,6 +45,8 @@
         "src/featureflagservice/assets/vendor/",
         "src/featureflagservice/priv/",
         "src/productcatalogservice/genproto/",
+        "src/recommendationservice/demo_pb2.py",
+        "src/recommendationservice/demo_pb2_grpc.py",
         "internal/tools/"
     ]
   }

CHANGELOG.md

@@ -21,6 +21,8 @@ the release.
   ([#1786](https://github.com/open-telemetry/opentelemetry-demo/pull/1786))
 * [chore] Add multi-platform build support
   ([#1785](https://github.com/open-telemetry/opentelemetry-demo/pull/1785))
+* [chore] Generate protobuf code for Go and Python services
+  ([#1794](https://github.com/open-telemetry/opentelemetry-demo/pull/1784))
 
 ## 1.12.0
 

Makefile

@@ -133,6 +133,25 @@ generate-kubernetes-manifests:
 	echo "  name: otel-demo" >> kubernetes/opentelemetry-demo.yaml
 	helm template opentelemetry-demo open-telemetry/opentelemetry-demo --namespace otel-demo | sed '/helm.sh\/chart\:/d' | sed '/helm.sh\/hook/d' | sed '/managed-by\: Helm/d' >> kubernetes/opentelemetry-demo.yaml
 
+.PHONY: docker-generate-protobuf
+docker-generate-protobuf:
+	./docker-gen-proto.sh
+
+.PHONY: clean
+clean:
+	rm -rf ./src/{checkoutservice,productcatalogservice}/genproto/oteldemo/
+	rm -rf ./src/recommendationservice/{demo_pb2,demo_pb2_grpc}.py
+
+.PHONY: check-clean-work-tree
+check-clean-work-tree:
+	@if ! git diff --quiet; then \
+	  echo; \
+	  echo 'Working tree is not clean, did you forget to run "make docker-generate-protobuf"?'; \
+	  echo; \
+	  git status; \
+	  exit 1; \
+	fi
+
 .PHONY: start
 start:
 	$(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) up --force-recreate --remove-orphans --detach

docker-gen-proto.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e # Exit immediately if a command exits with a non-zero status.
+set -x # Print commands and their arguments as they are executed
+
+# This script is used to generate protobuf files for all services with Docker.
+
+gen_proto_go() {
+  echo "Generating Go protobuf files for $1"
+  docker build -f "src/$1/genproto/Dockerfile" -t "$1-genproto" .
+  docker run --rm -v $(pwd):/build "$1-genproto" \
+    protoc -I /build/pb /build/pb/demo.proto --go_out="./src/$1/" --go-grpc_out="./src/$1/"
+}
+
+gen_proto_python() {
+  echo "Generating Python protobuf files for $1"
+  docker build -f "src/$1/genproto/Dockerfile" -t "$1-genproto" .
+  docker run --rm -v $(pwd):/build "$1-genproto" \
+    python -m grpc_tools.protoc -I /build/pb/ --python_out="./src/$1/" --grpc_python_out="./src/$1/" /build/pb/demo.proto
+}
+
+#gen_proto_dotnet accountingservice
+#gen_proto_java adservice
+#gen_proto_dotnet cartservice
+gen_proto_go checkoutservice
+#gen_proto_cpp currencyservice
+#gen_proto_ruby emailservice
+#gen_proto_ts frontend
+#gen_proto_js paymentservice
+gen_proto_go productcatalogservice
+#gen_proto_php quoteservice
+gen_proto_python recommendationservice
+#gen_proto_rust shippingservice

internal/tools/sanitycheck.py

@@ -84,7 +84,7 @@ def sanitycheck(pattern, allow_utf8 = False, allow_eol = (CRLF, LF), indent = 1)
 retval += sanitycheck('**/*.md', allow_eol = (LF,))
 retval += sanitycheck('**/*.proj', allow_eol = (LF,), indent = 2)
 retval += sanitycheck('**/*.props', allow_eol = (LF,), indent = 2)
-retval += sanitycheck('**/*.py', allow_eol = (LF,), indent = 4)
+retval += sanitycheck('**/[!demo_pb2]*.py', allow_eol = (LF,), indent = 4)
 retval += sanitycheck('**/*.sln', allow_utf8 = True, indent = 4)
 retval += sanitycheck('**/*.targets', allow_eol = (LF,), indent = 2)
 retval += sanitycheck('**/*.xml', allow_eol = (LF,), indent = 2)

src/checkoutservice/Dockerfile

@@ -6,22 +6,15 @@ FROM golang:1.22-alpine AS builder
 
 WORKDIR /usr/src/app/
 
-RUN apk update \
-    && apk add --no-cache make protobuf-dev
-
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=bind,source=./src/checkoutservice/go.sum,target=go.sum \
     --mount=type=bind,source=./src/checkoutservice/go.mod,target=go.mod \
-    --mount=type=bind,source=./src/checkoutservice/tools.go,target=tools.go \
-    go mod download \
-    && go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly
+    go mod download
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=bind,rw,source=./src/checkoutservice,target=. \
-    --mount=type=bind,rw,source=./pb,target=./pb \
-    protoc -I ./pb ./pb/demo.proto --go_out=./ --go-grpc_out=./ \
-    && go build -ldflags "-s -w" -o /go/bin/checkoutservice/ ./
+    go build -ldflags "-s -w" -o /go/bin/checkoutservice/ ./
 
 FROM alpine
 

src/checkoutservice/README.md

@@ -20,13 +20,10 @@ docker compose build checkoutservice
 
 ## Regenerate protos
 
-> [!NOTE]
-> [`protoc`](https://grpc.io/docs/protoc-installation/) is required.
-
-To regenerate gRPC code run:
+To build the protos, run from the root directory:
 
 ```sh
-go generate
+make docker-generate-protobuf
 ```
 
 ## Bump dependencies

src/checkoutservice/genproto/Dockerfile

@@ -0,0 +1,16 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+FROM golang:1.22-alpine
+
+WORKDIR /build
+
+RUN apk add --no-cache protobuf-dev
+
+COPY ./src/checkoutservice/go.mod ./
+COPY ./src/checkoutservice/go.sum ./
+COPY ./src/checkoutservice/tools.go ./
+
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly

src/checkoutservice/genproto/oteldemo/demo.pb.go

@@ -6,22 +6,15 @@ FROM golang:1.22-alpine AS builder
 
 WORKDIR /usr/src/app/
 
-RUN apk update \
-    && apk add --no-cache make protobuf-dev
-
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=bind,source=./src/productcatalogservice/go.sum,target=go.sum \
     --mount=type=bind,source=./src/productcatalogservice/go.mod,target=go.mod \
-    --mount=type=bind,source=./src/productcatalogservice/tools.go,target=tools.go \
-    go mod download \
-    && go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly
+    go mod download
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=bind,rw,source=./src/productcatalogservice,target=. \
-    --mount=type=bind,rw,source=./pb,target=./pb \
-    protoc -I ./pb ./pb/demo.proto --go_out=./ --go-grpc_out=./ \
-    && go build -ldflags "-s -w" -o /go/bin/productcatalogservice/ ./
+    go build -ldflags "-s -w" -o /go/bin/productcatalogservice/ ./
 
 FROM alpine AS release
 

src/checkoutservice/genproto/oteldemo/demo_grpc.pb.go

@@ -25,13 +25,10 @@ docker compose build productcatalogservice
 
 ## Regenerate protos
 
-> [!NOTE]
-> [`protoc`](https://grpc.io/docs/protoc-installation/) is required.
-
-To regenerate gRPC code run:
+To build the protos, run from the root directory:
 
 ```sh
-go generate
+make docker-generate-protobuf
 ```
 
 ## Bump dependencies

src/productcatalogservice/Dockerfile

@@ -0,0 +1,16 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+FROM golang:1.22-alpine
+
+WORKDIR /build
+
+RUN apk add --no-cache protobuf-dev
+
+COPY ./src/productcatalogservice/go.mod ./
+COPY ./src/productcatalogservice/go.sum ./
+COPY ./src/productcatalogservice/tools.go ./
+
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly \

src/productcatalogservice/README.md

@@ -18,23 +18,12 @@ COPY ./src/recommendationservice/requirements.txt ./
 RUN pip install --upgrade pip
 RUN pip install --prefix="/reqs" -r requirements.txt
 
-#
-# Build gRPC files
-#
-FROM base AS grpc-builder
-WORKDIR /usr/src/app/
-COPY ./pb/ ./proto/
-
-RUN python -m pip install grpcio-tools==1.59.2
-RUN python -m grpc_tools.protoc -I=./proto/ --python_out=./ --grpc_python_out=./ ./proto/demo.proto
-
 #
 # Runtime
 #
 FROM base AS runtime
 WORKDIR /usr/src/app/
 COPY --from=builder /reqs /usr/local
-COPY --from=grpc-builder /usr/src/app/ .
 COPY ./src/recommendationservice/ ./
 
 RUN opentelemetry-bootstrap -a install

src/productcatalogservice/genproto/Dockerfile

@@ -5,12 +5,10 @@ selected product.
 
 ## Local Build
 
-To build the protos, run:
+To build the protos, run from the root directory:
 
 ```sh
-pip install -r requirements.txt
-python -m pip install grpcio-tools==1.48.2
-python -m grpc_tools.protoc -I=../pb/ --python_out=./ --grpc_python_out=./ ../pb/demo.proto
+make docker-generate-protobuf
 ```
 
 ## Docker Build

src/productcatalogservice/genproto/oteldemo/demo.pb.go

@@ -0,0 +1,8 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+FROM python:3.12-slim-bookworm
+
+WORKDIR /build
+
+RUN python -m pip install grpcio-tools==1.59.2