fix(): add url to webhook (#216)

* fix(): add url to webhook

* fix(): fixing aws session config

examples/gateways/aws-sns-gateway-configmap.yaml

@@ -6,8 +6,10 @@ metadata:
 data:
   notification_1: |-
     topicArn: "topic-arn"
-    endpoint: "/"
-    port: "9600"
+    hook:
+     endpoint: "/"
+     port: "12000"
+     url: "placeholderurl"
     accessKey:
       name: aws-secret
       key: access

examples/gateways/aws-sns.yaml

@@ -27,6 +27,16 @@ spec:
           imagePullPolicy: "Always"
           command: ["/bin/aws-sns-gateway"]
       serviceAccountName: "argo-events-sa"
+  serviceSpec:
+    metadata:
+      name: aws-sns-gateway-svc
+    spec:
+      selector:
+        gateway-name: "aws-sns-gateway"
+      ports:
+        - port: 12000
+          targetPort: 12000
+      type: LoadBalancer
   configMap: "aws-sns-gateway-configmap"
   eventVersion: "1.0"
   type: "aws-sns"

examples/gateways/github-configmap.yaml

@@ -6,7 +6,10 @@ data:
   project_1: |-
     owner: "owner-example"
     repository: "repo-example"
-    url: "http://webhook-gateway-http-svc/push"
+    hook:
+     endpoint: "/push"
+     port: "12000"
+     url: "placeholderurl"
     events:
     - "*"
     apiToken:

examples/gateways/github.yaml

@@ -27,6 +27,16 @@ spec:
           imagePullPolicy: "Always"
           command: ["/bin/github-gateway"]
       serviceAccountName: "argo-events-sa"
+  serviceSpec:
+    metadata:
+      name: github-gateway-svc
+    spec:
+      selector:
+        gateway-name: "github-gateway"
+      ports:
+        - port: 12000
+          targetPort: 12000
+      type: LoadBalancer
   configMap: "github-gateway-configmap"
   type: "github"
   eventVersion: "1.0"

examples/gateways/gitlab-configmap.yaml

@@ -5,7 +5,10 @@ metadata:
 data:
   project_1: |-
     projectId: "1"
-    url: "YOUR_WEBHOOK_GATEWAY_EXTERNAL_URL"
+    hook:
+     endpoint: "/push"
+     port: "12000"
+     url: "placeholderurl"
     event: "PushEvents"
     accessToken:
       key: accesskey

examples/gateways/gitlab.yaml

@@ -27,6 +27,16 @@ spec:
           imagePullPolicy: "Always"
           command: ["/bin/gitlab-gateway"]
       serviceAccountName: "argo-events-sa"
+  serviceSpec:
+    metadata:
+      name: gitlab-gateway-svc
+    spec:
+      selector:
+        gateway-name: "gitlab-gateway"
+      ports:
+        - port: 12000
+          targetPort: 12000
+      type: LoadBalancer
   configMap: "gitlab-gateway-configmap"
   type: "gitlab"
   eventVersion: "1.0"

examples/gateways/trello-gateway-configmap.yaml

@@ -4,13 +4,14 @@ metadata:
   name: trello-gateway-configmap
 data:
   notification_1: |-
-    endpoint: "/"
-    port: "9600"
+    hook:
+     endpoint: "/test"
+     port: "12000"
+     url: "placeholderurl"
     apiKey:
       name: trello-secret
       key: apikey
     token:
       name trello-secret
       key: tokenkey
-    url: "URL_TO_REGISTER"
     description: "test webhook"

examples/gateways/trello.yaml

@@ -27,6 +27,16 @@ spec:
           imagePullPolicy: "Always"
           command: ["/bin/trello-gateway"]
       serviceAccountName: "argo-events-sa"
+  serviceSpec:
+    metadata:
+      name: trello-gateway-svc
+    spec:
+      selector:
+        gateway-name: "trello-gateway"
+      ports:
+        - port: 12000
+          targetPort: 12000
+      type: LoadBalancer
   configMap: "trello-gateway-configmap"
   type: "trello"
   eventVersion: "1.0"

gateways/common/webhook.go

@@ -36,6 +36,8 @@ type Webhook struct {
 	Method string `json:"method" protobuf:"bytes,2,name=method"`
 	// Port on which HTTP server is listening for incoming events.
 	Port string `json:"port" protobuf:"bytes,3,name=port"`
+	// URL is the url of the server.
+	URL string `json:"url" protobuf:"bytes,4,name=url"`
 	// ServerCertPath refers the file that contains the cert.
 	ServerCertPath string `json:"serverCertPath,omitempty" protobuf:"bytes,4,opt,name=serverCertPath"`
 	// ServerKeyPath refers the file that contains private key
@@ -236,17 +238,20 @@ func ProcessRoute(rc *RouteConfig, helper *WebhookHelper, eventStream gateways.E
 	return nil
 }
 
-func ValidateWebhook(endpoint, port string) error {
-	if endpoint == "" {
+func ValidateWebhook(w *Webhook) error {
+	if w == nil {
+		return fmt.Errorf("")
+	}
+	if w.Endpoint == "" {
 		return fmt.Errorf("endpoint can't be empty")
 	}
-	if port == "" {
+	if w.Port == "" {
 		return fmt.Errorf("port can't be empty")
 	}
-	if port != "" {
-		_, err := strconv.Atoi(port)
+	if w.Port != "" {
+		_, err := strconv.Atoi(w.Port)
 		if err != nil {
-			return fmt.Errorf("failed to parse server port %s. err: %+v", port, err)
+			return fmt.Errorf("failed to parse server port %s. err: %+v", w.Port, err)
 		}
 	}
 	return nil
@@ -258,3 +263,7 @@ func FormatWebhookEndpoint(endpoint string) string {
 	}
 	return endpoint
 }
+
+func GenerateFormattedURL(w *Webhook) string {
+	return fmt.Sprintf("%s%s", w.URL, FormatWebhookEndpoint(w.Endpoint))
+}

gateways/community/README.md

@@ -0,0 +1,34 @@
+# Community Gateways
+
+## How to set up gateways that run web server internally?
+For any gateway that run web server internally, you need to create the gateway resource before creating gateway configmap.
+Why? Because the `webhook` configuration in gateway configmap contains the `url` which the URL of the service that exposes the gateway pod, you need to have
+the gateway pod and service running before you can create the configmap. 
+
+  ### Example
+
+  Lets setup AWS SNS gateway,
+
+  1. Create the gateway resource, 
+     ```bash
+     kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/aws-sns.yaml
+     ```
+
+     Once the gateway resource is created, you should see the corresponding gateway pod and service being created.
+     Make sure the gateway service is being backed by the gateway pod.
+
+     How to get the URL for that resolves to your service is up to you. You can use Ingress controller, Route (if your installation is on Openshift) or other means.
+
+  2. Create the gateway configmap. The gateway configmap basically contains the event source configurations. You need to put the URL of your service. 
+
+     ```bash
+     kubectl -n argo-events create -f  ../../examples/gateway/aws-sns-gateway-configmap.yaml
+     ```
+
+  3. Create the sensor.
+
+     ```bash
+     kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/aws-sns.yaml
+     ```
+
+  Now your setup is complete.

gateways/community/aws-sns/config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package aws_sns
 
 import (
+	"github.com/argoproj/argo-events/gateways/common"
 	"k8s.io/client-go/kubernetes"
 	"time"
 
@@ -65,11 +66,9 @@ type httpNotification struct {
 
 // snsConfig contains configuration to subscribe to SNS topic
 type snsConfig struct {
-	TopicArn string `json:"topicArn"`
-	// Endpoint you wish to register
-	Endpoint string `json:"endpoint"`
-	// Port on which http server is running.
-	Port      string                    `json:"port"`
+	// Hook defines a webhook.
+	Hook      *common.Webhook           `json:"hook"`
+	TopicArn  string                    `json:"topicArn"`
 	AccessKey *corev1.SecretKeySelector `json:"accessKey" protobuf:"bytes,5,opt,name=accessKey"`
 	SecretKey *corev1.SecretKeySelector `json:"secretKey" protobuf:"bytes,6,opt,name=secretKey"`
 	Region    string                    `json:"region"`

gateways/community/aws-sns/start.go

@@ -121,8 +121,10 @@ func (ese *SNSEventSourceExecutor) PostActivate(rc *gwcommon.RouteConfig) error
 		AccessKeyID:     accessKey,
 		SecretAccessKey: secretKey,
 	})
+
+	formattedUrl := gwcommon.GenerateFormattedURL(sc.Hook)
+
 	awsSession, err := session.NewSession(&aws.Config{
-		Endpoint:    &rc.Webhook.Endpoint,
 		Region:      &sc.Region,
 		Credentials: creds,
 		HTTPClient:  &http.Client{},
@@ -137,7 +139,7 @@ func (ese *SNSEventSourceExecutor) PostActivate(rc *gwcommon.RouteConfig) error
 
 	logger.Info().Msg("subscribing to snsConfig topic")
 	if _, err := snsSession.Subscribe(&snslib.SubscribeInput{
-		Endpoint: &rc.Webhook.Endpoint,
+		Endpoint: &formattedUrl,
 		Protocol: &snsProtocol,
 		TopicArn: &sc.TopicArn,
 	}); err != nil {
@@ -173,10 +175,7 @@ func (ese *SNSEventSourceExecutor) StartEventSource(eventSource *gateways.EventS
 	sc := config.(*snsConfig)
 
 	return gwcommon.ProcessRoute(&gwcommon.RouteConfig{
-		Webhook: &gwcommon.Webhook{
-			Endpoint: sc.Endpoint,
-			Port:     sc.Port,
-		},
+		Webhook: sc.Hook,
 		Configs: map[string]interface{}{
 			LabelSNSConfig: sc,
 		},

gateways/community/aws-sns/validate.go

@@ -45,5 +45,5 @@ func validateSNSConfig(config interface{}) error {
 	if sc.SecretKey == nil {
 		return fmt.Errorf("must specify secret key")
 	}
-	return gwcommon.ValidateWebhook(sc.Endpoint, sc.Port)
+	return gwcommon.ValidateWebhook(sc.Hook)
 }

gateways/community/aws-sns/validate_test.go

@@ -27,8 +27,10 @@ var (
 	configKey   = "testConfig"
 	configId    = "1234"
 	validConfig = `
-endpoint: "/"
-port: "8080"
+hook:
+ endpoint: "/test"
+ port: "8080"
+ url: "myurl/test"
 topicArn: "test-arn"
 region: "us-east-1"
 accessKey:

gateways/community/gcp-pubsub/config.go

@@ -17,16 +17,13 @@ limitations under the License.
 package pubsub
 
 import (
-	"cloud.google.com/go/pubsub"
 	"github.com/ghodss/yaml"
 	"github.com/rs/zerolog"
 )
 
 // GcpPubSubEventSourceExecutor implements Eventing
 type GcpPubSubEventSourceExecutor struct {
 	Log zerolog.Logger
-	// client is the GCP PubSub Client
-	client *pubsub.Client
 }
 
 // pubSubConfig contains configuration to subscribe to GCP PubSub topic

gateways/community/gcp-pubsub/start.go

@@ -17,10 +17,8 @@ limitations under the License.
 package pubsub
 
 import (
-	"context"
-	"fmt"
-
 	"cloud.google.com/go/pubsub"
+	"context"
 	"github.com/argoproj/argo-events/gateways"
 	"google.golang.org/api/option"
 )
@@ -43,10 +41,6 @@ func (ese *GcpPubSubEventSourceExecutor) StartEventSource(eventSource *gateways.
 	sc := config.(*pubSubConfig)
 
 	ctx := eventStream.Context()
-	ese.client, err = pubsub.NewClient(ctx, sc.ProjectID, option.WithCredentialsFile(sc.CredentialsFile))
-	if err != nil {
-		return fmt.Errorf("failed to create new pubsub client, err: %+v", err)
-	}
 
 	dataCh := make(chan []byte)
 	errorCh := make(chan error)
@@ -60,15 +54,22 @@ func (ese *GcpPubSubEventSourceExecutor) StartEventSource(eventSource *gateways.
 func (ese *GcpPubSubEventSourceExecutor) listenEvents(ctx context.Context, sc *pubSubConfig, eventSource *gateways.EventSource, dataCh chan []byte, errorCh chan error, doneCh chan struct{}) {
 	// Create a new topic with the given name.
 	logger := ese.Log.With().Str("event-source", eventSource.Name).Str("topic", sc.Topic).Logger()
+
+	client, err := pubsub.NewClient(ctx, sc.ProjectID, option.WithCredentialsFile(sc.CredentialsFile))
+	if err != nil {
+		errorCh <- err
+		return
+	}
+
 	logger.Info().Msg("creating GCP PubSub topic")
-	topic, err := ese.client.CreateTopic(ctx, sc.Topic)
+	topic, err := client.CreateTopic(ctx, sc.Topic)
 	if err != nil {
 		errorCh <- err
 		return
 	}
 
 	logger.Info().Msg("subscribing to GCP PubSub topic")
-	sub, err := ese.client.CreateSubscription(ctx, eventSource.Id,
+	sub, err := client.CreateSubscription(ctx, eventSource.Id,
 		pubsub.SubscriptionConfig{Topic: topic})
 	if err != nil {
 		errorCh <- err
@@ -97,7 +98,7 @@ func (ese *GcpPubSubEventSourceExecutor) listenEvents(ctx context.Context, sc *p
 		panic(err)
 	}
 	logger.Info().Msg("closing GCP PubSub client")
-	if err = ese.client.Close(); err != nil {
+	if err = client.Close(); err != nil {
 		panic(err)
 	}
 }

gateways/community/github/config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package github
 
 import (
+	"github.com/argoproj/argo-events/gateways/common"
 	"github.com/ghodss/yaml"
 	"github.com/google/go-github/github"
 	"github.com/rs/zerolog"
@@ -37,18 +38,14 @@ type GithubEventSourceExecutor struct {
 
 // githubConfig contains information to setup a github project integration
 type githubConfig struct {
-	// REST API endpoint
-	Endpoint string `json:"endpoint"`
-	// Port on which HTTP server is listening for incoming events.
-	Port string `json:"port"`
+	// Webhook
+	Hook *common.Webhook `json:"hook"`
 	// GitHub owner name i.e. argoproj
 	Owner string `json:"owner"`
 	// GitHub repo name i.e. argo-events
 	Repository string `json:"repository"`
 	// Github events to subscribe to which the gateway will subscribe
 	Events []string `json:"events"`
-	// External URL for hooks
-	URL string `json:"url"`
 	// K8s secret containing github api token
 	APIToken *corev1.SecretKeySelector `json:"apiToken"`
 	// K8s secret containing WebHook Secret