@barnarddt @hperl feat: send emails via http api endpoint instead of …

…smtp (#1030) (#3341)


This change adds a new delivery method to the courier called `mailer`. Similar to SMS functionality it posts a templated Data model to a API endpoint.  This API can then send emails via a CRM or any other mechanism that it wants.

`Mailer` still uses the existing email data models so any new email added will automatically be sent to the API/CRM as well.

## Related issue(s)
Resolves #2825
Also see #1030 and #3008 
Documentation PR ory/docs#1298

courier/courier.go

@@ -22,6 +22,7 @@ import (
 type (
 	Dependencies interface {
 		PersistenceProvider
+		x.TracingProvider
 		x.LoggingProvider
 		ConfigProvider
 		x.HTTPClientProvider
@@ -51,6 +52,7 @@ type (
 	courier struct {
 		smsClient   *smsClient
 		smtpClient  *smtpClient
+		httpClient  *httpClient
 		deps        Dependencies
 		failOnError bool
 		backoff     backoff.BackOff
@@ -65,6 +67,7 @@ func NewCourier(ctx context.Context, deps Dependencies) (Courier, error) {
 	return &courier{
 		smsClient:  newSMS(ctx, deps),
 		smtpClient: smtp,
+		httpClient: newHTTP(ctx, deps),
 		deps:       deps,
 		backoff:    backoff.NewExponentialBackOff(),
 	}, nil

courier/http.go

@@ -0,0 +1,93 @@
+// Copyright © 2023 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package courier
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ory/kratos/request"
+	"github.com/ory/x/otelx"
+)
+
+type httpDataModel struct {
+	Recipient    string
+	Subject      string
+	Body         string
+	TemplateType TemplateType
+	TemplateData EmailTemplate
+}
+
+type httpClient struct {
+	RequestConfig json.RawMessage
+}
+
+func newHTTP(ctx context.Context, deps Dependencies) *httpClient {
+	return &httpClient{
+		RequestConfig: deps.CourierConfig().CourierEmailRequestConfig(ctx),
+	}
+}
+func (c *courier) dispatchMailerEmail(ctx context.Context, msg Message) (err error) {
+	ctx, span := c.deps.Tracer(ctx).Tracer().Start(ctx, "courier.http.dispatchMailerEmail")
+	defer otelx.End(span, &err)
+
+	builder, err := request.NewBuilder(c.httpClient.RequestConfig, c.deps)
+	if err != nil {
+		return err
+	}
+
+	tmpl, err := c.smtpClient.NewTemplateFromMessage(c.deps, msg)
+	if err != nil {
+		return err
+	}
+
+	td := httpDataModel{
+		Recipient:    msg.Recipient,
+		Subject:      msg.Subject,
+		Body:         msg.Body,
+		TemplateType: msg.TemplateType,
+		TemplateData: tmpl,
+	}
+
+	req, err := builder.BuildRequest(ctx, td)
+	if err != nil {
+		return err
+	}
+
+	res, err := c.deps.HTTPClient(ctx).Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+	case http.StatusCreated:
+	default:
+		err = fmt.Errorf(
+			"unable to dispatch mail delivery because upstream server replied with status code %d",
+			res.StatusCode,
+		)
+		c.deps.Logger().
+			WithField("message_id", msg.ID).
+			WithField("message_type", msg.Type).
+			WithField("message_template_type", msg.TemplateType).
+			WithField("message_subject", msg.Subject).
+			WithError(err).
+			Error("sending mail via HTTP failed.")
+		return err
+	}
+
+	c.deps.Logger().
+		WithField("message_id", msg.ID).
+		WithField("message_type", msg.Type).
+		WithField("message_template_type", msg.TemplateType).
+		WithField("message_subject", msg.Subject).
+		Debug("Courier sent out mailer.")
+
+	return nil
+}

courier/http_test.go

@@ -0,0 +1,125 @@
+// Copyright © 2023 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package courier_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ory/kratos/courier/template/email"
+	"github.com/ory/kratos/driver/config"
+	"github.com/ory/kratos/internal"
+	"github.com/ory/kratos/x"
+	"github.com/ory/x/resilience"
+)
+
+func TestQueueHTTPEmail(t *testing.T) {
+	ctx := context.Background()
+
+	type sendEmailRequestBody struct {
+		IdentityID       string
+		IdentityEmail    string
+		Recipient        string
+		TemplateType     string
+		To               string
+		RecoveryCode     string
+		RecoveryURL      string
+		VerificationURL  string
+		VerificationCode string
+		Body             string
+		Subject          string
+	}
+
+	expectedEmail := []*email.TestStubModel{
+		{
+			To:      "[email protected]",
+			Subject: "test-mailer-subject-1",
+			Body:    "test-mailer-body-1",
+		},
+		{
+			To:      "[email protected]",
+			Subject: "test-mailer-subject-2",
+			Body:    "test-mailer-body-2",
+		},
+	}
+
+	actual := make([]sendEmailRequestBody, 0, 2)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		rb, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+
+		var body sendEmailRequestBody
+
+		err = json.Unmarshal(rb, &body)
+		require.NoError(t, err)
+
+		assert.NotEmpty(t, r.Header["Authorization"])
+		assert.Equal(t, "Basic bWU6MTIzNDU=", r.Header["Authorization"][0])
+
+		actual = append(actual, body)
+	}))
+	t.Cleanup(srv.Close)
+
+	requestConfig := fmt.Sprintf(`{
+		"url": "%s",
+		"method": "POST",
+		"auth": {
+			"type": "basic_auth",
+			"config": {
+				"user":     "me",
+				"password": "12345"
+			}
+		}
+	}`, srv.URL)
+
+	conf, reg := internal.NewFastRegistryWithMocks(t)
+	conf.MustSet(ctx, config.ViperKeyCourierDeliveryStrategy, "http")
+	conf.MustSet(ctx, config.ViperKeyCourierHTTPRequestConfig, requestConfig)
+	conf.MustSet(ctx, config.ViperKeyCourierSMTPURL, "http://foo.url")
+	reg.Logger().Level = logrus.TraceLevel
+
+	courier, err := reg.Courier(ctx)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer t.Cleanup(cancel)
+
+	for _, message := range expectedEmail {
+		id, err := courier.QueueEmail(ctx, email.NewTestStub(reg, message))
+		require.NoError(t, err)
+		require.NotEqual(t, uuid.Nil, id)
+	}
+
+	go func() {
+		require.NoError(t, courier.Work(ctx))
+	}()
+
+	require.NoError(t, resilience.Retry(reg.Logger(), time.Millisecond*250, time.Second*10, func() error {
+		if len(actual) == len(expectedEmail) {
+			return nil
+		}
+		return errors.New("capacity not reached")
+	}))
+
+	for i, message := range actual {
+		expected := email.NewTestStub(reg, expectedEmail[i])
+
+		assert.Equal(t, x.Must(expected.EmailRecipient()), message.To)
+		assert.Equal(t, x.Must(expected.EmailBody(ctx)), message.Body)
+		assert.Equal(t, x.Must(expected.EmailSubject(ctx)), message.Subject)
+	}
+}

courier/smtp.go

@@ -157,6 +157,9 @@ func (c *courier) QueueEmail(ctx context.Context, t EmailTemplate) (uuid.UUID, e
 }
 
 func (c *courier) dispatchEmail(ctx context.Context, msg Message) error {
+	if c.deps.CourierConfig().CourierEmailStrategy(ctx) == "http" {
+		return c.dispatchMailerEmail(ctx, msg)
+	}
 	if c.smtpClient.Host == "" {
 		return errors.WithStack(herodot.ErrInternalServerError.WithErrorf("Courier tried to deliver an email but %s is not set!", config.ViperKeyCourierSMTPURL))
 	}

courier/stub/request.config.mailer.jsonnet

@@ -0,0 +1,11 @@
+function(ctx) {
+  recipient: ctx.Recipient,
+  template_type: ctx.TemplateType,
+  to: if "TemplateData" in ctx && "To" in ctx.TemplateData then ctx.TemplateData.To else null,
+  recovery_code: if "TemplateData" in ctx && "RecoveryCode" in ctx.TemplateData then ctx.TemplateData.RecoveryCode else null,
+  recovery_url: if "TemplateData" in ctx && "RecoveryURL" in ctx.TemplateData then ctx.TemplateData.RecoveryURL else null,
+  verification_url: if "TemplateData" in ctx && "VerificationURL" in ctx.TemplateData then ctx.TemplateData.VerificationURL else null,
+  verification_code: if "TemplateData" in ctx && "VerificationCode" in ctx.TemplateData then ctx.TemplateData.VerificationCode else null,
+  subject: if "TemplateData" in ctx && "Subject" in ctx.TemplateData then ctx.TemplateData.Subject else null,
+  body: if "TemplateData" in ctx && "Body" in ctx.TemplateData then ctx.TemplateData.Body else null
+}

driver/config/.snapshots/TestCourierEmailHTTP-case=configs_set.json

@@ -0,0 +1,15 @@
+{
+  "auth": {
+    "config": {
+      "password": "YourPass",
+      "user": "YourUsername"
+    },
+    "type": "basic_auth"
+  },
+  "body": "file://some.jsonnet",
+  "header": {
+    "Content-Type": "application/json"
+  },
+  "method": "POST",
+  "url": "https://example.com/email"
+}

driver/config/config.go

@@ -66,6 +66,8 @@ const (
 	ViperKeyCourierTemplatesVerificationValidEmail           = "courier.templates.verification.valid.email"
 	ViperKeyCourierTemplatesVerificationCodeInvalidEmail     = "courier.templates.verification_code.invalid.email"
 	ViperKeyCourierTemplatesVerificationCodeValidEmail       = "courier.templates.verification_code.valid.email"
+	ViperKeyCourierDeliveryStrategy                          = "courier.delivery_strategy"
+	ViperKeyCourierHTTPRequestConfig                         = "courier.http.request_config"
 	ViperKeyCourierSMTPFrom                                  = "courier.smtp.from_address"
 	ViperKeyCourierSMTPFromName                              = "courier.smtp.from_name"
 	ViperKeyCourierSMTPHeaders                               = "courier.smtp.headers"
@@ -255,6 +257,8 @@ type (
 		Config() *Config
 	}
 	CourierConfigs interface {
+		CourierEmailStrategy(ctx context.Context) string
+		CourierEmailRequestConfig(ctx context.Context) json.RawMessage
 		CourierSMTPURL(ctx context.Context) (*url.URL, error)
 		CourierSMTPClientCertPath(ctx context.Context) string
 		CourierSMTPClientKeyPath(ctx context.Context) string
@@ -976,6 +980,29 @@ func (p *Config) SelfServiceFlowLogoutRedirectURL(ctx context.Context) *url.URL
 	return p.GetProvider(ctx).RequestURIF(ViperKeySelfServiceLogoutBrowserDefaultReturnTo, p.SelfServiceBrowserDefaultReturnTo(ctx))
 }
 
+func (p *Config) CourierEmailStrategy(ctx context.Context) string {
+	return p.GetProvider(ctx).String(ViperKeyCourierDeliveryStrategy)
+}
+
+func (p *Config) CourierEmailRequestConfig(ctx context.Context) json.RawMessage {
+	if p.CourierEmailStrategy(ctx) != "http" {
+		return nil
+	}
+
+	out, err := p.GetProvider(ctx).Marshal(kjson.Parser())
+	if err != nil {
+		p.l.WithError(err).Warn("Unable to marshal mailer request configuration.")
+		return nil
+	}
+
+	config := gjson.GetBytes(out, ViperKeyCourierHTTPRequestConfig).Raw
+	if len(config) <= 0 {
+		return json.RawMessage("{}")
+	}
+
+	return json.RawMessage(config)
+}
+
 func (p *Config) CourierSMTPClientCertPath(ctx context.Context) string {
 	return p.GetProvider(ctx).StringF(ViperKeyCourierSMTPClientCertPath, "")
 }
@@ -1078,7 +1105,7 @@ func (p *Config) CourierSMSRequestConfig(ctx context.Context) json.RawMessage {
 
 	out, err := p.GetProvider(ctx).Marshal(kjson.Parser())
 	if err != nil {
-		p.l.WithError(err).Warn("Unable to marshal self service strategy configuration.")
+		p.l.WithError(err).Warn("Unable to marshal SMS request configuration.")
 		return nil
 	}
 

driver/config/config_test.go

@@ -1095,6 +1095,23 @@ func TestChangeMinPasswordLength(t *testing.T) {
 	})
 }
 
+func TestCourierEmailHTTP(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("case=configs set", func(t *testing.T) {
+		conf, _ := config.New(ctx, logrusx.New("", ""), os.Stderr,
+			configx.WithConfigFiles("stub/.kratos.courier.email.http.yaml"), configx.SkipValidation())
+		assert.Equal(t, "http", conf.CourierEmailStrategy(ctx))
+		snapshotx.SnapshotT(t, conf.CourierEmailRequestConfig(ctx))
+	})
+
+	t.Run("case=defaults", func(t *testing.T) {
+		conf, _ := config.New(ctx, logrusx.New("", ""), os.Stderr, configx.SkipValidation())
+
+		assert.Equal(t, "smtp", conf.CourierEmailStrategy(ctx))
+	})
+}
+
 func TestCourierSMS(t *testing.T) {
 	ctx := context.Background()
 

driver/config/stub/.kratos.courier.email.http.yaml

@@ -0,0 +1,24 @@
+dsn: sqlite://foo.db?mode=memory&_fk=true
+
+selfservice:
+  default_browser_return_url: https://example.com/return_to
+
+identity:
+  default_schema_id: default
+  schemas:
+    - id: default
+      url: base64://ewogICIkaWQiOiAib3J5Oi8vaWRlbnRpdHktdGVzdC1zY2hlbWEiLAogICIkc2NoZW1hIjogImh0dHA6Ly9qc29uLXNjaGVtYS5vcmcvZHJhZnQtMDcvc2NoZW1hIyIsCiAgInRpdGxlIjogIklkZW50aXR5U2NoZW1hIiwKICAidHlwZSI6ICJvYmplY3QiLAogICJwcm9wZXJ0aWVzIjogewogICAgInRyYWl0cyI6IHsKICAgICAgInR5cGUiOiAib2JqZWN0IiwKICAgICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJvYmplY3QiLAogICAgICAgICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICJmaXJzdCI6IHsKICAgICAgICAgICAgICAidHlwZSI6ICJzdHJpbmciCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJsYXN0IjogewogICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIKICAgICAgICAgICAgfQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfSwKICAgICAgInJlcXVpcmVkIjogWwogICAgICAgICJuYW1lIgogICAgICBdLAogICAgICAiYWRkaXRpb25hbFByb3BlcnRpZXMiOiB0cnVlCiAgICB9CiAgfQp9
+
+courier:
+  delivery_strategy: http
+  http:
+    request_config:
+      url: https://example.com/email
+      body: file://some.jsonnet
+      header:
+        'Content-Type': 'application/json'
+      auth:
+        type: basic_auth
+        config:
+          user: YourUsername
+          password: YourPass