feat(node): upload local email templates by hash on send

.changeset/node-emails-local-template.md

@@ -0,0 +1,5 @@
+---
+'posthog-node': minor
+---
+
+`posthog.emails.send()` also sends a local template: a send with `templateKey` and `templateData` names content from your code that PostHog fills in. The SDK uploads the template whenever PostHog doesn't hold its content yet, such as on its first send or after the content changed. A local template needs `html`.

packages/node/references/posthog-node-references-latest.json

@@ -2057,6 +2057,11 @@
           "type": "string",
           "name": "subject"
         },
+        {
+          "description": "Leave it out to send rendered content: the email goes out as is, and PostHog never stores it. Pass it to send a local template: PostHog fills its `{{ variables.* }}`, and the SDK uploads the template whenever PostHog doesn't hold its content yet. A local template needs `html`. Content that PostHog already replaced with newer content isn't uploaded again: the email goes out with the newer content, and the send returns a `template_superseded` warning.",
+          "type": "Record<string, unknown>",
+          "name": "templateData"
+        },
         {
           "description": "Names the email, such as `'password-reset'`, so its metrics, logs and status group under it in PostHog. Letters, digits, hyphens and underscores, at most 400.",
           "type": "string",
@@ -2070,7 +2075,7 @@
       "name": "CodeEmailForm",
       "properties": [],
       "path": "src/emails/types.ts",
-      "example": "CodeEmailFields & ({\n    html: string;\n    text?: string;\n} | {\n    html?: string;\n    text: string;\n})"
+      "example": "CodeEmailFields & ({\n    html: string;\n    text?: string;\n} | {\n    html?: string;\n    text: string;\n    templateData?: never;\n})"
     },
     {
       "id": "Coercer",
@@ -4382,7 +4387,7 @@
       "name": "SendCodeEmailOptions",
       "properties": [],
       "path": "src/emails/types.ts",
-      "example": "SendEmailBase & CodeEmailForm & Without<TemplateEmailForm>"
+      "example": "SendEmailBase & CodeEmailForm & Without<Omit<TemplateEmailForm, 'templateData'>>"
     },
     {
       "id": "SendEmailBase",

packages/node/src/__tests__/email-content-hash.spec.ts

@@ -0,0 +1,30 @@
+import { emailContentHash } from '@/emails/content-hash'
+
+describe('emailContentHash', () => {
+  it.each([
+    {
+      content: { subject: 'Welcome, {{ variables.first_name }}', html: '<p>Hi {{ variables.first_name }}</p>' },
+      hash: 'v1:f74cd913f3aa2e5667a588f2865634e56693df77793ca594d18e9b5f89e3d2e6',
+    },
+    {
+      content: { subject: 'Grüße', preheader: 'Vorschau', html: '<p>ä</p>', text: 'ä' },
+      hash: 'v1:75cf6fc3189ca26f60da672d29a1d4c89c7db3dd458df3a550a34a4d2a92c149',
+    },
+    {
+      content: {
+        subject: 'Welcome, {{ variables.first_name }}',
+        html: '<p>Hi {{ variables.first_name }}</p>',
+        text: '\r\n\r',
+      },
+      hash: 'v1:8112876ee1b2dd7327f571b04b546ed057bfff5a904d9f0ae507b6a449ab6453',
+    },
+  ])('matches the content_hash v1 test vector of the server: $hash', async ({ content, hash }) => {
+    expect(await emailContentHash(content)).toBe(hash)
+  })
+
+  it('hashes CRLF and lone CR line endings like LF', async () => {
+    const lf = await emailContentHash({ subject: 'Hi', html: '<p>\na\nb\n</p>' })
+
+    expect(await emailContentHash({ subject: 'Hi', html: '<p>\r\na\rb\r\n</p>' })).toBe(lf)
+  })
+})

packages/node/src/__tests__/emails.spec.ts

@@ -460,6 +460,21 @@ describe('emails', () => {
     expect(batchCallsDuringBackoff).toBe(1)
   })
 
+  it.each([
+    ['a template send', {}],
+    [
+      'a local template send',
+      { templateId: undefined, templateKey: 'welcome', subject: 'Hi', html: '<p>Hi</p>', templateData: {} },
+    ],
+  ])('sends %s that started before shutdown', async (_name, overrides) => {
+    replies = [accepted()]
+
+    const pending = send(overrides)
+    await posthog.shutdown()
+
+    expect((await pending).data?.id).toBe('msg_1')
+  })
+
   it('sends nothing after shutdown and returns client_closed', async () => {
     await posthog.shutdown()
 
@@ -510,7 +525,6 @@ describe('emails', () => {
       ['a code send without a subject', { templateKey: 'welcome', html: '<p>Hi</p>' }, 'subject'],
       ['a code send without html or text', { templateKey: 'welcome', subject: 'Hi' }, 'html'],
       ['a code send with html that is not a string', { ...CODE_SEND, html: 1 }, 'html'],
-      ['template data on a code send', { ...CODE_SEND, templateData: { name: 'Jane' } }, 'templateData'],
       [
         "Resend's scheduledAt on the template form",
         { templateId: TEMPLATE_ID, scheduledAt: 'in 1 hour' },
@@ -538,12 +552,6 @@ describe('emails', () => {
         "A template made in PostHog sets its own sender, subject and content, and follows the project's engagement setting.",
         'Remove captureEngagement, or send content from your code with templateKey.',
       ],
-      [
-        'template data on a code send',
-        { ...CODE_SEND, templateData: { name: 'Jane' } },
-        'A send with templateKey takes its content from your code.',
-        'Remove templateData, or send a template made in PostHog with templateId.',
-      ],
       [
         'a field no form takes',
         { ...CODE_SEND, scheduledAt: 'in 1 hour' },
@@ -572,6 +580,146 @@ describe('emails', () => {
     })
   })
 
+  describe('a local template', () => {
+    const UPLOAD_URL = 'https://us.posthog.com/api/projects/@current/messaging_templates/by_key/welcome/'
+    const WELCOME_HASH = 'v1:e78af62206b6fdc0bd87d56d2a70555a9bb8d4a37617dcbb8a746f86f7db7a63'
+    const WELCOME = { templateKey: 'welcome', subject: 'Welcome, {{ variables.name }}', html: '<p>Hi</p>' }
+    const uploaded = (): Reply => ({ status: 201, body: { id: 'tpl_1', key: 'welcome', result: 'created' } })
+
+    const sendWelcome = async (overrides: Record<string, unknown> = {}): ReturnType<PostHog['emails']['send']> => {
+      const pending = posthog.emails.send({
+        to: 'user@example.com',
+        distinctId: 'user_123',
+        ...WELCOME,
+        templateData: { name: 'Jane' },
+        ...overrides,
+      } as SendEmailOptions)
+      await vi.runAllTimersAsync()
+      return pending
+    }
+
+    const requests = (): string[] => fetch.mock.calls.map(([url, options]) => `${options.method} ${url}`)
+    const bodies = (): unknown[] => fetch.mock.calls.map(([, options]) => JSON.parse(options.body))
+
+    it('sends a template PostHog holds by its key and content hash in one request', async () => {
+      replies = [accepted()]
+
+      const { data } = await sendWelcome({ from: 'Acme <hi@example.com>', replyTo: 'help@example.com' })
+
+      expect(data?.id).toBe('msg_1')
+      expect(requests()).toEqual([`POST ${SEND_URL}`])
+      expect(bodies()).toEqual([
+        {
+          template: { key: 'welcome', hash: WELCOME_HASH },
+          to: 'user@example.com',
+          message_data: { name: 'Jane' },
+          identifiers: { distinct_id: 'user_123' },
+          from: 'Acme <hi@example.com>',
+          reply_to: ['help@example.com'],
+          capture_engagement: true,
+        },
+      ])
+    })
+
+    it('uploads a template PostHog does not hold, then sends it again with the same idempotency key', async () => {
+      makeClient({ migratedFrom: 'resend' })
+      replies = [rejected(409, 'template_outdated'), uploaded(), accepted()]
+
+      const { data } = await sendWelcome()
+
+      expect(data?.id).toBe('msg_1')
+      expect(requests()).toEqual([`POST ${SEND_URL}`, `PUT ${UPLOAD_URL}`, `POST ${SEND_URL}`])
+      const [firstSend, upload, secondSend] = bodies()
+      expect(upload).toEqual({
+        name: 'welcome',
+        subject: WELCOME.subject,
+        html: WELCOME.html,
+        if_code_managed: true,
+        if_not_superseded: true,
+      })
+      expect(secondSend).toEqual(firstSend)
+      const [firstHeaders, uploadHeaders, secondHeaders] = fetch.mock.calls.map(([, options]) => options.headers)
+      expect(secondHeaders['Idempotency-Key']).toBe(firstHeaders['Idempotency-Key'])
+      expect(uploadHeaders).toMatchObject({
+        Authorization: `Bearer ${SECRET_KEY}`,
+        'Content-Type': 'application/json',
+        'X-PostHog-Migrated-From': 'resend',
+      })
+    })
+
+    it.each([
+      ['a refused upload', [rejected(409, 'template_outdated'), rejected(409, 'template_edited_in_posthog')], 2],
+      [
+        'a second template_outdated',
+        [rejected(409, 'template_outdated'), uploaded(), rejected(409, 'template_outdated')],
+        3,
+      ],
+    ])('returns %s and uploads at most once per send', async (_name, queued, requestsMade) => {
+      replies = queued
+
+      const { data, error } = await sendWelcome()
+
+      expect(data).toBeNull()
+      expect(error).toMatchObject({ statusCode: 409, name: 'validation_error' })
+      expect(fetch).toHaveBeenCalledTimes(requestsMade)
+    })
+
+    it('never uploads content rendered for one recipient', async () => {
+      replies = [rejected(409, 'template_outdated')]
+
+      const { error } = await sendWelcome({ templateData: undefined })
+
+      expect(error?.code).toBe('template_outdated')
+      expect(requests()).toEqual([`POST ${SEND_URL}`])
+    })
+
+    it('sends the recipient and data it was called with, even when the caller changes them afterwards', async () => {
+      replies = [accepted(), accepted()]
+      const options = {
+        to: 'jane@example.com',
+        ...WELCOME,
+        templateData: { name: 'Jane' },
+        replyTo: ['jane-help@example.com'],
+      }
+
+      const toJane = posthog.emails.send(options)
+      options.to = 'bob@example.com'
+      options.templateData.name = 'Bob'
+      options.replyTo[0] = 'bob-help@example.com'
+      const toBob = posthog.emails.send(options)
+      await vi.runAllTimersAsync()
+      await Promise.all([toJane, toBob])
+
+      expect(bodies()).toHaveLength(2)
+      expect(bodies()).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            to: 'jane@example.com',
+            message_data: { name: 'Jane' },
+            reply_to: ['jane-help@example.com'],
+          }),
+          expect.objectContaining({
+            to: 'bob@example.com',
+            message_data: { name: 'Bob' },
+            reply_to: ['bob-help@example.com'],
+          }),
+        ])
+      )
+    })
+
+    it.each([
+      ['template data that is not an object', { templateData: ['Jane'] }, 'invalid_template_data', 'templateData'],
+      ['a local template without html', { html: undefined, text: 'Hi {{ variables.name }}' }, 'invalid_form', 'html'],
+      ['a local template with empty html', { html: '' }, 'invalid_form', 'html'],
+      ['a local template with an empty subject', { subject: '' }, 'invalid_form', 'subject'],
+    ])('refuses %s before sending', async (_name, overrides, code, field) => {
+      const { error } = await sendWelcome(overrides)
+
+      expect(error).toMatchObject({ code, name: 'validation_error', field })
+      expect(fetch).not.toHaveBeenCalled()
+    })
+  })
+
   describe('get', () => {
     const statusReply = (fields: Record<string, unknown> = {}): Reply => ({
       status: 200,

packages/node/src/__tests__/types/emails-forms.ts

@@ -7,6 +7,7 @@ export const forms: SendEmailOptions[] = [
   { to, templateId, templateData: { name: 'Jane' } },
   { to, templateKey: 'reset', subject: 'Reset', html: '<p>Reset</p>', replyTo: ['help@example.com'] },
   { to, templateKey: 'reset', subject: 'Reset', text: 'Reset', from: 'security@example.com', captureEngagement: false },
+  { to, templateKey: 'welcome', subject: 'Hi {{ variables.name }}', html: '<p>Hi</p>', templateData: { name: 'J' } },
 ]
 
 export const wrongCombinations: SendEmailOptions[] = [
@@ -20,6 +21,6 @@ export const wrongCombinations: SendEmailOptions[] = [
   { to, templateKey: 'reset', subject: 'Reset' },
   // @ts-expect-error the code form needs a subject
   { to, templateKey: 'reset', html: '<p>Reset</p>' },
-  // @ts-expect-error content from code takes no template data
-  { to, templateKey: 'reset', subject: 'Reset', html: '<p>Reset</p>', templateData: { name: 'J' } },
+  // @ts-expect-error a local template needs html
+  { to, templateKey: 'welcome', subject: 'Hi {{ variables.name }}', text: 'Hi', templateData: { name: 'J' } },
 ]

packages/node/src/emails/content-hash.ts

@@ -0,0 +1,39 @@
+/// <reference lib="dom" />
+
+export interface EmailContent {
+  subject?: string
+  preheader?: string
+  html?: string
+  text?: string
+}
+
+const HASH_PREFIX = 'v1:'
+const ENCODING_HEADER = 'posthog-email-template-v1\n'
+const HASHED_PARTS = ['subject', 'preheader', 'html', 'text'] as const
+
+export async function emailContentHash(content: EmailContent): Promise<string> {
+  const subtle = globalThis.crypto?.subtle
+  if (!subtle) {
+    throw new Error('SubtleCrypto API not available')
+  }
+  const digest = await subtle.digest('SHA-256', encode(content))
+  return HASH_PREFIX + toHex(new Uint8Array(digest))
+}
+
+function encode(content: EmailContent): Uint8Array<ArrayBuffer> {
+  const encoder = new TextEncoder()
+  let encoded = ENCODING_HEADER
+  for (const part of HASHED_PARTS) {
+    const value = normalizeLineEndings(content[part] ?? '')
+    encoded += `${encoder.encode(value).length}:${value}\n`
+  }
+  return encoder.encode(encoded)
+}
+
+function normalizeLineEndings(value: string): string {
+  return value.replace(/\r\n?/g, '\n')
+}
+
+function toHex(bytes: Uint8Array): string {
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('')
+}