fix(html_escape): name code points, not bytes, in numeric references

dsl_test.go

@@ -169,6 +169,54 @@ func TestDSLURLEncodeDecode(t *testing.T) {
 	})
 }
 
+func TestDSLHTMLEscapeNumericEntities(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "ascii is one reference per character",
+			input:    "ab",
+			expected: "ab",
+		},
+		{
+			name:     "cjk",
+			input:    "こ",
+			expected: "こ",
+		},
+		{
+			name:     "outside the basic multilingual plane",
+			input:    "🚀",
+			expected: "🚀",
+		},
+		{
+			name:     "multi-byte characters stay separate",
+			input:    "こん",
+			expected: "こん",
+		},
+		{
+			// The escaping pass already has a named entity for this one, so it
+			// reaches the loop as an entity and is copied across untouched.
+			name:     "named entity from the escaping pass is left alone",
+			input:    "<é>",
+			expected: "&lt;&eacute;&gt;",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			escaped, err := DefaultHelperFunctions["html_escape"](tc.input, true)
+			require.NoError(t, err, "html_escape should not error")
+			require.Equal(t, tc.expected, escaped, "html_escape should name the code point")
+
+			unescaped, err := DefaultHelperFunctions["html_unescape"](escaped)
+			require.NoError(t, err, "html_unescape should not error")
+			require.Equal(t, tc.input, unescaped, "html_unescape should reverse html_escape")
+		})
+	}
+}
+
 func TestDSLTimeComparison(t *testing.T) {
 	compiled, err := govaluate.NewEvaluableExpressionWithFunctions("unixtime() > not_after", DefaultHelperFunctions)
 	require.Nil(t, err, "could not compare time")

util.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/kataras/jwt"
 	"github.com/pkg/errors"
@@ -372,9 +373,17 @@ func strToNumEntities(s string) string {
 			}
 		}
 
-		r := rune(escaped[i])
+		// A numeric reference names a code point, so decode one rune rather
+		// than one byte. Converting a byte on its own split every multi-byte
+		// character into a run of references that a browser reads back as
+		// mojibake. A byte that is not valid UTF-8 has no code point to name,
+		// so it keeps the value it had before.
+		r, size := utf8.DecodeRuneInString(escaped[i:])
+		if r == utf8.RuneError && size == 1 {
+			r = rune(escaped[i])
+		}
 		fmt.Fprintf(&result, "&#%d;", int(r))
-		i++
+		i += size
 	}
 
 	return result.String()