fix(3p/lisp/mime4cl): make MIME-BODY-STREAM always return characters r/9026

Because OPEN-DECODED-FILE-PORTION only knows about transfer encodings it
would only return a character stream for 7bit encoded bodies. This
causes inconsistent behavior where some bodies would return binary and
some character streams. To fix this, we specialize MIME-BODY-STREAM for
MIME-TEXT parts which may or may not be a good enough solution.

We may actually want to make MIME-BODY-STREAM binary always and let the
user handle decoding?! This may be a good idea to take care after yet
another stream machinery redesign.

Since the mime4cl test suite doesn't test MIME-BODY-STREAM (much), add a
message generated by notemap that hits this issue to the mblog golden
test suite.

Change-Id: Ie340c42ced6c693af9b3c84b177408d6b6d2c9c4
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12913
Reviewed-by: sterni <sternenseemann@systemli.org>
Autosubmit: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

4 files changed, 33 insertions, 2 deletions

diff --git a/third_party/lisp/mime4cl/endec.lisp b/third_party/lisp/mime4cl/endec.lisp
index 2e282c237822..3f1fdb3bc634 100644
--- a/third_party/lisp/mime4cl/endec.lisp
+++ b/third_party/lisp/mime4cl/endec.lisp
@@ -656,6 +656,7 @@ sequence, a charset string indicating the original coding."
              (cons (flexi-streams:octets-to-string
                     (car part)
                     :external-format (flexi-streams:make-external-format
+                                      ;; TODO(sterni): sanitize charset before interning
                                       (intern (string-upcase (cdr part)) 'keyword))))
              (string part))))
     (apply #'concatenate
diff --git a/third_party/lisp/mime4cl/mime.lisp b/third_party/lisp/mime4cl/mime.lisp
index 657c1fd86238..da13a1126354 100644
--- a/third_party/lisp/mime4cl/mime.lisp
+++ b/third_party/lisp/mime4cl/mime.lisp
@@ -201,8 +201,25 @@ because they are stored in dedicated slots in MIME-PART.")
                :test #'string=)
        (mime= (mime-body part1) (mime-body part2))))
 
-(defun mime-body-stream (mime-part)
-  (make-input-adapter (mime-body mime-part)))
+(defgeneric mime-body-stream (mime-part)
+  (:documentation
+   "Returns stream that allows reading the decoded body of the given part.
+STREAM-ELEMENT-TYPE depends on part type."))
+
+;; TODO(sterni): Allow accessing underlying binary stream?
+;;               Would need matching behavior with :7bit
+(defmethod mime-body-stream ((part mime-text))
+  (let ((underlying-stream (call-next-method)))
+    (if (eq (stream-element-type underlying-stream) 'character)
+        underlying-stream
+        (make-flexi-stream underlying-stream
+                           :external-format
+                           ;; TODO(sterni): sanitize charset before interning
+                           (intern (string-upcase (mime-text-charset part))
+                                   'keyword)))))
+
+(defmethod mime-body-stream ((part mime-part))
+  (make-input-adapter (mime-body part)))
 
 (defmacro with-input-from-mime-body-stream ((stream part) &body forms)
   `(with-open-stream (,stream (mime-body-stream ,part))
diff --git a/third_party/lisp/mime4cl/test/samples/mail-note-from-notemap.msg b/third_party/lisp/mime4cl/test/samples/mail-note-from-notemap.msg
new file mode 100644
index 000000000000..2f74313b11ec
--- /dev/null
+++ b/third_party/lisp/mime4cl/test/samples/mail-note-from-notemap.msg
@@ -0,0 +1,13 @@
+From: <sterni>
+Date: Wed, 25 Dec 2024 23:54:39 +0100
+X-Universally-Unique-Identifier: d2ccdc74-830c-41ee-9d64-2221f1c35449
+X-Uniform-Type-Identifier: com.apple.mail-note
+X-Mailer: notemap
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+Subject: =?utf-8?Q?test.txt?=
+X-TUID: clxSN23/djqG
+
+gr=C3=BC=C3=9Fe from notemap
+
diff --git a/third_party/lisp/mime4cl/test/samples/mail-note.msg b/third_party/lisp/mime4cl/test/samples/mail-note-from-notes-app.msg
index 577fceb21f70..577fceb21f70 100644
--- a/third_party/lisp/mime4cl/test/samples/mail-note.msg
+++ b/third_party/lisp/mime4cl/test/samples/mail-note-from-notes-app.msg