fix extract-message-util preserve line break for multi line HTML text (#13983)

Upon more testing with titan email I found emails with multi line text did not preserve line break in HTML, this fixes it.
2025-08-20 07:49:05 +02:00
parent 2fc29d41a8
commit 44916afcdd
5 changed files with 158 additions and 5 deletions
@@ -117,6 +117,7 @@
    "graphql-upload": "13.0.0",
    "graphql-yoga": "4.0.5",
    "handlebars": "^4.7.8",
+    "html-to-text": "^9.0.5",
    "imapflow": "1.0.187",
    "ioredis": "5.6.0",
    "jsdom": "^26.1.0",
@@ -199,6 +200,7 @@
    "@types/express-session": "^1.18.0",
    "@types/graphql-upload": "^8.0.12",
    "@types/handlebars": "4.1.0",
+    "@types/html-to-text": "^9.0.4",
    "@types/imapflow": "^1.0.21",
    "@types/lodash.chunk": "^4.2.9",
    "@types/lodash.differencewith": "^4.5.9",
@@ -172,6 +172,146 @@ Developer Support`);
    expect(result).toBe('');
  });

+  it('should preserve new lines in html email', () => {
+    const parsed: ParsedMail = {
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+      html: `<html><head><style>
+  html, body {
+    font-size: 14.5px;
+    line-height: 1.5;
+    color: #333;
+    background-color: #fff;
+    border: 0;
+    margin: 0;
+    padding: 0;
+    overflow-x: auto;
+    font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Ubuntu, Helvetica, Arial, sans-serif;
+    margin: 0;
+    -webkit-text-size-adjust: auto;
+    word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;
+  }
+
+  strong, b, .bold {
+    font-weight: 600;
+  }
+
+  body {
+    overflow-y: hidden;
+    word-break: break-word;
+    filter: invert(0.88);
+    --image-filter: grayscale(20%);
+  }
+  .theme-emoji {
+    filter: var(--image-filter) invert(0.88);
+  }
+
+  a {
+    color: #925b00;
+  }
+
+  a:hover {
+    color: #925b00;
+  }
+
+  a:visited {
+    color: #925b00;
+  }
+  
+  a img {
+    border-bottom: 0;
+  }
+
+  body.heightDetermined {
+    overflow-y: hidden;
+  }
+
+  div,pre {
+    max-width: 100%;
+  }
+
+  pre {
+    white-space: pre-wrap;
+    word-wrap: break-word;
+  }
+
+  pre.flockmail-plaintext {
+    white-space: pre-wrap;
+    word-wrap: break-word;
+  }
+
+  img {
+    border: 0;
+  }
+  img:not([src*='.svg']) {
+    filter: var(--image-filter) invert(0.88);
+  }
+  .grayscale {
+    filter: var(--image-filter) invert(0.88);
+  }
+
+  search-match, .search-match {
+    background: #fff000;
+    border-radius: 4px;
+    box-shadow: 0 0.5px 0.5px rgba(0,0,0,0.25);
+    &.current-match {
+      background: #ff8b1a;
+    }
+  }
+
+  table {
+    word-break: initial;
+    border-collapse: collapse;
+  }
+
+  a.mk-unsubscribe:not([href]), a[fr-original-class=mk-unsubscribe]:not([href]) {
+    pointer-events: none !important;
+  }
+
+  p.MsoNormal, li.MsoNormal, div.MsoNormal {margin: 0px;}
+
+  ::-webkit-scrollbar-corner {
+    background-color: transparent;
+  }
+  
+  ::-webkit-scrollbar {
+    width: 14px;
+    height: 14px;
+    cursor: default;
+  }
+  
+  ::-webkit-scrollbar-thumb {
+    border-radius: 14px;
+    background-clip: content-box;
+    border: 3px solid transparent;
+    background: transparent;
+    box-shadow: inset 0 0 15px 15px rgba(136, 136, 136, 0.4);
+  }
+  ::-webkit-scrollbar-thumb:hover {
+    box-shadow: inset 0 0 15px 15px #bdbdbd;
+  }
+
+  </style></head><body><div id="inbox-html-wrapper"><div id="isPasted" fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Hi Sarah,</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">I wanted to quickly follow up regarding the Q3 marketing campaign results. &nbsp;</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">We’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target. &nbsp;</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Let’s schedule a short call early next week to discuss adjustments before the Q4 push. &nbsp;</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Would Monday 10 AM work for you?</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Best regards, &nbsp;</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">John</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><img class="flm-open" width="0" height="0" style="border: 0px; width: 0px; height: 0px; max-width: 100vw;" data-open-tracking-src="{{track-read-receipt}}"></div></body></html>`,
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toEqual(`Hi Sarah,
+
+
+I wanted to quickly follow up regarding the Q3 marketing campaign results.  
+We’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target.  
+
+
+Let’s schedule a short call early next week to discuss adjustments before the Q4 push.  
+Would Monday 10 AM work for you?
+
+
+Best regards,  
+John`);
+  });
+
  it('should prefer text over html when both are available', () => {
    const parsed: ParsedMail = {
      text: 'Plain text content\n\nOn 2023-01-01, user@example.com wrote:\n> Reply',
@@ -1,4 +1,5 @@
 import DOMPurify from 'dompurify';
+import { convert } from 'html-to-text';
 import { JSDOM } from 'jsdom';
 import { type ParsedMail } from 'mailparser';
 import * as planer from 'planer';
@@ -21,10 +22,12 @@ export const extractTextWithoutReplyQuotations = (
      dom.window.document,
    );

-    const textContent = new JSDOM(cleanedHtml, { runScripts: 'outside-only' })
-      .window.document.body?.textContent;
+    const text = convert(cleanedHtml, {
+      wordwrap: false,
+      preserveNewlines: true,
+    }).trim();

-    return textContent ?? '';
+    return text.replace(/\u00A0/g, ' ');
  }

  return '';
@@ -48,7 +48,6 @@
    "dist",
    "assets",
    "accessibility",
-    "assets",
    "components",
    "display",
    "feedback",
@@ -19662,6 +19662,13 @@ __metadata:
  languageName: node
  linkType: hard

+"@types/html-to-text@npm:^9.0.4":
+  version: 9.0.4
+  resolution: "@types/html-to-text@npm:9.0.4"
+  checksum: 10c0/8e0e548f280d1c1107f70db01ff6331a295f3097f535f4d07c30ced271818fc3803a8f03bb2117c1207c46a3d40da256a03a17bd05fc9f5f1b4821c5bcbf98d8
+  languageName: node
+  linkType: hard
+
 "@types/http-assert@npm:*":
  version: 1.5.5
  resolution: "@types/http-assert@npm:1.5.5"
@@ -34227,7 +34234,7 @@ __metadata:
  languageName: node
  linkType: hard

-"html-to-text@npm:9.0.5":
+"html-to-text@npm:9.0.5, html-to-text@npm:^9.0.5":
  version: 9.0.5
  resolution: "html-to-text@npm:9.0.5"
  dependencies:
@@ -50867,6 +50874,7 @@ __metadata:
    "@types/express-session": "npm:^1.18.0"
    "@types/graphql-upload": "npm:^8.0.12"
    "@types/handlebars": "npm:4.1.0"
+    "@types/html-to-text": "npm:^9.0.4"
    "@types/imapflow": "npm:^1.0.21"
    "@types/lodash.chunk": "npm:^4.2.9"
    "@types/lodash.differencewith": "npm:^4.5.9"
@@ -50936,6 +50944,7 @@ __metadata:
    graphql-upload: "npm:13.0.0"
    graphql-yoga: "npm:4.0.5"
    handlebars: "npm:^4.7.8"
+    html-to-text: "npm:^9.0.5"
    imapflow: "npm:1.0.187"
    ioredis: "npm:5.6.0"
    jsdom: "npm:^26.1.0"