From 44916afcdd73e7afb7884bf0663465de33212ecb Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:19:05 +0530 Subject: [PATCH] fix `extract-message-util` preserve line break for multi line HTML text (#13983) Upon more testing with titan email I found emails with multi line text did not preserve line break in HTML, this fixes it. --- packages/twenty-server/package.json | 2 + .../extract-message-text.util.spec.ts | 140 ++++++++++++++++++ .../imap/utils/extract-message-text.util.ts | 9 +- packages/twenty-ui/package.json | 1 - yarn.lock | 11 +- 5 files changed, 158 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 520219debb3..6dd72f72080 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -117,6 +117,7 @@ "graphql-upload": "13.0.0", "graphql-yoga": "4.0.5", "handlebars": "^4.7.8", + "html-to-text": "^9.0.5", "imapflow": "1.0.187", "ioredis": "5.6.0", "jsdom": "^26.1.0", @@ -199,6 +200,7 @@ "@types/express-session": "^1.18.0", "@types/graphql-upload": "^8.0.12", "@types/handlebars": "4.1.0", + "@types/html-to-text": "^9.0.4", "@types/imapflow": "^1.0.21", "@types/lodash.chunk": "^4.2.9", "@types/lodash.differencewith": "^4.5.9", diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts index 7ddafe78731..02c064d87d8 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts @@ -172,6 +172,146 @@ Developer Support`); expect(result).toBe(''); }); + it('should preserve new lines in html email', () => { + const parsed: ParsedMail = { + attachments: [], + headers: new Map(), + headerLines: [], + html: `
Hi Sarah,

I wanted to quickly follow up regarding the Q3 marketing campaign results.  
We’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target.  

Let’s schedule a short call early next week to discuss adjustments before the Q4 push.  
Would Monday 10 AM work for you?

Best regards,  
John

`, + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toEqual(`Hi Sarah, + + +I wanted to quickly follow up regarding the Q3 marketing campaign results. +We’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target. + + +Let’s schedule a short call early next week to discuss adjustments before the Q4 push. +Would Monday 10 AM work for you? + + +Best regards, +John`); + }); + it('should prefer text over html when both are available', () => { const parsed: ParsedMail = { text: 'Plain text content\n\nOn 2023-01-01, user@example.com wrote:\n> Reply', diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts index e2be81ed497..9c3450985d6 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts @@ -1,4 +1,5 @@ import DOMPurify from 'dompurify'; +import { convert } from 'html-to-text'; import { JSDOM } from 'jsdom'; import { type ParsedMail } from 'mailparser'; import * as planer from 'planer'; @@ -21,10 +22,12 @@ export const extractTextWithoutReplyQuotations = ( dom.window.document, ); - const textContent = new JSDOM(cleanedHtml, { runScripts: 'outside-only' }) - .window.document.body?.textContent; + const text = convert(cleanedHtml, { + wordwrap: false, + preserveNewlines: true, + }).trim(); - return textContent ?? ''; + return text.replace(/\u00A0/g, ' '); } return ''; diff --git a/packages/twenty-ui/package.json b/packages/twenty-ui/package.json index 9a0cad138de..3867bba9ee1 100644 --- a/packages/twenty-ui/package.json +++ b/packages/twenty-ui/package.json @@ -48,7 +48,6 @@ "dist", "assets", "accessibility", - "assets", "components", "display", "feedback", diff --git a/yarn.lock b/yarn.lock index 7edf6fd4b9b..e7184b9b325 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19662,6 +19662,13 @@ __metadata: languageName: node linkType: hard +"@types/html-to-text@npm:^9.0.4": + version: 9.0.4 + resolution: "@types/html-to-text@npm:9.0.4" + checksum: 10c0/8e0e548f280d1c1107f70db01ff6331a295f3097f535f4d07c30ced271818fc3803a8f03bb2117c1207c46a3d40da256a03a17bd05fc9f5f1b4821c5bcbf98d8 + languageName: node + linkType: hard + "@types/http-assert@npm:*": version: 1.5.5 resolution: "@types/http-assert@npm:1.5.5" @@ -34227,7 +34234,7 @@ __metadata: languageName: node linkType: hard -"html-to-text@npm:9.0.5": +"html-to-text@npm:9.0.5, html-to-text@npm:^9.0.5": version: 9.0.5 resolution: "html-to-text@npm:9.0.5" dependencies: @@ -50867,6 +50874,7 @@ __metadata: "@types/express-session": "npm:^1.18.0" "@types/graphql-upload": "npm:^8.0.12" "@types/handlebars": "npm:4.1.0" + "@types/html-to-text": "npm:^9.0.4" "@types/imapflow": "npm:^1.0.21" "@types/lodash.chunk": "npm:^4.2.9" "@types/lodash.differencewith": "npm:^4.5.9" @@ -50936,6 +50944,7 @@ __metadata: graphql-upload: "npm:13.0.0" graphql-yoga: "npm:4.0.5" handlebars: "npm:^4.7.8" + html-to-text: "npm:^9.0.5" imapflow: "npm:1.0.187" ioredis: "npm:5.6.0" jsdom: "npm:^26.1.0"