feat: fetch and parse full gmail message

.vscode/launch.json

@@ -33,6 +33,23 @@
       "internalConsoleOptions": "openOnSessionStart",
       "console": "internalConsole",
       "cwd": "${workspaceFolder}/packages/twenty-server/"
+    },
+    {
+      "name": "twenty-server - gmail fetch debug",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "yarn",
+      "runtimeVersion": "18",
+      "runtimeArgs": [
+        "nx",
+        "run",
+        "twenty-server:command",
+        "cron:messaging:gmail-fetch-messages-from-cache",
+      ],
+      "outputCapture": "std",
+      "internalConsoleOptions": "openOnSessionStart",
+      "console": "internalConsole",
+      "cwd": "${workspaceFolder}/packages/twenty-server/"
     }
   ]
 }

package.json

@@ -58,10 +58,10 @@
     "@types/lodash.camelcase": "^4.3.7",
     "@types/lodash.merge": "^4.6.7",
     "@types/lodash.pick": "^4.3.7",
-    "@types/mailparser": "^3.4.4",
     "@types/nodemailer": "^6.4.14",
     "@types/passport-microsoft": "^1.0.3",
     "add": "^2.0.6",
+    "addressparser": "^1.0.1",
     "afterframe": "^1.0.2",
     "apollo-server-express": "^3.12.0",
     "apollo-upload-client": "^17.0.0",
@@ -123,7 +123,6 @@
     "lodash.snakecase": "^4.1.1",
     "lodash.upperfirst": "^4.3.1",
     "luxon": "^3.3.0",
-    "mailparser": "^3.6.5",
     "microdiff": "^1.3.2",
     "nest-commander": "^3.12.0",
     "next": "14.0.4",
@@ -228,6 +227,7 @@
     "@swc/helpers": "~0.5.2",
     "@testing-library/jest-dom": "^6.1.5",
     "@testing-library/react": "14.0.0",
+    "@types/addressparser": "^1.0.3",
     "@types/apollo-upload-client": "^17.0.2",
     "@types/bcrypt": "^5.0.0",
     "@types/better-sqlite3": "^7.6.8",

packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts

@@ -1,14 +1,15 @@
 import { Injectable, Logger } from '@nestjs/common';
 
 import { AxiosResponse } from 'axios';
-import { simpleParser } from 'mailparser';
 import planer from 'planer';
+import addressparser from 'addressparser';
 
 import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
 import { MessageQuery } from 'src/modules/messaging/types/message-or-thread-query';
 import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response';
 import { FetchByBatchesService } from 'src/modules/messaging/services/fetch-by-batch/fetch-by-batch.service';
 import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util';
+import { assert } from 'src/utils/assert';
 
 @Injectable()
 export class FetchMessagesByBatchesService {
@@ -73,28 +74,25 @@ export class FetchMessagesByBatchesService {
           return;
         }
 
-        const { historyId, id, threadId, internalDate, raw } = message;
-
-        const body = atob(raw?.replace(/-/g, '+').replace(/_/g, '/'));
-
         try {
-          const parsed = await simpleParser(body, {
-            skipHtmlToText: true,
-            skipImageLinks: true,
-            skipTextToHtml: true,
-            maxHtmlLengthToParse: 0,
-          });
-
-          const { subject, messageId, from, to, cc, bcc, text, attachments } =
-            parsed;
+          const {
+            historyId,
+            id,
+            threadId,
+            internalDate,
+            subject,
+            from,
+            to,
+            headerMessageId,
+            text,
+            attachments,
+          } = this.parseGmailMessage(message);
 
           if (!from) throw new Error('From value is missing');
 
           const participants = [
             ...formatAddressObjectAsParticipants(from, 'from'),
             ...formatAddressObjectAsParticipants(to, 'to'),
-            ...formatAddressObjectAsParticipants(cc, 'cc'),
-            ...formatAddressObjectAsParticipants(bcc, 'bcc'),
           ];
 
           let textWithoutReplyQuotations = text;
@@ -115,12 +113,12 @@ export class FetchMessagesByBatchesService {
           const messageFromGmail: GmailMessage = {
             historyId,
             externalId: id,
-            headerMessageId: messageId || '',
+            headerMessageId,
             subject: subject || '',
             messageThreadExternalId: threadId,
             internalDate,
-            fromHandle: from.value[0].address || '',
-            fromDisplayName: from.value[0].name || '',
+            fromHandle: from[0].address || '',
+            fromDisplayName: from[0].name || '',
             participants,
             text: sanitizeString(textWithoutReplyQuotations || ''),
             attachments,
@@ -157,4 +155,62 @@ export class FetchMessagesByBatchesService {
 
     return { messages, errors };
   }
+
+  private parseGmailMessage(message: GmailMessageParsedResponse) {
+    const subject = this.getPropertyFromHeaders(message, 'Subject');
+    const rawFrom = this.getPropertyFromHeaders(message, 'From');
+    const rawTo = this.getPropertyFromHeaders(message, 'To');
+    const messageId = this.getPropertyFromHeaders(message, 'Message-ID');
+    const id = message.id;
+    const threadId = message.threadId;
+    const historyId = message.historyId;
+    const internalDate = message.internalDate;
+
+    assert(id);
+    assert(threadId);
+    assert(historyId);
+    assert(internalDate);
+
+    const bodyData = this.getBodyData(message);
+    const text = bodyData ? Buffer.from(bodyData, 'base64').toString() : '';
+
+    return {
+      id,
+      headerMessageId: messageId,
+      threadId,
+      historyId,
+      internalDate,
+      subject,
+      from: addressparser(rawFrom),
+      to: addressparser(rawTo),
+      text,
+      attachments: [],
+    };
+  }
+
+  private getBodyData(message: GmailMessageParsedResponse) {
+    const firstPart = message.payload?.parts?.[0];
+
+    if (firstPart?.mimeType === 'text/plain') {
+      return firstPart?.body?.data;
+    }
+
+    return firstPart?.parts?.find((part) => part.mimeType === 'text/plain')
+      ?.body?.data;
+  }
+
+  private getPropertyFromHeaders(
+    message: GmailMessageParsedResponse,
+    property: string,
+  ) {
+    const value = message.payload?.headers?.find(
+      (header) => header.name === property,
+    )?.value;
+
+    if (value === undefined || value === null) {
+      throw new Error(`Cannot find property "${property}" in message headers`);
+    }
+
+    return value;
+  }
 }

packages/twenty-server/src/modules/messaging/services/utils/format-address-object-as-participants.util.ts

@@ -1,10 +1,10 @@
-import { AddressObject } from 'mailparser';
+import addressparser from 'addressparser';
 
 import { Participant } from 'src/modules/messaging/types/gmail-message';
 
 const formatAddressObjectAsArray = (
-  addressObject: AddressObject | AddressObject[],
-): AddressObject[] => {
+  addressObject: addressparser.EmailAddress | addressparser.EmailAddress[],
+): addressparser.EmailAddress[] => {
   return Array.isArray(addressObject) ? addressObject : [addressObject];
 };
 
@@ -13,24 +13,23 @@ const removeSpacesAndLowerCase = (email: string): string => {
 };
 
 export const formatAddressObjectAsParticipants = (
-  addressObject: AddressObject | AddressObject[] | undefined,
+  addressObject:
+    | addressparser.EmailAddress
+    | addressparser.EmailAddress[]
+    | undefined,
   role: 'from' | 'to' | 'cc' | 'bcc',
 ): Participant[] => {
   if (!addressObject) return [];
   const addressObjects = formatAddressObjectAsArray(addressObject);
 
   const participants = addressObjects.map((addressObject) => {
-    const emailAdresses = addressObject.value;
+    const address = addressObject.address;
 
-    return emailAdresses.map((emailAddress) => {
-      const { name, address } = emailAddress;
-
-      return {
-        role,
-        handle: address ? removeSpacesAndLowerCase(address) : '',
-        displayName: name || '',
-      };
-    });
+    return {
+      role,
+      handle: address ? removeSpacesAndLowerCase(address) : '',
+      displayName: addressObject.name || '',
+    };
   });
 
   return participants.flat();

packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts

@@ -1,12 +1,6 @@
-export type GmailMessageParsedResponse = {
-  id: string;
-  threadId: string;
-  labelIds: string[];
-  snippet: string;
-  sizeEstimate: number;
-  raw: string;
-  historyId: string;
-  internalDate: string;
+import { gmail_v1 } from 'googleapis';
+
+export type GmailMessageParsedResponse = gmail_v1.Schema$Message & {
   error?: {
     code: number;
     message: string;

packages/twenty-server/src/modules/messaging/types/gmail-message.ts

@@ -1,5 +1,3 @@
-import { Attachment } from 'mailparser';
-
 export type GmailMessage = {
   historyId: string;
   externalId: string;
@@ -25,3 +23,10 @@ export type ParticipantWithMessageId = Participant & { messageId: string };
 export type ParticipantWithId = Participant & {
   id: string;
 };
+
+export type Attachment = {
+  id: string;
+  filename: string;
+  size: number;
+  mimeType: string;
+};

packages/twenty-server/src/modules/messaging/utils/create-queries-from-message-ids.util.ts

@@ -4,6 +4,6 @@ export const createQueriesFromMessageIds = (
   messageExternalIds: string[],
 ): MessageQuery[] => {
   return messageExternalIds.map((messageId) => ({
-    uri: '/gmail/v1/users/me/messages/' + messageId + '?format=RAW',
+    uri: '/gmail/v1/users/me/messages/' + messageId + '?format=FULL',
   }));
 };