Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: gmail attachments #5081

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -122,5 +122,15 @@ export const mapFieldMetadataToGraphqlQuery = (
addressLng
}
`;
} else if (fieldType === FieldMetadataType.FILE) {
return `
${field.name}
{
fileName
fileExtension
uploadedAt
storageType
}
`;
}
};
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ const getSchemaComponentsProperties = (
case FieldMetadataType.CURRENCY:
case FieldMetadataType.FULL_NAME:
case FieldMetadataType.ADDRESS:
case FieldMetadataType.FILE:
itemProperty = {
type: 'object',
properties: compositeTypeDefintions
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { LocalDriverOptions } from 'src/engine/integrations/file-storage/drivers

export enum StorageDriverType {
S3 = 's3',
Gmail = 'gmail',
Local = 'local',
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { CompositeType } from 'src/engine/metadata-modules/field-metadata/interfaces/composite-type.interface';

import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';

export const fileCompositeType: CompositeType = {
type: FieldMetadataType.FILE,
properties: [
{
name: 'fileName',
type: FieldMetadataType.TEXT,
hidden: false,
isRequired: true,
},
{
name: 'fileExtension',
type: FieldMetadataType.TEXT,
hidden: false,
isRequired: true,
},
{
name: 'uploadedAt',
type: FieldMetadataType.DATE_TIME,
hidden: false,
isRequired: false,
},
{
name: 'storageType',
type: FieldMetadataType.TEXT,
hidden: false,
isRequired: false,
},
],
};

export type FileMetadata = {
fileName: string;
fileExtension: string;
uploadedAt: Date;
storageType: 'server' | 'gmail';
};
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { fullNameCompositeType } from 'src/engine/metadata-modules/field-metadat
import { linkCompositeType } from 'src/engine/metadata-modules/field-metadata/composite-types/link.composite-type';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { addressCompositeType } from 'src/engine/metadata-modules/field-metadata/composite-types/address.composite-type';
import { fileCompositeType } from 'src/engine/metadata-modules/field-metadata/composite-types/file.composite-type';

export type CompositeFieldsDefinitionFunction = (
fieldMetadata?: FieldMetadataInterface,
Expand All @@ -19,4 +20,5 @@ export const compositeTypeDefintions = new Map<
[FieldMetadataType.CURRENCY, currencyCompositeType],
[FieldMetadataType.FULL_NAME, fullNameCompositeType],
[FieldMetadataType.ADDRESS, addressCompositeType],
[FieldMetadataType.FILE, fileCompositeType],
]);
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,17 @@ export class FieldMetadataDefaultValueAddress {
@IsNumber()
addressLng: number | null;
}

export class FieldMetadataDefaultValueFile {
@ValidateIf((_object, value) => value !== null)
@IsString()
fileName: string | null;

@ValidateIf((_object, value) => value !== null)
@IsString()
fileExtension: string | null;

@ValidateIf((_object, value) => value !== null)
@IsString()
storageType: string | null;
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export enum FieldMetadataType {
RELATION = 'RELATION',
POSITION = 'POSITION',
ADDRESS = 'ADDRESS',
FILE = 'FILE',
RAW_JSON = 'RAW_JSON',
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
FieldMetadataDefaultValueString,
FieldMetadataDefaultValueUuidFunction,
FieldMetadataDefaultValueNowFunction,
FieldMetadataDefaultValueFile,
} from 'src/engine/metadata-modules/field-metadata/dtos/default-value.input';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';

Expand Down Expand Up @@ -39,6 +40,7 @@ type FieldMetadataDefaultValueMapping = {
[FieldMetadataType.CURRENCY]: FieldMetadataDefaultValueCurrency;
[FieldMetadataType.FULL_NAME]: FieldMetadataDefaultValueFullName;
[FieldMetadataType.ADDRESS]: FieldMetadataDefaultValueAddress;
[FieldMetadataType.FILE]: FieldMetadataDefaultValueFile;
[FieldMetadataType.RATING]: FieldMetadataDefaultValueString;
[FieldMetadataType.SELECT]: FieldMetadataDefaultValueString;
[FieldMetadataType.MULTI_SELECT]: FieldMetadataDefaultValueString;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ export function generateDefaultValue(
url: "''",
label: "''",
};
case FieldMetadataType.FILE:
return {
fileName: "''",
fileExtension: "''",
storageType: "'server'",
};
case FieldMetadataType.CURRENCY:
return {
amountMicros: null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ export const isCompositeFieldMetadataType = (
type === FieldMetadataType.LINK ||
type === FieldMetadataType.CURRENCY ||
type === FieldMetadataType.FULL_NAME ||
type === FieldMetadataType.ADDRESS
type === FieldMetadataType.ADDRESS ||
type === FieldMetadataType.FILE
);
};
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
FieldMetadataDefaultValueNowFunction,
FieldMetadataDefaultValueUuidFunction,
FieldMetadataDefaultValueDate,
FieldMetadataDefaultValueFile,
} from 'src/engine/metadata-modules/field-metadata/dtos/default-value.input';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';

Expand Down Expand Up @@ -48,6 +49,7 @@ export const defaultValueValidatorsMap = {
[FieldMetadataType.SELECT]: [FieldMetadataDefaultValueString],
[FieldMetadataType.MULTI_SELECT]: [FieldMetadataDefaultValueStringArray],
[FieldMetadataType.ADDRESS]: [FieldMetadataDefaultValueAddress],
[FieldMetadataType.FILE]: [FieldMetadataDefaultValueFile],
[FieldMetadataType.RAW_JSON]: [FieldMetadataDefaultValueRawJson],
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { compositeTypeDefintions } from 'src/engine/metadata-modules/field-metad

export type CompositeFieldMetadataType =
| FieldMetadataType.ADDRESS
| FieldMetadataType.FILE
| FieldMetadataType.CURRENCY
| FieldMetadataType.FULL_NAME
| FieldMetadataType.LINK;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ export class WorkspaceMigrationFactory {
FieldMetadataType.ADDRESS,
{ factory: this.compositeColumnActionFactory },
],
[FieldMetadataType.FILE, { factory: this.compositeColumnActionFactory }],
[
FieldMetadataType.FULL_NAME,
{ factory: this.compositeColumnActionFactory },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ import { MessageThreadRepository } from 'src/modules/messaging/repositories/mess
import { MessageRepository } from 'src/modules/messaging/repositories/message.repository';
import { PersonRepository } from 'src/modules/person/repositories/person.repository';
import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
import { AttachmentRepository } from 'src/modules/attachment/repositories/attachment.repository';

export const metadataToRepositoryMapping = {
AttachmentObjectMetadata: AttachmentRepository,
AuditLogObjectMetadata: AuditLogRepository,
BlocklistObjectMetadata: BlocklistRepository,
CalendarChannelEventAssociationObjectMetadata:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,13 @@ export const ATTACHMENT_STANDARD_FIELD_IDS = {
name: '20202020-87a5-48f8-bbf7-ade388825a57',
fullPath: '20202020-0d19-453d-8e8d-fbcda8ca3747',
type: '20202020-a417-49b8-a40b-f6a7874caa0d',
file: '20202020-e32c-41c1-9e45-1c5a34cc1f9e',
author: '20202020-6501-4ac5-a4ef-b2f8522ef6cd',
activity: '20202020-b569-481b-a13f-9b94e47e54fe',
person: '20202020-0158-4aa2-965c-5cdafe21ffa2',
company: '20202020-ceab-4a28-b546-73b06b4c08d5',
opportunity: '20202020-7374-499d-bea3-9354890755b5',
message: '20202020-b505-4257-9efa-1dd72aa43c8e',
custom: '20202020-302d-43b3-9aea-aa4f89282a9f',
};

Expand Down Expand Up @@ -231,6 +233,7 @@ export const MESSAGE_STANDARD_FIELD_IDS = {
text: '20202020-d2ee-4e7e-89de-9a0a9044a143',
receivedAt: '20202020-140a-4a2a-9f86-f13b6a979afc',
messageParticipants: '20202020-7cff-4a74-b63c-73228448cbd9',
attachments: '20202020-afe1-4d5e-b0ae-fcaa099e2683',
messageChannelMessageAssociations: '20202020-3cef-43a3-82c6-50e7cfbc9ae4',
};

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { Injectable } from '@nestjs/common';

import { EntityManager } from 'typeorm';

import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { StorageDriverType } from 'src/engine/integrations/file-storage/interfaces';

@Injectable()
export class AttachmentRepository {
rostaklein marked this conversation as resolved.
Show resolved Hide resolved
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}

public async insert({
id,
messageId,
name,
personId,
storageDriverType,
type,
authorId,
workspaceId,
transactionManager,
}: {
id: string;
workspaceId: string;
personId: string | null;
messageId: string;
storageDriverType: StorageDriverType;
name: string;
type: string;
authorId: string;
transactionManager?: EntityManager;
}): Promise<void> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);

await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."attachment" ("id", "personId", "messageId", "storageDriverType", "name", "type", "authorId") VALUES ($1, $2, $3, $4, $5, $6, $7)`,
[id, personId, messageId, storageDriverType, name, type, authorId],
workspaceId,
transactionManager,
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import { OpportunityObjectMetadata } from 'src/modules/opportunity/standard-obje
import { PersonObjectMetadata } from 'src/modules/person/standard-objects/person.object-metadata';
import { WorkspaceMemberObjectMetadata } from 'src/modules/workspace-member/standard-objects/workspace-member.object-metadata';
import { IsNotAuditLogged } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-not-audit-logged.decorator';
import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/message.object-metadata';
import { FileMetadata } from 'src/engine/metadata-modules/field-metadata/composite-types/file.composite-type';

@ObjectMetadata({
standardId: STANDARD_OBJECT_IDS.attachment,
Expand Down Expand Up @@ -55,6 +57,16 @@ export class AttachmentObjectMetadata extends BaseObjectMetadata {
})
type: string;

@FieldMetadata({
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the change, but after discussion with the team, we now want to create a new composite FieldMetadataType File (you can take example on FieldMetadataType.Link in packages/twenty-server/src/engine/metadata-modules/field-metadata/composite-types/link.composite-type.ts) instead of creating a new storage driver type, since gmail is not really a storage driver type.
This composite field should have the following properties:

  • fileName: string
  • fileExtension: string
  • uploadedAt: DateTime
  • storageType: 'server' | 'gmail'
  • We also want to add a virtual property fullPath which will be computed inside query-result-getters.factory.

The table Attachment will only have one column of type File (in addition to the id, createdAt, updatedAt ... and the relations).

In packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts we want to create a new endpoint @Get(gmail/:filename) which will call users.messages.attachments.get to get the fileStream.

@charlesBochet can help you if you have any questions regarding this change.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense 馃憤 Ive actually already added one composite field :D #4492 This time it should be hopefully a bit easier though as I wont aim to add it as a field assignable to an object + read/write table, right?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great! It will actually be very similar, with an additional tweak on the getters logic! I can assist you if you have any question :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feel free to ping on Discord direclty if you need real time inputs!

standardId: ATTACHMENT_STANDARD_FIELD_IDS.file,
type: FieldMetadataType.FILE,
label: 'File',
description: 'Attachment file',
icon: 'IconCloud',
})
@IsNullable()
file: FileMetadata;

@FieldMetadata({
standardId: ATTACHMENT_STANDARD_FIELD_IDS.author,
type: FieldMetadataType.RELATION,
Expand Down Expand Up @@ -109,6 +121,17 @@ export class AttachmentObjectMetadata extends BaseObjectMetadata {
@IsNullable()
opportunity: Relation<OpportunityObjectMetadata>;

@FieldMetadata({
standardId: ATTACHMENT_STANDARD_FIELD_IDS.message,
type: FieldMetadataType.RELATION,
label: 'Message',
description: 'Attachment message',
icon: 'IconMessage',
joinColumn: 'messageId',
})
@IsNullable()
message: Relation<MessageObjectMetadata>;

@DynamicRelationFieldMetadata((oppositeObjectMetadata) => ({
standardId: ATTACHMENT_STANDARD_FIELD_IDS.custom,
name: oppositeObjectMetadata.nameSingular,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';

import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { AttachmentObjectMetadata } from 'src/modules/attachment/standard-objects/attachment.object-metadata';
import { MessageThreadModule } from 'src/modules/messaging/services/message-thread/message-thread.module';
import { MessageService } from 'src/modules/messaging/services/message/message.service';
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
Expand All @@ -17,6 +18,7 @@ import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/me
MessageObjectMetadata,
MessageChannelObjectMetadata,
MessageThreadObjectMetadata,
AttachmentObjectMetadata,
]),
MessageThreadModule,
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ import { MessageChannelRepository } from 'src/modules/messaging/repositories/mes
import { MessageThreadService } from 'src/modules/messaging/services/message-thread/message-thread.service';
import { MessageThreadObjectMetadata } from 'src/modules/messaging/standard-objects/message-thread.object-metadata';
import { MessageThreadRepository } from 'src/modules/messaging/repositories/message-thread.repository';
import { AttachmentObjectMetadata } from 'src/modules/attachment/standard-objects/attachment.object-metadata';
import { AttachmentRepository } from 'src/modules/attachment/repositories/attachment.repository';
import { StorageDriverType } from 'src/engine/integrations/file-storage/interfaces';

@Injectable()
export class MessageService {
Expand All @@ -30,6 +33,8 @@ export class MessageService {
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
@InjectObjectMetadataRepository(MessageObjectMetadata)
private readonly messageRepository: MessageRepository,
@InjectObjectMetadataRepository(AttachmentObjectMetadata)
private readonly attachmentRepository: AttachmentRepository,
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
private readonly messageChannelRepository: MessageChannelRepository,
@InjectObjectMetadataRepository(MessageThreadObjectMetadata)
Expand Down Expand Up @@ -77,6 +82,14 @@ export class MessageService {
transactionManager,
);

await this.saveAttachmentsToMessage(
message,
savedOrExistingMessageId,
connectedAccount,
workspaceId,
transactionManager,
);

messageExternalIdsAndIdsMap.set(
message.externalId,
savedOrExistingMessageId,
Expand Down Expand Up @@ -228,6 +241,30 @@ export class MessageService {
return Promise.resolve(newMessageId);
}

private async saveAttachmentsToMessage(
message: GmailMessage,
savedMessageId: string,
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
workspaceId: string,
manager: EntityManager,
): Promise<void> {
await Promise.all(
message.attachments.map((attachment) =>
this.attachmentRepository.insert({
id: v4(),
messageId: savedMessageId,
name: attachment.filename ?? '',
personId: null,
storageDriverType: StorageDriverType.Gmail,
type: attachment.contentType,
authorId: connectedAccount.accountOwnerId,
workspaceId,
transactionManager: manager,
}),
),
);
}

public async deleteMessages(
messagesDeletedMessageExternalIds: string[],
gmailMessageChannelId: string,
Expand Down