Skip to content

Commit

Permalink
Merge pull request #301 from mfts/feat/s3
Browse files Browse the repository at this point in the history 
feat: add aws s3 as an alternative storage
  • Loading branch information
@mfts
mfts committed Feb 21, 2024
2 parents c049fbd + 70d0862 commit 2fdd60b
Show file tree
Hide file tree
Showing 27 changed files with 4,349 additions and 2,112 deletions.
120 changes: 32 additions & 88 deletions components/documents/add-document-modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,19 @@ import { Label } from "@/components/ui/label";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { FormEvent, useState } from "react";
import { useRouter } from "next/router";
import { type PutBlobResult } from "@vercel/blob";
import { upload } from "@vercel/blob/client";
import DocumentUpload from "@/components/document-upload";
import { pdfjs } from "react-pdf";
import { copyToClipboard, getExtension } from "@/lib/utils";
import { copyToClipboard } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import { usePlausible } from "next-plausible";
import { toast } from "sonner";
import { useTeam } from "@/context/team-context";
import { parsePageId } from "notion-utils";

pdfjs.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjs.version}/pdf.worker.js`;
import { putFile } from "@/lib/files/put-file";
import {
DocumentData,
createDocument,
createNewDocumentVersion,
} from "@/lib/documents/create-document";

export function AddDocumentModal({
newVersion,
Expand All @@ -37,7 +38,9 @@ export function AddDocumentModal({
const [notionLink, setNotionLink] = useState<string | null>(null);
const teamInfo = useTeam();

const handleBrowserUpload = async (
const teamId = teamInfo?.currentTeam?.id as string;

const handleFileUpload = async (
event: FormEvent<HTMLFormElement>,
): Promise<void> => {
event.preventDefault();
Expand All @@ -51,28 +54,30 @@ export function AddDocumentModal({
try {
setUploading(true);

const newBlob = await upload(currentFile.name, currentFile, {
access: "public",
handleUploadUrl: "/api/file/browser-upload",
const { type, data, numPages } = await putFile({
file: currentFile,
teamId,
});

const documentData: DocumentData = {
name: currentFile.name,
key: data!,
storageType: type!,
};
let response: Response | undefined;
let numPages: number | undefined;
// create a document or new version in the database if the document is a pdf
if (getExtension(newBlob.pathname).includes("pdf")) {
numPages = await getTotalPages(newBlob.url);
if (!newVersion) {
// create a document in the database
response = await saveDocumentToDatabase(newBlob, numPages);
} else {
// create a new version for existing document in the database
const documentId = router.query.id;
response = await saveNewVersionToDatabase(
newBlob,
documentId as string,
numPages,
);
}
// create a document or new version in the database
if (!newVersion) {
// create a document in the database
response = await createDocument({ documentData, teamId, numPages });
} else {
// create a new version for existing document in the database
const documentId = router.query.id as string;
response = await createNewDocumentVersion({
documentData,
documentId,
numPages,
teamId,
});
}

if (response) {
Expand Down Expand Up @@ -114,67 +119,6 @@ export function AddDocumentModal({
}
};

async function saveDocumentToDatabase(
blob: PutBlobResult,
numPages?: number,
) {
// create a document in the database with the blob url
const response = await fetch(
`/api/teams/${teamInfo?.currentTeam?.id}/documents`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
name: blob.pathname,
url: blob.url,
numPages: numPages,
}),
},
);

if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}

return response;
}

// create a new version in the database
async function saveNewVersionToDatabase(
blob: PutBlobResult,
documentId: string,
numPages?: number,
) {
const response = await fetch(
`/api/teams/${teamInfo?.currentTeam?.id}/documents/${documentId}/versions`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
url: blob.url,
numPages: numPages,
type: "pdf",
}),
},
);

if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}

return response;
}

// get the number of pages in the pdf
async function getTotalPages(url: string): Promise<number> {
const pdf = await pdfjs.getDocument(url).promise;
return pdf.numPages;
}

const createNotionFileName = () => {
// Extract Notion file name from the URL
const urlSegments = (notionLink as string).split("/")[3];
Expand Down Expand Up @@ -291,7 +235,7 @@ export function AddDocumentModal({
<CardContent className="space-y-2">
<form
encType="multipart/form-data"
onSubmit={handleBrowserUpload}
onSubmit={handleFileUpload}
className="flex flex-col"
>
<div className="space-y-1">
Expand Down
37 changes: 17 additions & 20 deletions components/documents/document-card.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,29 +105,26 @@ export default function DocumentsCard({
return;
}

const response = await fetch(
`/api/teams/${teamInfo?.currentTeam?.id}/documents/${documentId}`,
{
toast.promise(
fetch(`/api/teams/${teamInfo?.currentTeam?.id}/documents/${documentId}`, {
method: "DELETE",
}).then(() => {
mutate(`/api/teams/${teamInfo?.currentTeam?.id}/documents`, null, {
populateCache: (_, docs) => {
return docs.filter(
(doc: DocumentWithLinksAndLinkCountAndViewCount) =>
doc.id !== documentId,
);
},
revalidate: false,
});
}),
{
loading: "Deleting document...",
success: "Document deleted successfully.",
error: "Failed to delete document. Try again.",
},
);

if (response.ok) {
// remove the document from the cache
mutate(`/api/teams/${teamInfo?.currentTeam?.id}/documents`, null, {
populateCache: (_, docs) => {
return docs.filter(
(doc: DocumentWithLinksAndLinkCountAndViewCount) =>
doc.id !== documentId,
);
},
revalidate: false,
});
toast.success("Document deleted successfully.");
} else {
const { message } = await response.json();
toast.error(message);
}
};

const handleMenuStateChange = (open: boolean) => {
Expand Down
49 changes: 35 additions & 14 deletions components/view/PagesViewer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,20 +191,41 @@ export default function PagesViewer({

<div className="flex justify-center mx-auto relative h-full w-full">
{pages && loadedImages[pageNumber - 1] ? (
pages.map((page, index) => (
<Image
key={index}
className={`object-contain mx-auto ${
pageNumber - 1 === index ? "block" : "hidden"
}`}
src={loadedImages[index] ? page.file : BlankImg}
alt={`Page ${index + 1}`}
priority={loadedImages[index] ? true : false}
fill
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 75vw, 50vw"
quality={100}
/>
))
pages.map((page, index) => {
// contains cloudfront.net in the file path, then use img tag otherwise use next/image
if (page.file.toLowerCase().includes("cloudfront.net")) {
return (
<img
key={index}
className={`object-contain mx-auto ${
pageNumber - 1 === index ? "block" : "hidden"
}`}
src={
loadedImages[index]
? page.file
: "https://www.papermark.io/_static/blank.gif"
}
alt={`Page ${index + 1}`}
fetchPriority={loadedImages[index] ? "high" : "auto"}
/>
);
}

return (
<Image
key={index}
className={`object-contain mx-auto ${
pageNumber - 1 === index ? "block" : "hidden"
}`}
src={loadedImages[index] ? page.file : BlankImg}
alt={`Page ${index + 1}`}
priority={loadedImages[index] ? true : false}
fill
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 75vw, 50vw"
quality={100}
/>
);
})
) : (
<LoadingSpinner className="h-20 w-20 text-foreground" />
)}
Expand Down
68 changes: 47 additions & 21 deletions jobs/convert-pdf-to-image.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { client } from "@/trigger";
import { eventTrigger, retry } from "@trigger.dev/sdk";
import { z } from "zod";
import prisma from "@/lib/prisma";
import { getFile } from "@/lib/files/get-file";

client.defineJob({
id: "convert-pdf-to-image",
Expand All @@ -13,19 +14,22 @@ client.defineJob({
documentVersionId: z.string(),
versionNumber: z.number().int().optional(),
documentId: z.string().optional(),
teamId: z.string().optional(),
}),
}),
run: async (payload, io, ctx) => {
const { documentVersionId } = payload;

// get file url from document version
// 1. get file url from document version
const documentUrl = await io.runTask("get-document-url", async () => {
return prisma.documentVersion.findUnique({
where: {
id: documentVersionId,
},
select: {
file: true,
storageType: true,
numPages: true,
},
});
});
Expand All @@ -36,32 +40,52 @@ client.defineJob({
return;
}

// send file to api/convert endpoint in a task and get back number of pages
const muDocument = await io.runTask("get-number-of-pages", async () => {
const response = await fetch(
`${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/get-pages`,
{
method: "POST",
body: JSON.stringify({ url: documentUrl.file }),
headers: {
"Content-Type": "application/json",
let numPages = documentUrl.numPages;

// skip if the numPages are already defined
if (!numPages) {
// 2. send file to api/convert endpoint in a task and get back number of pages
const muDocument = await io.runTask("get-number-of-pages", async () => {
const response = await fetch(
`${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/get-pages`,
{
method: "POST",
body: JSON.stringify({ url: documentUrl.file }),
headers: {
"Content-Type": "application/json",
},
},
},
);
await io.logger.info("log response", { response });
);
await io.logger.info("log response", { response });

const { numPages } = (await response.json()) as { numPages: number };
return { numPages };
const { numPages } = (await response.json()) as { numPages: number };
return { numPages };
});

if (!muDocument || muDocument.numPages < 1) {
await io.logger.error("Failed to get number of pages", { payload });
return;
}

numPages = muDocument.numPages;
}

// 3. get signed url from file
const signedUrl = await io.runTask("get-signed-url", async () => {
return await getFile({
type: documentUrl.storageType,
data: documentUrl.file,
});
});

if (!muDocument || muDocument.numPages < 1) {
await io.logger.error("Failed to get number of pages", { payload });
if (!signedUrl) {
await io.logger.error("Failed to get signed url", { payload });
return;
}

// iterate through pages and upload to blob in a task
// 4. iterate through pages and upload to blob in a task
let currentPage = 0;
for (var i = 0; i < muDocument.numPages; ++i) {
for (var i = 0; i < numPages; ++i) {
currentPage = i + 1;
await io.runTask(
`upload-page-${currentPage}`,
Expand All @@ -74,10 +98,12 @@ client.defineJob({
body: JSON.stringify({
documentVersionId: documentVersionId,
pageNumber: currentPage,
url: documentUrl.file,
url: signedUrl,
teamId: payload.teamId,
}),
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${process.env.INTERNAL_API_KEY}`,
},
},
);
Expand All @@ -104,7 +130,7 @@ client.defineJob({
);
}

// after all pages are uploaded, update document version to hasPages = true
// 5. after all pages are uploaded, update document version to hasPages = true
await io.runTask("enable-pages", async () => {
return prisma.documentVersion.update({
where: {
Expand Down

0 comments on commit 2fdd60b

Please sign in to comment.