Issue for repeat upload file

api/controllers/console/datasets/file.py

@@ -1,6 +1,7 @@
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
+from werkzeug.exceptions import Forbidden
 
 import services
 from controllers.console import api
@@ -57,6 +58,8 @@ def post(self):
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
+        except services.errors.file.FileUploadedError as file_is_uploaded_error:
+            raise Forbidden(file_is_uploaded_error.description)
 
         return upload_file, 201
 

api/controllers/console/workspace/workspace.py

@@ -3,7 +3,7 @@
 from flask import request
 from flask_login import current_user
 from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
-from werkzeug.exceptions import Unauthorized
+from werkzeug.exceptions import Unauthorized, Forbidden
 
 import services
 from controllers.console import api
@@ -201,7 +201,9 @@ def post(self):
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
-
+        except services.errors.file.FileUploadedError as file_is_uploaded_error:
+            raise Forbidden(file_is_uploaded_error.description)
+
         return { 'id': upload_file.id }, 201
 
 

api/controllers/service_api/app/file.py

@@ -1,5 +1,6 @@
 from flask import request
 from flask_restful import Resource, marshal_with
+from werkzeug.exceptions import Forbidden
 
 import services
 from controllers.service_api import api
@@ -39,6 +40,8 @@ def post(self, app_model: App, end_user: EndUser):
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
+        except services.errors.file.FileUploadedError as file_is_uploaded_error:
+            raise Forbidden(file_is_uploaded_error.description)
 
         return upload_file, 201
 

api/controllers/web/file.py

@@ -1,5 +1,6 @@
 from flask import request
 from flask_restful import marshal_with
+from werkzeug.exceptions import Forbidden
 
 import services
 from controllers.web import api
@@ -28,6 +29,8 @@ def post(self, app_model, end_user):
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
+        except services.errors.file.FileUploadedError as file_is_uploaded_error:
+            raise Forbidden(file_is_uploaded_error.description)
 
         return upload_file, 201
 

api/services/dataset_service.py

@@ -245,9 +245,19 @@ def delete_dataset(dataset_id, user):
 
         DatasetService.check_dataset_permission(dataset, user)
 
+        uploaded_files_id = []
+        documents = DocumentService.get_document_by_dataset_id_and_data_source_type(dataset_id, 'upload_file')
+        for document in documents:
+            document_was_deleted.send(document.id, dataset_id=document.dataset_id, doc_form=document.doc_form)
+            db.session.delete(document)
+            uploaded_files_id.append(json.loads(document.data_source_info)['upload_file_id'])
+
         dataset_was_deleted.send(dataset)
 
         db.session.delete(dataset)
+
+        db.session.query(UploadFile).filter(UploadFile.id.in_(uploaded_files_id)).delete(synchronize_session=False)
+
         db.session.commit()
         return True
 
@@ -423,6 +433,14 @@ def get_error_documents_by_dataset_id(dataset_id: str) -> list[Document]:
 
         return documents
 
+    @staticmethod
+    def get_document_by_dataset_id_and_data_source_type(dataset_id: str, data_source_type: str) -> list[Document]:
+        document = db.session.query(Document).filter(
+            Document.dataset_id == dataset_id,
+            Document.data_source_type == data_source_type
+        ).all()
+        return document
+
     @staticmethod
     def get_batch_documents(dataset_id: str, batch: str) -> list[Document]:
         documents = db.session.query(Document).filter(
@@ -451,7 +469,9 @@ def check_archived(document):
     def delete_document(document):
         # trigger document_was_deleted signal
         document_was_deleted.send(document.id, dataset_id=document.dataset_id, doc_form=document.doc_form)
-
+        if document.data_source_type == 'upload_file':
+            file_id = json.loads(document.data_source_info)['upload_file_id']
+            db.session.query(UploadFile).filter(UploadFile.id == file_id).delete(synchronize_session=False)
         db.session.delete(document)
         db.session.commit()
 

api/services/errors/file.py

@@ -9,5 +9,9 @@ class FileTooLargeError(BaseServiceError):
     description = "{message}"
 
 
+class FileUploadedError(BaseServiceError):
+    description = "{message}"
+
+
 class UnsupportedFileTypeError(BaseServiceError):
     pass

api/services/file_service.py

@@ -15,7 +15,7 @@
 from extensions.ext_storage import storage
 from models.account import Account
 from models.model import EndUser, UploadFile
-from services.errors.file import FileTooLargeError, UnsupportedFileTypeError
+from services.errors.file import FileTooLargeError, UnsupportedFileTypeError, FileUploadedError
 
 IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'webp', 'gif', 'svg']
 IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
@@ -64,13 +64,24 @@ def upload_file(file: FileStorage, user: Union[Account, EndUser], only_image: bo
             # end_user
             current_tenant_id = user.tenant_id
 
+        config = current_app.config
+        # Check if the file has been uploaded.
+        # Unique check field：tenant_id + name + hash
+        uploaded_file = db.session.query(UploadFile) \
+            .filter(UploadFile.tenant_id == current_tenant_id) \
+            .filter(UploadFile.name == file.filename) \
+            .filter(UploadFile.hash == hashlib.sha3_256(file_content).hexdigest())\
+            .first()
+        if uploaded_file:
+            message = f'File has uploaded. {file.filename}'
+            raise FileUploadedError(message)
+
         file_key = 'upload_files/' + current_tenant_id + '/' + file_uuid + '.' + extension
 
         # save file to storage
         storage.save(file_key, file_content)
 
         # save file to db
-        config = current_app.config
         upload_file = UploadFile(
             tenant_id=current_tenant_id,
             storage_type=config['STORAGE_TYPE'],

web/app/components/datasets/create/file-uploader/index.tsx

@@ -129,7 +129,10 @@ const FileUploader = ({
         return Promise.resolve({ ...completeFile })
       })
       .catch((e) => {
-        notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
+        // Add delay for each file error.
+        setTimeout(() => {
+          notify ({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
+        }, 500 * fileListRef.current.findIndex(item => item.fileID === fileItem.fileID))
         onFileUpdate(fileItem, -2, fileListRef.current)
         return Promise.resolve({ ...fileItem })
       })