feat: upload all fields to qdrant

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.13.6-dev0
+
+### Enhancements
+* **Upload all element fields to Qdrant** Since Qdrant allows any content that can be represented as JSON as the "payload", upload all element fields to the destination. Previous approach included redundant parsing.
+
+### Features
+
+### Fixes
+
 ## 0.13.5
 
 ### Enhancements

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.13.5"  # pragma: no cover
+__version__ = "0.13.6-dev0"  # pragma: no cover

unstructured/ingest/connector/qdrant.py

@@ -1,4 +1,3 @@
-import json
 import multiprocessing as mp
 import typing as t
 import uuid
@@ -16,7 +15,6 @@
 )
 from unstructured.ingest.logger import logger
 from unstructured.ingest.utils.data_prep import chunk_generator
-from unstructured.staging.base import flatten_dict
 from unstructured.utils import requires_dependencies
 
 if t.TYPE_CHECKING:
@@ -133,13 +131,7 @@ def normalize_dict(self, element_dict: dict) -> dict:
         return {
             "id": str(uuid.uuid4()),
             "vector": element_dict.pop("embeddings", {}),
-            "payload": {
-                "text": element_dict.pop("text", None),
-                "element_serialized": json.dumps(element_dict),
-                **flatten_dict(
-                    element_dict,
-                    separator="-",
-                    flatten_lists=True,
-                ),
-            },
+            # In Qdrant, "payload" can be any information that can be represented using JSON.
+            # https://qdrant.tech/documentation/concepts/payload/#payload
+            "payload": element_dict,
         }