I don't understand how I can fix this code with llava hosted locally with ollama

[startakovsky]

I was reading this and using the llava model which works with a normal example.

Then I tried to do something I read in the Instructor Documentation which gave me an error.

My questions are like this:

1. Please explain the purpose of putting example. When I would have it as written in the documentation the returned identified product would be a list that contained an embellished headphone that didn't make sense.
2. Am I generally using this the right way?
3. What is the purpose of message in IdentifiedProduct?

Code

from pydantic import BaseModel, Field
from typing import List, Optional
import openai
import instructor

client = instructor.patch(
    openai.OpenAI(
        base_url="http://localhost:11434/v1",
        api_key="ollama",  # required, but unused
    ),
    mode=instructor.Mode.JSON,
)

class Product(BaseModel):
    """
    Represents a product extracted from an image using AI.

    The product attributes are dynamically determined based on the content
    of the image and the AI's interpretation. This class serves as a structured
    representation of the identified product characteristics.
    """

    name: str = Field(
        description="A generic name for the product.", example="Headphones"
    )
    key_features: Optional[List[str]] = Field(
        description="A list of key features of the product that stand out.",
        default=None,
    )

    description: Optional[str] = Field(
        description="A description of the product.",
        default=None,
    )

class IdentifiedProduct(BaseModel):
    """
    Represents a list of products identified in the images.
    """

    products: Optional[List[Product]] = Field(
        description="A list of products identified by the AI.",
        default=None,
    )

    error: bool = Field(default=False)
    message: Optional[str] = Field(default=None)

response = client.chat.completions.create(
    model="llava",
    response_model=IdentifiedProduct,
    messages=[
            {
                "role": "user",
                "content": "Identify products using the given images and generate key features for each product.",
                "images": [filepath]
            }
        ],
    )

Error

---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
Cell In[3], line 49
     46     error: bool = Field(default=False)
     47     message: Optional[str] = Field(default=None)
---> 49 response = client.chat.completions.create(
     50     model="llava",
     51     response_model=IdentifiedProduct,
     52     messages=[
     53             {
     54                 "role": "user",
     55                 "content": "Identify products using the given images and generate key features for each product.",
     56                 "images": [filepath]
     57             }
     58         ],
     59     )

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py:588](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py#line=587), in patch.<locals>.new_create_sync(response_model, validation_context, max_retries, *args, **kwargs)
    577 @wraps(func)
    578 def new_create_sync(
    579     response_model: Type[T_Model] = None,
   (...)
    583     **kwargs: T_ParamSpec.kwargs,
    584 ) -> T_Model:
    585     response_model, new_kwargs = handle_response_model(
    586         response_model=response_model, mode=mode, **kwargs
    587     )
--> 588     response = retry_sync(
    589         func=func,
    590         response_model=response_model,
    591         validation_context=validation_context,
    592         max_retries=max_retries,
    593         args=args,
    594         kwargs=new_kwargs,
    595         mode=mode,
    596     )
    597     return response

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py:405](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py#line=404), in retry_sync(func, response_model, validation_context, args, kwargs, max_retries, strict, mode)
    402     raise ValueError("max_retries must be an int or a `tenacity.Retrying` object")
    404 try:
--> 405     for attempt in max_retries:
    406         with attempt:
    407             try:

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py:347](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py#line=346), in BaseRetrying.__iter__(self)
    345 retry_state = RetryCallState(self, fn=None, args=(), kwargs={})
    346 while True:
--> 347     do = self.iter(retry_state=retry_state)
    348     if isinstance(do, DoAttempt):
    349         yield AttemptManager(retry_state=retry_state)

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py:325](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py#line=324), in BaseRetrying.iter(self, retry_state)
    323     retry_exc = self.retry_error_cls(fut)
    324     if self.reraise:
--> 325         raise retry_exc.reraise()
    326     raise retry_exc from fut.exception()
    328 if self.wait:

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py:158](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/tenacity/__init__.py#line=157), in RetryError.reraise(self)
    156 def reraise(self) -> t.NoReturn:
    157     if self.last_attempt.failed:
--> 158         raise self.last_attempt.result()
    159     raise self

File [~/.pyenv/versions/3.11.4/lib/python3.11/concurrent/futures/_base.py:449](http://localhost:8888/lab/workspaces/auto-W/tree/~/.pyenv/versions/3.11.4/lib/python3.11/concurrent/futures/_base.py#line=448), in Future.result(self, timeout)
    447     raise CancelledError()
    448 elif self._state == FINISHED:
--> 449     return self.__get_result()
    451 self._condition.wait(timeout)
    453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File [~/.pyenv/versions/3.11.4/lib/python3.11/concurrent/futures/_base.py:401](http://localhost:8888/lab/workspaces/auto-W/tree/~/.pyenv/versions/3.11.4/lib/python3.11/concurrent/futures/_base.py#line=400), in Future.__get_result(self)
    399 if self._exception:
    400     try:
--> 401         raise self._exception
    402     finally:
    403         # Break a reference cycle with the exception in self._exception
    404         self = None

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py:459](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py#line=458), in retry_sync(func, response_model, validation_context, args, kwargs, max_retries, strict, mode)
    452                 if mode == Mode.MD_JSON:
    453                     kwargs["messages"].append(
    454                         {
    455                             "role": "assistant",
    456                             "content": "```json",
    457                         },
    458                     )
--> 459                 raise e
    460 except RetryError as e:
    461     logger.exception(f"Failed after retries: {e.last_attempt.exception}")

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py:420](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py#line=419), in retry_sync(func, response_model, validation_context, args, kwargs, max_retries, strict, mode)
    418         total_usage.total_tokens += response.usage.total_tokens or 0
    419         response.usage = total_usage  # Replace each response usage with the total usage
--> 420     return process_response(
    421         response,
    422         response_model=response_model,
    423         stream=stream,
    424         validation_context=validation_context,
    425         strict=strict,
    426         mode=mode,
    427     )
    428 except (ValidationError, JSONDecodeError) as e:
    429     logger.debug(f"Error response: {response}")

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py:213](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/patch.py#line=212), in process_response(response, response_model, stream, validation_context, strict, mode)
    207     model = response_model.from_streaming_response(
    208         response,
    209         mode=mode,
    210     )
    211     return model
--> 213 model = response_model.from_response(
    214     response,
    215     validation_context=validation_context,
    216     strict=strict,
    217     mode=mode,
    218 )
    220 # ? This really hints at the fact that we need a better way of
    221 # ? attaching usage data and the raw response to the model we return.
    222 if isinstance(model, IterableBase):

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/function_calls.py:131](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/instructor/function_calls.py#line=130), in OpenAISchema.from_response(cls, completion, validation_context, strict, mode)
    125     return cls.model_validate_json(
    126         tool_call.function.arguments,
    127         context=validation_context,
    128         strict=strict,
    129     )
    130 elif mode in {Mode.JSON, Mode.JSON_SCHEMA, Mode.MD_JSON}:
--> 131     return cls.model_validate_json(
    132         message.content,
    133         context=validation_context,
    134         strict=strict,
    135     )
    136 else:
    137     raise ValueError(f"Invalid patch mode: {mode}")

File [~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/pydantic/main.py:538](http://localhost:8888/lab/workspaces/auto-W/tree/~/.local/share/virtualenvs/llm_playground--RZ6LqXE/lib/python3.11/site-packages/pydantic/main.py#line=537), in BaseModel.model_validate_json(cls, json_data, strict, context)
    536 # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks
    537 __tracebackhide__ = True
--> 538 return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)

ValidationError: 1 validation error for IdentifiedProduct
products
  Input should be a valid array [type=list_type, input_value={'anyOf': [{'items': {'$r... in case of an error.'}}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.6/v/list_type