how to send model name to agent via api call in langserve?

[octadion]

so currently I use langserve like this:

class Input(BaseModel):
    input: str
    tool_llm_name: str
    agent_llm_name: str

class Output(BaseModel):
    output: Any

# Initialize agent
agent = create_agent()

app = FastAPI(
    title="LangChain Server",
    version="1.0",
    description="Spin up a simple api server using LangChain's Runnable interfaces",
)

add_routes(
    app,
    agent.with_types(input_type=Input, output_type=Output).with_config(
        {"run_name": "agent"}
    ),
)

if __name__ == "__main__":

    import uvicorn

    uvicorn.run(app, host="localhost", port=8000)

I want to include the model / agent name in the api call which will later be processed in agent.py so that the model change process can be done dynamically, how do I do that?

In agent.py:

def create_agent(
    tool_llm_name: str = "gemini-pro",
    agent_llm_name: str = "gemini-pro",
):

example api call:

{
    "input":{
        "input":"input",
        "tool_llm_name":"gpt-3.5-turbo",
        "agent_llm_name":"gpt-3.5-turbo"
    },
    "config":{},
    "kwargs":{}
}

Thankyou!

[eyurtsev]

You can use Runnable Generator or Runnable Lambda to create the agent at run time.

e.g.,
https://github.com/langchain-ai/langserve/blob/main/examples/agent_custom_streaming/server.py

Or you can create a custom agent executor:

https://github.com/langchain-ai/langserve/blob/main/examples/configurable_agent_executor/server.py

[octadion]

Thank you for the advice, it worked but I had a few problems, here's what I did:

class Input(BaseModel):
    input: str
    tool_llm_name: str
    agent_llm_name: str

async def custom_stream(input: Input) -> AsyncIterator[str]:
    agent_executor = create_agent(input["tool_llm_name"], input["agent_llm_name"])
    async for event in agent_executor.astream_events(
        {
            "input": input["input"],
        },
    ):  
        print(f"Received event data: {event["data"]}")
        kind = event["event"]
        if kind == "on_chain_start":
            if (
                event["name"] == "agent"
            ):  # matches `.with_config({"run_name": "Agent"})` in agent_executor
                yield "\n"
                yield (
                    f"Starting agent: {event['name']} "
                    f"with input: {event['data'].get('input')}"
                )
                yield "\n"
        elif kind == "on_chain_end":
            if (
                event["name"] == "agent"
            ):  # matches `.with_config({"run_name": "Agent"})` in agent_executor
                yield "\n"
                yield (
                    f"Done agent: {event['name']} "
                    f"with output: {event['data'].get('output')['output']}"
                )
                yield "\n"
        if kind == "on_chat_model_stream":
            content = event["data"]["chunk"].content
            if content:
                # Empty content in the context of OpenAI means
                # that the model is asking for a tool to be invoked.
                # So we only print non-empty content
                yield content
        elif kind == "on_tool_start":
            yield "\n"
            yield (
                f"Starting tool: {event['name']} "
                f"with inputs: {event['data'].get('input')}"
            )
            yield "\n"
        elif kind == "on_tool_end":
            yield "\n"
            yield (
                f"Done tool: {event['name']} "
                f"with output: {event['data'].get('output')}"
            )
            yield {"output": event["data"].get("output")}

class Output(BaseModel):
    output: Any

app = FastAPI(
    title="LangChain Server",
    version="1.0",
    description="Spin up a simple api server using LangChain's Runnable interfaces",
)

agent = create_agent()

add_routes(
    app,
    RunnableLambda(custom_stream).with_types(input_type=Input, output_type=Output).with_config(
        {"run_name": "agent"}
    ),
    # agent.with_types(input_type=Input, output_type=Output).with_config(
    #     {"run_name": "agent"}
    # ),
)

"output": {
        "output": "[('OUTPUT', 8)]"
    },
"expected_output": {
        "output": "The head of the community with the most members is X with a total of 8 members."
    },

It seems that the output produced is the result of the tool, whereas what I expected and is in accordance with my previous code is the result of the agent whose output is in the form of a conversation. is there something wrong? Thank You!