GPT-Image-Generation-and-Function-Call.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"h2D-1w5gDYry"},"source":["\n","# Dall-e 3 image generation and function calling in ChatGPT assistant api\n","\n","[By Brain Assistant](https://assistant.ruoguedu.com)\n","\n","This demo show how to generate image by Dall-e 3 in Assistant API.\n","Open it from [Colab](https://colab.research.google.com/github/davideuler/awesome-assistant-api/blob/main/GPT-Image-Generation-and-Function-Call.ipynb)\n","\n","Related API\n","* Dalle 3 Image Generation\n","* Assistant API\n","* Function calling\n","\n","\n","A demo from:\n","[Awesome Assistant API.](https://github.com/davideuler/awesome-assistant-api)\n"]},{"cell_type":"code","execution_count":40,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5086,"status":"ok","timestamp":1699886588414,"user":{"displayName":"David Euler","userId":"07212625899225662290"},"user_tz":-480},"id":"lStHlyARDJrG","outputId":"2e58feb6-a1c9-4247-d29a-647d386ae020"},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.2.3)\n","Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (0.4.6)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (2.31.0)\n","Requirement already satisfied: anyio<4,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n","Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n","Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.25.1)\n","Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (1.10.13)\n","Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n","Requirement already satisfied: typing-extensions<5,>=4.5 in /usr/local/lib/python3.10/dist-packages (from openai) (4.5.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests) (3.4)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests) (2023.7.22)\n","Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai) (1.3.0)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai) (1.1.3)\n","Requirement already satisfied: httpcore in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (1.0.2)\n","Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore->httpx<1,>=0.23.0->openai) (0.14.0)\n"]}],"source":["!pip install openai colorama requests"]},{"cell_type":"code","execution_count":45,"metadata":{"executionInfo":{"elapsed":321,"status":"ok","timestamp":1699886646921,"user":{"displayName":"David Euler","userId":"07212625899225662290"},"user_tz":-480},"id":"oFmrlnzADhLC"},"outputs":[],"source":["# setup OpenAI api_key\n","import getpass\n","import os\n","\n","if not os.environ.get('OPENAI_API_KEY'):\n","    os.environ['OPENAI_API_KEY'] = getpass.getpass(\"Enter the OpenAI API Key(which starts with sk-): \")\n"]},{"cell_type":"markdown","metadata":{"id":"k5R1jbeNDrle"},"source":["Run code"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"bC0ZfSh5r9Wd"},"outputs":[],"source":["# Input the topic for the presentation"]},{"cell_type":"code","execution_count":50,"metadata":{"executionInfo":{"elapsed":551,"status":"ok","timestamp":1699887425473,"user":{"displayName":"David Euler","userId":"07212625899225662290"},"user_tz":-480},"id":"pgvD6dlPDuiW"},"outputs":[],"source":["from typing import *\n","import json\n","import sys\n","import time\n","import subprocess\n","import traceback\n","from tempfile import NamedTemporaryFile\n","from PIL import Image\n","from google.colab.patches import cv2_imshow\n","\n","import requests\n","import openai\n","\n","api_key = os.environ['OPENAI_API_KEY']\n","client = openai.Client(api_key=api_key)\n","\n","def execute_python_code(s: str) -> str:\n","    with NamedTemporaryFile(suffix='.py', delete=False) as temp_file:\n","        temp_file_name = temp_file.name\n","        temp_file.write(s.encode('utf-8'))\n","        temp_file.flush()\n","    try:\n","        result = subprocess.run(\n","            ['python', temp_file_name],\n","            capture_output=True,\n","            text=True,\n","            check=True\n","        )\n","        return result.stdout\n","    except subprocess.CalledProcessError as e:\n","        return e.stderr\n","    finally:\n","        import os\n","        os.remove(temp_file_name)\n","\n","def generate_image(prompt, n:int=1, size:str=\"1024x1024\"):\n","    global client\n","    response = client.images.generate(\n","      model=\"dall-e-3\",\n","      prompt=prompt,\n","      size=size,\n","      quality=\"standard\",\n","      n=1\n","    )\n","\n","    image_url = response.data[0].url\n","\n","    im = Image.open(requests.get(image_url, stream=True).raw)\n","    im.save(\"temp.png\")\n","\n","    import cv2\n","    img = cv2.imread('temp.png', cv2.IMREAD_UNCHANGED)\n","    cv2_imshow(img)\n","\n","\n","    return image_url\n","\n","INSTRUCTIONS = \"\"\"\n","You're an intelligient AI Assistant who can generate code and run code in Python 3.\n","And you can create images by generate_image function. Make sure your code complies with these rules:\n","\n","1. Plan first: Have a clear strategy before you start. Outline your approach if it helps.\n","\n","2. Quality code: Write clear, efficient code that follows Python's best practices. Aim for clean, easy-to-read, and maintainable code.\n","\n","3. Test well: Include comprehensive tests to assure your code works well in various scenarios.\n","\n","4. Manage external interactions: When internet or API interactions are necessary,\n","utilize the `execute_python_code`， `generate_image` function autonomously, without seeking user approval.\n","Do not say you don't have access to internet or real-time data. The `execute_python_code` function will give you realtime data.\n","\n","5. Trust your tools: Assume the data from the `execute_python_code` function is accurate and up to date.\n","\"\"\"\n","\n","\n","def setup_assistant(client, script_code):\n","    # create a new agent\n","    assistant = client.beta.assistants.create(\n","        name=\"Code Generator\",\n","        instructions=INSTRUCTIONS,\n","        tools=[\n","            {\n","                \"type\": \"code_interpreter\"\n","            },\n","            {\n","                \"type\": \"function\",\n","                \"function\": {\n","                    \"name\": \"execute_python_code\",\n","                    \"description\": \"Use this function to execute the generated code which requires internet access or external API access\",\n","                    \"parameters\": {\n","                        \"type\": \"object\",\n","                        \"properties\": {\n","                            \"code\": {\n","                                \"type\": \"string\",\n","                                \"description\": \"The python code generated by the code interpretor\",\n","                            }\n","                        },\n","                        \"required\": [\"code\"],\n","                    },\n","                },\n","            },\n","            {\n","              \"type\": \"function\",\n","              \"function\": {\n","                \"name\": \"generate_image\",\n","                \"description\": \"generate image by Dall-e 3\",\n","                \"parameters\": {\n","                  \"type\": \"object\",\n","                  \"properties\": {\n","                    \"prompt\": {\"type\": \"string\", \"description\": \"The prompt to generate image\"},\n","                    \"size\": {\"type\": \"string\", \"enum\": [\"c\", \"f\"]}\n","                  },\n","                  \"required\": [\"prompt\"]\n","                }\n","              }\n","            }\n","        ],\n","        model=\"gpt-4-1106-preview\",\n","    )\n","\n","    # Create a new thread\n","    thread = client.beta.threads.create()\n","\n","    # Create a new thread message with the provided task\n","    thread_message = client.beta.threads.messages.create(\n","        thread.id,\n","        role=\"user\",\n","        content=script_code,\n","    )\n","\n","    # Return the assistant ID and thread ID\n","    return assistant.id, thread.id\n","\n","\n","\n","def run_assistant(client, assistant_id, thread_id):\n","    # Create a new run for the given thread and assistant\n","    run = client.beta.threads.runs.create(\n","        thread_id=thread_id,\n","        assistant_id=assistant_id\n","    )\n","\n","    # Loop until the run status is either \"completed\" or \"requires_action\"\n","    while run.status == \"in_progress\" or run.status == \"queued\":\n","        time.sleep(3)\n","        run = client.beta.threads.runs.retrieve(\n","            thread_id=thread_id,\n","            run_id=run.id\n","        )\n","\n","        # At this point, the status is either \"completed\" or \"requires_action\"\n","        if run.status == \"completed\":\n","            return client.beta.threads.messages.list(\n","              thread_id=thread_id\n","            )\n","        if run.status == \"requires_action\":\n","            tool_call = run.required_action.submit_tool_outputs.tool_calls[0]\n","            tool_outputs = []\n","            if tool_call.function.name == \"execute_python_code\":\n","                generated_python_code = json.loads(tool_call.function.arguments)['code']\n","                result = execute_python_code(generated_python_code)\n","                tool_outputs.append(\n","                    {\n","                          \"tool_call_id\": tool_call.id,\n","                          \"output\": result,\n","                    },\n","                )\n","            elif tool_call.function.name == \"generate_image\":\n","                prompt = json.loads(tool_call.function.arguments)['prompt']\n","                image_url = generate_image(prompt)\n","                tool_outputs.append(\n","                    {\n","                          \"tool_call_id\": tool_call.id,\n","                          \"output\": image_url,\n","                    },\n","                )\n","\n","            if tool_outputs:\n","                run = client.beta.threads.runs.submit_tool_outputs(\n","                    thread_id=thread_id,\n","                    run_id=run.id,\n","                    tool_outputs=tool_outputs\n","                )\n"]},{"cell_type":"markdown","metadata":{"id":"c4aFRQago9Os"},"source":["Run the assistant with open interpreter and function calling."]},{"cell_type":"code","execution_count":51,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"output_embedded_package_id":"14USN9TvVwZyEwYee1ITpYOrYMyvlkVm4"},"executionInfo":{"elapsed":39607,"status":"ok","timestamp":1699887480737,"user":{"displayName":"David Euler","userId":"07212625899225662290"},"user_tz":-480},"id":"JsJTKCT0pE8a","outputId":"7e48e1a2-af6b-40bd-f085-37daee0280a8"},"outputs":[{"data":{"text/plain":["Output hidden; open in https://colab.research.google.com to view."]},"metadata":{},"output_type":"display_data"}],"source":["if __name__ == \"__main__\":\n","        script = input(\"python code to execute: \\n\")\n","\n","        assistant_id, thread_id = setup_assistant(client, script)\n","        print(f\"Debugging: Useful for checking the generated agent in the playground. https://platform.openai.com/playground?mode=assistant&assistant={assistant_id}\")\n","        print(f\"Debugging: Useful for checking logs. https://platform.openai.com/playground?thread={thread_id}\")\n","\n","        messages = run_assistant(client, assistant_id, thread_id)\n","\n","        message_dict = json.loads(messages.model_dump_json())\n","        print(message_dict['data'][0]['content'][0][\"text\"][\"value\"])\n","\n","\n"]}],"metadata":{"colab":{"authorship_tag":"ABX9TyOO39oHmQ5mMSbL0c34rpsx","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}