diff --git a/agenthub/codeact_agent/codeact_agent.py b/agenthub/codeact_agent/codeact_agent.py index 59b8ba131c1..3154db5dafb 100644 --- a/agenthub/codeact_agent/codeact_agent.py +++ b/agenthub/codeact_agent/codeact_agent.py @@ -122,6 +122,12 @@ def step(self, state: State) -> Action: self.messages.append({'role': 'user', 'content': content}) elif isinstance(obs, IPythonRunCellObservation): content = 'OBSERVATION:\n' + obs.content + # replace base64 images with a placeholder + splited = content.split('\n') + for i, line in enumerate(splited): + if '![image](data:image/png;base64,' in line: + splited[i] = '![image](data:image/png;base64, ...) already displayed to user' + content = '\n'.join(splited) self.messages.append({'role': 'user', 'content': content}) else: raise NotImplementedError( diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 231da53a4bc..e4fe6f11616 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -26,6 +26,7 @@ "react": "^18.2.0", "react-accessible-treeview": "^2.8.3", "react-dom": "^18.2.0", + "react-highlight": "^0.15.0", "react-hot-toast": "^2.4.1", "react-i18next": "^14.1.0", "react-icons": "^5.0.1", @@ -44,6 +45,7 @@ "@types/node": "^18.0.0 ", "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", + "@types/react-highlight": "^0.12.8", "@types/react-syntax-highlighter": "^15.5.11", "@typescript-eslint/eslint-plugin": "^7.4.0", "@typescript-eslint/parser": "^7.0.0", @@ -4878,6 +4880,15 @@ "@types/react": "*" } }, + "node_modules/@types/react-highlight": { + "version": "0.12.8", + "resolved": "https://registry.npmjs.org/@types/react-highlight/-/react-highlight-0.12.8.tgz", + "integrity": "sha512-V7O7zwXUw8WSPd//YUO8sz489J/EeobJljASGhP0rClrvq+1Y1qWEpToGu+Pp7YuChxhAXSgkLkrOYpZX5A62g==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/react-syntax-highlighter": { "version": "15.5.11", "resolved": "https://registry.npmjs.org/@types/react-syntax-highlighter/-/react-syntax-highlighter-15.5.11.tgz", @@ -12799,6 +12810,14 @@ "react": "^18.2.0" } }, + "node_modules/react-highlight": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/react-highlight/-/react-highlight-0.15.0.tgz", + "integrity": "sha512-5uV/b/N4Z421GSVVe05fz+OfTsJtFzx/fJBdafZyw4LS70XjIZwgEx3Lrkfc01W/RzZ2Dtfb0DApoaJFAIKBtA==", + "dependencies": { + "highlight.js": "^10.5.0" + } + }, "node_modules/react-hot-toast": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.4.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 154310d5a80..56f2d04f155 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,6 +25,7 @@ "react": "^18.2.0", "react-accessible-treeview": "^2.8.3", "react-dom": "^18.2.0", + "react-highlight": "^0.15.0", "react-hot-toast": "^2.4.1", "react-i18next": "^14.1.0", "react-icons": "^5.0.1", @@ -64,6 +65,7 @@ "@types/node": "^18.0.0 ", "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", + "@types/react-highlight": "^0.12.8", "@types/react-syntax-highlighter": "^15.5.11", "@typescript-eslint/eslint-plugin": "^7.4.0", "@typescript-eslint/parser": "^7.0.0", diff --git a/frontend/src/components/Jupyter.tsx b/frontend/src/components/Jupyter.tsx new file mode 100644 index 00000000000..22b27e80c02 --- /dev/null +++ b/frontend/src/components/Jupyter.tsx @@ -0,0 +1,77 @@ +import React from "react"; +import { useSelector } from "react-redux"; +import SyntaxHighlighter from "react-syntax-highlighter"; +import Markdown from "react-markdown"; +import { atomOneDark } from "react-syntax-highlighter/dist/esm/styles/hljs"; +import { RootState } from "#/store"; +import { Cell } from "#/state/jupyterSlice"; + +interface IJupyterCell { + cell: Cell; +} + +function JupyterCell({ cell }: IJupyterCell): JSX.Element { + const code = cell.content; + + if (cell.type === "input") { + return ( +
+
EXECUTE
+ 
+          
+            {code}
+          
+
+
+ ); + } + return ( +
+
STDOUT/STDERR
+ 
+        {/* split code by newline and render each line as a plaintext, except it starts with `![image]` so we render it as markdown */}
+        {code.split("\n").map((line, index) => {
+          if (line.startsWith("![image](data:image/png;base64,")) {
+            // add new line before and after the image
+            return (
+              

+                 value}>
+                  {line}
+                
+                

+              

+            );
+          }
+          return (
+            

+              
+                {line}
+              
+              

+            

+          );
+        })}
+
+
+ ); +} + +function Jupyter(): JSX.Element { + const { cells } = useSelector((state: RootState) => state.jupyter); + + return ( +
+ {cells.map((cell, index) => ( + + ))} +
+ ); +} + +export default Jupyter; diff --git a/frontend/src/components/Workspace.tsx b/frontend/src/components/Workspace.tsx index 7d12b64d0f2..b291e18c0bc 100644 --- a/frontend/src/components/Workspace.tsx +++ b/frontend/src/components/Workspace.tsx @@ -12,6 +12,7 @@ import { AllTabs, TabOption, TabType } from "#/types/TabOption"; import Browser from "./Browser"; import CodeEditor from "./CodeEditor"; import Planner from "./Planner"; +import Jupyter from "./Jupyter"; function Workspace() { const { t } = useTranslation(); @@ -20,12 +21,13 @@ function Workspace() { const screenshotSrc = useSelector( (state: RootState) => state.browser.screenshotSrc, ); - + const jupyterCells = useSelector((state: RootState) => state.jupyter.cells); const [activeTab, setActiveTab] = useState(TabOption.CODE); const [changes, setChanges] = useState>({ [TabOption.PLANNER]: false, [TabOption.CODE]: false, [TabOption.BROWSER]: false, + [TabOption.JUPYTER]: false, }); const tabData = useMemo( @@ -45,6 +47,11 @@ function Workspace() { icon: , component: , }, + [TabOption.JUPYTER]: { + name: t(I18nKey.WORKSPACE$JUPYTER_TAB_LABEL), + icon: , + component: , + }, }), [t], ); @@ -73,6 +80,14 @@ function Workspace() { // eslint-disable-next-line react-hooks/exhaustive-deps }, [screenshotSrc]); + useEffect(() => { + if (activeTab !== TabOption.JUPYTER && jupyterCells.length > 0) { + // FIXME: This is a temporary solution to show the jupyter tab when the first cell is added + // Only need to show the tab only when a cell is added + setChanges((prev) => ({ ...prev, [TabOption.JUPYTER]: true })); + } + }, [jupyterCells]); + return (
{ getPlan().then((fetchedPlan) => store.dispatch(setPlan(fetchedPlan))); diff --git a/frontend/src/services/observations.ts b/frontend/src/services/observations.ts index c5a5cbab7bb..956fc906697 100644 --- a/frontend/src/services/observations.ts +++ b/frontend/src/services/observations.ts @@ -3,6 +3,7 @@ import { setUrl, setScreenshotSrc } from "#/state/browserSlice"; import store from "#/store"; import { ObservationMessage } from "#/types/Message"; import { appendOutput } from "#/state/commandSlice"; +import { appendJupyterOutput } from "#/state/jupyterSlice"; import ObservationType from "#/types/ObservationType"; export function handleObservationMessage(message: ObservationMessage) { @@ -12,7 +13,7 @@ export function handleObservationMessage(message: ObservationMessage) { break; case ObservationType.RUN_IPYTHON: // FIXME: render this as markdown - store.dispatch(appendOutput(message.content)); + store.dispatch(appendJupyterOutput(message.content)); break; case ObservationType.BROWSE: if (message.extras?.screenshot) { diff --git a/frontend/src/state/jupyterSlice.ts b/frontend/src/state/jupyterSlice.ts new file mode 100644 index 00000000000..241a8df6f95 --- /dev/null +++ b/frontend/src/state/jupyterSlice.ts @@ -0,0 +1,27 @@ +import { createSlice } from "@reduxjs/toolkit"; + +export type Cell = { + content: string; + type: "input" | "output"; +}; + +const initialCells: Cell[] = []; + +export const cellSlice = createSlice({ + name: "cell", + initialState: { + cells: initialCells, + }, + reducers: { + appendJupyterInput: (state, action) => { + state.cells.push({ content: action.payload, type: "input" }); + }, + appendJupyterOutput: (state, action) => { + state.cells.push({ content: action.payload, type: "output" }); + }, + }, +}); + +export const { appendJupyterInput, appendJupyterOutput } = cellSlice.actions; + +export default cellSlice.reducer; diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 05ea2fe509a..f675f929ec5 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -7,6 +7,7 @@ import commandReducer from "./state/commandSlice"; import errorsReducer from "./state/errorsSlice"; import planReducer from "./state/planSlice"; import taskReducer from "./state/taskSlice"; +import jupyterReducer from "./state/jupyterSlice"; export const rootReducer = combineReducers({ browser: browserReducer, @@ -17,6 +18,7 @@ export const rootReducer = combineReducers({ errors: errorsReducer, plan: planReducer, agent: agentReducer, + jupyter: jupyterReducer, }); const store = configureStore({ diff --git a/frontend/src/types/TabOption.tsx b/frontend/src/types/TabOption.tsx index b114c0da285..a732acb2c74 100644 --- a/frontend/src/types/TabOption.tsx +++ b/frontend/src/types/TabOption.tsx @@ -2,10 +2,20 @@ enum TabOption { PLANNER = "planner", CODE = "code", BROWSER = "browser", + JUPYTER = "jupyter", } -type TabType = TabOption.PLANNER | TabOption.CODE | TabOption.BROWSER; +type TabType = + | TabOption.PLANNER + | TabOption.CODE + | TabOption.BROWSER + | TabOption.JUPYTER; -const AllTabs = [TabOption.CODE, TabOption.BROWSER, TabOption.PLANNER]; +const AllTabs = [ + TabOption.CODE, + TabOption.BROWSER, + TabOption.PLANNER, + TabOption.JUPYTER, +]; export { AllTabs, TabOption, type TabType }; diff --git a/opendevin/sandbox/plugins/jupyter/execute_server b/opendevin/sandbox/plugins/jupyter/execute_server index 828ac520e8b..c7212e907a0 100755 --- a/opendevin/sandbox/plugins/jupyter/execute_server +++ b/opendevin/sandbox/plugins/jupyter/execute_server @@ -187,8 +187,7 @@ class JupyterKernel: outputs.append(msg['content']['data']['text/plain']) if 'image/png' in msg['content']['data']: # use markdone to display image (in case of large image) - # outputs.append(f"\n\n") - outputs.append(f"![image](data:image/png;base64,{msg['content']['data']['image/png']})") + outputs.append(f"\n![image](data:image/png;base64,{msg['content']['data']['image/png']})\n") elif msg_type == 'execute_reply': execution_done = True