Implement Jupyter Frontend

agenthub/codeact_agent/codeact_agent.py

@@ -122,6 +122,12 @@ def step(self, state: State) -> Action:
                     self.messages.append({'role': 'user', 'content': content})
                 elif isinstance(obs, IPythonRunCellObservation):
                     content = 'OBSERVATION:\n' + obs.content
+                    # replace base64 images with a placeholder
+                    splited = content.split('\n')
+                    for i, line in enumerate(splited):
+                        if '![image](data:image/png;base64,' in line:
+                            splited[i] = '![image](data:image/png;base64, ...) already displayed to user'
+                    content = '\n'.join(splited)
                     self.messages.append({'role': 'user', 'content': content})
                 else:
                     raise NotImplementedError(

frontend/package.json

@@ -25,6 +25,7 @@
     "react": "^18.2.0",
     "react-accessible-treeview": "^2.8.3",
     "react-dom": "^18.2.0",
+    "react-highlight": "^0.15.0",
     "react-hot-toast": "^2.4.1",
     "react-i18next": "^14.1.0",
     "react-icons": "^5.0.1",
@@ -64,6 +65,7 @@
     "@types/node": "^18.0.0 ",
     "@types/react": "^18.2.66",
     "@types/react-dom": "^18.2.22",
+    "@types/react-highlight": "^0.12.8",
     "@types/react-syntax-highlighter": "^15.5.11",
     "@typescript-eslint/eslint-plugin": "^7.4.0",
     "@typescript-eslint/parser": "^7.0.0",

frontend/src/components/Jupyter.tsx

@@ -0,0 +1,77 @@
+import React from "react";
+import { useSelector } from "react-redux";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import Markdown from "react-markdown";
+import { atomOneDark } from "react-syntax-highlighter/dist/esm/styles/hljs";
+import { RootState } from "#/store";
+import { Cell } from "#/state/jupyterSlice";
+
+interface IJupyterCell {
+  cell: Cell;
+}
+
+function JupyterCell({ cell }: IJupyterCell): JSX.Element {
+  const code = cell.content;
+
+  if (cell.type === "input") {
+    return (
+      <div className="rounded-lg bg-gray-800 dark:bg-gray-900 p-2 text-xs">
+        <div className="mb-1 text-gray-400">EXECUTE</div>
+        <pre
+          className="scrollbar-custom scrollbar-thumb-gray-500 hover:scrollbar-thumb-gray-400 dark:scrollbar-thumb-white/10 dark:hover:scrollbar-thumb-white/20 overflow-auto px-5"
+          style={{ padding: 0, marginBottom: 0, fontSize: "0.75rem" }}
+        >
+          <SyntaxHighlighter language="python" style={atomOneDark}>
+            {code}
+          </SyntaxHighlighter>
+        </pre>
+      </div>
+    );
+  }
+  return (
+    <div className="rounded-lg bg-gray-800 dark:bg-gray-900 p-2 text-xs">
+      <div className="mb-1 text-gray-400">STDOUT/STDERR</div>
+      <pre
+        className="scrollbar-custom scrollbar-thumb-gray-500 hover:scrollbar-thumb-gray-400 dark:scrollbar-thumb-white/10 dark:hover:scrollbar-thumb-white/20 overflow-auto px-5 max-h-[60vh] bg-gray-800"
+        style={{ padding: 0, marginBottom: 0, fontSize: "0.75rem" }}
+      >
+        {/* split code by newline and render each line as a plaintext, except it starts with `![image]` so we render it as markdown */}
+        {code.split("\n").map((line, index) => {
+          if (line.startsWith("![image](data:image/png;base64,")) {
+            // add new line before and after the image
+            return (
+              <div key={index}>
+                <Markdown urlTransform={(value: string) => value}>
+                  {line}
+                </Markdown>
+                <br />
+              </div>
+            );
+          }
+          return (
+            <div key={index}>
+              <SyntaxHighlighter language="plaintext" style={atomOneDark}>
+                {line}
+              </SyntaxHighlighter>
+              <br />
+            </div>
+          );
+        })}
+      </pre>
+    </div>
+  );
+}
+
+function Jupyter(): JSX.Element {
+  const { cells } = useSelector((state: RootState) => state.jupyter);
+
+  return (
+    <div className="flex-1 overflow-y-auto flex flex-col">
+      {cells.map((cell, index) => (
+        <JupyterCell key={index} cell={cell} />
+      ))}
+    </div>
+  );
+}
+
+export default Jupyter;

frontend/src/components/Workspace.tsx

@@ -12,6 +12,7 @@
 import Browser from "./Browser";
 import CodeEditor from "./CodeEditor";
 import Planner from "./Planner";
+import Jupyter from "./Jupyter";
 
 function Workspace() {
   const { t } = useTranslation();
@@ -20,12 +21,13 @@
   const screenshotSrc = useSelector(
     (state: RootState) => state.browser.screenshotSrc,
   );
-
+  const jupyterCells = useSelector((state: RootState) => state.jupyter.cells);
   const [activeTab, setActiveTab] = useState<TabType>(TabOption.CODE);
   const [changes, setChanges] = useState<Record<TabType, boolean>>({
     [TabOption.PLANNER]: false,
     [TabOption.CODE]: false,
     [TabOption.BROWSER]: false,
+    [TabOption.JUPYTER]: false,
   });
 
   const tabData = useMemo(
@@ -45,6 +47,11 @@
         icon: <IoIosGlobe size={18} />,
         component: <Browser key="browser" />,
       },
+      [TabOption.JUPYTER]: {
+        name: t(I18nKey.WORKSPACE$JUPYTER_TAB_LABEL),
+        icon: <VscCode size={18} />,
+        component: <Jupyter key="jupyter" />,
+      },
     }),
     [t],
   );
@@ -73,6 +80,14 @@
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [screenshotSrc]);
 
+  useEffect(() => {
+    if (activeTab !== TabOption.JUPYTER && jupyterCells.length > 0) {
+      // FIXME: This is a temporary solution to show the jupyter tab when the first cell is added
+      // Only need to show the tab only when a cell is added
+      setChanges((prev) => ({ ...prev, [TabOption.JUPYTER]: true }));
+    }
+  }, [jupyterCells]);
+
   return (
     <div className="flex flex-col min-h-0 grow">
       <div

frontend/src/i18n/translation.json

@@ -39,6 +39,19 @@
     "pt": "Planejador",
     "es": "Planificador"
   },
+  "WORKSPACE$JUPYTER_TAB_LABEL": {
+    "en": "Jupyter IPython",
+    "zh-CN": "Jupyter IPython",
+    "de": "Jupyter IPython",
+    "ko-KR": "Jupyter IPython",
+    "no": "Jupyter IPython",
+    "zh-TW": "Jupyter IPython",
+    "ar": "Jupyter IPython",
+    "fr": "Jupyter IPython",
+    "it": "Jupyter IPython",
+    "pt": "Jupyter IPython",
+    "es": "Jupyter IPython"
+  },
   "WORKSPACE$CODE_EDITOR_TAB_LABEL": {
     "en": "Code Editor",
     "zh-CN": "代码编辑器",

frontend/src/services/actions.ts

@@ -3,6 +3,7 @@ import { setScreenshotSrc, setUrl } from "#/state/browserSlice";
 import { appendAssistantMessage } from "#/state/chatSlice";
 import { setCode, updatePath } from "#/state/codeSlice";
 import { appendInput } from "#/state/commandSlice";
+import { appendJupyterInput } from "#/state/jupyterSlice";
 import { setPlan } from "#/state/planSlice";
 import { setInitialized } from "#/state/taskSlice";
 import store from "#/store";
@@ -45,7 +46,7 @@ const messageActions = {
     if (message.args.thought) {
       store.dispatch(appendAssistantMessage(message.args.thought));
     }
-    store.dispatch(appendInput(message.args.code));
+    store.dispatch(appendJupyterInput(message.args.code));
   },
   [ActionType.ADD_TASK]: () => {
     getPlan().then((fetchedPlan) => store.dispatch(setPlan(fetchedPlan)));

frontend/src/services/observations.ts

@@ -3,6 +3,7 @@ import { setUrl, setScreenshotSrc } from "#/state/browserSlice";
 import store from "#/store";
 import { ObservationMessage } from "#/types/Message";
 import { appendOutput } from "#/state/commandSlice";
+import { appendJupyterOutput } from "#/state/jupyterSlice";
 import ObservationType from "#/types/ObservationType";
 
 export function handleObservationMessage(message: ObservationMessage) {
@@ -12,7 +13,7 @@ export function handleObservationMessage(message: ObservationMessage) {
       break;
     case ObservationType.RUN_IPYTHON:
       // FIXME: render this as markdown
-      store.dispatch(appendOutput(message.content));
+      store.dispatch(appendJupyterOutput(message.content));
       break;
     case ObservationType.BROWSE:
       if (message.extras?.screenshot) {

frontend/src/state/jupyterSlice.ts

@@ -0,0 +1,27 @@
+import { createSlice } from "@reduxjs/toolkit";
+
+export type Cell = {
+  content: string;
+  type: "input" | "output";
+};
+
+const initialCells: Cell[] = [];
+
+export const cellSlice = createSlice({
+  name: "cell",
+  initialState: {
+    cells: initialCells,
+  },
+  reducers: {
+    appendJupyterInput: (state, action) => {
+      state.cells.push({ content: action.payload, type: "input" });
+    },
+    appendJupyterOutput: (state, action) => {
+      state.cells.push({ content: action.payload, type: "output" });
+    },
+  },
+});
+
+export const { appendJupyterInput, appendJupyterOutput } = cellSlice.actions;
+
+export default cellSlice.reducer;

frontend/src/store.ts

@@ -7,6 +7,7 @@ import commandReducer from "./state/commandSlice";
 import errorsReducer from "./state/errorsSlice";
 import planReducer from "./state/planSlice";
 import taskReducer from "./state/taskSlice";
+import jupyterReducer from "./state/jupyterSlice";
 
 export const rootReducer = combineReducers({
   browser: browserReducer,
@@ -17,6 +18,7 @@ export const rootReducer = combineReducers({
   errors: errorsReducer,
   plan: planReducer,
   agent: agentReducer,
+  jupyter: jupyterReducer,
 });
 
 const store = configureStore({

frontend/src/types/TabOption.tsx

@@ -2,10 +2,20 @@ enum TabOption {
   PLANNER = "planner",
   CODE = "code",
   BROWSER = "browser",
+  JUPYTER = "jupyter",
 }
 
-type TabType = TabOption.PLANNER | TabOption.CODE | TabOption.BROWSER;
+type TabType =
+  | TabOption.PLANNER
+  | TabOption.CODE
+  | TabOption.BROWSER
+  | TabOption.JUPYTER;
 
-const AllTabs = [TabOption.CODE, TabOption.BROWSER, TabOption.PLANNER];
+const AllTabs = [
+  TabOption.CODE,
+  TabOption.BROWSER,
+  TabOption.PLANNER,
+  TabOption.JUPYTER,
+];
 
 export { AllTabs, TabOption, type TabType };

opendevin/sandbox/plugins/jupyter/execute_server

@@ -187,8 +187,7 @@ class JupyterKernel:
                     outputs.append(msg['content']['data']['text/plain'])
                     if 'image/png' in msg['content']['data']:
                         # use markdone to display image (in case of large image)
-                        # outputs.append(f"\n<img src=\'data:image/png;base64,{msg['content']['data']['image/png']}\'/>\n")
-                        outputs.append(f"![image](data:image/png;base64,{msg['content']['data']['image/png']})")
+                        outputs.append(f"\n![image](data:image/png;base64,{msg['content']['data']['image/png']})\n")
 
                 elif msg_type == 'execute_reply':
                     execution_done = True