Merge pull request #1881 from open-webui/dev

0.1.123

.env.example

@@ -10,7 +10,8 @@ OPENAI_API_KEY=''
 # DO NOT TRACK
 SCARF_NO_ANALYTICS=true
 DO_NOT_TRACK=true
+ANONYMIZED_TELEMETRY=false
 
 # Use locally bundled version of the LiteLLM cost map json
 # to avoid repetitive startup connections
-LITELLM_LOCAL_MODEL_COST_MAP="True"
+LITELLM_LOCAL_MODEL_COST_MAP="True"

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/backend"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"

.github/workflows/format-build-frontend.yaml

@@ -29,6 +29,9 @@ jobs:
       - name: Format Frontend
         run: npm run format
 
+      - name: Run i18next
+        run: npm run i18n:parse
+
       - name: Check for Changes After Format
         run: git diff --exit-code
 

.github/workflows/integration-test.yml

@@ -53,3 +53,134 @@ jobs:
           name: compose-logs
           path: compose-logs.txt
           if-no-files-found: ignore
+
+  migration_test:
+    name: Run Migration Tests
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+#      mysql:
+#        image: mysql
+#        env:
+#          MYSQL_ROOT_PASSWORD: mysql
+#          MYSQL_DATABASE: mysql
+#        options: >-
+#          --health-cmd "mysqladmin ping -h localhost"
+#          --health-interval 10s
+#          --health-timeout 5s
+#          --health-retries 5
+#        ports:
+#          - 3306:3306
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up uv
+        uses: yezz123/setup-uv@v4
+        with:
+          uv-venv: venv
+
+      - name: Activate virtualenv
+        run: |
+          . venv/bin/activate
+          echo PATH=$PATH >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          uv pip install -r backend/requirements.txt
+
+      - name: Test backend with SQLite
+        id: sqlite
+        env:
+          WEBUI_SECRET_KEY: secret-key
+          GLOBAL_LOG_LEVEL: debug
+        run: |
+          cd backend
+          uvicorn main:app --port "8080" --forwarded-allow-ips '*' &
+          UVICORN_PID=$!
+          # Wait up to 20 seconds for the server to start
+          for i in {1..20}; do
+              curl -s http://localhost:8080/api/config > /dev/null && break
+              sleep 1
+              if [ $i -eq 20 ]; then
+                  echo "Server failed to start"
+                  kill -9 $UVICORN_PID
+                  exit 1
+              fi
+          done
+          # Check that the server is still running after 5 seconds
+          sleep 5
+          if ! kill -0 $UVICORN_PID; then
+              echo "Server has stopped"
+              exit 1
+          fi
+          
+
+      - name: Test backend with Postgres
+        if: success() || steps.sqlite.conclusion == 'failure'
+        env:
+          WEBUI_SECRET_KEY: secret-key
+          GLOBAL_LOG_LEVEL: debug
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+        run: |
+          cd backend
+          uvicorn main:app --port "8081" --forwarded-allow-ips '*' &
+          UVICORN_PID=$!
+          # Wait up to 20 seconds for the server to start
+          for i in {1..20}; do
+              curl -s http://localhost:8081/api/config > /dev/null && break
+              sleep 1
+              if [ $i -eq 20 ]; then
+                  echo "Server failed to start"
+                  kill -9 $UVICORN_PID
+                  exit 1
+              fi
+          done
+          # Check that the server is still running after 5 seconds
+          sleep 5
+          if ! kill -0 $UVICORN_PID; then
+              echo "Server has stopped"
+              exit 1
+          fi
+
+#      - name: Test backend with MySQL
+#        if: success() || steps.sqlite.conclusion == 'failure' || steps.postgres.conclusion == 'failure'
+#        env:
+#          WEBUI_SECRET_KEY: secret-key
+#          GLOBAL_LOG_LEVEL: debug
+#          DATABASE_URL: mysql://root:mysql@localhost:3306/mysql
+#        run: |
+#          cd backend
+#          uvicorn main:app --port "8083" --forwarded-allow-ips '*' &
+#          UVICORN_PID=$!
+#          # Wait up to 20 seconds for the server to start
+#          for i in {1..20}; do
+#              curl -s http://localhost:8083/api/config > /dev/null && break
+#              sleep 1
+#              if [ $i -eq 20 ]; then
+#                  echo "Server failed to start"
+#                  kill -9 $UVICORN_PID
+#                  exit 1
+#              fi
+#          done
+#          # Check that the server is still running after 5 seconds
+#          sleep 5
+#          if ! kill -0 $UVICORN_PID; then
+#              echo "Server has stopped"
+#              exit 1
+#          fi

CHANGELOG.md

@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.123] - 2024-05-02
+
+### Added
+
+- **🎨 New Landing Page Design**: Refreshed design for a more modern look and optimized use of screen space.
+- **📹 Youtube RAG Pipeline**: Introduces dedicated RAG pipeline for Youtube videos, enabling interaction with video transcriptions directly.
+- **🔧 Enhanced Admin Panel**: Streamlined user management with options to add users directly or in bulk via CSV import.
+- **👥 '@' Model Integration**: Easily switch to specific models during conversations; old collaborative chat feature phased out.
+- **🌐 Language Enhancements**: Swedish translation added, plus improvements to German, Spanish, and the addition of Doge translation.
+
+### Fixed
+
+- **🗑️ Delete Chat Shortcut**: Addressed issue where shortcut wasn't functioning.
+- **🖼️ Modal Closing Bug**: Resolved unexpected closure of modal when dragging from within.
+- **✏️ Edit Button Styling**: Fixed styling inconsistency with edit buttons.
+- **🌐 Image Generation Compatibility Issue**: Rectified image generation compatibility issue with third-party APIs.
+- **📱 iOS PWA Icon Fix**: Corrected iOS PWA home screen icon shape.
+- **🔍 Scroll Gesture Bug**: Adjusted gesture sensitivity to prevent accidental activation when scrolling through code on mobile; now requires scrolling from the leftmost side to open the sidebar.
+
+### Changed
+
+- **🔄 Unlimited Context Length**: Advanced settings now allow unlimited max context length (previously limited to 16000).
+- **👑 Super Admin Assignment**: The first signup is automatically assigned a super admin role, unchangeable by other admins.
+- **🛡️ Admin User Restrictions**: User action buttons from the admin panel are now disabled for users with admin roles.
+- **🔝 Default Model Selector**: Set as default model option now exclusively available on the landing page.
+
 ## [0.1.122] - 2024-04-27
 
 ### Added

Dockerfile

@@ -51,7 +51,8 @@ ENV OLLAMA_BASE_URL="/ollama" \
 ENV OPENAI_API_KEY="" \
     WEBUI_SECRET_KEY="" \
     SCARF_NO_ANALYTICS=true \
-    DO_NOT_TRACK=true
+    DO_NOT_TRACK=true \
+    ANONYMIZED_TELEMETRY=false
 
 # Use locally bundled version of the LiteLLM cost map json
 # to avoid repetitive startup connections
@@ -74,6 +75,10 @@ ENV HF_HOME="/app/backend/data/cache/embedding/models"
 
 WORKDIR /app/backend
 
+ENV HOME /root
+RUN mkdir -p $HOME/.cache/chroma
+RUN echo -n 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry_user_id
+
 RUN if [ "$USE_OLLAMA" = "true" ]; then \
         apt-get update && \
         # Install pandoc and netcat
@@ -129,4 +134,4 @@ COPY ./backend .
 
 EXPOSE 8080
 
-CMD [ "bash", "start.sh"]
+CMD [ "bash", "start.sh"]

backend/apps/images/main.py

@@ -24,6 +24,7 @@
 from typing import Optional
 from pydantic import BaseModel
 from pathlib import Path
+import mimetypes
 import uuid
 import base64
 import json
@@ -315,38 +316,50 @@ class GenerateImageForm(BaseModel):
 
 
 def save_b64_image(b64_str):
-    image_id = str(uuid.uuid4())
-    file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.png")
-
     try:
-        # Split the base64 string to get the actual image data
-        img_data = base64.b64decode(b64_str)
+        header, encoded = b64_str.split(",", 1)
+        mime_type = header.split(";")[0]
+
+        img_data = base64.b64decode(encoded)
 
-        # Write the image data to a file
+        image_id = str(uuid.uuid4())
+        image_format = mimetypes.guess_extension(mime_type)
+
+        image_filename = f"{image_id}{image_format}"
+        file_path = IMAGE_CACHE_DIR / f"{image_filename}"
         with open(file_path, "wb") as f:
             f.write(img_data)
-
-        return image_id
+        return image_filename
     except Exception as e:
-        log.error(f"Error saving image: {e}")
+        log.exception(f"Error saving image: {e}")
         return None
 
 
 def save_url_image(url):
     image_id = str(uuid.uuid4())
-    file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.png")
-
     try:
         r = requests.get(url)
         r.raise_for_status()
+        if r.headers["content-type"].split("/")[0] == "image":
+
+            mime_type = r.headers["content-type"]
+            image_format = mimetypes.guess_extension(mime_type)
+
+            if not image_format:
+                raise ValueError("Could not determine image type from MIME type")
 
-        with open(file_path, "wb") as image_file:
-            image_file.write(r.content)
+            file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}{image_format}")
+            with open(file_path, "wb") as image_file:
+                for chunk in r.iter_content(chunk_size=8192):
+                    image_file.write(chunk)
+            return image_id, image_format
+        else:
+            log.error(f"Url does not point to an image.")
+            return None, None
 
-        return image_id
     except Exception as e:
         log.exception(f"Error saving image: {e}")
-        return None
+        return None, None
 
 
 @app.post("/generations")
@@ -385,8 +398,8 @@ def generate_image(
             images = []
 
             for image in res["data"]:
-                image_id = save_b64_image(image["b64_json"])
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_filename = save_b64_image(image["b64_json"])
+                images.append({"url": f"/cache/image/generations/{image_filename}"})
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:
@@ -422,8 +435,10 @@ def generate_image(
             images = []
 
             for image in res["data"]:
-                image_id = save_url_image(image["url"])
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_id, image_format = save_url_image(image["url"])
+                images.append(
+                    {"url": f"/cache/image/generations/{image_id}{image_format}"}
+                )
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:
@@ -460,8 +475,8 @@ def generate_image(
             images = []
 
             for image in res["images"]:
-                image_id = save_b64_image(image)
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_filename = save_b64_image(image)
+                images.append({"url": f"/cache/image/generations/{image_filename}"})
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:

backend/apps/openai/main.py

@@ -171,6 +171,7 @@ async def fetch_url(url, key):
 
 
 def merge_models_lists(model_lists):
+    log.info(f"merge_models_lists {model_lists}")
     merged_list = []
 
     for idx, models in enumerate(model_lists):
@@ -199,14 +200,16 @@ async def get_all_models():
         ]
 
         responses = await asyncio.gather(*tasks)
+        log.info(f"get_all_models:responses() {responses}")
+
         models = {
             "data": merge_models_lists(
                 list(
                     map(
                         lambda response: (
                             response["data"]
-                            if response and "data" in response
-                            else None
+                            if (response and "data" in response)
+                            else (response if isinstance(response, list) else None)
                         ),
                         responses,
                     )