settings-local.yaml

# poetry install --extras "ui llms-llama-cpp vector-stores-qdrant embeddings-huggingface"
server:
  env_name: ${APP_ENV:local}

llm:
  mode: llamacpp
  # Should be matching the selected model
  max_new_tokens: 512
  context_window: 3900
  tokenizer: mistralai/Mistral-7B-Instruct-v0.2

llamacpp:
  prompt_style: "mistral"
  llm_hf_repo_id: TheBloke/Mistral-7B-Instruct-v0.2-GGUF
  llm_hf_model_file: mistral-7b-instruct-v0.2.Q4_K_M.gguf

embedding:
  mode: huggingface

huggingface:
  embedding_hf_model_name: BAAI/bge-small-en-v1.5

vectorstore:
  database: qdrant

qdrant:
  path: local_data/private_gpt/qdrant