.env

MODEL_PATH = ""
# if MODEL_PATH is "", default llama.cpp/gptq models 
# will be downloaded to: ./models

# Example ggml path:
# MODEL_PATH = "./models/llama-2-7b-chat.ggmlv3.q4_0.bin"

# options: llama.cpp, gptq, transformers
BACKEND_TYPE = "llama.cpp"

# only for transformers bitsandbytes 8 bit
LOAD_IN_8BIT = False

MAX_MAX_NEW_TOKENS = 2048
DEFAULT_MAX_NEW_TOKENS = 1024
MAX_INPUT_TOKEN_LENGTH = 4000

DEFAULT_SYSTEM_PROMPT = ""