[Bug] Error when evaluate using LightLLM api

[andakai]

先决条件

• 我已经搜索过 问题 和 讨论 但未得到预期的帮助。
• 错误在 最新版本 中尚未被修复。

问题类型

我正在使用官方支持的任务/模型/数据集进行评估。

环境

{'CUDA available': True,
 'CUDA_HOME': '/usr/local/cuda-12.3',
 'GCC': 'gcc (Debian 12.2.0-14) 12.2.0',
 'GPU 0,1': 'NVIDIA A100-PCIE-40GB',
 'MMEngine': '0.10.3',
 'MUSA available': False,
 'NVCC': 'Cuda compilation tools, release 12.3, V12.3.107',
 'OpenCV': '4.9.0',
 'PyTorch': '2.2.1+cu121',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 9.3\n'
                              '  - C++ Version: 201703\n'
                              '  - Intel(R) oneAPI Math Kernel Library Version '
                              '2022.2-Product Build 20220804 for Intel(R) 64 '
                              'architecture applications\n'
                              '  - Intel(R) MKL-DNN v3.3.2 (Git Hash '
                              '2dc95a2ad0841e29db8b22fbccaf3e5da7992b01)\n'
                              '  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: AVX2\n'
                              '  - CUDA Runtime 12.1\n'
                              '  - NVCC architecture flags: '
                              '-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_90,code=sm_90\n'
                              '  - CuDNN 8.9.2\n'
                              '  - Magma 2.6.1\n'
                              '  - Build settings: BLAS_INFO=mkl, '
                              'BUILD_TYPE=Release, CUDA_VERSION=12.1, '
                              'CUDNN_VERSION=8.9.2, '
                              'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, '
                              'CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 '
                              '-fabi-version=11 -fvisibility-inlines-hidden '
                              '-DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO '
                              '-DLIBKINETO_NOROCTRACER -DUSE_FBGEMM '
                              '-DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK '
                              '-DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE '
                              '-O2 -fPIC -Wall -Wextra -Werror=return-type '
                              '-Werror=non-virtual-dtor -Werror=bool-operation '
                              '-Wnarrowing -Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wno-unused-parameter '
                              '-Wno-unused-function -Wno-unused-result '
                              '-Wno-strict-overflow -Wno-strict-aliasing '
                              '-Wno-stringop-overflow -Wsuggest-override '
                              '-Wno-psabi -Wno-error=pedantic '
                              '-Wno-error=old-style-cast -Wno-missing-braces '
                              '-fdiagnostics-color=always -faligned-new '
                              '-Wno-unused-but-set-variable '
                              '-Wno-maybe-uninitialized -fno-math-errno '
                              '-fno-trapping-math -Werror=format '
                              '-Wno-stringop-overflow, LAPACK_INFO=mkl, '
                              'PERF_WITH_AVX=1, PERF_WITH_AVX2=1, '
                              'PERF_WITH_AVX512=1, TORCH_VERSION=2.2.1, '
                              'USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, '
                              'USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, '
                              'USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, '
                              'USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, '
                              'USE_ROCM_KERNEL_ASSERT=OFF, \n',
 'Python': '3.10.14 (main, Mar 21 2024, 16:24:04) [GCC 11.2.0]',
 'TorchVision': '0.17.1+cu121',
 'numpy_random_seed': 2147483648,
 'opencompass': '0.2.3+',
 'sys.platform': 'linux'}

重现问题 - 代码/配置示例

from mmengine.config import read_base
from opencompass.models import LightllmAPI
from opencompass.partitioners import NaivePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask

with read_base():
    from .summarizers.leaderboard import summarizer
    # from .datasets.humaneval.humaneval_gen import humaneval_datasets
    from .datasets.mmlu.mmlu_gen import mmlu_datasets

datasets = [*mmlu_datasets]

'''
# Prompt template for InternLM2-Chat
# https://github.com/InternLM/InternLM/blob/main/chat/chat_format.md

_meta_template = dict(
    begin='<|im_start|>system\nYou are InternLM2-Chat, a harmless AI assistant<|im_end|>\n',
    round=[
        dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
        dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
    ]
)
'''

_meta_template = None

models = [
    dict(
        abbr='LightllmAPI',
        type=LightllmAPI,
        url='http://localhost:8080/generate',
        meta_template=_meta_template,
        batch_size=128,
        rate_per_worker=64,
        retry=4,
        generation_kwargs=dict(
            do_sample=False,
            ignore_eos=False,
            max_new_tokens=2
        ),
    ),
]

infer = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=LocalRunner,
        max_num_workers=64,
        task=dict(type=OpenICLInferTask),
    ),
)

重现问题 - 命令或脚本

python run.py configs/eval_lightllm.py

重现问题 - 错误信息

In terminal:

03/28 05:32:53 - OpenCompass - ERROR - /home/adk/opencompass/opencompass/runners/local.py - _launch - 192 - task OpenICLInfer[LightllmAPI/lukaemon_mmlu_high_school_european_history] fail, see
./outputs/default/20240328_052846/logs/infer/LightllmAPI/lukaemon_mmlu_high_school_european_history.out
  2%|██▏                                                                                                                       | 1/57 [04:06<3:49:39, 246.06s/it]

In the log file:

03/28 05:19:05 - OpenCompass - INFO - Task [LightllmAPI/lukaemon_mmlu_high_school_european_history]
03/28 05:19:08 - OpenCompass - INFO - Start inferencing [LightllmAPI/lukaemon_mmlu_high_school_european_history]

  0%|          | 0/165 [00:00<?, ?it/s]
100%|██████████| 165/165 [00:00<00:00, 1855389.17it/s]
[2024-03-28 05:19:09,235] [opencompass.openicl.icl_inferencer.icl_gen_inferencer] [INFO] Starting inference process...

  0%|          | 0/2 [00:00<?, ?it/s]--- Logging error ---
Traceback (most recent call last):
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/site-packages/requests/models.py", line 971, in json
    return complexjson.loads(self.text, **kwargs)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/adk/opencompass/opencompass/models/lightllm_api.py", line 76, in _generate
    response = raw_response.json()
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/site-packages/requests/models.py", line 975, in json
    raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 1100, in emit
    msg = self.format(record)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 943, in format
    return fmt.format(record)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/site-packages/mmengine/logging/logger.py", line 124, in format
    result = logging.Formatter.format(self, record)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 678, in format
    record.message = record.getMessage()
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 368, in getMessage
    msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/threading.py", line 973, in _bootstrap
    self._bootstrap_inner()
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/concurrent/futures/thread.py", line 83, in _worker
    work_item.run()
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/adk/opencompass/opencompass/models/lightllm_api.py", line 83, in _generate
    self.logger.error('JsonDecode error, got',
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 1506, in error
    self._log(ERROR, msg, args, **kwargs)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 1624, in _log
    self.handle(record)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/logging/__init__.py", line 1634, in handle
    self.callHandlers(record)
  File "/home/adk/miniconda3/envs/opencompass/lib/python3.10/site-packages/mmengine/logging/logger.py", line 306, in callHandlers
    handler.handle(record)
Message: 'JsonDecode error, got'
Arguments: ("b'Internal Server Error'",)

其他信息

No response

[xuanyuc]

一般来说是服务器的问题，你可以把request改成其它openai的接口调用就行