help request: When I was testing with forward-auth.lua, I found that enabling keepalive would cause the authentication server to fail when parsing the HTTP protocol

[546454170]

Description

When I disabled keepalive, no issues were encountered. But after enabling keepalive, there were instances where HTTP 400 Bad Request errors were returned after several requests, with some requests in between remaining unaffected.

My authentication server is using Spring Boot, with the version being 2.6.13. tomcat-embed-core:9.0.68;

During internal parsing, Tomcat will throw an exception with the message: 'invalid character found in method name [xxxx]. HTTP method names must be tokens'. (The source code: org.apache.coyote.http11.Http11InputBuffer#419)

I am using the Docker Compose method for deployment, with the following configuration:

docker-compose.yml

version: "3"

services:
  apisix-dashboard:
    image: apache/apisix-dashboard:3.0.1-alpine
    restart: always
    volumes:
    - ./dashboard_conf/conf.yaml:/usr/local/apisix-dashboard/conf/conf.yaml
    ports:
    - "9000:9000"
    networks:
      apisix:

  apisix:
    image: apache/apisix:${APISIX_IMAGE_TAG:-3.9.0-debian}
    restart: always
    volumes:
      - ./apisix_conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro 
      - ./apisix_conf/debug.yaml:/usr/local/apisix/conf/debug.yaml:ro
      - ./apisix/:/usr/local/apisix/apisix/:rw
    depends_on:
      - etcd
    ##network_mode: host
    ports:
      - "9180:9180/tcp"
      - "9080:9080/tcp"
      - "9091:9091/tcp"
      - "9443:9443/tcp"
      - "9092:9092/tcp"
    networks:
      apisix:

  etcd:
    image: bitnami/etcd:3.5.11
    restart: always
    volumes:
      - etcd_data:/bitnami/etcd
    environment:
      ETCD_ENABLE_V2: "true"
      ALLOW_NONE_AUTHENTICATION: "yes"
      ETCD_ADVERTISE_CLIENT_URLS: "http://etcd:2379"
      ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
    ports:
      - "2379:2379/tcp"
    networks:
      apisix:

networks:
  apisix:
    driver: bridge

volumes:
  etcd_data:
    driver: local

apisix_conf/config.yaml

apisix:
  node_listen: 9080              # APISIX listening port
  enable_ipv6: false

  enable_control: true
  control:
    ip: "0.0.0.0"
    port: 9092

deployment:
  admin:
    allow_admin:               # https://nginx.org/en/docs/http/ngx_http_access_module.html#allow
      - 0.0.0.0/0              # We need to restrict ip access rules for security. 0.0.0.0/0 is for test.

    admin_key:
      - name: "admin"
        key: xxxxxx
        role: admin                 # admin: manage all configuration data

      - name: "viewer"
        key: xxxxxx
        role: viewer

  etcd:
    host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
      - "http://etcd:2379"          # multiple etcd address
    prefix: "/apisix"               # apisix configurations prefix
    timeout: 30                     # 30 seconds

nginx_config:
  error_log_level: "info"


plugin_attr:
  prometheus:
    export_addr:
      ip: "0.0.0.0"
      port: 9091

apisix_conf/debug.yaml

basic:
  enable: true
http_filter:
  enable: false         # enable or disable this feature
  enable_header_name: X-APISIX-Dynamic-Debug # the header name of dynamic enable
hook_conf:
  enable: false                 # enable or disable this feature
  name: hook_phase              # the name of module and function list
  log_level: warn               # log level
  is_print_input_args: true     # print the input arguments
  is_print_return_value: true   # print the return value

hook_phase:                     # module and function list, name: hook_phase
  apisix:                       # required module name
    - http_access_phase         # function name
    - http_header_filter_phase
    - http_body_filter_phase
    - http_log_phase

dashboard_conf/conf.yaml

conf:
  listen:
    host: 0.0.0.0     # `manager api` listening ip or host name
    port: 9000          # `manager api` listening port
  allow_list:           # If we don't set any IP list, then any IP access is allowed by default.
    - 0.0.0.0/0
  etcd:
    endpoints:          # supports defining multiple etcd host addresses for an etcd cluster
      - "http://etcd:2379"
                          # yamllint disable rule:comments-indentation
                          # etcd basic auth info
    # username: "root"    # ignore etcd username if not enable etcd auth
    # password: "123456"  # ignore etcd password if not enable etcd auth
    mtls:
      key_file: ""          # Path of your self-signed client side key
      cert_file: ""         # Path of your self-signed client side cert
      ca_file: ""           # Path of your self-signed ca cert, the CA is used to sign callers' certificates
    # prefix: /apisix     # apisix config's prefix in etcd, /apisix by default
  log:
    error_log:
      level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal
      file_path:
        logs/error.log  # supports relative path, absolute path, standard output
                        # such as: logs/error.log, /tmp/logs/error.log, /dev/stdout, /dev/stderr
    access_log:
      file_path:
        logs/access.log  # supports relative path, absolute path, standard output
                         # such as: logs/access.log, /tmp/logs/access.log, /dev/stdout, /dev/stderr
                         # log example: 2020-12-09T16:38:09.039+0800	INFO	filter/logging.go:46	/apisix/admin/routes/r1	{"status": 401, "host": "127.0.0.1:9000", "query": "asdfsafd=adf&a=a", "requestId": "3d50ecb8-758c-46d1-af5b-cd9d1c820156", "latency": 0, "remoteIP": "127.0.0.1", "method": "PUT", "errs": []}
  security:
      # access_control_allow_origin: "http://httpbin.org"
      # access_control_allow_credentials: true          # support using custom cors configration
      # access_control_allow_headers: "Authorization"
      # access_control-allow_methods: "*"
      # x_frame_options: "deny"
      content_security_policy: "default-src 'self'; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; frame-src *"  # You can set frame-src to provide content for your grafana panel.

authentication:
  secret:
    secret              # secret for jwt token generation.
                        # NOTE: Highly recommended to modify this value to protect `manager api`.
                        # if it's default value, when `manager api` start, it will generate a random string to replace it.
  expire_time: 3600     # jwt token expire time, in second
  users:                # yamllint enable rule:comments-indentation
    - username: admin   # username and password for login `manager api`
      password: admin
    - username: user
      password: user

plugins:                          # plugin list (sorted in alphabetical order)
  - api-breaker
  - authz-keycloak
  - basic-auth
  - forward-auth
  - batch-requests
  - consumer-restriction
  - cors
  # - dubbo-proxy
  - echo
  # - error-log-logger
  # - example-plugin
  - fault-injection
  - grpc-transcode
  - hmac-auth
  - http-logger
  - ip-restriction
  - jwt-auth
  - kafka-logger
  - key-auth
  - limit-conn
  - limit-count
  - limit-req
  # - log-rotate
  # - node-status
  - openid-connect
  - prometheus
  - proxy-cache
  - proxy-mirror
  - proxy-rewrite
  - redirect
  - referer-restriction
  - request-id
  - request-validation
  - response-rewrite
  - serverless-post-function
  - serverless-pre-function
  # - skywalking
  - sls-logger
  - syslog
  - tcp-logger
  - udp-logger
  - uri-blocker
  - wolf-rbac
  - zipkin
  - server-info
  - traffic-split

Apisix router

{
  "uri": "/manager/*",
  "name": "route_manager_test",
  "methods": [
    "GET",
    "POST",
    "PUT",
    "DELETE",
    "PATCH",
    "HEAD",
    "OPTIONS",
    "CONNECT",
    "TRACE",
    "PURGE"
  ],
  "plugins": {
    "forward-auth": {
      "_meta": {
        "disable": false,
        "priority": 101
      },
      "client_headers": [
        "x-api-requestId"
      ],
      "keepalive": true,
      "request_headers": [
        "x-api-requestId"
      ],
      "request_method": "GET",
      "ssl_verify": false,
      "timeout": 6000,
      "upstream_headers": [
        "x-api-requestId"
      ],
      "uri": "http://192.168.110.80:8080/manager/permission/preauthorize"
    }
  },
  "upstream_id": "512453398356820674",
  "status": 1
}

Apisix upstream

# upstream_id:  512453398356820674
{
  "nodes": [
    {
      "host": "192.168.110.80",
      "port": 8080,
      "weight": 1
    }
  ],
  "timeout": {
    "connect": 6,
    "send": 6,
    "read": 6
  },
  "type": "least_conn",
  "scheme": "http",
  "pass_host": "pass",
  "name": "upstream_manager_test",
  "keepalive_pool": {
    "idle_timeout": 60000,
    "requests": 1000,
    "size": 320
  }
}

request:

curl --location 'http://localhost:9080/manager/queryPage' \
--header 'x-api-requestId: 1715049834000' \
--header 'Content-Type: application/json' \
--data '{
    "pageSize": 10,
    "pageNum": 1
}'

Environment

• APISIX version (run apisix version): 3.9.0
• Operating system (run uname -a): Linux 2006dce01339 5.15.49-linuxkit-pr change: added doc of how to load plugin. #1 SMP Thu May 25 07:17:40 UTC 2023 x86_64 GNU/Linux
• OpenResty / Nginx version (run openresty -V or nginx -V): nginx version: openresty/1.25.3.1
• etcd version, if relevant (run curl http://127.0.0.1:9090/v1/server_info):
• APISIX Dashboard version, if relevant: 3.5.11
• Plugin runner version, for issues related to plugin runners:
• LuaRocks version, for installation issues (run luarocks --version):