Authentication exception (401) when connecting to elasticsearch: 'unable to authenticate user [elastic] for REST request [/]'

[d3vr10]

PGSync version: 3.1.0

Postgres version:

Elasticsearch/OpenSearch version: elastic search 8.11.1

Redis version: 7.2.3

Python version: 3.11.7

Problem Description:
I'm having trouble with elastic search authentication though I'm setting the same password and SSL certificates, the three of them: client, CA and key certificates. If I turn off ssl and https as schema everything works fine, but I need to use secure connection. Here below is a reproduceable example. I can login to elasticsearch through kibana and curl just fine (I haven't tested yet with curl supplying the ca and client certificates).

Any feedback on this would be appreciated.

Docker YAML File:

version: '3.9'

services:
  setup:
    networks:
      - elastic
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: '0'
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ['CMD-SHELL', '[ -f config/certs/es01/es01.crt ]']
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
        
    networks:
      - elastic
      - frontend
      - data-center
      
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          'CMD-SHELL',
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
        
    networks:
      - elastic
      
    image: kibana:${STACK_VERSION}
    
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
      
    ports:
      - ${KIBANA_PORT}:5601
      
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          'CMD-SHELL',
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  postgres:
    ports:
      - 5432:5432
    image: postgres:16.1
    volumes:
      - "./init_db:/docker-entrypoint-initdb.d"
      # - "../postgres/data:/var/lib/postgresql/data"
    environment:
      - "POSTGRES_USER=revista"
      - "POSTGRES_PASSWORD=${ELASTIC_PASSWORD}"
    restart: always

    networks:
      - data-center
    
    command: 
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "max_replication_slots=3"
    
  redis:
    image: redis:7.2.3
    ports:
      - 6379:6379
    command: redis-server --requirepass ${ELASTIC_PASSWORD}
    networks:
      - data-center
    
  pgsync:
    build:
      context: ../pgsync
      dockerfile: ./Dockerfile-pgsync

    restart: on-failure
    
    labels:
      org.label-schema.name: "pgsync"
      org.label-schema.description: "Postgres to Elasticsearch sync"
      com.label-schema.service-type: "daemon"
      
    sysctls:
      - net.ipv4.tcp_keepalive_time=200
      - net.ipv4.tcp_keepalive_intvl=200
      - net.ipv4.tcp_keepalive_probes=5
      
    depends_on:
      - postgres
      - redis
      
    environment:
      - CHECKPOINT_PATH=/pgsync
      - PG_USER=revista
      - PG_HOST=postgres
      - PG_PORT=5432
      - PG_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTICSEARCH_SCHEME=https
      - ELASTICSEARCH_HOST=es01
      - ELASTICSEARCH_PORT=9200
      - ELASTICSEARCH_USER=elastic
      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTICSEARCH_VERIFY_CERTS=True
      - ELASTICSEARCH_USE_SSL=True
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - REDIS_AUTH=${ELASTIC_PASSWORD}
      - ELASTICSEARCH=True
      - OPENSEARCH=False
      - SCHEMA=/pgsync/schema.json
      - LOG_LEVEL=INFO
      - ELASTICSEARCH_CA_CERTS=/usr/share/elasticsearch/config/certs/ca/ca.crt
      - ELASTICSEARCH_CLIENT_CERT=/usr/share/elasticsearch/config/certs/es01/es01.crt
      - ELASTICSEARCH_CLIENT_KEY=/usr/share/elasticsearch/config/certs/es01/es01.key


    command: ./runserver.sh && sleep inf

    volumes:
      # - ../pgsync/example-schema.json:/pgsync/schema.json
      - ../pgsync/schema.json:/pgsync/schema.json
      - certs:/usr/share/elasticsearch/config/certs

    networks:
      - data-center

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local

networks:
  networks:
  data-center:
  frontend:
  elastic:

Error Message (if any):

pgsync-1  | Traceback (most recent call last):
pgsync-1  |   File "/usr/local/bin/pgsync", line 7, in <module>
pgsync-1  |  0:00:00.202205 (0.20 sec)
pgsync-1  |     sync.main()
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1157, in __call__
pgsync-1  |     return self.main(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1078, in main
pgsync-1  |     rv = self.invoke(ctx)
pgsync-1  |          ^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1434, in invoke
pgsync-1  |     return ctx.invoke(self.callback, **ctx.params)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 783, in invoke
pgsync-1  |     return __callback(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 1480, in main
pgsync-1  |     sync: Sync = Sync(
pgsync-1  |                  ^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/singleton.py", line 36, in __call__
pgsync-1  |     cls._instances[key] = super(Singleton, cls).__call__(
pgsync-1  |                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 88, in __init__
pgsync-1  |     self.search_client: SearchClient = SearchClient()
pgsync-1  |                                        ^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/search_client.py", line 48, in __init__
pgsync-1  |     self.__client.info()["version"]["number"].split(".")[0]
pgsync-1  |     ^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/utils.py", line 402, in wrapped
pgsync-1  |     return api(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/__init__.py", line 2278, in info
pgsync-1  |     return self.perform_request(  # type: ignore[return-value]
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/_base.py", line 320, in perform_request
pgsync-1  |     raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
pgsync-1  | elasticsearch.AuthenticationException: AuthenticationException(401, 'security_exception', 'unable to authenticate user [elastic] for REST request [/]')