Merge remote-tracking branch 'upstream/unstable' into replica-redirect

.github/actions/generate-package-build-matrix/action.yml

@@ -0,0 +1,44 @@
+name: Generate target matrix.
+description: Matrix creation for building Valkey for different architectures and platforms.
+
+inputs:
+  ref:
+    description: The commit, tag or branch of Valkey to checkout to determine what version to use.
+    required: true
+outputs:
+  x86_64-build-matrix:
+    description: The x86_64 build matrix.
+    value: ${{ steps.set-matrix.outputs.x86matrix }}
+  arm64-build-matrix:
+    description: The arm64 build matrix.
+    value: ${{ steps.set-matrix.outputs.armmatrix }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout code for version check
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        path: version-check
+
+    - name: Get targets
+      run: |
+        x86_arch=$(jq -c '[.linux_targets[] | select(.arch=="x86_64")]' utils/releasetools/build-config.json)
+        x86_matrix=$(echo "{ \"distro\" : $x86_arch }" | jq -c .)
+        echo "X86_MATRIX=$x86_matrix" >> $GITHUB_ENV
+
+        arm_arch=$(jq -c '[.linux_targets[] | select(.arch=="arm64")]' utils/releasetools/build-config.json)
+        arm_matrix=$(echo "{ \"distro\" : $arm_arch }" | jq -c .)
+        echo "ARM_MATRIX=$arm_matrix" >> $GITHUB_ENV
+      shell: bash
+
+    - id: set-matrix
+      run: |
+        echo $X86_MATRIX
+        echo $X86_MATRIX| jq .
+        echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT
+        echo $ARM_MATRIX
+        echo $ARM_MATRIX| jq .
+        echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT
+      shell: bash

.github/workflows/build-release-packages.yml

@@ -0,0 +1,89 @@
+name: Build Release Packages
+
+on:
+  release:
+    types: [published]
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version of Valkey to build
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  # This job provides the version metadata from the tag for the other jobs to use.
+  release-build-get-meta:
+    name: Get metadata to build
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
+    steps:
+
+      - run: |
+          echo "Version: ${{ inputs.version || github.ref_name }}"
+        shell: bash
+
+      # This step is to consolidate the three different triggers into a single "version"
+      # 1. If manual dispatch - use the version provided.
+      # 3. If tag trigger, use that tag.
+      - name: Get the version
+        id: get_version
+        run: |
+          VERSION="${INPUT_VERSION}"
+          if [ -z "${VERSION}" ]; then
+            exit 1
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+        shell: bash
+        env:
+          # Use the dispatch variable in preference, if empty use the context ref_name which should
+          # only ever be a tag
+          INPUT_VERSION: ${{ inputs.version || github.ref_name }}
+
+  generate-build-matrix:
+      name: Generating build matrix
+      runs-on: ubuntu-latest
+      outputs:
+        x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }}
+        arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }}
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+        # Set up the list of target to build so we can pass the JSON to the reusable job
+        - uses: ./.github/actions/generate-package-build-matrix
+          id: set-matrix
+          with:
+            ref: ${{ inputs.version || github.ref_name }}
+
+  release-build-linux-x86-packages:
+    needs:
+      - release-build-get-meta
+      - generate-build-matrix
+    uses: ./.github/workflows/call-build-linux-x86-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+
+  release-build-linux-arm-packages:
+    needs:
+      - release-build-get-meta
+      - generate-build-matrix
+    uses: ./.github/workflows/call-build-linux-arm-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}

.github/workflows/call-build-linux-arm-packages.yml

@@ -0,0 +1,79 @@
+name: Builds Linux arm binary packages into S3 bucket.
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Valkey to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.version }}
+
+      - name: Make Valkey
+        uses: uraimo/run-on-arch-action@v2
+        with:
+          arch: aarch64
+          distro: ${{matrix.distro.target}}
+          install: apt-get update && apt-get install -y build-essential libssl-dev
+          run: make -C src all BUILD_TLS=yes
+
+      - name: Create Tarball and SHA256sums
+        run: |
+          TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
+          mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
+          cp -rfv src/valkey-* $TAR_FILE_NAME/bin
+          cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
+          tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
+          sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
+          mkdir -p packages-files
+          cp -rfv $TAR_FILE_NAME.tar* packages-files/
+      
+      - name: Install AWS cli.
+        run: |
+          sudo apt-get install -y awscli
+
+      - name: Configure AWS credentials
+        run: |
+          aws configure set region us-west-2
+          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
+          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
+
+      - name: Sync to S3
+        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/

.github/workflows/call-build-linux-x86-packages.yml

@@ -0,0 +1,73 @@
+name: Builds Linux X86 binary packages into S3 bucket.
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Valkey to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.version }}
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev jq wget awscli
+
+      - name: Make Valkey
+        run: make -C src all BUILD_TLS=yes
+
+      - name: Create Tarball and SHA256sums
+        run: |
+          TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
+          mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
+          cp -rfv src/valkey-* $TAR_FILE_NAME/bin
+          cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
+          tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
+          sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
+          mkdir -p packages-files
+          cp -rfv $TAR_FILE_NAME.tar* packages-files/
+
+      - name: Configure AWS credentials
+        run: |
+          aws configure set region us-west-2
+          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
+          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
+
+      - name: Sync to S3
+        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/

.github/workflows/ci.yml

@@ -10,11 +10,11 @@ jobs:
   test-ubuntu-latest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: make
       # Fail build if there are warnings
       # build with TLS just for compilation coverage
-      run: make SERVER_CFLAGS='-Werror' BUILD_TLS=yes
+      run: make all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes
     - name: test
       run: |
         sudo apt-get install tcl8.6 tclx
@@ -27,11 +27,14 @@ jobs:
         make commands.def
         dirty=$(git diff)
         if [[ ! -z  $dirty ]]; then echo $dirty; exit 1; fi
+    - name: unit tests
+      run: |
+        ./src/valkey-unit-tests
 
   test-sanitizer-address:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: make
         # build with TLS module just for compilation coverage
         run: make SANITIZER=address SERVER_CFLAGS='-Werror -DDEBUG_ASSERTIONS' BUILD_TLS=module
@@ -49,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     container: debian:buster
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: make
       run: |
         apt-get update && apt-get install -y build-essential
@@ -58,14 +61,14 @@ jobs:
   build-macos-latest:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: make
       run: make SERVER_CFLAGS='-Werror'
 
   build-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: make
       run: |
         sudo apt-get update && sudo apt-get install libc6-dev-i386
@@ -74,7 +77,7 @@ jobs:
   build-libc-malloc:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: make
       run: make SERVER_CFLAGS='-Werror' MALLOC=libc
 
@@ -84,7 +87,7 @@ jobs:
     steps:
     # on centos7, actions/checkout@v4 does not work, so we use v3
     # ref. https://github.com/actions/checkout/issues/1487
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
     - name: make
       run: |

.github/workflows/codecov.yml

@@ -0,0 +1,24 @@
+name: "Codecov"
+
+# Enabling on each push is to display the coverage changes in every PR, 
+# where each PR needs to be compared against the coverage of the head commit
+on: [push, pull_request]
+
+jobs:
+  code-coverage:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Install lcov and run test
+      run: |
+        sudo apt-get install lcov
+        make lcov
+
+    - name: Upload code coverage
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: ./src/valkey.info