Merge commit 'b15802c2328898bb5da63a156f7b56b134868d1d' into releases…

…/2.11.0

.buildkite/cicd.rayci.yml

@@ -3,12 +3,14 @@ depends_on:
   - forge
 steps:
   - label: "CI/CD: ray_ci tooling"
+    key: ray-ci-tooling-tests
     commands:
       # TODO(aslonnie): wrap this in a script, and upload test telemetry.
       - bazel test --test_tag_filters=ci_unit //ci/ray_ci/...
     instance_type: small
 
   - label: "CI/CD: release test infra"
+    key: ray-release-infra-tests
     commands:
       # TODO(aslonnie): wrap this in a script, and upload test telemetry.
       - bazel test --test_tag_filters=release_unit //release/...

.buildkite/release-automation/pre_release.rayci.yml

@@ -18,9 +18,23 @@ steps:
       - bash .buildkite/release-automation/check-commit-hash.sh
 
   - label: "Trigger Postmerge test"
+    if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") != "1"
     trigger: "postmerge"
     key: trigger-postmerge
     depends_on: check-commit-hash
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      message: "Triggered by release-automation build #${BUILDKITE_BUILD_NUMBER}"
+      env:
+        RAYCI_CONTINUOUS_BUILD: 1
+        RAYCI_WEEKLY_RELEASE: 1
+
+  - label: "Trigger Postmerge nightly build & test"
+    if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") == "1"
+    trigger: "postmerge"
+    key: trigger-postmerge-nightly
+    depends_on: check-commit-hash
     build:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
@@ -89,7 +103,8 @@ steps:
         RELEASE_FREQUENCY: "weekly"
 
   - label: "Check Ray commit in {{matrix}} nightly images"
-    depends_on: trigger-postmerge
+    if: build.branch !~ /^releases\// && build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") == "1"
+    depends_on: trigger-postmerge-nightly
     allow_dependency_failure: true
     job_env: forge
     commands:

.buildkite/release.rayci.yml

@@ -8,7 +8,8 @@ steps:
       - skip-on-premerge
       - oss
     commands:
-      - ./release/run_release_test.sh microbenchmark.aws --report
+      - bazel run //release:run -- microbenchmark.aws --report 
+        --test-definition-root /ray/release --test-collection-file release_tests.yaml
     instance_type: release
     job_env: oss-ci-base_build
     depends_on:

bazel/conftest.py

@@ -13,3 +13,12 @@ def shutdown_ray():
 def preserve_block_order():
     ray.data.context.DataContext.get_current().execution_options.preserve_order = True
     yield
+
+
+@pytest.fixture(autouse=True)
+def disable_start_message():
+    context = ray.data.context.DataContext.get_current()
+    original_value = context.print_on_execution_start
+    context.print_on_execution_start = False
+    yield
+    context.print_on_execution_start = original_value

ci/ray_ci/automation/test_db_bot.py

@@ -4,7 +4,7 @@
 
 from ci.ray_ci.utils import logger, ci_init
 from ci.ray_ci.tester_container import TesterContainer
-from ray_release.configs.global_config import BRANCH_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 @click.command()
@@ -17,7 +17,10 @@ def main(team: str, bazel_log_dir: str) -> None:
         logger.info("Skip upload test results. We only upload on master branch.")
         return
 
-    if os.environ.get("BUILDKITE_PIPELINE_ID") not in BRANCH_PIPELINES:
+    if (
+        os.environ.get("BUILDKITE_PIPELINE_ID")
+        not in get_global_config()["ci_pipeline_postmerge"]
+    ):
         logger.info("Skip upload test results. We only upload on postmerge pipeline.")
         return
 

ci/ray_ci/builder.py

@@ -14,7 +14,7 @@
 from ci.ray_ci.ray_docker_container import RayDockerContainer
 from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer
 from ci.ray_ci.container import _DOCKER_ECR_REPO
-from ci.ray_ci.utils import logger, docker_login
+from ci.ray_ci.utils import logger, docker_login, ci_init
 
 
 @click.command()
@@ -85,6 +85,7 @@ def main(
     Build a wheel or jar artifact
     """
     docker_login(_DOCKER_ECR_REPO.split("/")[0])
+    ci_init()
     if artifact_type == "wheel":
         logger.info(f"Building wheel for {python_version}")
         build_wheel(python_version, build_type, architecture, operating_system, upload)

ci/ray_ci/oss_config.yaml

@@ -6,6 +6,13 @@ release_byod:
   aws_cr: 029272617770.dkr.ecr.us-west-2.amazonaws.com
   gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci
   aws2gce_credentials: release/aws2gce_iam.json
+ci_pipeline:
+  premerge:
+    - 0189942e-0876-4b8f-80a4-617f988ec59b  # premerge
+  postmerge:
+    - 0189e759-8c96-4302-b6b5-b4274406bf89  # postmerge
+    - 018e0f94-ccb6-45c2-b072-1e624fe9a404  # postmerge-macos
+    - 018af6d3-58e1-463f-90ec-d9aa4a4f57f1  # release
 state_machine:
   pr:
     aws_bucket: ray-ci-pr-results

ci/ray_ci/pipeline/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_library(
+    name = "pipeline",
+    srcs = glob(
+        ["*.py"],
+        exclude = [
+            "test_*.py",
+        ],
+    ),
+    visibility = ["//ci/ray_ci/pipeline:__subpackages__"],
+    deps = [
+        "//ci/ray_ci:ray_ci_lib",
+    ],
+)
+
+py_test(
+    name = "test_gap_filling_scheduler",
+    size = "small",
+    srcs = ["test_gap_filling_scheduler.py"],
+    exec_compatible_with = ["//:hermetic_python"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ":pipeline",
+        ci_require("pytest"),
+    ],
+)

ci/ray_ci/pipeline/gap_filling_scheduler.py

@@ -0,0 +1,67 @@
+import subprocess
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional, Any
+
+from pybuildkite.buildkite import Buildkite
+
+
+BRANCH = "master"
+
+
+class GapFillingScheduler:
+    """
+    This buildkite pipeline scheduler is responsible for scheduling gap filling builds
+    when the latest build is failing.
+    """
+
+    def __init__(
+        self,
+        buildkite_organization: str,
+        buildkite_pipeline: str,
+        buildkite_access_token: str,
+        repo_checkout: str,
+        days_ago: int = 1,
+    ):
+        self.buildkite_organization = buildkite_organization
+        self.buildkite_pipeline = buildkite_pipeline
+        self.buildkite = Buildkite()
+        self.buildkite.set_access_token(buildkite_access_token)
+        self.repo_checkout = repo_checkout
+        self.days_ago = days_ago
+
+    def _get_latest_commit_for_build_state(self, build_state: str) -> Optional[str]:
+        latest_commits = self._get_latest_commits()
+        commit_to_index = {commit: index for index, commit in enumerate(latest_commits)}
+        builds = []
+        for build in self._get_builds():
+            if build["state"] == build_state and build["commit"] in latest_commits:
+                builds.append(build)
+        if not builds:
+            return None
+
+        builds = sorted(builds, key=lambda build: commit_to_index[build["commit"]])
+        return builds[0]["commit"]
+
+    def _get_latest_commits(self) -> List[str]:
+        return (
+            subprocess.check_output(
+                [
+                    "git",
+                    "log",
+                    "--pretty=tformat:'%H'",
+                    f"--since={self.days_ago}.days",
+                ],
+                cwd=self.repo_checkout,
+            )
+            .decode("utf-8")
+            .strip()
+            .split("\n")
+        )
+
+    def _get_builds(self) -> List[Dict[str, Any]]:
+        return self.buildkite.builds().list_all_for_pipeline(
+            self.buildkite_organization,
+            self.buildkite_pipeline,
+            created_from=datetime.now() - timedelta(days=self.days_ago),
+            branch=BRANCH,
+        )

ci/ray_ci/pipeline/test_gap_filling_scheduler.py

@@ -0,0 +1,41 @@
+import sys
+from unittest import mock
+
+import pytest
+
+from ci.ray_ci.pipeline.gap_filling_scheduler import GapFillingScheduler
+
+
+@mock.patch(
+    "ci.ray_ci.pipeline.gap_filling_scheduler.GapFillingScheduler._get_latest_commits"
+)
+@mock.patch("ci.ray_ci.pipeline.gap_filling_scheduler.GapFillingScheduler._get_builds")
+def test_get_latest_commit_for_build_state(mock_get_builds, mock_get_latest_commits):
+    mock_get_builds.return_value = [
+        {
+            "state": "failed",
+            "commit": "000",
+        },
+        {
+            "state": "passed",
+            "commit": "111",
+        },
+        {
+            "state": "failed",
+            "commit": "222",
+        },
+    ]
+    mock_get_latest_commits.return_value = [
+        "222",
+        "111",
+        "000",
+        "/ray",
+    ]
+    scheduler = GapFillingScheduler("org", "pipeline", "token", "/ray")
+    assert scheduler._get_latest_commit_for_build_state("failed") == "222"
+    assert scheduler._get_latest_commit_for_build_state("passed") == "111"
+    assert scheduler._get_latest_commit_for_build_state("something") is None
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

ci/ray_ci/ray_docker_container.py

@@ -5,7 +5,7 @@
 from ci.ray_ci.docker_container import DockerContainer
 from ci.ray_ci.builder_container import PYTHON_VERSIONS, DEFAULT_ARCHITECTURE
 from ci.ray_ci.utils import docker_pull, RAY_VERSION
-from ray_release.configs.global_config import BRANCH_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 class RayDockerContainer(DockerContainer):
@@ -59,7 +59,10 @@ def run(self) -> None:
     def _should_upload(self) -> bool:
         if not self.upload:
             return False
-        if os.environ.get("BUILDKITE_PIPELINE_ID") not in BRANCH_PIPELINES:
+        if (
+            os.environ.get("BUILDKITE_PIPELINE_ID")
+            not in get_global_config()["ci_pipeline_postmerge"]
+        ):
             return False
         if os.environ.get("BUILDKITE_BRANCH", "").startswith("releases/"):
             return True

ci/ray_ci/test_base.py

@@ -4,10 +4,12 @@
 
 from ci.ray_ci.builder_container import PYTHON_VERSIONS
 from ci.ray_ci.builder import DEFAULT_PYTHON_VERSION
+from ci.ray_ci.utils import ci_init
 
 
 class RayCITestBase(unittest.TestCase):
     def setUp(self) -> None:
+        ci_init()
         self.patcher = patch.dict(
             os.environ,
             {

ci/ray_ci/test_linux_tester_container.py

@@ -8,9 +8,12 @@
 from typing import List, Optional
 
 from ci.ray_ci.linux_tester_container import LinuxTesterContainer
-from ci.ray_ci.utils import chunk_into_n
+from ci.ray_ci.utils import chunk_into_n, ci_init
 from ci.ray_ci.container import _DOCKER_ECR_REPO, _RAYCI_BUILD_ID
-from ray_release.configs.global_config import BRANCH_PIPELINES, PR_PIPELINES
+from ray_release.configs.global_config import get_global_config
+
+
+ci_init()
 
 
 class MockPopen:
@@ -47,7 +50,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "non-master",
-            "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")
@@ -57,7 +60,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "non-master",
-            "BUILDKITE_PIPELINE_ID": PR_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_premerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")
@@ -67,7 +70,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "master",
-            "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")

ci/ray_ci/test_ray_docker_container.py

@@ -10,7 +10,7 @@
 from ci.ray_ci.ray_docker_container import RayDockerContainer
 from ci.ray_ci.test_base import RayCITestBase
 from ci.ray_ci.utils import RAY_VERSION
-from ray_release.configs.global_config import BRANCH_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 class TestRayDockerContainer(RayCITestBase):
@@ -320,14 +320,18 @@ def test_should_upload(self) -> None:
             # environment_variables, expected_result (with upload flag on)
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][0],
                     "BUILDKITE_BRANCH": "releases/1.0.0",
                 },
                 True,  # satisfy upload requirements
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][0],
                     "BUILDKITE_BRANCH": "master",
                     "RAYCI_SCHEDULE": "nightly",
                 },
@@ -343,14 +347,18 @@ def test_should_upload(self) -> None:
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][-1],
                     "BUILDKITE_BRANCH": "non-release/1.2.3",
                 },
                 False,  # not satisfied: branch is not release/master
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][-1],
                     "BUILDKITE_BRANCH": "123",
                     "RAYCI_SCHEDULE": "nightly",
                 },