Merge branch '4.3.9'

PowerJob · Mar 2, 2024 · d441311 · d441311
2 parents 9c0731f + f4c0a03
commit d441311
Show file tree

Hide file tree

Showing 36 changed files with 416 additions and 351 deletions.
diff --git a/others/powerjob-mysql.sql b/others/powerjob-mysql.sql
diff --git a/others/sql/upgrade/v4.3.7-v4.3.8.sql b/others/sql/upgrade/v4.3.7-v4.3.8.sql
@@ -0,0 +1,6 @@
+-- Upgrade SQL FROM 4.3.7 to 4.3.8
+-- ----------------------------
+-- Table change for job_info
+-- ----------------------------
+alter table job_info add dispatch_strategy_config varchar(255) comment 'dispatch_strategy_config' default null;
+alter table job_info add advanced_runtime_config varchar(255) comment 'advanced_runtime_config' default null;
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>tech.powerjob</groupId>
     <artifactId>powerjob</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <packaging>pom</packaging>
     <name>powerjob</name>
     <url>http://www.powerjob.tech</url>

diff --git a/powerjob-client/pom.xml b/powerjob-client/pom.xml
@@ -5,18 +5,18 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-client</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <packaging>jar</packaging>
 
     <properties>
         <junit.version>5.9.1</junit.version>
         <fastjson.version>1.2.83</fastjson.version>
-        <powerjob.common.version>4.3.8</powerjob.common.version>
+        <powerjob.common.version>4.3.9</powerjob.common.version>
 
         <mvn.shade.plugin.version>3.2.4</mvn.shade.plugin.version>
     </properties>

diff --git a/powerjob-common/pom.xml b/powerjob-common/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-common</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <packaging>jar</packaging>
 
     <properties>

diff --git a/powerjob-official-processors/pom.xml b/powerjob-official-processors/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-official-processors</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <packaging>jar</packaging>
 
     <properties>
@@ -20,7 +20,7 @@
         <!-- 不会被打包的部分，scope 只能是 test 或 provide -->
         <junit.version>5.9.1</junit.version>
         <logback.version>1.2.13</logback.version>
-        <powerjob.worker.version>4.3.8</powerjob.worker.version>
+        <powerjob.worker.version>4.3.9</powerjob.worker.version>
         <h2.db.version>2.2.224</h2.db.version>
         <mysql.version>8.0.28</mysql.version>
         <spring.version>5.3.31</spring.version>

diff --git a/powerjob-remote/pom.xml b/powerjob-remote/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>

diff --git a/powerjob-remote/powerjob-remote-benchmark/pom.xml b/powerjob-remote/powerjob-remote-benchmark/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-remote</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -21,8 +21,8 @@
 
         <logback.version>1.2.13</logback.version>
         <springboot.version>2.7.18</springboot.version>
-        <powerjob-remote-impl-http.version>4.3.8</powerjob-remote-impl-http.version>
-        <powerjob-remote-impl-akka.version>4.3.8</powerjob-remote-impl-akka.version>
+        <powerjob-remote-impl-http.version>4.3.9</powerjob-remote-impl-http.version>
+        <powerjob-remote-impl-akka.version>4.3.9</powerjob-remote-impl-akka.version>
 
         <gatling.version>3.9.0</gatling.version>
         <gatling-maven-plugin.version>4.2.9</gatling-maven-plugin.version>

diff --git a/powerjob-remote/powerjob-remote-framework/pom.xml b/powerjob-remote/powerjob-remote-framework/pom.xml
@@ -5,19 +5,19 @@
     <parent>
         <artifactId>powerjob-remote</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <artifactId>powerjob-remote-framework</artifactId>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
-        <powerjob-common.version>4.3.8</powerjob-common.version>
+        <powerjob-common.version>4.3.9</powerjob-common.version>
         <reflections.version>0.10.2</reflections.version>
 
 

diff --git a/powerjob-remote/powerjob-remote-impl-akka/pom.xml b/powerjob-remote/powerjob-remote-impl-akka/pom.xml
@@ -5,19 +5,19 @@
     <parent>
         <artifactId>powerjob-remote</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>powerjob-remote-impl-akka</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
-        <powerjob-remote-framework.version>4.3.8</powerjob-remote-framework.version>
+        <powerjob-remote-framework.version>4.3.9</powerjob-remote-framework.version>
 
         <akka.version>2.6.13</akka.version>
     </properties>

diff --git a/powerjob-remote/powerjob-remote-impl-http/pom.xml b/powerjob-remote/powerjob-remote-impl-http/pom.xml
@@ -5,20 +5,20 @@
     <parent>
         <artifactId>powerjob-remote</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>powerjob-remote-impl-http</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
         <vertx.version>4.3.7</vertx.version>
-        <powerjob-remote-framework.version>4.3.8</powerjob-remote-framework.version>
+        <powerjob-remote-framework.version>4.3.9</powerjob-remote-framework.version>
     </properties>
 
     <dependencies>

diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-server</artifactId>
-    <version>4.3.8</version>
+    <version>4.3.9</version>
     <packaging>pom</packaging>
 
     <modules>
@@ -50,12 +50,12 @@
         <groovy.version>3.0.10</groovy.version>
         <cron-utils.version>9.2.1</cron-utils.version>
 
-        <powerjob-common.version>4.3.8</powerjob-common.version>
-        <powerjob-remote-impl-http.version>4.3.8</powerjob-remote-impl-http.version>
-        <powerjob-remote-impl-akka.version>4.3.8</powerjob-remote-impl-akka.version>
+        <powerjob-common.version>4.3.9</powerjob-common.version>
+        <powerjob-remote-impl-http.version>4.3.9</powerjob-remote-impl-http.version>
+        <powerjob-remote-impl-akka.version>4.3.9</powerjob-remote-impl-akka.version>
         <springdoc-openapi-ui.version>1.6.14</springdoc-openapi-ui.version>
         <aliyun-sdk-oss.version>3.17.1</aliyun-sdk-oss.version>
-        <minio.version>8.5.2</minio.version>
+        <aws-java-sdk-s3.version>1.12.665</aws-java-sdk-s3.version>
         <commons-collections4.version>4.4</commons-collections4.version>
     </properties>
 
@@ -114,12 +114,13 @@
                 <artifactId>aliyun-sdk-oss</artifactId>
                 <version>${aliyun-sdk-oss.version}</version>
             </dependency>
-            <!-- 存储扩展-Minio，未使用可移除 -->
+            <!-- 存储扩展-Minio/S3，未使用可移除（minio-client 依赖 OKHTTP4.x 版本，强制引入 kotlin 标准库，为了防止引入更多问题放弃） -->
             <dependency>
-                <groupId>io.minio</groupId>
-                <artifactId>minio</artifactId>
-                <version>${minio.version}</version>
+                <groupId>com.amazonaws</groupId>
+                <artifactId>aws-java-sdk-s3</artifactId>
+                <version>${aws-java-sdk-s3.version}</version>
             </dependency>
+
             <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
             <dependency>
                 <groupId>org.apache.commons</groupId>

diff --git a/powerjob-server/powerjob-server-common/pom.xml b/powerjob-server/powerjob-server-common/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/powerjob-server/powerjob-server-core/pom.xml b/powerjob-server/powerjob-server-core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/...erjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java b/...erjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java
@@ -317,40 +317,39 @@ public String fetchDeployedInfo(Long appId, Long containerId) {
                 .collect(Collectors.toSet());
 
         Set<String> deployedList = Sets.newLinkedHashSet();
-        List<String> unDeployedList = Lists.newLinkedList();
-        Multimap<String, String> version2Address = ArrayListMultimap.create();
+        Multimap<String, DeployedContainerInfo> version2DeployedContainerInfoList = ArrayListMultimap.create();
         infoList.forEach(info -> {
             String targetWorkerAddress = info.getWorkerAddress();
             if (aliveWorkers.contains(targetWorkerAddress)) {
                 deployedList.add(targetWorkerAddress);
-                version2Address.put(info.getVersion(), targetWorkerAddress);
-            }else {
-                unDeployedList.add(targetWorkerAddress);
+                version2DeployedContainerInfoList.put(info.getVersion(), info);
             }
         });
 
+        Set<String> unDeployedList = Sets.newHashSet(aliveWorkers);
+        unDeployedList.removeAll(deployedList);
+
         StringBuilder sb = new StringBuilder("========== DeployedInfo ==========").append(System.lineSeparator());
+
         // 集群分裂，各worker版本不统一，问题很大
-        if (version2Address.keySet().size() > 1) {
+        if (version2DeployedContainerInfoList.keySet().size() > 1) {
             sb.append("WARN: there exists multi version container now, please redeploy to fix this problem").append(System.lineSeparator());
-            sb.append("divisive version ==> ").append(System.lineSeparator());
-            version2Address.forEach((v, addressList) -> {
-                sb.append("version: ").append(v).append(System.lineSeparator());
-                sb.append(addressList);
-            });
-            sb.append(System.lineSeparator());
         }
+
+        sb.append("divisive version ==> ").append(System.lineSeparator());
+        version2DeployedContainerInfoList.asMap().forEach((version, deployedContainerInfos) -> {
+            sb.append("[version] ").append(version).append(System.lineSeparator());
+            deployedContainerInfos.forEach(deployedContainerInfo -> sb.append(String.format("Address: %s, DeployedTime: %s", deployedContainerInfo.getWorkerAddress(), CommonUtils.formatTime(deployedContainerInfo.getDeployedTime()))).append(System.lineSeparator()));
+        });
+
         // 当前在线未部署机器
         if (!CollectionUtils.isEmpty(unDeployedList)) {
-            sb.append("WARN: there exists unDeployed worker(OhMyScheduler will auto fix when some job need to process)").append(System.lineSeparator());
-            sb.append("unDeployed worker list ==> ").append(System.lineSeparator());
+            sb.append("WARN: there exists unDeployed worker(PowerJob will auto fix when some job need to process)").append(System.lineSeparator());
+            sb.append("unDeployed worker list ==> ").append(unDeployedList).append(System.lineSeparator());
         }
-        // 当前部署机器
-        sb.append("deployed worker list ==> ").append(System.lineSeparator());
+
         if (CollectionUtils.isEmpty(deployedList)) {
-            sb.append("no worker deployed now~");
-        }else {
-            sb.append(deployedList);
+            sb.append("no worker deployed this container now~");
         }
 
         return sb.toString();

diff --git a/...er-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java b/...er-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java
@@ -145,7 +145,11 @@ public void checkRunningInstance() {
         log.info("[InstanceStatusChecker] RunningInstance status check using {}.", stopwatch.stop());
     }
 
-    private void handleWaitingDispatchInstance(List<Long> partAppIds) {
+    private void handleWaitingDispatchInstance(List<Long> appIds) {
+
+        // 存在移除操作，需要重新创建集合，否则会导致外层抛出 NoSuchElementException: null
+        List<Long> partAppIds = Lists.newArrayList(appIds);
+
         // 1. 检查等待 WAITING_DISPATCH 状态的任务
         long threshold = System.currentTimeMillis() - DISPATCH_TIMEOUT_MS;
         List<InstanceInfoDO> waitingDispatchInstances = instanceInfoRepository.findAllByAppIdInAndStatusAndExpectedTriggerTimeLessThan(partAppIds, InstanceStatus.WAITING_DISPATCH.getV(), threshold, PageRequest.of(0, MAX_BATCH_NUM_INSTANCE));

diff --git a/powerjob-server/powerjob-server-extension/pom.xml b/powerjob-server/powerjob-server-extension/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/powerjob-server/powerjob-server-migrate/pom.xml b/powerjob-server/powerjob-server-migrate/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/powerjob-server/powerjob-server-monitor/pom.xml b/powerjob-server/powerjob-server-monitor/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>powerjob-server</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.8</version>
+        <version>4.3.9</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -41,8 +41,8 @@
             <artifactId>aliyun-sdk-oss</artifactId>
         </dependency>
         <dependency>
-            <groupId>io.minio</groupId>
-            <artifactId>minio</artifactId>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
         </dependency>
     </dependencies>