Merge branch '4.3.8_zjlab_v2'

PowerJob · Feb 25, 2024 · 9c0731f · 9c0731f
2 parents 4793c19 + bd725aa
commit 9c0731f
Show file tree

Hide file tree

Showing 103 changed files with 8,507 additions and 6,499 deletions.
diff --git a/others/dev/publish_docker_apple_silicon.sh b/others/dev/publish_docker_apple_silicon.sh
@@ -41,20 +41,31 @@ if [ "$rebuild" = "y" ] || [  "$rebuild" = "Y" ]; then
   docker rmi -f powerjob/powerjob-worker-samples:$version
   echo "================== 构建 powerjob-server 镜像(tjqq) =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-server:$version powerjob-server/docker/. --push || exit
-  docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-server:latest powerjob-server/docker/. --push || exit
   echo "================== 构建 powerjob-server 镜像(powerjob) =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-server:$version powerjob-server/docker/. --push || exit
-  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-server:latest powerjob-server/docker/. --push || exit
   echo "================== 构建 powerjob-agent 镜像(tjqq) =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-agent:$version powerjob-worker-agent/. --push|| exit
-  docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-agent:latest powerjob-worker-agent/. --push|| exit
   echo "================== 构建 powerjob-agent 镜像(powerjob) =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-agent:$version powerjob-worker-agent/. --push|| exit
-  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-agent:latest powerjob-worker-agent/. --push|| exit
   echo "================== 构建 powerjob-mysql 镜像 =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-mysql:$version others/. --push|| exit
-  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-mysql:latest others/. --push|| exit
   echo "================== 构建 powerjob-worker-samples 镜像 =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-worker-samples:$version powerjob-worker-samples/. --push|| exit
+fi
+
+read -r -p "是否推送LATEST（y/n）:" push_latest
+if [ "$push_latest" = "y" ] || [  "$push_latest" = "Y" ]; then
+
+  echo "==================  powerjob-server LATEST (tjqq) =================="
+  docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-server:latest powerjob-server/docker/. --push || exit
+  echo "==================  powerjob-server LATEST (powerjob) =================="
+  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-server:latest powerjob-server/docker/. --push || exit
+  echo "==================  powerjob-agent LATEST (tjqq) =================="
+  docker buildx build --platform=linux/amd64,linux/arm64 --tag tjqq/powerjob-agent:latest powerjob-worker-agent/. --push|| exit
+  echo "==================  powerjob-agent LATEST (powerjob) =================="
+  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-agent:latest powerjob-worker-agent/. --push|| exit
+  echo "==================  powerjob-mysql LATEST =================="
+  docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-mysql:latest others/. --push|| exit
+  echo "==================  powerjob-worker-samples LATEST =================="
   docker buildx build --platform=linux/amd64,linux/arm64 --tag powerjob/powerjob-worker-samples:latest powerjob-worker-samples/. --push|| exit
 fi
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>tech.powerjob</groupId>
     <artifactId>powerjob</artifactId>
-    <version>4.3.7</version>
+    <version>4.3.8</version>
     <packaging>pom</packaging>
     <name>powerjob</name>
     <url>http://www.powerjob.tech</url>

diff --git a/powerjob-client/pom.xml b/powerjob-client/pom.xml
@@ -5,18 +5,18 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.7</version>
+        <version>4.3.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-client</artifactId>
-    <version>4.3.7</version>
+    <version>4.3.8</version>
     <packaging>jar</packaging>
 
     <properties>
         <junit.version>5.9.1</junit.version>
         <fastjson.version>1.2.83</fastjson.version>
-        <powerjob.common.version>4.3.7</powerjob.common.version>
+        <powerjob.common.version>4.3.8</powerjob.common.version>
 
         <mvn.shade.plugin.version>3.2.4</mvn.shade.plugin.version>
     </properties>

diff --git a/powerjob-common/pom.xml b/powerjob-common/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <artifactId>powerjob</artifactId>
         <groupId>tech.powerjob</groupId>
-        <version>4.3.7</version>
+        <version>4.3.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <artifactId>powerjob-common</artifactId>
-    <version>4.3.7</version>
+    <version>4.3.8</version>
     <packaging>jar</packaging>
 
     <properties>

diff --git a/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java b/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java
@@ -55,4 +55,13 @@ public class PowerJobDKey {
      */
     public static final String FREQUENCY_JOB_MAX_INTERVAL = "powerjob.server.frequency-job.max-interval";
 
+    /* ******************* 不太可能有人用的参数，主要方便内部测试 ******************* */
+
+    /**
+     * 最大活跃任务数量，超出部分 SWAP 到磁盘以提升性能
+     */
+    public static final String WORKER_RUNTIME_SWAP_MAX_ACTIVE_TASK_NUM = "powerjob.worker.swap.max-active-task-num";
+
+    public static final String WORKER_RUNTIME_SWAP_TASK_SCHEDULE_INTERVAL_MS = "powerjob.worker.swap.scan-interval";
+
 }
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/enums/DispatchStrategy.java b/powerjob-common/src/main/java/tech/powerjob/common/enums/DispatchStrategy.java
@@ -13,8 +13,19 @@
 @AllArgsConstructor
 public enum DispatchStrategy {
 
+    /**
+     * 健康度优先
+     */
     HEALTH_FIRST(1),
-    RANDOM(2);
+    /**
+     * 随机
+     */
+    RANDOM(2),
+    /**
+     * 指定执行
+     */
+    SPECIFY(11)
+    ;
 
     private final int v;
 

diff --git a/powerjob-common/src/main/java/tech/powerjob/common/enums/TaskTrackerBehavior.java b/powerjob-common/src/main/java/tech/powerjob/common/enums/TaskTrackerBehavior.java
@@ -0,0 +1,42 @@
+package tech.powerjob.common.enums;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * TaskTracker 行为枚举
+ *
+ * @author tjq
+ * @since 2024/2/24
+ */
+@Getter
+@AllArgsConstructor
+public enum TaskTrackerBehavior {
+
+    /**
+     * 普通：不特殊处理，参与集群计算，会导致 TaskTracker 负载比常规节点高。适用于节点数不那么多，任务不那么繁重的场景
+     */
+    NORMAL(1),
+    /**
+     * 划水：只负责管理节点，不参与计算，稳定性最优。适用于节点数量非常多的大规模计算场景，少一个计算节点来换取稳定性提升
+     */
+    PADDLING(11)
+    ;
+
+
+    private final Integer v;
+
+    public static TaskTrackerBehavior of(Integer type) {
+
+        if (type == null) {
+            return NORMAL;
+        }
+
+        for (TaskTrackerBehavior t : values()) {
+            if (t.v.equals(type)) {
+                return t;
+            }
+        }
+        return NORMAL;
+    }
+}
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/model/InstanceDetail.java b/powerjob-common/src/main/java/tech/powerjob/common/model/InstanceDetail.java
@@ -1,8 +1,8 @@
 package tech.powerjob.common.model;
 
-import tech.powerjob.common.PowerSerializable;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import tech.powerjob.common.PowerSerializable;
 
 import java.util.List;
 
@@ -51,8 +51,15 @@ public class InstanceDetail implements PowerSerializable {
 
     /**
      * Task detail, used by MapReduce or Broadcast tasks.
+     * 命名有点问题，实际是 task 统计信息
      */
     private TaskDetail taskDetail;
+
+    /**
+     * 查询出来的 Task 详细结果
+     */
+    private List<TaskDetailInfo> queriedTaskDetailInfoList;
+
     /**
      * Sub instance details, used by frequent tasks.
      */
@@ -92,5 +99,14 @@ public static class TaskDetail implements PowerSerializable {
         private long totalTaskNum;
         private long succeedTaskNum;
         private long failedTaskNum;
+
+        // 等待派发状态（仅存在 TaskTracker 数据库中）
+        protected Long waitingDispatchTaskNum;
+        // 已派发，但 ProcessorTracker 未确认，可能由于网络错误请求未送达，也有可能 ProcessorTracker 线程池满，拒绝执行
+        protected Long workerUnreceivedTaskNum;
+        // ProcessorTracker确认接收，存在与线程池队列中，排队执行
+        protected Long receivedTaskNum;
+        // ProcessorTracker正在执行
+        protected Long runningTaskNum;
     }
 }
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/model/JobAdvancedRuntimeConfig.java b/powerjob-common/src/main/java/tech/powerjob/common/model/JobAdvancedRuntimeConfig.java
@@ -0,0 +1,25 @@
+package tech.powerjob.common.model;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+/**
+ * 任务运行时高级配置
+ *
+ * @author tjq
+ * @since 2024/2/24
+ */
+@Getter
+@Setter
+@ToString
+@Accessors(chain = true)
+public class JobAdvancedRuntimeConfig {
+
+    /**
+     * MR 任务专享参数，TaskTracker 行为 {@link tech.powerjob.common.enums.TaskTrackerBehavior}
+     */
+    private Integer taskTrackerBehavior;
+
+}
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/model/TaskDetailInfo.java b/powerjob-common/src/main/java/tech/powerjob/common/model/TaskDetailInfo.java
@@ -0,0 +1,43 @@
+package tech.powerjob.common.model;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+import tech.powerjob.common.PowerSerializable;
+
+/**
+ * Task 详细信息
+ *
+ * @author tjq
+ * @since 2024/2/25
+ */
+@Data
+@Accessors(chain = true)
+public class TaskDetailInfo implements PowerSerializable {
+
+    private String taskId;
+    private String taskName;
+    /**
+     * 任务对象（map 的 subTask）
+     */
+    private String taskContent;
+    /**
+     * 处理器地址
+     */
+    private String processorAddress;
+    private Integer status;
+    private String statusStr;
+    private String result;
+    private Integer failedCnt;
+    /**
+     * 创建时间
+     */
+    private Long createdTime;
+    /**
+     * 最后修改时间
+     */
+    private Long lastModifiedTime;
+    /**
+     * ProcessorTracker 最后上报时间
+     */
+    private Long lastReportTime;
+}
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/request/ServerQueryInstanceStatusReq.java b/powerjob-common/src/main/java/tech/powerjob/common/request/ServerQueryInstanceStatusReq.java
@@ -17,4 +17,11 @@
 public class ServerQueryInstanceStatusReq implements PowerSerializable {
     private Long instanceId;
 
+    /**
+     * 自定义查询
+     * 针对高阶用户，直接开放底库查询，便于运维和排查问题
+     * 此处只传递查询条件，前置拼接 select *，后置拼接 limit
+     */
+    private String customQuery;
+
 }
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java b/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java
@@ -97,4 +97,9 @@ public class ServerScheduleJobReq implements PowerSerializable {
      * 日志配置
      */
     private String logConfig;
+
+    /**
+     * 高级运行时配置
+     */
+    private String advancedRuntimeConfig;
 }
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/request/http/SaveJobInfoRequest.java b/powerjob-common/src/main/java/tech/powerjob/common/request/http/SaveJobInfoRequest.java
@@ -5,6 +5,7 @@
 import tech.powerjob.common.enums.ProcessorType;
 import tech.powerjob.common.enums.TimeExpressionType;
 import tech.powerjob.common.model.AlarmConfig;
+import tech.powerjob.common.model.JobAdvancedRuntimeConfig;
 import tech.powerjob.common.model.LogConfig;
 import tech.powerjob.common.model.LifeCycle;
 import tech.powerjob.common.utils.CommonUtils;
@@ -134,6 +135,11 @@ public class SaveJobInfoRequest {
 
     private DispatchStrategy dispatchStrategy;
 
+    /**
+     * 某种派发策略背后的具体配置，值取决于 dispatchStrategy
+     */
+    private String dispatchStrategyConfig;
+
     private LifeCycle lifeCycle;
     /**
      * alarm config
@@ -150,6 +156,10 @@ public class SaveJobInfoRequest {
      */
     private LogConfig logConfig;
 
+    /**
+     * 高级运行时配置
+     */
+    private JobAdvancedRuntimeConfig advancedRuntimeConfig;
 
     /**
      * Check non-null properties.

diff --git a/powerjob-common/src/main/java/tech/powerjob/common/response/JobInfoDTO.java b/powerjob-common/src/main/java/tech/powerjob/common/response/JobInfoDTO.java
@@ -2,6 +2,7 @@
 
 import lombok.Data;
 import tech.powerjob.common.model.AlarmConfig;
+import tech.powerjob.common.model.JobAdvancedRuntimeConfig;
 import tech.powerjob.common.model.LogConfig;
 
 import java.util.Date;
@@ -121,7 +122,14 @@ public class JobInfoDTO {
 
     private String extra;
 
+    /**
+     * 派发策略
+     */
     private Integer dispatchStrategy;
+    /**
+     * 某种派发策略背后的具体配置，值取决于 dispatchStrategy
+     */
+    private String dispatchStrategyConfig;
 
     private String lifecycle;
 
@@ -137,4 +145,5 @@ public class JobInfoDTO {
      */
     private LogConfig logConfig;
 
+    private JobAdvancedRuntimeConfig advancedRuntimeConfig;
 }
diff --git a/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java b/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java
@@ -1,8 +1,10 @@
 package tech.powerjob.common.serialize;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.MapperFeature;
 import com.fasterxml.jackson.databind.json.JsonMapper;
 import lombok.extern.slf4j.Slf4j;
@@ -28,15 +30,23 @@ public class JsonUtils {
             .configure(MapperFeature.PROPAGATE_TRANSIENT_MARKER, true)
             .configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
             .configure(JsonParser.Feature.IGNORE_UNDEFINED, true)
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
             .build();
 
+    static {
+        JSON_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    }
+
     private static final TypeReference<Map<String, Object>>  MAP_TYPE_REFERENCE  = new TypeReference<Map<String, Object>> () {};
 
     private JsonUtils(){
 
     }
 
     public static String toJSONString(Object obj) {
+        if (obj == null) {
+            return null;
+        }
         if (obj instanceof String) {
             return (String) obj;
         }