Elastic-Job-Lite分布式调度（springboot）

最新推荐文章于 2025-03-18 22:39:21 发布

原创最新推荐文章于 2025-03-18 22:39:21 发布 · 1.1k 阅读

本内容遵循CC 4.0 BY-SA版权协议

本文介绍Elastic-Job-Lite的分布式任务调度与SpringBoot集成，包括POM配置、job配置和关键技术原理，适合Spring开发者理解和使用分布式作业中间件。

概述

Elastic-Job 是一个开源的分布式调度中间件，由两个相互独立的子项目 Elastic-Job-Lite 和 Elastic- Job-Cloud 组成。

Elastic-Job-Lite 为轻量级无中心化解决方案，使用 jar 包提供分布式任务的调度和治理。

Elastic-Job-Lite 定位为纯粹的作业中间件，仅关注分布式调度、协调以及分片等核心功能，将作业部署、资源分配等功能交于 Mesos 或 Kubernetes 处理；

项目架构

ElasticJob-Lite 定位为轻量级无中心化解决方案，使用 jar 的形式提供分布式任务的协调服务。

如何开发（SpringBoot项目为例）

POM配置

<dependency> 2.

<version>${elastic-job.latest.release.version}</version>

3. <artifactId>elasticjob-lite-core</artifactId> 4.

5. </dependency> 6.

<artifactId>elasticjob-lite-spring-boot-starter</artifactId>

7. <groupId>org.apache.shardingsphere.elasticjob</groupId> 8.

</dependency>

9. <version>${elastic-job.latest.release.version}</version> 10.

<groupId>org.apache.shardingsphere.elasticjob</groupId>

11. <dependency> 12.

<version>${latest.release.version}</version>

13. <artifactId>elasticjob-error-handler-dingtalk</artifactId> 14.

</dependency>

properties配置

elasticjob.reg-center.namespace=hsy-bz-elasticjob

# elastic 2.

3. elasticjob.reg-center.server-lists=192.168.0.6:2181, 192.168.0.7:2182,192.168.0.8:2183

elasticjob.reg-center.base-sleep-time-milliseconds=20000

elasticjob.reg-center.connection-timeout-milliseconds=60000

5. elasticjob.reg-center.max-sleep-time-milliseconds=50000 6.

7. elasticjob.reg-center.max-retries=6 8.

elasticjob.jobs.UserChallenge.elastic-job-class=com.damei.hsydata.bz.datasync.job.UserCh

allenge

9. ### simple job 10.

elasticjob.jobs.UserChallenge.sharding-total-count=1

11. elasticjob.jobs.UserChallenge.cron=55 * * * * ? 12.

13.

# dataflow job

14.

15. elasticjob.jobs.KolDayDataProcess.elastic-job-class=com.damei.hsydata.bz.datasync.job.Ko lDayDataProcess

elasticjob.jobs.KolDayDataProcess.cron=0 40 9 * * ?

16.

elasticjob.jobs.UserChallenge.sharding-item-parameters= 0=Beijing,1=Shanghai

17. elasticjob.jobs.KolDayDataProcess.sharding-total-count=2 18.

elasticjob.jobs.KolDayDataProcess.overwrite=true

19. elasticjob.jobs.KolDayDataProcess.props.streaming.process=true 20.

21. elasticjob.dump.port=9111 22.

elasticjob.jobs.myScriptJob.elastic-job-type=SCRIPT

23. # scrip job 24.

elasticjob.jobs.myScriptJob.sharding-total-count=2

25. elasticjob.jobs.myScriptJob.cron=0/10 * * * * ? 26.

elasticjob.jobs.myScriptJob.props.script.command.line= echo sharding execution context is $*

简单作业

1. package com.damei.hsydata.bz.datasync.job; 2.

import com.cms.cache.Cache;

3. import com.alibaba.fastjson.JSONObject; 4.

import com.damei.hsydata.bz.datasync.constant.RedisKeys;

5. import com.damei.common.util.DateUtil; 6.

import com.damei.hsydata.model.hsy.UserChallengeInfo;

7. import com.damei.hsydata.model.hsy.KolInfo; 8.

import com.google.common.collect.Maps;

9. import com.damei.hsydata.service.hsy.*; 10.

import org.apache.shardingsphere.elasticjob.simple.job.SimpleJob;

11. import org.apache.shardingsphere.elasticjob.api.ShardingContext; 12.

import org.slf4j.LoggerFactory;

13. import org.slf4j.Logger; 14.

import org.springframework.stereotype.Component;

15. import org.springframework.beans.factory.annotation.Autowired; 16.

17.

import java.util.Date;

18.

import java.util.Map;

19. import java.util.List; 20.

21.

@Component

22.

private final Logger logger = LoggerFactory.getLogger(UserChallenge.class);

23. public class UserChallenge implements SimpleJob { 24.

private IUserChallengeInfoService userChallengeInfoService;

25. @Autowired 26.

private IKolInfoService kolInfoService;

27. @Autowired 28.

Cache cache;

29. @Autowired 30.

public void execute(ShardingContext context) {

31. @Override 32.

case 0:

33. switch (context.getShardingItem()) { 34.

setUserChallengeCacheTask();

35. // do something by sharding item 0 36.

case 1:

37. break; 38.

break;

39. // do something by sharding item 1 40.

// do something by sharding item 2

41. case 2: 42.

// case n: ...

break; 44.

}

45. } 46.

47. }

流式作业

1. package com.damei.hsydata.bz.datasync.job; 2.

import com.damei.hsydata.bz.datasync.constant.RedisKeys;

3. import com.cms.cache.Cache; 4.

import com.damei.hsydata.bz.datasync.util.CommonUtil;

5. import com.damei.hsydata.bz.datasync.factory.KolDayDataProcessFactory; 6.

import org.apache.shardingsphere.elasticjob.dataflow.job.DataflowJob;

7. import org.apache.shardingsphere.elasticjob.api.ShardingContext; 8.

9. import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

10.

import org.springframework.stereotype.Component;

11. import org.springframework.beans.factory.annotation.Autowired; 12.

13.

import java.util.*;

14.

15.

import java.util.concurrent.ThreadPoolExecutor;

16.

17.

@Component

18.

private final Logger logger = LoggerFactory.getLogger(KolDayDataProcess.class);

19. public class KolDayDataProcess implements DataflowJob { 20.

21.

@Autowired

22.

@Autowired

23. private KolDayDataProcessFactory kolDayDataProcessFactory; 24.

private ThreadPoolExecutor threadPoolExecutor= CommonUtil.initDefaultThreadPool();

25. Cache cache; 26.

27.

28.

public List fetchData(ShardingContext shardingContext) {

29. @Override 30.

31. try {

32.	return cache.mRpoplpush(RedisKeys.KolInfoIdCacheSourceKey,RedisKeys.KolInfoIdCacheDestinationKey
	,	RedisKeys.SRandomNumber);
33.
34.		} catch (Exception ex) {
35.		ex.printStackTrace();
36.		logger.error("执行设置KOL日数据缓存任务失败：", ex);
37.		}
38.		return null;
39.		}
40.
41.		@Override
42.		public void processData(ShardingContext shardingContext, List list) {
43.		for (Object id : list) {
44.		threadPoolExecutor.execute(new Runnable() {
45.		@Override
46.		public void run() {
47.		try {
48.		final Long kolId = Long.parseLong(id.toString());