多线程消费音讯

关键词：Java，多线程，音讯行列，rocketmq

多线程一个用例之一就是音讯的快速消费，比方咱们有一个音讯行列咱们期望以更快的速度消费音讯，假如咱们用的是rocketmq，咱们从中获取音讯，然后使用多线程处理。

代码地址Github

完成思路

1. 不停的拉取音讯
2. 将拉取的音讯分片
3. 多个线程一起消费每一片音讯
4. 将一切音讯消费完成后，接着拉取新的音讯

代码

CrazyTask

这是一个抽象类，针对不同的使命可能有不同的处理逻辑，对于不同的使命去承继这个CrazyTask 完成他的process方法。

package crazyConsumer;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
/**
 * {@code @author:} keboom
 * {@code @date:} 2023/11/17
 * {@code @description:}
 */
public abstract class CrazyTask {
    String taskName;
    int threadNum;
    volatile boolean isTerminated;
    // every partition data num.
    // for example: I receive 5 messages, partitionDataNum is 2, then i will partition 5 messages to 3 parts, 2,2,1
    int partitionDataCount = 2;
    abstract void process(Message message);
    void doExecute(SimpleConsumer consumer) {
        while (true) {
            // 此顾客每次自动拉取音讯行列中音讯
            List<Message> messages = consumer.receive();
            if (messages.isEmpty()) {
                try {
                    Thread.sleep(5000);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
                continue;
            }
            // 获取处理此topic或者说处理此类型task的线程池
            ExecutorService executor = CrazyTaskUtil.getOrInitExecutor(taskName, threadNum);
            // 将音讯分片，每个线程处理一部分音讯
            List<List<Message>> partition = Lists.partition(messages, partitionDataCount);
            // 以音讯分片数初始化CountDownLatch，每个线程处理完一片音讯，countDown一次
            // 当countDownLatch为0时，阐明一切音讯都处理完了，countDownLatch.await();持续向下履行
            CountDownLatch countDownLatch = new CountDownLatch(partition.size());
            partition.forEach(messageList -> {
                executor.execute(() -> {
                    messageList.forEach(message -> {
                        process(message);
                        consumer.ack(message);
                    });
                    countDownLatch.countDown();
                });
            });
            try {
                countDownLatch.await();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            if (isTerminated) {
                break;
            }
        }
        // 释放线程池
        CrazyTaskUtil.shutdownThreadPool(taskName);
    }
    void terminate() {
        isTerminated = true;
        System.out.println();
        System.out.println(taskName + " shut down");
    }
    public String getTaskName() {
        return taskName;
    }
}

PhoneTask

package crazyConsumer;
/**
 * {@code @author:} keboom
 * {@code @date:} 2023/11/17
 * {@code @description:}
 */
public class PhoneTask extends CrazyTask {
    public PhoneTask(String taskName, int threadNum) {
        this.taskName = taskName;
        // default thread num
        this.threadNum = threadNum;
        this.isTerminated = false;
    }
    @Override
    void process(Message message) {
        System.out.println(Thread.currentThread().getName() +"  process  "+ message.toString());
        try {
            Thread.sleep(30);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
    @Override
    public String toString() {
        return "PhoneTask{" +
                "taskName='" + taskName + ''' +
                ", threadNum=" + threadNum +
                ", isTerminated=" + isTerminated +
                '}';
    }
}

EmailTask

package crazyConsumer;
/**
 * {@code @author:} keboom
 * {@code @date:} 2023/11/17
 * {@code @description:}
 */
public class EmailTask extends CrazyTask{
    public EmailTask(String taskName, int threadNum) {
        this.taskName = taskName;
        // default thread num
        this.threadNum = threadNum;
        this.isTerminated = false;
    }
    @Override
    void process(Message message) {
        // do something
        System.out.println(Thread.currentThread().getName() +"  process  "+ message.toString());
        try {
            Thread.sleep(20);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
    @Override
    public String toString() {
        return "EmailTask{" +
                "taskName='" + taskName + ''' +
                ", threadNum=" + threadNum +
                ", isTerminated=" + isTerminated +
                '}';
    }
}

CrazyTaskUtil

创立毁掉线程池的东西类

package crazyConsumer;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.Map;
import java.util.concurrent.*;
/**
 * {@code @author:} keboom
 * {@code @date:} 2023/11/17
 * {@code @description:}
 */
public class CrazyTaskUtil {
    private static final Map<String, ExecutorService> executors = new ConcurrentHashMap<>();
    public static ExecutorService getOrInitExecutor(String taskName, int threadNum) {
        ExecutorService executorService = executors.get(taskName);
        if (executorService == null) {
            synchronized (CrazyTaskUtil.class) {
                executorService = executors.get(taskName);
                if (executorService == null) {
                    executorService = initPool(taskName, threadNum);
                    executors.put(taskName, executorService);
                }
            }
        }
        return executorService;
    }
    private static ExecutorService initPool(String taskName, int threadNum) {
        // init pool
        return new ThreadPoolExecutor(threadNum, threadNum,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<>(),
                new ThreadFactoryBuilder().setNameFormat(taskName + "-%d").build());
    }
    public static void shutdownThreadPool(String taskName) {
        ExecutorService remove = executors.remove(taskName);
        if (remove != null) {
            remove.shutdown();
        }
    }
}

Main

程序入口

package crazyConsumer;
import java.util.ArrayList;
/**
 * {@code @author:} keboom
 * {@code @date:} 2023/11/17
 * {@code @description:}
 */
public class Main {
    /**
     * 一种多线程消费场景。比方咱们有一个消费行列，里边有各种音讯，咱们需求赶快的消费他们，不同的音讯对应不同的业务
     *
     * @param args
     */
    public static void main(String[] args) throws InterruptedException {
        // 比方说咱们这个有rocketmq不同主题的consumer
        /*
        List<MessageView> messageViewList = null;
        try {
            messageViewList = simpleConsumer.receive(10, Duration.ofSeconds(30));
            messageViewList.forEach(messageView -> {
                System.out.println(messageView);
                //消费处理完成后，需求自动调用ACK提交消费成果。
                try {
                    simpleConsumer.ack(messageView);
                } catch (ClientException e) {
                    e.printStackTrace();
                }
            });
        } catch (ClientException e) {
            //假如遇到体系流控等原因形成拉取失利，需求重新发起获取音讯请求。
            e.printStackTrace();
        }
         */
        // 想要完成多线程消费音讯，咱们期望有一个使命，此使命能够不停的拉取音讯，然后创立子线程池去消费音讯。
        // 停止使命后，需求将使命中的音讯消费完后，再关闭使命。
        ArrayList<CrazyTask> tasks = new ArrayList<>();
        tasks.add(new PhoneTask("phoneTask", 2));
        tasks.add(new EmailTask("emailTask", 3));
        for (CrazyTask task : tasks) {
            new Thread(() -> {
                task.doExecute(new SimpleConsumer("topic"+task.getTaskName().charAt(0), "tagA"));
            }).start();
        }
        // task running
        Thread.sleep(150);
        // task terminated
        tasks.forEach(CrazyTask::terminate);
    }
}

终究履行成果

receive message: [Message{messageBody='topice-tagA-0-1700470193487'}, Message{messageBody='topice-tagA-1-1700470193487'}, Message{messageBody='topice-tagA-2-1700470193487'}, Message{messageBody='topice-tagA-3-1700470193487'}, Message{messageBody='topice-tagA-4-1700470193487'}]
receive message: [Message{messageBody='topicp-tagA-0-1700470193487'}, Message{messageBody='topicp-tagA-1-1700470193487'}, Message{messageBody='topicp-tagA-2-1700470193487'}, Message{messageBody='topicp-tagA-3-1700470193487'}, Message{messageBody='topicp-tagA-4-1700470193487'}]
phoneTask-0  process  Message{messageBody='topicp-tagA-0-1700470193487'}
emailTask-1  process  Message{messageBody='topice-tagA-2-1700470193487'}
emailTask-0  process  Message{messageBody='topice-tagA-0-1700470193487'}
phoneTask-1  process  Message{messageBody='topicp-tagA-2-1700470193487'}
emailTask-2  process  Message{messageBody='topice-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-2-1700470193487'}
emailTask-1  process  Message{messageBody='topice-tagA-3-1700470193487'}
ack message: Message{messageBody='topice-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-0-1700470193487'}
emailTask-0  process  Message{messageBody='topice-tagA-1-1700470193487'}
ack message: Message{messageBody='topicp-tagA-2-1700470193487'}
ack message: Message{messageBody='topicp-tagA-0-1700470193487'}
phoneTask-0  process  Message{messageBody='topicp-tagA-1-1700470193487'}
phoneTask-1  process  Message{messageBody='topicp-tagA-3-1700470193487'}
ack message: Message{messageBody='topice-tagA-1-1700470193487'}
ack message: Message{messageBody='topice-tagA-3-1700470193487'}
receive message: [Message{messageBody='topice-tagA-0-1700470193570'}, Message{messageBody='topice-tagA-1-1700470193570'}, Message{messageBody='topice-tagA-2-1700470193570'}, Message{messageBody='topice-tagA-3-1700470193570'}, Message{messageBody='topice-tagA-4-1700470193570'}]
emailTask-0  process  Message{messageBody='topice-tagA-2-1700470193570'}
emailTask-2  process  Message{messageBody='topice-tagA-0-1700470193570'}
emailTask-1  process  Message{messageBody='topice-tagA-4-1700470193570'}
ack message: Message{messageBody='topicp-tagA-3-1700470193487'}
ack message: Message{messageBody='topicp-tagA-1-1700470193487'}
phoneTask-1  process  Message{messageBody='topicp-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-0-1700470193570'}
ack message: Message{messageBody='topice-tagA-4-1700470193570'}
ack message: Message{messageBody='topice-tagA-2-1700470193570'}
emailTask-0  process  Message{messageBody='topice-tagA-3-1700470193570'}
emailTask-2  process  Message{messageBody='topice-tagA-1-1700470193570'}
ack message: Message{messageBody='topicp-tagA-4-1700470193487'}
receive message: [Message{messageBody='topicp-tagA-0-1700470193618'}, Message{messageBody='topicp-tagA-1-1700470193618'}, Message{messageBody='topicp-tagA-2-1700470193618'}, Message{messageBody='topicp-tagA-3-1700470193618'}, Message{messageBody='topicp-tagA-4-1700470193618'}]
phoneTask-0  process  Message{messageBody='topicp-tagA-0-1700470193618'}
phoneTask-1  process  Message{messageBody='topicp-tagA-2-1700470193618'}
ack message: Message{messageBody='topice-tagA-1-1700470193570'}
ack message: Message{messageBody='topice-tagA-3-1700470193570'}
receive message: [Message{messageBody='topice-tagA-0-1700470193634'}, Message{messageBody='topice-tagA-1-1700470193634'}, Message{messageBody='topice-tagA-2-1700470193634'}, Message{messageBody='topice-tagA-3-1700470193634'}, Message{messageBody='topice-tagA-4-1700470193634'}]
emailTask-1  process  Message{messageBody='topice-tagA-0-1700470193634'}
emailTask-0  process  Message{messageBody='topice-tagA-4-1700470193634'}
emailTask-2  process  Message{messageBody='topice-tagA-2-1700470193634'}
ack message: Message{messageBody='topicp-tagA-2-1700470193618'}
ack message: Message{messageBody='topicp-tagA-0-1700470193618'}
phoneTask-0  process  Message{messageBody='topicp-tagA-1-1700470193618'}
phoneTask-1  process  Message{messageBody='topicp-tagA-3-1700470193618'}
phoneTask shut down
emailTask shut down
ack message: Message{messageBody='topice-tagA-0-1700470193634'}
ack message: Message{messageBody='topice-tagA-2-1700470193634'}
emailTask-1  process  Message{messageBody='topice-tagA-1-1700470193634'}
ack message: Message{messageBody='topice-tagA-4-1700470193634'}
emailTask-2  process  Message{messageBody='topice-tagA-3-1700470193634'}
ack message: Message{messageBody='topicp-tagA-3-1700470193618'}
ack message: Message{messageBody='topicp-tagA-1-1700470193618'}
phoneTask-1  process  Message{messageBody='topicp-tagA-4-1700470193618'}
ack message: Message{messageBody='topice-tagA-3-1700470193634'}
ack message: Message{messageBody='topice-tagA-1-1700470193634'}
ack message: Message{messageBody='topicp-tagA-4-1700470193618'}

可以看到成果是，当每次收到的音讯消费完后会拉取新的音讯。当履行shutdown使命时，会将当前使命履行完后再毁掉线程池。