使用CompletionService进行多个文件打包为zip下载

最近没怎么写博客了，因为前段时间在准备软考复习，昨天考完试，现在总算轻松一点了，有更多自由的时间了，总结一下JUC包下的一些并发工具类，主要是从使用场景入手。

CompletionService可以用于实现任务并行化，提高任务处理的效率。以下是CompletionService在项目中的应用场景：

多个任务需要并行执行，但是它们的执行时间不同，需要优先处理先执行完成的任务。
需要获取多个任务的执行结果，但是不关心任务的执行顺序。
处理任务的线程池数量有限，需要充分利用线程池资源，提高任务处理的效率。

举例来说，假设有一个需要下载多个文件的任务，使用CompletionService可以将文件下载任务分配给多个线程并行执行，而不需要等待所有文件都下载完成才开始处理文件。同时，由于每个线程的下载时间可能不同，CompletionService可以优先获取下载完成的文件，并在下载完成后立即处理文件，提高任务处理的效率。

以下就是使用CompletionService从minio文件服务器下载文件，并进行打包压缩的使用场景；因为我们对文件进行打包压缩，并不关心下载的多个文件的下载顺序，哪个文件先下载完，就先处理哪个文件，然后最后统一放入到一个文件夹进行打包压缩，提供下载

业务场景如下：

需求是选中多个附件，然后批量下载，下载下来后是一个zip文件，附件使用的是minio文件存储服务

业务实现代码如下：

import cn.hutool.core.io.FileUtil;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.dcboot.module.common.dto.request.ContractDocumentPageQueryRequestDTO;
import com.dcboot.module.common.mapper.ContractAttachmentMidMapper;
import com.dcboot.module.common.service.ContractAttachmentMidService;
import com.dcboot.module.common.util.CaseFormatUtil;
import com.dcboot.module.entity.ContractAttachmentMid;
import com.dcboot.module.minio.configuration.MinioConfig;
import com.dcboot.module.minio.entity.MinioFile;
import com.dcboot.module.minio.service.MinioFileService;
import com.dcboot.module.util.MinioClientUtils;
import com.dcboot.module.util.ZipUtil;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.Setter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.concurrent.DelegatingSecurityContextExecutorService;
import org.springframework.stereotype.Service;

import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

/**
 * @author xiaomifeng1010
 * @version 1.0
 * @date: 2023/4/24 16:46
 * @Description
 */
@Service
@Setter(onMethod_ = @Autowired)
public class ContractAttachmentMidServiceImpl extends ServiceImpl<ContractAttachmentMidMapper, ContractAttachmentMid> implements ContractAttachmentMidService {

    private MinioClientUtils minioClientUtils;

    private MinioConfig minioConfig;

    private MinioFileService minioFileService;


    @Override
    public void export2Zip(List<Long> fileIds, HttpServletResponse response) throws IOException {
        if (CollectionUtils.isNotEmpty(fileIds)) {
            List<File> fileList = new ArrayList<>();
            ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("zip-file-%d").build();
            ExecutorService threadPoolExecutor = new ThreadPoolExecutor(12, 20, 0L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100),
                    threadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
            threadPoolExecutor = new DelegatingSecurityContextExecutorService(threadPoolExecutor);
            CompletionService<File> completionService = new ExecutorCompletionService<>(threadPoolExecutor);
            List<Callable<File>> callableList = Lists.newArrayList();
            fileIds.forEach(fileId -> {
                MinioFile minioFile = minioFileService.getById(fileId);
                if (minioFile != null) {
                    callableList.add(() -> {
                        String fileName = minioFile.getFileName();
                        String originalFileName = minioFile.getOriginalFileName();
                        originalFileName = StringUtils.substringBeforeLast(originalFileName, ".");
                        String fileExtName = minioFile.getFileExtName();
                        Integer access = minioFile.getAccess();
                        String bucketName = minioConfig.getBucketName();
//                   值为1 默认公有桶, 2 私有桶
                        if (access == 2) {
                            bucketName = minioConfig.getBucketNamePrivate();
                        }
                        try {
                            InputStream inputStream = minioClientUtils.getObject(bucketName, fileName);
                            if (inputStream != null) {
                                File javaIoTmpDir = SystemUtils.getJavaIoTmpDir();
                                String finalFilePath = javaIoTmpDir.getAbsolutePath() + File.separator + originalFileName + "." + fileExtName;
                                File newFile = FileUtil.newFile(finalFilePath);
                                if (newFile.exists()) {
                                    originalFileName = originalFileName + RandomStringUtils.randomNumeric(5);
                                }
                                if (StringUtils.length(originalFileName) < 3) {
                                    originalFileName = originalFileName + RandomStringUtils.randomNumeric(3);
                                }
                                File tempFile = FileUtil.createTempFile(originalFileName, "." + fileExtName, javaIoTmpDir, true);
                                tempFile = FileUtil.writeFromStream(inputStream, tempFile);
                                return tempFile;
                            }
                        } catch (Exception e) {
                            log.error("从minio获取文件失败：", e);
                        }
                        return null;
                    });


                }
            });
            if (CollectionUtils.isNotEmpty(callableList)) {
                for (Callable<File> fileCallable : callableList) {
                    completionService.submit(fileCallable);

                }

                for (int i = 0; i < callableList.size(); i++) {

                    try {
                        File temFile = completionService.take().get();
                        if (temFile != null) {
                            fileList.add(temFile);
                        }

                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (ExecutionException e) {
                        log.error("completionService获取文件对象失败:{}", e);
                    }

                }

            }
            threadPoolExecutor.shutdown();
            String zipFileName = "合同管理";
            File tempZipFile = FileUtil.createTempFile(zipFileName, ".zip", true);

            response.setContentType("application/zip");
            response.setHeader("Location", tempZipFile.getName());
            response.setHeader("Content-Disposition", "attachment; filename=" + tempZipFile.getName());
            OutputStream outputStream = response.getOutputStream();
            ZipUtil.filesToZip(fileList, outputStream);
            tempZipFile.delete();
            outputStream.flush();
            outputStream.close();
//            清理临时文件
            fileList.forEach(File::delete);

        }

    }



}

minio工具类，可以参照我的另一篇安装使用minio的博文中，有非常详细的用法MinIO文件服务器，从安装到使用

还有一个ziputil工具类

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ZipUtil {
    private static final Logger log = LoggerFactory.getLogger(ZipUtil.class);
    private static String destFileName;

    public ZipUtil() {
    }

    public static List<String> unZipFiles(File zipFile, String descDir) throws IOException {
        List<String> fileNames = new ArrayList();
        ZipFile zip = new ZipFile(zipFile, Charset.forName("GBK"));
        String name = zip.getName().substring(zip.getName().lastIndexOf(92) + 1, zip.getName().lastIndexOf(46));
        File pathFile = new File(descDir + name);
        if (!pathFile.exists()) {
            pathFile.mkdirs();
        }

        String outPath = "";
        Enumeration entries = zip.entries();

        while(true) {
            InputStream in;
            do {
                if (!entries.hasMoreElements()) {
                    pathFile.delete();
                    return fileNames;
                }

                ZipEntry entry = (ZipEntry)entries.nextElement();
                String zipEntryName = entry.getName();
                fileNames.add(zipEntryName);
                in = zip.getInputStream(entry);
                outPath = (descDir + name + "/" + zipEntryName).replaceAll("\\*", "/");
                File file = new File(outPath.substring(0, outPath.lastIndexOf(47)));
                if (!file.exists()) {
                    file.mkdirs();
                }
            } while((new File(outPath)).isDirectory());

            FileOutputStream out = new FileOutputStream(outPath);
            byte[] buf1 = new byte[1024];

            int len;
            while((len = in.read(buf1)) > 0) {
                out.write(buf1, 0, len);
            }

            in.close();
            out.close();
        }
    }

    public static void docToZip(String srcDir, String targetFile, boolean KeepDirStructure) throws RuntimeException, FileNotFoundException {
        long start = System.currentTimeMillis();
        ZipOutputStream zos = null;
        FileOutputStream out = new FileOutputStream(new File(targetFile));

        try {
            zos = new ZipOutputStream(out);
            File sourceFile = new File(srcDir);
            destFileName = targetFile.substring(targetFile.lastIndexOf(File.separator) + 1, targetFile.length());
            System.out.println(destFileName);
            compress(sourceFile, zos, sourceFile.getName(), KeepDirStructure);
            long end = System.currentTimeMillis();
            System.out.println("压缩完成，耗时：" + (end - start) + " ms");
        } catch (Exception var20) {
            throw new RuntimeException("zip error from ZipUtils", var20);
        } finally {
            if (zos != null) {
                try {
                    zos.close();
                } catch (IOException var19) {
                    log.error(var19.getMessage(), var19);
                }
            }

            if (out != null) {
                try {
                    out.close();
                } catch (IOException var18) {
                    log.error(var18.getMessage(), var18);
                }
            }

        }

    }

    public static void filesToZip(List<File> srcFiles, OutputStream out) throws RuntimeException {
        long start = System.currentTimeMillis();
        ZipOutputStream zos = null;
        short BUFFER_SIZE = 2048;

        try {
            zos = new ZipOutputStream(out);
            Iterator var6 = srcFiles.iterator();

            while(var6.hasNext()) {
                File srcFile = (File)var6.next();
                byte[] buf = new byte[BUFFER_SIZE];
                zos.putNextEntry(new ZipEntry(srcFile.getName()));
                FileInputStream in = new FileInputStream(srcFile);

                int len;
                while((len = in.read(buf)) != -1) {
                    zos.write(buf, 0, len);
                }

                zos.closeEntry();
                in.close();
            }

            long end = System.currentTimeMillis();
            System.out.println("压缩完成，耗时：" + (end - start) + " ms");
        } catch (Exception var18) {
            throw new RuntimeException("zip error from ZipUtils", var18);
        } finally {
            if (zos != null) {
                try {
                    zos.close();
                } catch (IOException var17) {
                    log.error(var17.getMessage(), var17);
                }
            }

        }
    }

    private static void compress(File sourceFile, ZipOutputStream zos, String name, boolean KeepDirStructure) throws Exception {
        int BUFFER_SIZE = 2048;
        byte[] buf = new byte[BUFFER_SIZE];
        System.out.println(name);
        if (sourceFile.isFile()) {
            if (!destFileName.equals(sourceFile.getName())) {
                zos.putNextEntry(new ZipEntry(name));
                FileInputStream in = new FileInputStream(sourceFile);

                int len;
                while((len = in.read(buf)) != -1) {
                    zos.write(buf, 0, len);
                }

                zos.closeEntry();
                in.close();
            }
        } else {
            File[] listFiles = sourceFile.listFiles();
            if (listFiles != null && listFiles.length != 0) {
                File[] var12 = listFiles;
                int var8 = listFiles.length;

                for(int var9 = 0; var9 < var8; ++var9) {
                    File file = var12[var9];
                    if (KeepDirStructure) {
                        compress(file, zos, name + "/" + file.getName(), KeepDirStructure);
                    } else {
                        compress(file, zos, file.getName(), KeepDirStructure);
                    }
                }
            } else if (KeepDirStructure) {
                zos.putNextEntry(new ZipEntry(name + "/"));
                zos.closeEntry();
            }
        }

    }
}

CompletionService调用线程池异步从minio下载文件，下载好的文件放到List集合，然后使用ziputil进行压缩，有个注意事项，就是在创建临时文件的时候，文件名的字符长度不能小于3，否则会抛出异常。所以在代码中有个文件名字符长度的判断。