关于minio的简单使用

最近在学习minio相关知识，小小的记录一下学习内容

MinIO 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。

MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL。目前支持JavaScript 、Java、Python、Golang、.NET。

官网：https://min.io/ 中文文档：http://www.minio.org.cn/

源码地址：https://github.com/minio/minio

由于使用的是windows10的电脑，因此下载的是windows版本的可查考官方提供文档进行操作https://min.io/download#/windows

也可以直接下载可执行exehttps://dl.min.io/server/minio/release/windows-amd64/minio.exe

下载完成后不要不要双击minio.exe文件，在存放minio.exe文件夹中新建一个minioData文件夹，用来储存minio上传的文件目录，在地址栏里输入cmd按下回车键输入minio.exe server E:\minio\minioData(启动minio的命令)

至此就可以在浏览器上进行创建bucket，文件的上传、下载、删除等一系列操作了。

使用springboot进行minio的相关操作，首先需要添加minio依耐

<dependency>
    <groupId>io.minio</groupId>
    <artifactId>minio</artifactId>
    <version>8.5.4</version>
</dependency>

在yml或properties进行minio配置

minio:
  endpoint: http://192.168.1.102:9000 #Minio服务所在地址
  bucketName: test #存储桶名称(文件夹名称)
  accessKey: minioadmin #访问的key(类似用户名)
  secretKey: minioadmin #访问的秘钥(类似密码)

minio基本配置

@Component
@ConfigurationProperties(prefix = "minio")
@Data
public class MinioProperties {
    private String endpoint;
    private String bucketName;
    private String accessKey;
    private String secretKey;

}

简单的上传、下载文件及压缩包、查询列表等访问操作

@Slf4j
@RestController
@RequestMapping("/minio")
public class MinioController {
    @Autowired
    private MinioClient minioClient;
    
    @Autowired
    private MinioProperties minioProperties;

    private List<String> docList = new ArrayList<>();

    /**
     * 获取文件列表
     * @param map
     * @return
     * @throws Exception
     */
    @GetMapping("/list")
    public List<Object> list(ModelMap map) throws Exception {
        ListObjectsArgs listObjectsArgs = ListObjectsArgs.builder().bucket(minioProperties.getBucketName()).build();
        Iterable<Result<Item>> myObjects = minioClient.listObjects(listObjectsArgs);
//        Iterable<Result<Item>> myObjects = minioClient.listObjects(minioProperties.getBucketName());
        Iterator<Result<Item>> iterator = myObjects.iterator();
        List<Object> items = new ArrayList<>();
        String format = "{'fileName':'%s','fileSize':'%s'}";
        while (iterator.hasNext()) {
            Item item = iterator.next().get();
            items.add(JSON.parse(String.format(format, item.objectName(), formatFileSize(item.size()))));
        }
        return items;
    }

    /**
     * 查看所有的上传文件列表
     * @param pageNum
     * @param pageSize
     * @return
     * @throws Exception
     */
    @ResponseBody
    @RequestMapping(value = "/fileList", method = RequestMethod.GET, produces = {"application/json;charset=utf-8"})
    public String getFileList(Integer pageNum, Integer pageSize) throws Exception {
        List<Bucket> buckets = minioClient.listBuckets();
        List<FileVo> list = new ArrayList<>();
        if (!buckets.isEmpty()) {
            for (int i = 0; i < buckets.size(); i++) {
                Bucket s = buckets.get(i);
                listDocs(s.name(),"", list);
            }
        }
        JSONObject res = new JSONObject();
        res.put("code", 200);
        res.put("message", "获取文件列表成功");
        // 按最后上传时间排序
        list.sort(new Comparator<FileVo>() {
            @Override
            public int compare(FileVo o1, FileVo o2) {
                return o2.getUpdateTime().compareTo(o1.getUpdateTime());
            }
        });
        // 分页
        List returnList = PageUtil.startPage(list, pageNum, pageSize);
        res.put("list", returnList);
        ObjectMapper mapper = new ObjectMapper();
        String s = mapper.writeValueAsString(res);
        return s;
    }

    public void listDocs(String bucketName,String prefix, List<FileVo> list){
        DecimalFormat df = new DecimalFormat("0.00");
        Iterable<Result<Item>> listObjects=new ArrayList<>();
        if(StringUtils.hasLength(prefix)){
            listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                    .bucket(bucketName)
                    .prefix(prefix)
                    .build());
        }else {
            listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                    .bucket(bucketName)
                    .build());
        }
        try{
            for (Result<Item> result : listObjects) {
                Item item = result.get();
                if(item.isDir()){
                    docList.add(item.objectName());
                    listDocs(bucketName,item.objectName(),list);
                }else{
                    FileVo fileVo = new FileVo();
                    fileVo.setBucketName(bucketName);  // 文件夹名称
                    String name = URLDecoder.decode(item.objectName(),"utf-8");
                    fileVo.setFileName(name);  // 文件名称
                    fileVo.setUpdateTime(localDateTime2Date(item.lastModified().toLocalDateTime()));  // 文件上传时间
                    Long size = item.size();
                    if (size > (1024 * 1024)) {
                        fileVo.setFileSize(df.format(((double) size / 1024 / 1024)) + "MB");  // 文件大小，如果超过1M，则把单位换成MB
                    } else if (size > 1024) {
                        fileVo.setFileSize(df.format(((double) size / 1024)) + "KB"); // 文件大小，如果没超过1M但是超过1000字节，则把单位换成KB
                    } else {
                        fileVo.setFileSize(size + "bytes");  // // 文件大小，如果没超过1000字节，则把单位换成bytes
                    }
                    list.add(fileVo);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    public Iterable<Result<Item>> listObjects(String bucketName,String prefix,String title,boolean recursive){
        try {
            Iterable<Result<Item>> listObjects=new ArrayList<>();
            if(StringUtils.hasLength(title)){
                listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                        .bucket(bucketName)
                        .prefix(prefix)
                        .startAfter(title)
                        .build());
            }
            else if(StringUtils.hasLength(prefix)){
                listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                        .bucket(bucketName)
                        .prefix(prefix)
                        .recursive(recursive) // 递归
                        .build());
            }else {
                listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                        .bucket(bucketName)
                        .recursive(recursive) // 递归
                        .build());
            }
            return listObjects;
        } catch (Exception e) {
            System.out.println("Error occurred: " + e);
            e.printStackTrace();
            return null;
        }
    }

    public Date localDateTime2Date( LocalDateTime localDateTime){
        ZoneId zoneId = ZoneId.systemDefault();
        ZonedDateTime zdt = localDateTime.atZone(zoneId);
        Date date = Date.from(zdt.toInstant());
        Calendar cal=Calendar.getInstance();
        cal.setTime(date);
        // 由于获取的时间存在时间差，我这里手动加上16小时
        cal.add(Calendar.HOUR_OF_DAY, 16);
        date = cal.getTime();
        return date;
    }


    /**
     * 上传文件
     * @param file 要上传的文件
     * @return
     */
    @PostMapping("/upload")
    public Res upload(@RequestParam(name = "file", required = false) MultipartFile[] file) throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException, RegionConflictException, ServerException {
        Res res = new Res();
        res.setCode(500);

        if (file == null || file.length == 0) {
            res.setMessage("上传文件不能为空");
            return res;
        }

        //存入bucket不存在则创建，并设置为只读
        if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(minioProperties.getBucketName()).build())){
            minioClient.makeBucket(MakeBucketArgs.builder().bucket(minioProperties.getBucketName()).build());
//        if (!minioClient.bucketExists(minioProperties.getBucketName())) {
//            minioClient.makeBucket(minioProperties.getBucketName());
            //minioClient.setBucketPolicy(minioProperties.getBucketName(), "*.*", PolicyType.READ_ONLY);
        }

        List<String> orgfileNameList = new ArrayList<>(file.length);

        for (MultipartFile multipartFile : file) {
            // 获取真实文件名
            String orgfileName = multipartFile.getOriginalFilename();
            // 父目录名
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
            // 文件存储的目录结构
            String objectName = sdf.format(new Date()) + "/" + orgfileName;

            try {
                InputStream in = multipartFile.getInputStream();
                // 开始上传
                PutObjectArgs objectArgs = PutObjectArgs.builder().object(objectName).bucket(minioProperties.getBucketName()).contentType("application/octet-stream").stream(in,in.available(),-1).build();
                minioClient.putObject(objectArgs);
//                minioClient.putObject(minioProperties.getBucketName(), objectName, in, new PutObjectOptions(in.available(), -1));
                in.close();
                // 完成上传以后再保存文件完整路径
                String fullFilePath = minioProperties.getEndpoint() + "/" + minioProperties.getBucketName() + "/" + objectName;
                orgfileNameList.add(fullFilePath);
            } catch (Exception e) {
                log.error(e.getMessage());
                res.setMessage("上传失败");
                return res;
            }
        }

        Map<String, Object> data = new HashMap<String, Object>();
        data.put("bucketName", minioProperties.getBucketName());
        data.put("fileName", orgfileNameList);
        res.setCode(200);
        res.setMessage("上传成功");
        res.setData(data);
        return res;
    }

    /**
     * 将指定文件以压缩包下载
     * @param response
     * @param fileName 要下载的文件名
     */
    @RequestMapping("/downLoadZip")
    public void downLoadZip(HttpServletResponse response, @RequestParam("fileName") String fileName){
        ZipOutputStream zipos = null;
        DataOutputStream os = null;
        InputStream is = null;
        try {
            response.reset();
            String zipName = new String(URLEncoder.encode(fileName, "UTF-8").getBytes(), StandardCharsets.ISO_8859_1);
            response.setHeader("Content-Disposition", "attachment;fileName=\"" + zipName + ".zip\"");
            response.setContentType("application/octet-stream; charset=UTF-8");
            zipos = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()));
            zipos.setMethod(ZipOutputStream.DEFLATED); //设置压缩方法
            try {
                zipos.putNextEntry(new ZipEntry(fileName));
                os = new DataOutputStream(zipos);
                GetObjectArgs getObjectArgs = GetObjectArgs.builder().object(fileName).bucket(minioProperties.getBucketName()).build();
                is = minioClient.getObject(getObjectArgs);
//                is = minioClient.getObject(minioProperties.getBucketName(), fileName);
                byte[] b = new byte[1024];
                int length = 0;
                while((length = is.read(b))!= -1){
                    os.write(b, 0, length);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            is.close();
            os.flush();
            os.close();
            zipos.close();
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            if (is != null) {
                try {
                    is.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            if (os != null) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (zipos != null) {
                try {
                    zipos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 将bucket下所有文件以压缩包下载
     * @param response
     * @param bucket 要下载的bucket
     */
    @RequestMapping("/batchDownLoadZip")
    public void batchDownLoadZip(HttpServletResponse response, @RequestParam("bucket") String bucket) {
        List<String> filePaths = new ArrayList<>();
        listZips(bucket,"2023",filePaths);
        ZipOutputStream zipos = null;
        DataOutputStream os = null;
        InputStream is = null;
        try {
            response.reset();
            String zipName = new String(URLEncoder.encode(bucket, "UTF-8").getBytes(), StandardCharsets.ISO_8859_1);
            response.setHeader("Content-Disposition", "attachment;fileName=\"" + zipName + ".zip\"");
            response.setContentType("application/octet-stream; charset=UTF-8");
            zipos = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()));
            zipos.setMethod(ZipOutputStream.DEFLATED); //设置压缩方法
            for(int i = 0;i < filePaths.size();i++) {
                String file = filePaths.get(i);
                String packageName = filePaths.get(i).replace(bucket+"/", "");
                try {
                    zipos.putNextEntry(new ZipEntry(packageName));
                    os = new DataOutputStream(zipos);
                    GetObjectArgs getObjectArgs = GetObjectArgs.builder().object(file).bucket(minioProperties.getBucketName()).build();
                    is = minioClient.getObject(getObjectArgs);
                    byte[] b = new byte[1024];
                    int length = 0;
                    while((length = is.read(b))!= -1){
                        os.write(b, 0, length);
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            is.close();
            os.flush();
            os.close();
            zipos.close();
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            if (is != null) {
                try {
                    is.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            if (os != null) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (zipos != null) {
                try {
                    zipos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 遍历bucket下所有文件名
     * @param bucketName 要下载的bucket
     * @param prefix 前缀
     * @param filePaths 文件名列表
     */
    public void listZips(String bucketName,String prefix, List<String> filePaths){
        Iterable<Result<Item>> listObjects=new ArrayList<>();
        if(StringUtils.hasLength(prefix)){
            listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                    .bucket(bucketName)
                    .prefix(prefix)
                    .build());
        }else {
            listObjects = minioClient.listObjects(ListObjectsArgs.builder()
                    .bucket(bucketName)
                    .build());
        }
        try{
            for (Result<Item> result : listObjects) {
                Item item = result.get();
                if(item.isDir()){
                    listZips(bucketName,item.objectName(),filePaths);
                }else{
                    String name = URLDecoder.decode(item.objectName(),"utf-8");
                    filePaths.add(name);  // 文件名称
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    /**
     * 下载指定文件
     * @param response
     * @param fileName 要下载的文件名
     */
    @RequestMapping("/download")
    public void download(HttpServletResponse response, @RequestParam("fileName") String fileName) {
        InputStream in = null;
        try {
            String name = "";
            //文件名截取
            if(fileName.lastIndexOf("/") != -1 ){
                name = fileName.substring(fileName.lastIndexOf("/") + 1);
            }else{
                name = fileName;
            }
            StatObjectArgs statObjectArgs = StatObjectArgs.builder().object(fileName).bucket(minioProperties.getBucketName()).build();
//            ObjectStat stat = minioClient.statObject(minioProperties.getBucketName(), fileName);
            ObjectStat stat = minioClient.statObject(statObjectArgs);
            response.setContentType(stat.contentType());
            response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(name, "UTF-8"));

            GetObjectArgs getObjectArgs = GetObjectArgs.builder().object(fileName).bucket(minioProperties.getBucketName()).build();
//            in = minioClient.getObject(minioProperties.getBucketName(), fileName);
            in = minioClient.getObject(getObjectArgs);
            IOUtils.copy(in, response.getOutputStream());

        } catch (Exception e) {
            log.error(e.getMessage());
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    log.error(e.getMessage());
                }
            }
        }
    }

    /**
     * 删除文件
     * @param fileName 要删除的文件名
     * @return
     */
    @DeleteMapping("/delete")
    public Res delete(@RequestParam("fileName") String fileName) {
        Res res = new Res();
        res.setCode(200);
        try {
            RemoveObjectArgs removeObjectArgs = RemoveObjectArgs.builder().object(fileName).bucket(minioProperties.getBucketName()).build();
            minioClient.removeObject(removeObjectArgs);
//            minioClient.removeObject(minioProperties.getBucketName(), fileName);
        } catch (Exception e) {
            res.setCode(500);
            log.error(e.getMessage());
        }
        return res;
    }


    /**
     * 生成可以预览的文件链接
     * @return
     * @throws XmlParserException
     * @throws NoSuchAlgorithmException
     * @throws InsufficientDataException
     * @throws InternalException
     * @throws InvalidResponseException
     * @throws InvalidKeyException
     * @throws InvalidBucketNameException
     * @throws ErrorResponseException
     * @throws IOException
     * @throws InvalidExpiresRangeException
     */
    @GetMapping("/previewList")
    public List<Object> getPreviewList() throws XmlParserException, NoSuchAlgorithmException, InsufficientDataException, InternalException, InvalidResponseException, InvalidKeyException, InvalidBucketNameException, ErrorResponseException, IOException, InvalidExpiresRangeException, ServerException {
        ListObjectsArgs listObjectsArgs = ListObjectsArgs.builder().bucket(minioProperties.getBucketName()).build();
        Iterable<Result<Item>> myObjects = minioClient.listObjects(listObjectsArgs);
//        Iterable<Result<Item>> myObjects = minioClient.listObjects(minioProperties.getBucketName());
        Iterator<Result<Item>> iterator = myObjects.iterator();
        List<Object> items = new ArrayList<>();
        String format = "{'fileName':'%s'}";
        while (iterator.hasNext()) {
            Item item = iterator.next().get();
            // TODO 根据文件后缀名，过滤哪些是可以预览的文件
            //String bucketName, 桶名称
            // String objectName, 文件路径
            // Integer expires, 链接过期时间
            // Map<String, String> reqParams 请求参数
            // 开始生成
            String filePath = minioClient.presignedGetObject(minioProperties.getBucketName(), item.objectName());
            items.add(JSON.parse(String.format(format, filePath)));
        }
        return  items;
    }



    /**
     * 显示文件大小信息单位
     * @param fileS
     * @return
     */
    private static String formatFileSize(long fileS) {
        DecimalFormat df = new DecimalFormat("#.00");
        String fileSizeString = "";
        String wrongSize = "0B";
        if (fileS == 0) {
            return wrongSize;
        }
        if (fileS < 1024) {
            fileSizeString = df.format((double) fileS) + " B";
        } else if (fileS < 1048576) {
            fileSizeString = df.format((double) fileS / 1024) + " KB";
        } else if (fileS < 1073741824) {
            fileSizeString = df.format((double) fileS / 1048576) + " MB";
        } else {
            fileSizeString = df.format((double) fileS / 1073741824) + " GB";
        }
        return fileSizeString;
    }
}