【OSS对象存储】Springboot集成阿里云OSS + 私有化部署Minio
- 一、摘要
- 二、POM依赖
- 三、配置文件
- 四、表结构设计
- 五、代码实现
- 5.1 代码包结构
- 5.2 API封装
- 5.3 增删改查
- 六、扩展
- 6.1 Minio配置https访问
一、摘要
- 掌握阿里云OSS、私有化部署Minio两种对象存储的使用方式
- 运用工厂+策略模式，封装OSS对象存储API，可实现动态切换
- Docker部署Minio
二、POM依赖
<!-- 阿里云对象存储 -->
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.15.1</version>
</dependency>
<!-- Minio私有化对象存储 -->
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.5.6</version>
</dependency>
三、配置文件
ai-wxapp:
oss:
# strategy: ALI_OSS
# endpoint: oss-cn-beijing.aliyuncs.com
# accessKeyId: xxxxxx
# accessKeySecret: xxxxxx
# bucketName: xxxx
strategy: MinIO
endpoint: http://localhost:9000
accessKeyId: xxxxxx
accessKeySecret: xxxxxx
bucketName: xxxx
OssConfig.java：
@Data
@Component
@RefreshScope
public class OssConfig {
@Value("${ai-wxapp.oss.strategy}")
private String ossStrategy;
@Value("${ai-wxapp.oss.bucketName}")
private String bucketName;
@Value("${ai-wxapp.oss.endpoint}")
private String endpoint;
@Value("${ai-wxapp.oss.accessKeyId}")
private String accessKeyId;
@Value("${ai-wxapp.oss.accessKeySecret}")
private String accessKeySecret;
}
四、表结构设计
CREATE TABLE `oss_file` (
`id` bigint NOT NULL AUTO_INCREMENT,
`file_id` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '文件id',
`original_file_name` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT '原始文件名',
`file_ext` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT '文件扩展名',
`file_path` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT '文件路径',
`create_by` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT '创建人',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_by` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL COMMENT '修改人',
`update_time` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
PRIMARY KEY (`id`),
UNIQUE KEY `oss_file_unique` (`file_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='对象存储信息表';
五、代码实现
5.1 代码包结构
5.2 API封装
ObjectStorageEnum.java：
```java
/**
* @ClassName ObjectStorageEnum
* @Description 对象存储策略枚举
* @Author shn
* @Date 2023/11/2 18:17
* @Version 1.0
**/
@Getter
public enum ObjectStorageEnum {
/**
* 阿里云OSS对象存储
*/
ALI_OSS,
/**
* MinIO私有化对象存储
*/
MinIO
}
IOssFileRepository.java
public interface IOssFileRepository {
OssFileEntity getByFileId(String fileId);
List<OssFileEntity> getByFileIds(List<String> fileIds);
void saveEntity(OssFileEntity build);
}
ObjectStorageFactory.java
/**
* @ClassName ObjectStorageFactory
* @Description 对象存储策略工厂
* @Author shn
* @Date 2023/11/2 18:23
* @Version 1.0
**/
@Component
public class ObjectStorageFactory implements ApplicationContextAware, InitializingBean {
private ApplicationContext applicationContext;
private static final Map<ObjectStorageEnum, ObjectStorage> SERVICES = new HashMap<>();
/**
* 根据枚举获取策略实现类实例
*/
public ObjectStorage getInstance(ObjectStorageEnum storageEnum) {
return SERVICES.get(storageEnum);
}
@Override
public void afterPropertiesSet() {
Map<String, ObjectStorage> beans = applicationContext.getBeansOfType(ObjectStorage.class);
for (ObjectStorage bean : beans.values()) {
SERVICES.put(bean.getStrategyEnum(), bean);
}
}
@Override
public void setApplicationContext(@NotNull ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}
ObjectStorage.java：
/**
* @ClassName ObjectStorage
* @Description 对象存储接口，异常全部对外抛出
* @Author shn
* @Date 2023/11/2 17:51
* @Version 1.0
**/
public interface ObjectStorage {
/**
* 获取策略枚举
*/
ObjectStorageEnum getStrategyEnum();
/**
* 判断对象是否存在
*
* @param bucketName 桶名称
* @param objectName 对象存储全路径，不包含桶名称
*/
boolean objectExists(String bucketName, String objectName) throws Exception;
/**
* 上传对象
*
* @param bucketName 桶名称
* @param objectName 对象存储全路径，不包含桶名称
* @param in 输入流
*/
void uploadObject(String bucketName, String objectName, InputStream in) throws Exception;
/**
* 下载对象
*
* @param bucketName 桶名称
* @param objectName 对象存储全路径，不包含桶名称
*/
byte[] downloadObject(String bucketName, String objectName) throws Exception;
/**
* 删除对象
*
* @param bucketName 桶名称
* @param objectName 对象存储全路径，不包含桶名称
*/
void deleteObject(String bucketName, String objectName) throws Exception;
/**
* 删除对象
*
* @param bucketName 桶名称
* @param objectNames 对象存储全路径，不包含桶名称
*/
void deleteObjects(String bucketName, List<String> objectNames);
/**
* 判断桶是否存在
*
* @param bucketName 桶名称
*/
boolean bucketExists(String bucketName) throws Exception;
/**
* 创建桶
*
* @param bucketName 桶名称
*/
void createBucket(String bucketName) throws Exception;
/**
* 删除桶
*
* @param bucketName 桶名称
*/
void deleteBucket(String bucketName) throws Exception;
/**
* 列出桶中的对象
*
* @param bucketName 桶名称
*/
List<String> listObjects(String bucketName) throws Exception;
/**
* 获取临时授权访问url
*
* @param bucketName 桶名称
* @param objectName 对象存储全路径，不包含桶名称
* @param expireSeconds 过期时间（单位：秒）
* @return java.lang.String
* @author shn 2023/11/8 13:27
*/
String getTempAccessUrl(String bucketName, String objectName, int expireSeconds);
}
AliOssService.java
/**
* @ClassName AliOssService
* @Description 阿里云OSS对象存储
* @Author shn
* @Date 2023/11/2 18:10
* @Version 1.0
**/
@Slf4j
@Service("aliOssService")
public class AliOssService implements ObjectStorage {
@Resource
private OssConfig ossConfig;
/**
* 获取ossClient <br>
* 1）每次请求都创建一个新的ossClient <br>
* 2）虽然ossClient是线程安全的，但是考虑到OssConfig支持动态刷新，所以这里没有设计为单例。<br>
* 3）因此ossClient需要手动关闭
* 参考：<a href="https://help.aliyun.com/zh/oss/developer-reference/initialization-3">...</a> <br>
*/
private OSS getOssClient() {
return new OSSClientBuilder().build(ossConfig.getEndpoint(),
ossConfig.getAccessKeyId(), ossConfig.getAccessKeySecret());
}
/**
* 关闭ossClient
*/
private void close(OSS ossClient) {
if (ossClient != null) {
ossClient.shutdown();
}
}
@Override
public ObjectStorageEnum getStrategyEnum() {
return ObjectStorageEnum.ALI_OSS;
}
@Override
public boolean objectExists(String bucketName, String objectName) {
OSS ossClient = getOssClient();
try {
boolean bucketExist = ossClient.doesBucketExist(bucketName);
if (!bucketExist) {
return false;
}
boolean exists = ossClient.doesObjectExist(bucketName, objectName);
if (!exists) {
return false;
}
} finally {
close(ossClient);
}
return true;
}
@Override
public void uploadObject(String bucketName, String objectName, InputStream in) throws Exception {
if (in.available() <= 0) {
throw new IOException("输入流为空");
}
OSS ossClient = getOssClient();
try {
boolean bucketExist = ossClient.doesBucketExist(bucketName);
if (!bucketExist) {
throw new ServiceException("bucket不存在");
}
ossClient.putObject(bucketName, objectName, in);
log.info("文件上传OSS成功，文件路径: {}", objectName);
} finally {
close(ossClient);
in.close();
}
}
@Override
public byte[] downloadObject(String bucketName, String objectName) {
OSS ossClient = getOssClient();
try {
boolean exists = ossClient.doesObjectExist(bucketName, objectName);
if (!exists) {
throw new ServiceException("文件不存在");
}
// ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
OSSObject ossObject = ossClient.getObject(bucketName, objectName);
InputStream in = ossObject.getObjectContent();
// 该API默认会关闭流
return IoUtil.readBytes(in);
} finally {
close(ossClient);
}
}
@Override
public void deleteObject(String bucketName, String objectName) {
OSS ossClient = getOssClient();
try {
// 删除文件或目录。如果要删除目录，目录必须为空。
ossClient.deleteObject(bucketName, objectName);
} finally {
close(ossClient);
}
}
@Override
public void deleteObjects(String bucketName, List<String> objectNames) {
OSS ossClient = getOssClient();
try {
DeleteObjectsRequest request = new DeleteObjectsRequest(bucketName);
request.setKeys(objectNames);
// 静默删除，删除操作不返回删除结果信息，效率高，需结合业务场景选择是否开启
request.setQuiet(true);
ossClient.deleteObjects(request);
} finally {
close(ossClient);
}
}
@Override
public boolean bucketExists(String bucketName) {
OSS ossClient = getOssClient();
try {
// 判断存储空间是否存在
boolean exists = ossClient.doesBucketExist(bucketName);
if (!exists) {
return false;
}
} finally {
close(ossClient);
}
return true;
}
@Override
public void createBucket(String bucketName) {
OSS ossClient = getOssClient();
try {
// 创建CreateBucketRequest对象。
CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);
// 如果创建存储空间的同时需要指定存储类型、存储空间的读写权限、数据容灾类型, 请参考如下代码。
// 此处以设置存储空间的存储类型为标准存储为例介绍。
//createBucketRequest.setStorageClass(StorageClass.Standard);
// 数据容灾类型默认为本地冗余存储，即DataRedundancyType.LRS。如果需要设置数据容灾类型为同城冗余存储，请设置为DataRedundancyType.ZRS。
//createBucketRequest.setDataRedundancyType(DataRedundancyType.ZRS);
// 设置存储空间读写权限为公共读，默认为私有。
//createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
// 在支持资源组的地域创建Bucket时，您可以为Bucket配置资源组。
//createBucketRequest.setResourceGroupId(rsId);
// 创建存储空间
ossClient.createBucket(createBucketRequest);
} finally {
close(ossClient);
}
}
@Override
public void deleteBucket(String bucketName) {
OSS ossClient = getOssClient();
try {
// 删除存储空间
ossClient.deleteBucket(bucketName);
} finally {
close(ossClient);
}
}
@Override
public List<String> listObjects(String bucketName) {
return null;
}
@Override
public String getTempAccessUrl(String bucketName, String objectName, int expireSeconds) {
OSS ossClient = getOssClient();
try {
boolean exists = ossClient.doesObjectExist(bucketName, objectName);
if (!exists) {
throw new ServiceException("文件不存在");
}
String url = ossClient.generatePresignedUrl(
bucketName, objectName, DateUtil.offsetSecond(new Date(), expireSeconds)).toString();
return StringUtil.replaceHttp2Https(url);
} finally {
close(ossClient);
}
}
}
MinIoService.java：
@Slf4j
@Service("minIoService")
public class MinIoService implements ObjectStorage {
@Resource
private OssConfig ossConfig;
/**
* 获取 MinioClient，每次请求都创建一个新的 MinioClient
*/
private MinioClient getMinioClient() {
return MinioClient.builder()
.endpoint(ossConfig.getEndpoint())
.credentials(ossConfig.getAccessKeyId(), ossConfig.getAccessKeySecret())
.build();
}
@Override
public ObjectStorageEnum getStrategyEnum() {
return ObjectStorageEnum.MinIO;
}
@SneakyThrows(Exception.class)
@Override
public boolean objectExists(String bucketName, String objectName) {
return false;
}
@SneakyThrows(Exception.class)
@Override
public void uploadObject(String bucketName, String objectName, InputStream in) {
if (in.available() <= 0) {
throw new IOException("输入流为空");
}
try {
getMinioClient().putObject(
PutObjectArgs.builder().bucket(bucketName).object(objectName)
.stream(in, in.available(), -1).build());
log.info("文件上传Minio成功，文件路径: {}", objectName);
} finally {
in.close();
}
}
@SneakyThrows(Exception.class)
@Override
public byte[] downloadObject(String bucketName, String objectName) {
InputStream in = getMinioClient().getObject(
GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
// 该API默认会关闭流
return IoUtil.readBytes(in);
}
@SneakyThrows(Exception.class)
@Override
public void deleteObject(String bucketName, String objectName) {
getMinioClient().removeObject(
RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
@SneakyThrows(Exception.class)
@Override
public void deleteObjects(String bucketName, List<String> objectNames) {
if (CollectionUtil.isEmpty(objectNames)) {
return;
}
getMinioClient().removeObjects(
RemoveObjectsArgs.builder()
.bucket(bucketName)
.objects(objectNames.stream().map(DeleteObject::new).collect(Collectors.toList()))
.build());
}
@SneakyThrows(Exception.class)
@Override
public boolean bucketExists(String bucketName) {
return getMinioClient().bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
}
@SneakyThrows(Exception.class)
@Override
public void createBucket(String bucketName) {
MinioClient minioClient = getMinioClient();
boolean exist = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
if (!exist) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
}
}
@SneakyThrows(Exception.class)
@Override
public void deleteBucket(String bucketName) {
getMinioClient().removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
}
@SneakyThrows(Exception.class)
@Override
public List<String> listObjects(String bucketName) {
return null;
}
@SneakyThrows(Exception.class)
@Override
public String getTempAccessUrl(String bucketName, String objectName, int expireSeconds) {
String url = getMinioClient().getPresignedObjectUrl(
GetPresignedObjectUrlArgs.builder()
.method(Method.GET)
.bucket(bucketName)
.object(objectName)
.expiry(expireSeconds).build());
log.info("Minio origin url: {}", url);
return url;
}
}
5.3 增删改查
OssFileEntity.java：
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class OssFileEntity {
private String fileId;
private String originalFileName;
private String fileExt;
private String filePath;
}
IOssFileService.java
```java
public interface IOssFileService {
OssFileEntity getByFileId(String fileId);
byte[] downloadFile(String fileId) throws Exception;
void deleteFiles(List<String> fileIds);
String uploadFile(byte[] fileBytes, String originalFileName);
String getTempAccessUrl(String fileId, int expireSeconds);
}
OssFileService.java
@Slf4j
@Service
public class OssFileService implements IOssFileService {
@Resource
private OssConfig ossConfig;
@Resource
private ObjectStorageFactory objectStorageFactory;
@Resource
private RedisUtils redisUtils;
@Resource
private IOssFileRepository ossFileRepository;
private ObjectStorage getObjectStorage() {
return objectStorageFactory.getInstance(ObjectStorageEnum.valueOf(ossConfig.getOssStrategy()));
}
public OssFileEntity getByFileId(String fileId) {
OssFileEntity entity = ossFileRepository.getByFileId(fileId);
if (entity == null) {
throw new ServiceException(CommonError.OSS_FILE_NOT_EXIST);
}
return entity;
}
public byte[] downloadFile(String fileId) throws Exception {
OssFileEntity entity = getByFileId(fileId);
return getObjectStorage().downloadObject(ossConfig.getBucketName(), entity.getFilePath());
}
public void deleteFiles(List<String> fileIds) {
List<OssFileEntity> list = ossFileRepository.getByFileIds(fileIds);
if (CollectionUtil.isEmpty(list)) {
return;
}
getObjectStorage().deleteObjects(ossConfig.getBucketName(),
list.stream().map(OssFileEntity::getFilePath).collect(Collectors.toList()));
}
/**
* 文件上传oss
*
* @param fileBytes 文件
* @param originalFileName 文件原始名称
* @return java.lang.String 文件id
* @author shn 2023/12/13 11:42
*/
public String uploadFile(byte[] fileBytes, String originalFileName) {
if (ArrayUtil.isEmpty(fileBytes)
|| StringUtils.isBlank(originalFileName)) {
throw new ServiceException(CommonError.OSS_FILE_UPLOAD_ERROR);
}
// 上传oss
String fileId = this.generateFileId();
String fileExtension = FileUtil.getFileExtension(originalFileName);
String filePath = buildFilePath(fileId, fileExtension);
try {
InputStream inputStream = new ByteArrayInputStream(fileBytes);
getObjectStorage().uploadObject(ossConfig.getBucketName(), filePath, inputStream);
} catch (Exception e) {
log.error("文档上传失败", e);
throw new ServiceException(CommonError.OSS_FILE_UPLOAD_ERROR);
}
// 保存数据库
ossFileRepository.saveEntity(OssFileEntity.builder()
.fileId(fileId)
.originalFileName(originalFileName)
.fileExt(fileExtension)
.filePath(filePath)
.build());
return fileId;
}
/**
* 获取临时url
*
* @param fileId 文件id
* @param expireSeconds 过期时间（秒）
* @return java.lang.String
* @author shn 2024/05/29 15:36
*/
public String getTempAccessUrl(String fileId, int expireSeconds) {
OssFileEntity entity = getByFileId(fileId);
// 先查缓存
String key = String.format(RedisKeys.OSS_FILE_URL_KEY, fileId);
if (redisUtils.hasKey(key) && redisUtils.getExpire(key) > 0L) {
return String.valueOf(redisUtils.get(key));
}
// 缓存不存在，则从OSS获取临时URL。设置OSS过期时间比 Redis晚2秒，保证 URL一直可用。
String url = getObjectStorage().getTempAccessUrl(
ossConfig.getBucketName(), entity.getFilePath(), expireSeconds + 2);
// 保存临时URL到Redis
redisUtils.set(key, url, expireSeconds, TimeUnit.SECONDS);
return url;
}
/**
* 构建文件存储路径
*/
@NotNull
private static String buildFilePath(String fileId, String fileExtension) {
return DateUtil.today() + "/" + fileId + fileExtension;
}
/**
* 生成文件id
*/
private String generateFileId() {
String fileId = "file_" + IdUtil.simpleUUID();
log.info("generate new file id: {}", fileId);
return fileId;
}
}
六、扩展
6.1 Minio配置https访问
目标：实现https域名访问minio控制台、资源
步骤：
1）前往阿里云下载Apache证书文件
2）修改公钥和私钥文件名为private.key、public.crt 并上传至服务端
3）Nginx配置
upstream minio_s3 {
#least_conn;
server 127.0.0.1:9000;
}
upstream minio_console {
#least_conn;
server 127.0.0.1:9001;
}
server {
listen 80;
server_name huiling.xxxx.com;
listen 443 ssl;
server_name huiling.xxxx.com;
# SSL-START SSL相关配置，请勿删除或修改下一行带注释的404规则
# error_page 404/404.html;
ssl_certificate /etc/nginx/ssl/huiling/huiling.xxxx.com.pem; # pem文件的路径
ssl_certificate_key /etc/nginx/ssl/huiling/huiling.xxxx.com.key; # key文件的路径
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:HIGH:!aNULL:!MD5:!RC4:!DHE;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 10m;
# error_page 497 https://$host$request_uri;
proxy_buffering off;
proxy_request_buffering off;
location / {
client_max_body_size 10m;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass https://minio_s3; # This uses the upstream directive definition to load balance
}
location /minio/ui/ {
rewrite ^/minio/ui/(.*) /$1 break;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websockets in MinIO versions released after January 2023
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Some environments may encounter CORS errors (Kubernetes + Nginx Ingress)
# Uncomment the following line to set the Origin request to an empty string
# proxy_set_header Origin '';
chunked_transfer_encoding off;
proxy_pass https://minio_console; # This uses the upstream directive definition to load balance
}
}
4）Docker启动配置
# 定义compose语义版本
version: '3.8'
# 定义服务
services:
minio:
image: minio/minio:latest
container_name: minio
restart: unless-stopped
command: server /data --console-address ":9001" -address ":9000"
environment:
TZ: Asia/Shanghai
LANG: en_US.UTF-8
MINIO_SERVER_URL: https://huiling.xxx.com
MINIO_BROWSER_REDIRECT_URL: https://huiling.xxx.com/minio/ui
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin9339
volumes:
- "/usr/local/docker/minio/data:/data"
- "/usr/local/docker/minio/config:/root/.minio"
ports:
- "9000:9000"
- "9001:9001"
5）验证
![[Vulnhub] Acid-Reloaded SQLI+图片数据隐写提取+Pkexec权限提升+Overlayfs权限提升](https://img-blog.csdnimg.cn/img_convert/d9709e4c2c8ce39e883012e2f64a51e5.jpeg)
