分片上传则是将一个大文件分割成多个小块分别上传，最后再由服务器合并成完整的文件。这种做法的好处是可以并行处理多个小文件，提高上传效率；同时，如果某一部分上传失败，只需要重传这一部分，不影响其他部分。
初步实现
后端代码
/**
* 分片上传
*
* @param file 上传的文件
* @param start 文件开始上传的位置
* @param fileName 文件名称
* @return 上传结果
*/
@PostMapping("/fragmentUpload")
@ResponseBody
public AjaxResult fragmentUpload(@RequestParam("file") MultipartFile file, @RequestParam("start") long start, @RequestParam("fileName") String fileName) {
try {
// 检查上传目录是否存在，如果不存在则创建
File directory = new File(uploadPath);
if (!directory.exists()) {
directory.mkdirs();
}
// 设置上传文件的目标路径
File targetFile = new File(uploadPath +File.separator+ fileName);
// 创建 RandomAccessFile 对象以便进行文件的随机读写操作
RandomAccessFile randomAccessFile = new RandomAccessFile(targetFile, "rw");
// 获取 RandomAccessFile 对应的 FileChannel
FileChannel channel = randomAccessFile.getChannel();
// 设置文件通道的位置，即从哪里开始写入文件内容
channel.position(start);
// 从 MultipartFile 对象的资源通道中读取文件内容，并写入到指定位置
channel.transferFrom(file.getResource().readableChannel(), start, file.getSize());
// 关闭文件通道和 RandomAccessFile 对象
channel.close();
randomAccessFile.close();
// 返回上传成功的响应
return AjaxResult.success("上传成功");
} catch (Exception e) {
// 捕获异常并返回上传失败的响应
return AjaxResult.error("上传失败");
}
}
/**
* 检测文件是否存在
* 如果文件存在，则返回已经存在的文件大小。
* 如果文件不存在，则返回 0，表示前端从头开始上传该文件。
* @param filename
* @return
*/
@GetMapping("/checkFile")
@ResponseBody
public AjaxResult checkFile(@RequestParam("filename") String filename) {
File file = new File(uploadPath+File.separator + filename);
if (file.exists()) {
return AjaxResult.success(file.length());
} else {
return AjaxResult.success(0L);
}
}
前端
var prefix = ctx + "/kuroshiro/file-upload";
// 每次上传大小
const chunkSize = 1 * 1024 * 1024;
/**
* 开始上传
*/
function startUpload(type) {
const fileInput = document.getElementById('fileInput');
const file = fileInput.files[0];
if (!file) {
alert("请选择文件");
return;
}
if(type == 1){
checkFile(filename).then(start => {
uploadFile(file, start,Math.min(start + chunkSize, file.size));
})
}
}
/**
* 检查是否上传过
* @param filename
* @returns {Promise<unknown>}
*/
function checkFile(filename) {
return $fetch(prefix+`/checkFile?filename=${filename}`);
}
/**
* 开始分片上传
* @param file 文件
* @param start 开始位置
* @param end 结束位置
*/
function uploadFile(file, start,end) {
if(start < end){
const chunk = file.slice(start, end);
const formData = new FormData();
formData.append('file', chunk);
formData.append('start', start);
formData.append('fileName', file.name);
$fetch(prefix+'/fragmentUpload', {
method: 'POST',
body: formData
}).then(response => {
console.log(`分片 ${start} - ${end} 上传成功`);
// 递归调用
uploadFile(file,end,Math.min(end + chunkSize, file.size))
})
}
}
function $fetch(url,requestInit){
return new Promise((resolve, reject) => {
fetch(url,requestInit).then(response => {
if (!response.ok) {
throw new Error('请求失败');
}
return response.json();
}).then(data => {
if (data.code === 0) {
resolve(data.data);
} else {
console.error(data.msg);
reject(data.msg)
}
}).catch(error => {
console.error(error);
reject(error)
});
});
}
以上虽然实现的分片上传，但是它是某种意义上来说还是与整体上传差不多，它是一段一段的上传，某段上传失败后，后续的就不会再继续上传；不过比起整体上传来说，它会保存之前上传的内容，下一个上传时，从之前上传的位置接着上传。不用整体上传。下面进行优化。
优化
首先，之前的分片上传，后端是直接写入了一个文件中了，所以只能顺序的上传写入，虽然可以保存上传出错之前的内容，但是整体上看来是速度也不行。
优化逻辑：把分片按顺序单独保存下来，等到所有分片都上传成功后，把所有分片合并成文件。这样上传的时候就不用等着上一个上传成功才上传下一个了。
后端代码
/**
* 分片上传
* @param file 文件
* @param chunkIndex 分片下标
*/
@PostMapping("/uploadChunk")
@ResponseBody
public AjaxResult uploadChunk(@RequestParam("file") MultipartFile file, @RequestParam("chunkIndex") int chunkIndex,@RequestParam("fileName") String fileName) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
File directory = new File(uploadDirectory);
if (!directory.exists()||directory.isFile()) {
directory.mkdirs();
}
String filePath = uploadDirectory + File.separator + fileName+ "_" + chunkIndex;
try (OutputStream os = new FileOutputStream(filePath)) {
os.write(file.getBytes());
return AjaxResult.success("分片"+(chunkIndex+1)+"上传成功");
}catch (Exception e){
// 保存失败后如果文件建立了就删除，下次上传时重新保存，避免文件内容错误
File chunkFile = new File(filePath);
if(chunkFile.exists()) chunkFile.delete();
e.printStackTrace();
return AjaxResult.error("分片"+(chunkIndex+1)+"上传失败");
}
}
/**
* 检测分片是否存在
* 如果文件存在，则返回已经存在的分片下标集合。存在的就不上传
* 如果文件不存在，则返回空集合，表示前端从头开始上传该文件
* @param fileName
* @return
*/
@GetMapping("/checkChunk")
@ResponseBody
public AjaxResult checkChunk(@RequestParam("fileName") String fileName) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
List<Integer> list = new ArrayList<>();
File file = new File(uploadDirectory);
// 文件目录不存在
if(!file.exists()||file.isFile()) return AjaxResult.success(list);
File[] files = file.listFiles();
// 文件目录下没有分片文件
if(files == null) return AjaxResult.success(list);
// 返回存在分片下标集合
return AjaxResult.success(Arrays.stream(files).map(item->Integer.valueOf(item.getName().substring(item.getName().lastIndexOf("_")+1))).collect(Collectors.toList()));
}
// 合并文件分片
@PostMapping("/mergeChunks")
@ResponseBody
public AjaxResult mergeChunks(@RequestParam("fileName") String fileName, @RequestParam("totalChunks") int totalChunks) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
String mergedFilePath = uploadPath +File.separator+ fileName;
try (OutputStream os = new FileOutputStream(mergedFilePath, true)) {
for (int i = 0; i < totalChunks; i++) {
Path chunkFilePath = Paths.get(uploadDirectory +File.separator+ fileName + "_" + i);
Files.copy(chunkFilePath, os);
Files.delete(chunkFilePath);
}
return AjaxResult.success();
}catch (Exception e){
e.printStackTrace();
return AjaxResult.error(e.getMessage());
}
}
前端代码
/**
* 开始上传
*/
function startUpload(type) {
const fileInput = document.getElementById('fileInput');
const file = fileInput.files[0];
if (!file) {
alert("请选择文件");
return;
}
const filename = file.name;
if(type == 1){
checkFile(filename).then(start => {
uploadFile(file, start,Math.min(start + chunkSize, file.size));
})
}
if(type == 2){
checkChunk(filename).then(arr => {
uploadChunk(file, arr);
})
}
}
/**
* 切割文件为多个分片
* @param file
* @returns {*[]}
*/
function sliceFile(file) {
const chunks = [];
let offset = 0;
while (offset < file.size) {
const chunk = file.slice(offset, offset + chunkSize);
chunks.push(chunk);
offset += chunkSize;
}
return chunks;
}
/**
* 检查是否上传过
* @param filename
* @returns {Promise<unknown>}
*/
function checkChunk(filename) {
return $fetch(prefix+`/checkChunk?fileName=${filename}`);
}
/**
* 开始分片上传
* @param file 文件
* @param exists 存在的分片下标
*/
function uploadChunk(file,exists) {
const chunkArr = sliceFile(file);
Promise.all(chunkArr.map((chunk, index) => {
if(!exists.includes(index)){
const formData = new FormData();
formData.append('file', chunk);
formData.append('fileName', file.name);
formData.append('chunkIndex', index);
return $fetch(prefix+'/uploadChunk', {
method: 'POST',
body: formData
});
}
})).then(uploadRes=> {
// 合并分片
const formData = new FormData();
formData.append('fileName', file.name);
formData.append('totalChunks', chunkArr.length);
$fetch(prefix + '/mergeChunks', {
method: 'POST',
body:formData,
}).then(mergeRes=>{
console.log("合并成功")
});
});
}
以上优化后所有分片可以同时上传，所有分片上传都成功后进行合并。
最后是完整代码
@Controller()
@RequestMapping("/kuroshiro/file-upload")
public class FileUploadController {
private String prefix = "kuroshiro/fragmentUpload";
// 文件保存目录
private final String uploadPath = RuoYiConfig.getUploadPath();
// 分片保存目录
private final String chunkUploadPath = uploadPath+File.separator+"chunks";
/**
* demo
* @return
*/
@GetMapping("/demo")
public String demo() {
return prefix+"/demo";
}
/**
* 分片上传
*
* @param file 上传的文件
* @param start 文件开始上传的位置
* @param fileName 文件名称
* @return 上传结果
*/
@PostMapping("/fragmentUpload")
@ResponseBody
public AjaxResult fragmentUpload(@RequestParam("file") MultipartFile file, @RequestParam("start") long start, @RequestParam("fileName") String fileName) {
try {
// 检查上传目录是否存在，如果不存在则创建
File directory = new File(uploadPath);
if (!directory.exists()) {
directory.mkdirs();
}
// 设置上传文件的目标路径
File targetFile = new File(uploadPath +File.separator+ fileName);
// 创建 RandomAccessFile 对象以便进行文件的随机读写操作
RandomAccessFile randomAccessFile = new RandomAccessFile(targetFile, "rw");
// 获取 RandomAccessFile 对应的 FileChannel
FileChannel channel = randomAccessFile.getChannel();
// 设置文件通道的位置，即从哪里开始写入文件内容
channel.position(start);
// 从 MultipartFile 对象的资源通道中读取文件内容，并写入到指定位置
channel.transferFrom(file.getResource().readableChannel(), start, file.getSize());
// 关闭文件通道和 RandomAccessFile 对象
channel.close();
randomAccessFile.close();
// 返回上传成功的响应
return AjaxResult.success("上传成功");
} catch (Exception e) {
// 捕获异常并返回上传失败的响应
return AjaxResult.error("上传失败");
}
}
/**
* 检测文件是否存在
* 如果文件存在，则返回已经存在的文件大小。
* 如果文件不存在，则返回 0，表示前端从头开始上传该文件。
* @param filename
* @return
*/
@GetMapping("/checkFile")
@ResponseBody
public AjaxResult checkFile(@RequestParam("filename") String filename) {
File file = new File(uploadPath+File.separator + filename);
if (file.exists()) {
return AjaxResult.success(file.length());
} else {
return AjaxResult.success(0L);
}
}
/**
* 分片上传
* @param file 文件
* @param chunkIndex 分片下标
*/
@PostMapping("/uploadChunk")
@ResponseBody
public AjaxResult uploadChunk(@RequestParam("file") MultipartFile file, @RequestParam("chunkIndex") int chunkIndex,@RequestParam("fileName") String fileName) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
File directory = new File(uploadDirectory);
if (!directory.exists()||directory.isFile()) {
directory.mkdirs();
}
String filePath = uploadDirectory + File.separator + fileName+ "_" + chunkIndex;
try (OutputStream os = new FileOutputStream(filePath)) {
os.write(file.getBytes());
return AjaxResult.success("分片"+(chunkIndex+1)+"上传成功");
}catch (Exception e){
// 保存失败后如果文件建立了就删除，下次上传时重新保存，避免文件内容错误
File chunkFile = new File(filePath);
if(chunkFile.exists()) chunkFile.delete();
e.printStackTrace();
return AjaxResult.error("分片"+(chunkIndex+1)+"上传失败");
}
}
/**
* 检测分片是否存在
* 如果文件存在，则返回已经存在的分片下标集合。存在的就不上传
* 如果文件不存在，则返回空集合，表示前端从头开始上传该文件
* @param fileName
* @return
*/
@GetMapping("/checkChunk")
@ResponseBody
public AjaxResult checkChunk(@RequestParam("fileName") String fileName) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
List<Integer> list = new ArrayList<>();
File file = new File(uploadDirectory);
// 文件目录不存在
if(!file.exists()||file.isFile()) return AjaxResult.success(list);
File[] files = file.listFiles();
// 文件目录下没有分片文件
if(files == null) return AjaxResult.success(list);
// 返回存在分片下标集合
return AjaxResult.success(Arrays.stream(files).map(item->Integer.valueOf(item.getName().substring(item.getName().lastIndexOf("_")+1))).collect(Collectors.toList()));
}
// 合并文件分片
@PostMapping("/mergeChunks")
@ResponseBody
public AjaxResult mergeChunks(@RequestParam("fileName") String fileName, @RequestParam("totalChunks") int totalChunks) {
String uploadDirectory = chunkUploadPath+File.separator+fileName;
String mergedFilePath = uploadPath +File.separator+ fileName;
try (OutputStream os = new FileOutputStream(mergedFilePath, true)) {
for (int i = 0; i < totalChunks; i++) {
Path chunkFilePath = Paths.get(uploadDirectory +File.separator+ fileName + "_" + i);
Files.copy(chunkFilePath, os);
Files.delete(chunkFilePath);
}
File chunkDir = new File(uploadDirectory);
if (chunkDir.exists()) chunkDir.delete();
return AjaxResult.success();
}catch (Exception e){
e.printStackTrace();
return AjaxResult.error(e.getMessage());
}
}
}
<!DOCTYPE html>
<html lang="zh" xmlns:th="http://www.thymeleaf.org" xmlns:shiro="http://www.pollix.at/thymeleaf/shiro">
<head>
<th:block th:include="include :: header('分片上传')" />
</head>
<body class="gray-bg">
<div class="container-div" id="chunk-div">
<div class="row">
<div class="col-sm-12 search-collapse">
<form id="formId">
<div class="select-list">
<ul>
<li>
<label>选择文件：</label>
<input type="file" id="fileInput"/>
</li>
<li>
<a class="btn btn-primary btn-rounded btn-sm" @click="startUpload(1)"><i class="fa fa-upload"></i> 开始上传1</a>
<a class="btn btn-primary btn-rounded btn-sm" @click="startUpload(2)"><i class="fa fa-upload"></i> 开始上传2</a>
</li>
</ul>
</div>
</form>
</div>
<div class="col-sm-12" style="padding-left: 0;">
<div class="ibox">
<div class="ibox-content">
<h3>上传进度</h3>
<ul class="sortable-list connectList agile-list" v-if="uploadMsg">
<li v-for="item in uploadMsg" :class="item.status+'-element'">
{{item.title}}
<div class="agile-detail">
{{item.result}}
</div>
</li>
</ul>
</div>
</div>
</div>
</div>
</div>
<th:block th:include="include :: footer" />
<script th:inline="javascript">
var prefix = ctx + "/kuroshiro/file-upload";
new Vue({
el: '#chunk-div',
data: {
// 每次上传大小
chunkSize: 100 * 1024 * 1024,
uploadMsg:{},
startTime:0,
},
methods: {
/**
* 开始上传
*/
startUpload: function(type){
const fileInput = document.getElementById('fileInput');
const file = fileInput.files[0];
if (!file) {
alert("请选择文件");
return;
}
const filename = file.name;
this.uploadMsg = {};
this.startTime = (new Date()).getTime();
Vue.set(this.uploadMsg, 'checkMsg', {
title:`文件检测`,
result: "检测中... ...",
status:"info"
});
if(type == 1){
this.checkFile(filename).then(start => {
this.uploadMsg['checkMsg'].result = `检测成功:已存在文件，大小为 ${start}`
this.uploadFile(file, start,Math.min(start + this.chunkSize, file.size));
},err => {
this.uploadMsg['checkMsg'].result = `检测失败:${err}`
})
}
if(type == 2){
this.checkChunk(filename).then(arr => {
this.uploadMsg['checkMsg'].result = `检测成功:已存在文件分片 ${arr.length}`
this.uploadChunk(file, arr);
},err => {
this.uploadMsg['checkMsg'].result = `检测失败:${err}`
this.uploadMsg['checkMsg'].status = `info`
})
}
},
/**
* 检查是否上传过
* @param filename
* @returns {Promise<unknown>}
*/
checkFile: function(filename) {
return this.$fetch(prefix+`/checkFile?filename=${filename}`);
},
/**
* 开始分片上传
* @param file 文件
* @param start 开始位置
* @param end 结束位置
*/
uploadFile: function(file, start,end) {
if(start < end){
const chunk = file.slice(start, end);
const formData = new FormData();
formData.append('file', chunk);
formData.append('start', start);
formData.append('fileName', file.name);
Vue.set(this.uploadMsg, 'uploadMsg_'+start, {
title:`分片 ${start} - ${end} 上传`,
result: "上传中... ...",
status:"info"
});
this.$fetch(prefix+'/fragmentUpload', {
method: 'POST',
body: formData
}).then(response => {
this.uploadMsg['uploadMsg_'+start].result = `上传成功`;
// 递归调用
this.uploadFile(file,end,Math.min(end + this.chunkSize, file.size))
},err=>{
this.uploadMsg['uploadMsg_'+start].result = `上传失败：${err}`;
this.uploadMsg['uploadMsg_'+start].status = `danger`;
})
}else{
this.uploadMsg['uploadSuccess'] = {
title:`文件已上传`,
result:`耗时：`+((new Date()).getTime()-this.startTime),
status:"info"
};
}
},
/**
* 切割文件为多个分片
* @param file
* @returns {*[]}
*/
sliceFile: function(file) {
const chunks = [];
let offset = 0;
while (offset < file.size) {
const chunk = file.slice(offset, offset + this.chunkSize);
chunks.push(chunk);
offset += this.chunkSize;
}
return chunks;
},
/**
* 检查是否上传过
* @param filename
* @returns {Promise<unknown>}
*/
checkChunk: function(filename) {
return this.$fetch(prefix+`/checkChunk?fileName=${filename}`);
},
/**
* 开始分片上传
* @param file 文件
* @param exists 存在的分片下标
*/
uploadChunk: function(file,exists) {
const chunkArr = this.sliceFile(file);
Promise.all(chunkArr.map(async (chunk, index) => {
if (!exists.includes(index)) {
const formData = new FormData();
formData.append('file', chunk);
formData.append('fileName', file.name);
formData.append('chunkIndex', index);
Vue.set(this.uploadMsg, "upload_" + index, {
title: `分片 ${index + 1} 上传`,
result: "上传中... ...",
status: "info"
});
return new Promise((resolve, reject) => {
this.$fetch(prefix+'/uploadChunk', {
method: 'POST',
body: formData
}).then(res => {
resolve(res)
this.uploadMsg["upload_"+index].result = "上传成功";
},err => {
reject(err)
this.uploadMsg["upload_"+index].result = err;
this.uploadMsg["upload_"+index].status = "danger";
});
})
}
})).then(uploadRes=> {
this.uploadMsg["uploadSuccess"] = {
title:`上传成功`,
result: "耗时："+((new Date()).getTime()-this.startTime),
status:"info"
};
// 合并分片
const formData = new FormData();
formData.append('fileName', file.name);
formData.append('totalChunks', chunkArr.length);
Vue.set(this.uploadMsg, 'mergeChunks', {
title:`合并分片`,
result: "合并中... ...",
status:"info"
});
this.$fetch(prefix + '/mergeChunks', {
method: 'POST',
body:formData,
}).then(mergeRes=>{
this.uploadMsg["mergeChunks"].result = "合并成功";
},err => {
this.uploadMsg["mergeChunks"].result = `合并失败：${err}`;
this.uploadMsg["mergeChunks"].status = "danger";
});
});
},
$fetch: function(url,requestInit){
return new Promise((resolve, reject) => {
fetch(url,requestInit).then(response => {
if (!response.ok) {
throw new Error('请求失败');
}
return response.json();
}).then(data => {
if (data.code === 0) {
resolve(data.data);
} else {
reject(data.msg)
}
}).catch(error => {
reject(error)
});
});
},
}
});
</script>
</body>
</html>
