大文件上传之断点续传实现方案与原理详解

news2025/4/17 6:03:18

一、实现原理

文件分块：将大文件切割为固定大小的块（如5MB）
进度记录：持久化存储已上传分块信息
续传能力：上传中断后根据记录继续上传未完成块
块校验机制：通过哈希值验证块完整性
合并策略：所有块上传完成后进行有序合并

二、前端实现（JavaScript）

// 文件分块（示例使用Blob.slice）
// 默认 5M 一个片段
function createChunks(file, chunkSize = 5 * 1024 * 1024) {
  const chunks = [];
  let offset = 0;
  
  while (offset < file.size) {
    const chunk = file.slice(offset, offset + chunkSize);
    chunks.push({
      chunk,
      index: chunks.length,
      hash: file.name + '-' + chunks.length
    });
    offset += chunkSize;
  }
  return chunks;
}

// 上传控制器
class Uploader {
  constructor(file) {
    this.file = file
    this.chunks = createChunks(file)
    this.uploaded = new Set() // 已上传分块索引
  }

  async checkProgress() {
    // 查询服务端已上传分块
    const { data } = await axios.get('/progress', {
      params: { hash: this.fileHash }
    })
    this.uploaded = new Set(data.uploaded)
  }

  async upload() {
    await this.checkProgress()
    
    for (const chunk of this.chunks) {
      if (this.uploaded.has(chunk.index)) continue
      
      const formData = new FormData()
      formData.append('chunk', chunk.chunk)
      formData.append('hash', chunk.hash)
      formData.append('index', chunk.index)
      formData.append('total', this.chunks.length)
      
      await axios.post('/upload', formData, {
        onUploadProgress: progress => {
          console.log(`块${chunk.index}上传进度：`, progress)
        }
      })
      this.uploaded.add(chunk.index)
    }
    
    await axios.post('/merge', { 
      filename: this.file.name,
      total: this.chunks.length 
    })
  }
}

三、服务端实现（`Node.js + Express`）

const express = require('express')
const fs = require('fs-extra')
const path = require('path')
const app = express()

// 临时存储目录
const UPLOAD_DIR = path.resolve(__dirname, 'temp')

// 处理分块上传
app.post('/upload', async (req, res) => {
  const { chunk, hash, index } = req.files
  const chunkDir = path.resolve(UPLOAD_DIR, hash.split('-')[0])
  
  await fs.ensureDir(chunkDir)
  await fs.move(chunk.path, path.resolve(chunkDir, hash))
  
  res.json({ code: 0 })
})

// 合并分块
app.post('/merge', async (req, res) => {
  const { filename, total } = req.body
  const fileHash = filename + '-' + Date.now()
  const chunkDir = path.resolve(UPLOAD_DIR, fileHash)
  const destFile = path.resolve(UPLOAD_DIR, filename)
  
  // 按索引顺序合并
  await fs.ensureDir(chunkDir)
  for (let i = 0; i < total; i++) {
    const chunkPath = path.resolve(chunkDir, `${fileHash}-${i}`)
    await fs.appendFile(destFile, await fs.readFile(chunkPath))
    await fs.unlink(chunkPath)
  }
  
  await fs.rmdir(chunkDir)
  res.json({ code: 0 })
})

// 查询上传进度
app.get('/progress', async (req, res) => {
  const { hash } = req.query
  const chunkDir = path.resolve(UPLOAD_DIR, hash.split('-')[0])
  
  if (!await fs.pathExists(chunkDir)) {
    return res.json({ uploaded: [] })
  }
  
  const uploaded = (await fs.readdir(chunkDir))
    .map(name => parseInt(name.split('-').pop()))
    
  res.json({ uploaded })
})