Koa KoaMulter

2024年10月17日

柏拉文

越努力，越幸运

一、认识

---

并发上传基本思想---文件分片、断点续传

使用 Blob.slice 方法对大文件按照指定的大小进行切割，然后通过多线程进行分块上传，等所有分块都成功上传后，再通知服务端进行分块合并。

File 对象是特殊类型的 Blob，且可以用在任意的 Blob 类型的上下文中。比如说 FileReader、URL.createObjectURL() 及 XMLHttpRequest.send() 都能处理 Blob 和 File。在大文件上传的场景中，我们将使用 Blob.slice 方法对大文件按照指定的大小进行切割，然后对分块进行并行上传。

二、实现

---

const fs = require("fs");
const path = require("path");
const util = require("util");
const Koa = require("koa");
const cors = require("@koa/cors");
const multer = require("@koa/multer");
const Router = require("@koa/router");
const serve = require("koa-static");
const fse = require("fs-extra");
const readdir = util.promisify(fs.readdir);
const unlink = util.promisify(fs.unlink);

const app = new Koa();
const router = new Router();
const TEMP_DIR = path.join(__dirname, "public/temp"); // 临时目录
const UPLOAD_DIR = path.join(__dirname, "/public/upload");
const IGNORES = [".DS_Store"]; // 忽略的文件列表

const storage = multer.diskStorage({
  destination: async function (req, file, cb) {
    let fileMd5 = file.originalname.split("-")[0];
    const fileDir = path.join(TEMP_DIR, fileMd5);
    await fse.ensureDir(fileDir);
    cb(null, fileDir);
  },
  filename: function (req, file, cb) {
    let chunkIndex = file.originalname.split("-")[1];
    cb(null, `${chunkIndex}`);
  },
});

const multerUpload = multer({ storage });

router.get("/", async (ctx) => {
  ctx.body = "大型文件分片、断点续传";
});

router.get("/exists", async (ctx) => {
  const { fileName, fileMD5 } = ctx.query;
  const filePath = path.join(UPLOAD_DIR, fileName);
  const isExists = await fse.pathExists(filePath);
  if (isExists) {
    ctx.body = {
      code:200,
      msg:'成功',
      data: {
        isExists: true,
        url: `http://localhost:4000/${fileName}`,
      },
    };
  } else {
    let chunkFinishList = [];
    const chunksPath = path.join(TEMP_DIR, fileMD5);
    const hasChunksPath = await fse.pathExists(chunksPath);
    if (hasChunksPath) {
      let files = await readdir(chunksPath);
      chunkFinishList = files.filter((file) => {
        return IGNORES.indexOf(file) === -1;
      });
    }
    ctx.body = {
      code:200,
      msg:'成功',
      data: {
        isExists: false,
        chunkFinishList,
      },
    };
  }
});

router.post(
  "/useChunk",
  multerUpload.single("file"),
  async (ctx, next) => {
    ctx.body = {
      code: 200,
      msg:'成功',
      data: ctx.file,
    };
  }
);

router.get("/concatChunks", async (ctx) => {
  const { fileName, fileMD5 } = ctx.query;
  await concatFiles(
    path.join(TEMP_DIR, fileMD5),
    path.join(UPLOAD_DIR, fileName)
  );
  ctx.body = {
    code:200,
    msg:'成功',
    data: {
      url: `http://localhost:4000/${fileName}`,
    },
  };
});

async function concatFiles(sourceDir, targetPath) {
  const readFile = (file, ws) =>
    new Promise((resolve, reject) => {
      fs.createReadStream(file)
        .on("data", (data) => ws.write(data))
        .on("end", resolve)
        .on("error", reject);
    });
  const files = await readdir(sourceDir);
  const sortedFiles = files
    .filter((file) => {
      return IGNORES.indexOf(file) === -1;
    })
    .sort((a, b) => a - b);
  const writeStream = fs.createWriteStream(targetPath);
  for (const file of sortedFiles) {
    let filePath = path.join(sourceDir, file);
    await readFile(filePath, writeStream);
    await unlink(filePath); // 删除已合并的分块
  }
  writeStream.end();
}

// 注册中间件
app.use(cors());
app.use(serve(UPLOAD_DIR));
app.use(router.routes()).use(router.allowedMethods());

app.listen(4000, () => {
  console.log("服务启动成功！");
});

参考资料

---

阿宝哥-JavaScript 中如何实现大文件并发上传？