upload-plugin

2025年02月19日

柏拉文

越努力，越幸运

一、认识

UploadPlugin 插件在 Webpack 编译后上传构建产物，并对 sourceMap 文件进行特殊处理（上传后删除）。

上传策略（Strategy Pattern）: 采用策略模式将 CDN、S3、OSS 三种上传方式解耦，使得上传平台可以通过配置（uploadType）进行切换，便于后续扩展其它上传方式。
并发调度器（Scheduler）: 实现了一个简单的并发任务调度器，控制同时进行的上传任务数，并支持网络状态异常时暂停（pause）和恢复（resume）上传。
重试机制: 使用 requestWithRetry 方法实现了对单个上传任务的重试逻辑（默认重试 3 次）, 并加入指数退避, 每次重试等待更长时间, 以降低连续重试带来的压力
Source Map 管理: 针对 Source Map 文件，支持上传后在 Webpack 构建产物中删除（即不再生成 Source Map 文件到最终产物中）。
在 compiler.hooks.done.tap 上传构建产物, 在 compiler.hooks.done.tap 删除 sourceMap 产物

二、实现

const fs = require("fs");
const path = require("path");
const pluginName = "UploadPlugin";

const extMap = {
  js: ["js"],
  css: ["css"],
  images: ["png", "jpg", "jpeg", "gif", "svg"],
  fonts: ["woff", "woff2", "ttf", "eot"],
  worker: ["worker.js"],
  sourceMap: ["map"],
};

const contentTypeMap = {
  png: "image/png",
  jpg: "image/jpeg",
  jpeg: "image/jpeg",
  svg: "image/svg+xml",
  gif: "image/gif",
  css: "text/css",
  js: "text/javascript",
  html: "text/html",
  json: "application/json",
};

// 上传策略基类
class UploadStrategy {
  constructor(options) {
    this.options = options;
  }
  async upload(filename, content) {
    throw new Error("upload() must be implemented in subclasses");
  }
}

// CDN 上传策略
class CDNStrategy extends UploadStrategy {
  async upload(data) {
    console.log(`${data.filename} CDN 上传加入队列`);
    await new Promise((resolve) => setTimeout(resolve, 1000));
  }
}

// S3 上传策略
class S3Strategy extends UploadStrategy {
  async upload(data) {
    console.log(`${data.filename} S3 上传加入队列`);
    await new Promise((resolve) => setTimeout(resolve, 1000));
  }
}

// OSS 上传策略
class OSSStrategy extends UploadStrategy {
  async upload(data) {
    console.log(`${data.filename} OSS 上传加入队列`);
    await new Promise((resolve) => setTimeout(resolve, 1000));
  }
}

// Scheduler 调度器
class Scheduler {
  constructor(parallelism) {
    this.queue = [];
    this.paused = false;
    this.runningTask = 0;
    this.parallelism = parallelism;
  }

  add(task, callback) {
    return new Promise((resolve, reject) => {
      const taskItem = {
        reject,
        resolve,
        callback,
        processor: () => Promise.resolve().then(() => task()),
      };

      this.queue.push(taskItem);
      this.schedule();
    });
  }

  pause() {
    this.paused = true;
    console.log("⚠️ 网络状态不佳，上传已暂停");
  }

  resume() {
    if (this.paused) {
      this.paused = false;
      console.log("✅ 网络恢复，恢复上传");
      this.schedule();
    }
  }

  schedule() {
    while (
      !this.paused &&
      this.runningTask < this.parallelism &&
      this.queue.length
    ) {
      this.runningTask++;
      const taskItem = this.queue.shift();
      const { processor, resolve, reject, callback } = taskItem;

      processor()
        .then((res) => {
          resolve && resolve(res);
          callback && callback(null, res);
        })
        .catch((error) => {
          reject && reject(error);
          callback && callback(error, null);
        })
        .finally(() => {
          this.runningTask--;
          this.schedule();
        });
    }
  }
}

function uploadNormal(data) {
  let uploader = null;
  const { uploadStrategy } = data;

  switch (uploadStrategy) {
    case "S3":
      uploader = new S3Strategy(data);
      break;
    case "OSS":
      uploader = new OSSStrategy(data);
      break;
    case "CDN":
    default:
      uploader = new CDNStrategy(data);
      break;
  }
  return uploader.upload(data);
}

function uploadSourceMap(data) {
  const uploader = new CDNStrategy(data);
  return uploader.upload(data);
}

function wait(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function wait(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function requestWithRetry(
  upload,
  data,
  retries = 3,
  success = [],
  failed = []
) {
  return new Promise((resolve, reject) => {
    function attempt(remaining) {
      upload(data)
        .then((res) => {
          console.log(
            `📤 上传 ${data.filename} ${data.absoluteFilenamePath} 完成`
          );
          success.push(data.filename);
          resolve(res);
        })
        .catch(async (err) => {
          if (remaining > 0) {
            console.log(
              `⚠️ 任务 ${data.filename} 失败，正在重试 (${
                retries - remaining + 1
              }/${retries})...`
            );
            await wait(1000 * (retries - remaining + 1));
            attempt(remaining - 1);
          } else {
            console.log(`📤 上传 ${data.filename} 失败`);
            failed.push(data.filename);
            reject(err);
          }
        });
    }
    attempt(retries);
  });
}

class UploadPlugin {
  constructor(options = {}) {
    this.options = {
      retry: 3,
      concurrent: 5,
      uploadType: "CDN",
      deleteSourceMap: true,
      ...options,
    };

    this.failed = [];
    this.success = [];
    this.scheduler = new Scheduler(this.options.concurrent);
  }

  addTask(data) {
    this.scheduler.add(() => requestWithRetry(data));
  }

  isSourceMap(filename) {
    const ext = filename.split(".").pop();
    return extMap.sourceMap.includes(ext);
  }

  getAssetPath(compilation, name) {
    return path.join(
      compilation.getPath(compilation.compiler.outputPath),
      name.split("?")[0]
    );
  }

  deleteSourceMap(stats) {
    Object.keys(stats.compilation.assets)
      .filter((name) => {
        return this.isSourceMap(name);
      })
      .forEach((name) => {
        const filePath = this.getAssetPath(stats.compilation, name);
        if (filePath) {
          console.log("已移除 SourceMap Path", filePath);
          fs.unlinkSync(filePath);
        } else {
          console.log(`${filePath} 删除失败！！！ 可能 ${filePath} 不存在`);
        }
      });
  }

  apply(compiler) {
    compiler.hooks.afterEmit.tapPromise(pluginName, async (compilation) => {
      const { assets } = compilation;
      const uploadTasks = [];

      Object.entries(assets).forEach(([filename, source]) => {
        const ext = filename.split(".").pop();
        const contentType = contentTypeMap[ext];
        const isSourceMap = extMap.sourceMap.includes(ext);
        const absoluteFilenamePath = this.getAssetPath(compilation, filename);

        if (isSourceMap) {
          uploadTasks.push(
            this.scheduler.add(() =>
              requestWithRetry(
                uploadSourceMap,
                { filename, contentType, absoluteFilenamePath },
                this.options.retry,
                this.success,
                this.failed
              )
            )
          );
        } else {
          uploadTasks.push(
            this.scheduler.add(() =>
              requestWithRetry(
                uploadNormal,
                { filename, contentType, absoluteFilenamePath },
                this.options.retry,
                this.success,
                this.failed
              )
            )
          );
        }
      });

      await Promise.all(uploadTasks)
        .then(() => {
          console.log(
            "所有文件上传完成",
            `成功 ${this.success.length}`,
            `失败 ${this.failed.length}`
          );
        })
        .catch((error) => {
          console.log(
            `⚠️ 构建产物上传失败！！！`,
            `成功 ${this.success.length}`,
            `失败 ${this.failed.length}`
          );
          process.exit(1);
        });
    });

    compiler.hooks.done.tap(pluginName, (stats, callback) => {
      if (this.options.deleteSourceMap) {
        this.deleteSourceMap(stats);
      }

      callback?.(null);
    });
  }
}

module.exports = UploadPlugin;

三、配置

const { merge } = require("webpack-merge");
const common = require("./rspack.config.common");
const UploadPlugin = require("./plugin/upload-plugin");

module.exports = merge(common, {
  mode: "production",
  devtool: "source-map",
  plugins: [
    new UploadPlugin(),
  ],
});

一、认识​

二、实现​

三、配置​

一、认识

二、实现

三、配置