AudioWorkletNode

2024年02月06日

柏拉文

越努力，越幸运

一、认识

在 JavaScript 计算录制的音频的音量、获取音频样本数据通常涉及到使用 AudioContext API 和 ScriptProcessorNode 或者现代的 AudioWorkletNode。本文采用 AudioWorkletNode 精确实时的计算音频音量, 是一种先进的处理方式。

1.1 音频录制并处理

获取用户音频流: 通过 navigator.mediaDevices.getUserMedia({ audio: true })
创建音频上下文: 实例化一个新的 AudioContext 上下文, 用于处理和控制音频操作, 提供了一个用于音频处理和合成的工作环境。
创建音源节点(音频处理的输入源): 通过 audioContext.createMediaStreamSource(stream) 创建一个接口, 该接口可以从传入的媒体流（MediaStream）对象中抽取音频数据作为音频上下文（AudioContext）里的一个音源节点。
向音频渲染线程注入JavaScript代码，以创建自定义的音频处理节点: 通过 audioContext.audioWorklet.addModule('volumeProcessor.js'); 向音频渲染线程注入 JavaScript 代码，以创建自定义的音频处理节点
创建自定义的音频处理节点: 通过 new AudioWorkletNode(audioContext, 'volume-processor') 创建自定义的音频处理节点, volume-processor 为已经创建并注册的音频处理器名称。
音源节点连接音频处理节点并连接音频输出点（扬声器）: 这样，音源发出的音频信号在输出到扬声器前，会先经过自定义的音频处理器处理
通过自定义音频处理节点的 prot.onmessage 来接收自定义音频处理器处理结果

1.2 自定义音频处理器

自定义音频处理器 自定义一个类, 继承自 AudioWorkletProcessor, 在 process 核心方法中, 处理音频样本数据。process 接收一个参数 inputs, inputs 是一个二维数组, 包含了多个信道的音频样本数组, 通常在立体声中有两个信道。我们取出第一个信道的样本数组，通常是左声道的音频数据, 进行计算即可。在音频工作线程中，process方法会被频繁调用，每次调用会处理一小部分音频样本。process 处理如下:

取出第一个信道的样本数组: 取出第一个信道的样本数组, 通常是左声道的音频数据。
计算信道中音频样本均方根（RMS）: RMS 用于表示音频信号的振幅大小。获取输入缓冲区获取音频数据样本的均方根值。将每个输入缓冲区音频数据样本的值平方, 累加这些平方值，然后对累加值取平均（除以缓冲区长度），最后得出幅度的均方根值
将均方根 RMS 转换为分贝（DB）: 输入缓冲区获取音频数据样本均方根值转换为分贝值。公式: dB = 20 * log10(RMS)
计算分贝（DB）百分比: 计算分贝百分比以 IOS 为例, 提供了 80db 的动态范围, 所以 minDb 设置为 -80.0
通过 this.port.postMessage 传递处理结果

二、实现

2.1 index.html

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>绘制音频音量</title>
</head>
<body>
    <canvas id="audio-canvas"></canvas>
    
    <script type="module" src="./index.js"></script>
</body>
</html>

2.2 index.js

const audioCanvas = document.getElementById('audio-canvas');
audioCanvas.width = 110;
audioCanvas.height = 8;
const audioCanvasCtx = audioCanvas.getContext('2d');

const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);

function getColor(volumePercent) {
  if (volumePercent > 0.9 || volumePercent < 0.3) {
    return '#B80000';
  }
  if (volumePercent > 0.6 || volumePercent < 0.4) {
    return '#FAB400';
  }
  return '#21A564';
}

function drawVolume(volumePercent) {
  audioCanvasCtx.clearRect(0, 0, audioCanvas.width, audioCanvas.height);
  const xEndPos = volumePercent * audioCanvas.width;

  audioCanvasCtx.lineWidth = 20;
  const gradient = audioCanvasCtx.createLinearGradient(0, 0, xEndPos, 0);
  const color = getColor(volumePercent);
  gradient.addColorStop(0, color);
  gradient.addColorStop(0.8, `${color}88`);
  gradient.addColorStop(1, `${color}00`);
  audioCanvasCtx.beginPath();
  audioCanvasCtx.moveTo(0, 0);
  audioCanvasCtx.lineTo(xEndPos, 0);
  audioCanvasCtx.strokeStyle = gradient;
  audioCanvasCtx.stroke();
  audioCanvasCtx.closePath();
}

async function audioRecorder() {
  await audioContext.audioWorklet.addModule('volumeProcessor.js');
  const volumeNode = new AudioWorkletNode(audioContext, 'volume-processor');
  source.connect(volumeNode).connect(audioContext.destination);

  volumeNode.port.onmessage = event => {
    const {
      data: { volumePercent }
    } = event;

    drawVolume(volumePercent);
  };
}

audioRecorder();

2.3 volumeProcessor.js

function getRMS(samples) {
  const sum = samples.reduce((acc, curr) => acc + curr * curr, 0);
  return Math.sqrt(sum / samples.length);
}

function rmsToDb(gain) {
  return 20 * Math.log10(gain);
}

function getVolumePercent(dbValue) {
  const minDb = -80;

  if (dbValue < minDb) {
    return 0;
  } else if (dbValue > 1) {
    return 1;
  }

  const volumePercent = (Math.abs(minDb) - Math.abs(dbValue)) / Math.abs(minDb);
  return volumePercent;
}

class VolumeProcessor extends AudioWorkletProcessor {
  constructor(options) {
    super(options);
  }

  process(inputs) {
    const input = inputs[0];
    if (!input || input.length === 0) {
      return;
    }

    const samples = input[0];
    const rms = getRMS(samples);
    const db = rmsToDb(rms);
    const volumePercent = getVolumePercent(db);
    this.port.postMessage({ volumePercent });

    return true;
  }
}

registerProcessor('volume-processor', VolumeProcessor);

三、兼容

3.1 processor.js

function isSupportAudioWorklet(audioContext) {
  return (
    audioContext.audioWorklet &&
    typeof audioContext.audioWorklet.addModule === 'function' &&
    typeof AudioWorkletNode !== 'undefined'
  );
}

function isSupportCreateScriptProcessor(audioContext) {
  return typeof audioContext.createScriptProcessor === 'function';
}

function to16kHz(audioData, sampleRate = 44100) {
  const data = new Float32Array(audioData);
  const fitCount = Math.round(data.length * (16000 / sampleRate));
  const newData = new Float32Array(fitCount);
  const springFactor = (data.length - 1) / (fitCount - 1);
  newData[0] = data[0];
  for (let i = 1; i < fitCount - 1; i++) {
    const tmp = i * springFactor;
    const before = Math.floor(tmp).toFixed();
    const after = Math.ceil(tmp).toFixed();
    const atPoint = tmp - before;
    newData[i] = data[before] + (data[after] - data[before]) * atPoint;
  }
  newData[fitCount - 1] = data[data.length - 1];
  return newData;
}

function to16BitPCM(input) {
  const dataLength = input.length * (16 / 8);
  const dataBuffer = new ArrayBuffer(dataLength);
  const dataView = new DataView(dataBuffer);
  let offset = 0;
  for (let i = 0; i < input.length; i++, offset += 2) {
    const s = Math.max(-1, Math.min(1, input[i]));
    dataView.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
  }
  return dataView;
}

export default class Processor {
  constructor(options) {
    const { stream } = options;

    this.options = options;
    this.audioContext = new AudioContext();
    this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);

    this.init();
  }

  init() {
    if (isSupportAudioWorklet(this.audioContext)) {
      this.audioWorkletNodeDealAudioData();
    } else {
      this.scriptNodeDealAudioData();
    }
  }

  scriptNodeDealAudioData() {
    if (!isSupportCreateScriptProcessor(this.audioContext)) {
      return;
    }

    try {
      const scriptProcessor = this.audioContext.createScriptProcessor(
        1024,
        1,
        1
      );
      this.mediaStreamSource.connect(scriptProcessor);
      scriptProcessor.connect(this.audioContext.destination);

      scriptProcessor.onaudioprocess = event => {
        const samples = event.inputBuffer.getChannelData(0);
        const output = to16kHz(samples);
        const audioBuffer = to16BitPCM(output);

        const data = {
          buffer: audioBuffer
        };

        this.options.processRecord?.(data);
      };

    } catch (e) {
      console.log('scriptNodeDealAudioData 错误原因:', e);
    }
  }

  async audioWorkletNodeDealAudioData() {
    if (!isSupportAudioWorklet(this.audioContext)) {
      return;
    }

    try {
      await this.audioContext.audioWorklet.addModule('http://127.0.0.1:5502/test/javascript/audioRecord/022301/processor/custom-processor.js');

      const customNode = new AudioWorkletNode(
        this.audioContext,
        'custom-processor'
      );

      this.mediaStreamSource
        .connect(customNode)
        .connect(this.audioContext.destination);

      customNode.port.onmessage = event => {
        const { audioBuffer } = event.data;
        const data = {
          buffer: audioBuffer
        };

        this.options.processRecord?.(data);
      };
    } catch (e) {
      console.log('audioWorkletNodeDealAudioData 错误原因:', e);
    }
  }
}

3.2 custom-processor.js

function to16kHz(audioData, sampleRate = 44100) {
  const data = new Float32Array(audioData);
  const fitCount = Math.round(data.length * (16000 / sampleRate));
  const newData = new Float32Array(fitCount);
  const springFactor = (data.length - 1) / (fitCount - 1);
  newData[0] = data[0];
  for (let i = 1; i < fitCount - 1; i++) {
    const tmp = i * springFactor;
    const before = Math.floor(tmp).toFixed();
    const after = Math.ceil(tmp).toFixed();
    const atPoint = tmp - before;
    newData[i] = data[before] + (data[after] - data[before]) * atPoint;
  }
  newData[fitCount - 1] = data[data.length - 1];
  return newData;
}

function to16BitPCM(input) {
  const dataLength = input.length * (16 / 8);
  const dataBuffer = new ArrayBuffer(dataLength);
  const dataView = new DataView(dataBuffer);
  let offset = 0;
  for (let i = 0; i < input.length; i++, offset += 2) {
    const s = Math.max(-1, Math.min(1, input[i]));
    dataView.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
  }
  return dataView;
}

class CustomProcessor extends AudioWorkletProcessor {
  constructor(options) {
    super(options);
  }

  process(inputs) {
    const input = inputs[0];
    if (!input || input.length === 0) {
      return;
    }

    const samples = input[0];
    const output = to16kHz(samples);
    const audioBuffer = to16BitPCM(output);
    this.port.postMessage({ audioBuffer });

    return true;
  }
}

registerProcessor('custom-processor', CustomProcessor);

一、认识​

1.1 音频录制并处理​

1.2 自定义音频处理器​

二、实现​

2.1 index.html​

2.2 index.js​

2.3 volumeProcessor.js​

三、兼容​

3.1 processor.js​

3.2 custom-processor.js​