Chrome Extension Capture Tab Audio

Question

I'm trying to create a Chrome extension that captures the audio from the active tab and either sends it to another server or makes it accessible via a URL.

I'm using the chrome.tabCapture.capture API and can successfully get a MediaStream of the tab's audio, but I don't know what to do after that.

The Chrome docs have nothing about MediaStreams so I've looked through some documentation here and played with the JS debugger to see what methods are available, but can't find a way to send the MediaStream somewhere.

Erhart

The Mozilla Developer Network probably is your friend: developer.mozilla.org/en-US/docs/WebRTC/MediaStream_API – Halyard

You can send the audio via a webrtc peer connection. You can specify audio only on your capture and send only for your SDP options. – Melonymelos

Sorry, I should have mentioned that all this is taking place on a local network, so web rtc seems like overkill. Is there any simple way to make the stream available over a local URL or something? @bwtrent – Erhart

WebRTC is still very simple. However, you there may be a way to simply publish out the stream blob that is created, I am just not sure how. – Melonymelos

MediaStreamRecorder is being actively developed and has an experimental implementation. crbug.com/262211 – Mo

@BenjaminTrent I'm game with going with WebRTC to grab the data and process it on the server. I was trying to scour the internet to see if that actually works. I tried MediaStreamRecorder's solution, but it results in very poor video quality (taking frames from a video object every interval, passing it into a canvas element, and then taking a screenshot). – Korwun

Answer 1

It's now possible to record a stream locally in JS using MediaRecorder. There is a demo here and the w3c spec is here

The method startRecording in the demo requires window.stream to be set to the MediaStream instance.

// The nested try blocks will be simplified when Chrome 47 moves to Stable
var mediaRecorder;
var recordedBlobs;
window.stream = myMediaStreamInstance;
function startRecording() {
  var options = {mimeType: 'video/webm', bitsPerSecond: 100000};
  recordedBlobs = [];
  try {
    mediaRecorder = new MediaRecorder(window.stream, options);
  } catch (e0) {
    console.log('Unable to create MediaRecorder with options Object: ', e0);
    try {
      options = {mimeType: 'video/webm,codecs=vp9', bitsPerSecond: 100000};
      mediaRecorder = new MediaRecorder(window.stream, options);
    } catch (e1) {
      console.log('Unable to create MediaRecorder with options Object: ', e1);
      try {
        options = 'video/vp8'; // Chrome 47
        mediaRecorder = new MediaRecorder(window.stream, options);
      } catch (e2) {
        alert('MediaRecorder is not supported by this browser.\n\n' +
            'Try Firefox 29 or later, or Chrome 47 or later, with Enable experimental Web Platform features enabled from chrome://flags.');
        console.error('Exception while creating MediaRecorder:', e2);
        return;
      }
    }
  }
  console.log('Created MediaRecorder', mediaRecorder, 'with options', options);

  // do UI cleanup here
  mediaRecorder.onstop = function() {/** stop */};
  mediaRecorder.ondataavailable = function() {/** data avail */};
  mediaRecorder.start(10); // collect 10ms of data
  console.log('MediaRecorder started', mediaRecorder);
}

1. https://www.w3.org/TR/mediastream-recording/
2. https://simpl.info/mediarecorder/

Polyamide

Answer 2

Define the popup.js to capture audio in your active tab of chrome extension like this. First, you have to get current active tab's id and then audio capturing will be allowed on it. Of course you can capture audio in popup.js and background.js but audio capture in background.js is prevented in Manifest v3. In manifest v3, you have to capture audio in content.js created newly when capture is started as a new tab. By the way, I metion here how to capture audio and then how to play it. This is a sample code to capture audio in frontend(popup.js). And also you have to allow permission in manifest.json...

"permissions": ["tabCapture", "tabs", "downloads", "activeTab"]

// popup.js
let recorder;
let audioData = [];
let startTabId;
document.getElementById('btnRecord').addEventListener('click', ()=>{
  startCapture();
});

document.getElementById('btnStop').addEventListener('click', ()=>{
  stopCapture();
});

async function startCapture() {
  chrome.tabCapture.capture({ audio:true, video:false }, (stream)=> {
    chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => startTabId = tabs[0].id)
  
    context = new AudioContext();
    var newStream = context.createMediaStreamSource(stream);
    newStream.connect(context.destination);
    recorder = new MediaRecorder(stream);
    
    recorder.ondataavailable = async(e) => {
      audioData.push(e.data);
    }
    recorder.onpause = async(e) => {
      await recorder.requestData();
    }
    recorder.onstop = async(e) => {
      let blob = new Blob(audioData, { type: 'audio/wav' });
      audioURL = window.URL.createObjectURL(blob);
      await chrome.downloads.download({
        url: audioURL,
        filename: 'filename',
        saveAs: true
      });
    }
  });
}

async function stopCapture() {
  chrome.tabs.query({
    active: true,
    currentWindow: true
  }, async (tabs) => {
    let endTabId = tabs[0].id;
    if (startTabId === endTabId) {
      await recorder.requestData();
      await recorder.stop();
    }
  });
}

<button id="btnRecord">
  Record
</button>
<button id="btnStop">
  Stop
</button>

Addams

Answer 3

You can upload captured audio stream data to other server like cloud web server. Captured audio stream is stored as blob taking the file format of MP3 or WAV. Artem's answer is okay. https://mcmap.net/q/741206/-chrome-extension-capture-tab-audio

We can modify a little for upload to other server.

async function startCapture() {
  chrome.tabCapture.capture({ audio:true, video:false }, (stream)=> {
    chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => startTabId = tabs[0].id)
  
    context = new AudioContext();
    var newStream = context.createMediaStreamSource(stream);
    newStream.connect(context.destination);
    recorder = new MediaRecorder(stream);
    
    recorder.ondataavailable = async(e) => {
      audioData.push(e.data);
    }
    recorder.onpause = async(e) => {
      await recorder.requestData();
    }
    // when stopped audio capture in above answer
    recorder.onstop = async(e) => {
      let blob = new Blob(audioData, { type: 'audio/wav' });
      base64 = blobToBase64(blob);
      const newForm = new FormData();
      newForm.append('audioData', base64);
      // then ajax upload code is here
      ...
    }
  });
}
    

    function blobToBase64(blob) {
      return new Promise(resolve => {
        const reader = new FileReader();
        reader.onloadend = () => resolve(reader.result);
        reader.readAsDataURL(blob);
      })
    }
}

Garold