[TypeScript] Record MediaStream

Intro

This time, I will try recording video and audio stream.

Environments

Node.js ver.16.12.0
TypeScript ver.4.4.3
Webpack ver.5.56.0

Draw video streams on a canvas

Because I want to add images on a video stream, I try drawing it on a canvas.

index.html

<!DOCTYPE html>
<html lang="en">
    <head>
        <title>video sample</title>
        <meta charset="utf-8">
        <link rel="stylesheet" href="../css/month_picker.css" />
    </head>
    <body>
        <div>
            <canvas id="local_video_area">
                <video id="local_video"></video>
            </canvas>
        </div>
        <button id="record_button" onclick="Page.record()">Record</button>
        <a id="download_link"></a>
        <script src="./js/main.page.js"></script>
        <script>Page.init();</script>
    </body>
</html>

main.page.ts

import { MediaStreamController } from "./mediaStreamController";

let controller: MediaStreamController;
export function init(): void {
    controller = new MediaStreamController();
    controller.init();
}

mediaStreamController.ts

export class MediaStreamController {
    private canvas!: HTMLCanvasElement;

    public init(): void {        
        this.canvas = document.getElementById("local_video_area") as HTMLCanvasElement;
        const ctx = this.canvas.getContext("2d");
        if(ctx == null) {
            console.error("failed getting context");
            return;
        }
        navigator.mediaDevices.getUserMedia({ video: true, audio: true })
            .then(stream => {
                // add the video stream on a video element
                const localVideo = document.getElementById("local_video") as HTMLVideoElement;
                localVideo.srcObject = stream;
                localVideo.play();

                // must draw the video stream every frames because it is drawn as static images
                setInterval(() => {
                    // clear last images
                    ctx.clearRect(0, 0, localVideo.videoWidth, localVideo.videoHeight);
                    // set canvas size as same as the video element
                    this.canvas.width = localVideo.videoWidth;
                    this.canvas.height = localVideo.videoHeight;
                    ctx.drawImage(localVideo, 0, 0, localVideo.videoWidth, localVideo.videoHeight);
                }, 1000/60);
            })
            .catch(err => console.error(`An error occurred: ${err}`));
    }
}

Add static images

Because I draw the video capture image every frames, I also have to draw static images every frames.

mediaStreamController.ts

export class MediaStreamController {
    private canvas!: HTMLCanvasElement;

    public init(): void {        
...
        navigator.mediaDevices.getUserMedia({ video: true, audio: true })
            .then(stream => {
...
                const glassesImage = document.createElement("img");
                glassesImage.src = "../image/glasses.png";
...
                setInterval(() => {
...
                    ctx.drawImage(localVideo, 0, 0, localVideo.videoWidth / 2, localVideo.videoHeight / 2);
                    ctx.drawImage(glassesImage, 5, 5, glassesImage.width, glassesImage.height);
                }, 1000/60);
            })
            .catch(err => console.error(`An error occurred: ${err}`));
    }
}

Result

Change color

I can change the image color.

mediaStreamController.ts

export class MediaStreamController {
    private canvas!: HTMLCanvasElement;

    public init(): void {        
...
        navigator.mediaDevices.getUserMedia({ video: true, audio: true })
            .then(stream => {
...
                const glassesImage = document.createElement("img");
                glassesImage.src = "../image/glasses.png";
...
                setInterval(() => {
...
                    ctx.drawImage(localVideo, 0, 0, localVideo.videoWidth / 2, localVideo.videoHeight / 2);
                    // change color
                    const canvasImage = ctx.getImageData(0, 0, this.canvas.width / 3, this.canvas.height / 3);
                    for (let i = 0; i < canvasImage.data.length; i += 4) {
                        canvasImage.data[i]= canvasImage.data[i]! * 0;
                        canvasImage.data[i + 1] = canvasImage.data[i + 1]! * 0.1;        
                        canvasImage.data[i + 2] = canvasImage.data[i + 2]! * 1;
                        canvasImage.data[i + 3] = canvasImage.data[i + 3]! * 1;
                    }
                    ctx.putImageData(canvasImage,0,0);

                    ctx.drawImage(glassesImage, 5, 5, glassesImage.width, glassesImage.height);
                }, 1000/60);
            })
            .catch(err => console.error(`An error occurred: ${err}`));
    }
}

Result

Record the video

I can record the video stream by MediaStream Recording API.

MediaStream Recording API - Recording API - Web API | MDN

main.page.ts

...
let recording = false;
...
export function record(): void {
    if(recording === true) {
        controller.stopRecording();
        recording = false;
    } else {
        controller.startRecording();
        recording = true;
    }
}

mediaStreamController.ts

export class MediaStreamController {
    private canvas!: HTMLCanvasElement;
    private mediaRecorder: MediaRecorder|null = null;
    private dataAvailableEvent = (ev: BlobEvent) => this.handleDataAvailable(ev);
...
    public startRecording(): void {
        if(this.mediaRecorder != null) {
            this.mediaRecorder.removeEventListener("dataavailable", this.dataAvailableEvent);
            this.mediaRecorder = null;
        }
        const stream = this.canvas.captureStream(60);
        const options = { mimeType: "video/webm; codecs=vp9" };
        this.mediaRecorder = new MediaRecorder(stream, options);
        this.mediaRecorder.ondataavailable = this.dataAvailableEvent;
        this.mediaRecorder.start();
    }
    public stopRecording(): void {
        if(this.mediaRecorder == null) {
            return;
        }
        this.mediaRecorder.stop();
    }
    private handleDataAvailable(ev: BlobEvent) {
        if (ev.data.size <= 0) {
            return;
        }
        this.download(ev.data);
    }
    private download(blob: Blob): void {
        const url = URL.createObjectURL(blob);
        const a = document.getElementById("download_link") as HTMLAnchorElement;
        a.style.display = "none";
        a.href = url;
        a.download = "test.webm";
        a.click();
        window.URL.revokeObjectURL(url);
    }
}

MIME TYPE

I want to save data as "mp4".

But when I changed the options of "MediaRecorder" like below, I got an error.

options

const options = { mimeType: 'video/mp4; codecs="avc1.424028, mp4a.40.2"' };

error

Uncaught DOMException: Failed to construct 'MediaRecorder': Failed to initialize native MediaRecorder the type provided (video/mp4;) is not supported.
    at MediaStreamController.startRecording