Playful webcam

Playful webcam

by Zarema Khalilova

from Saint-Petersburg, Russia

founder of the community

currently working with RealAdvisor

previously worked with

Media Capture and Stream API

Stream

Get stream

			const constraints = {video: true, audio: true}
			 
			navigator.mediaDevices.getUserMedia(constraints)
				.then(stream => {...})
				.catch(error => {...})
		

Stream to video element

			stream => {
				const $video = document.querySelector('video')
				 
				$video.srcObject = stream
				$video.onloadedmetadata = () => $video.play()
			}
		

Stream to canvas

    stream => {
      const $video = document.createElement('video')
      
      $video.srcObject = stream
      $video.onloadedmetadata = () => {
        $video.play()
        ...
  

Stream to canvas

        ...
        const $canvas = document.querySelector('canvas')
        
        $canvas.width = $video.videoWidth
        $canvas.height = $video.videoHeight
        ...
  

Stream to canvas

        ...
        const context = $canvas.getContext('2d')
        
        const draw = () => {
          context.drawImage($video, 0, 0)
          requestAnimationFrame(draw)
        }
        ...
  

Stream to canvas

        ...
        requestAnimationFrame(draw)
      }
    }
  

Stream to canvas

Performance problems:

1. Frequency of drawImage calls is higher than the frame rate of the video
2. The more pixels, the more resource-intensive it gets

Stream to canvas

Solution of performance problems:

    let frameDrawed = true
    const runDraw = () => {
      if (!frameDrawed) return
      frameDrawed = false
      requestAnimationFrame(draw)
    }
  

Stream to canvas

Solution of performance problems:

    const streamTrack = stream.getVideoTracks()[0]
    const {frameRate} = streamTrack.getSettings()
    const delay = 1000 / frameRate
      
    let frameInterval = setInterval(runDraw, delay)
  

Stream to canvas

Solution of performance problems:

    const draw = () => {
      ...
      requestAnimationFrame(draw)
      frameDrawed = true
    }
  

Constraints

Constraints

    const constraints = {video: true, audio: true}
     
    navigator.mediaDevices.getUserMedia(constraints)
    	.then(stream => {...})
    	.catch(error => {...})
  

Supported constraints

    const supportedConstraints =
      navigator.mediaDevices.getSupportedConstraints()
  

Demo: Supported constraints

Video constraints

    const constraints = {
      video: {
        height: 200,
        aspectRatio: 1,
      },
    }
     
    navigator.mediaDevices.getUserMedia(constraints)
  

Capabilities

Capabilities

    const videoTrack = stream.getVideoTracks()[0]
    const capabilities = videoTrack.getCapabilities()
      
    if (Object.keys(capabilities).length) {
      // You can configure stream
    }
  

Capabilities: torch

    if (capabilities.torch) {
      const $torch = document.querySelector('button.torch')
      $torch.onclick = () => {
        const torch = (videoTrack.getSettings()).torch
        videoTrack.applyConstraints({
          advanced: [{torch: !currentTorch}]
        })
  

Capabilities: zoom

    if (capabilities.zoom) {
      const $zoom = document.querySelector('input[type=range]')
      
      $zoom.min = capabilities.zoom.min
      $zoom.max = capabilities.zoom.max
      $zoom.step = capabilities.zoom.step
      ...
  

Capabilities: zoom

      ...
      $zoom.value = (videoTrack.getSettings()).zoom
      
      $zoom.onchange = (e) => videoTrack.applyConstraints({
        advanced: [{zoom: e.target.value}]
      })
  

Take photo

Tip: save blob to image

    const blobToImage = blob => {
      const $img = document.createElement('img')
      
      $img.src = URL.createObjectURL(blob)
      $img.onload = () => URL.revokeObjectURL($img.src)
      
      document.querySelector('.output').append($img)
    }
  

Take photo

Take photo: drawImage

    stream => {
      ... // Get video element and put stream to that
      ... // Create canvas and context
      context.drawImage($video, 0, 0)
      ...
  

Take photo: drawImage

      ...
      $canvas.toBlob(blob => blobToImage(blob))
    }
  

Take photo

Take photo: grabFrame

    stream => {
      const streamTrack = stream.getVideoTracks()[0]
      const imageCapture = new ImageCapture(streamTrack)
      ...
  

Take photo: grabFrame

      ...
      imageCapture.grabFrame()
        .then(imageBitmap => {
          ... // Create canvas and context
          context.drawImage(imageBitmap, 0, 0)
          $canvas.toBlob(blob => blobToImage(blob))
        }
    }
  

Take photo

Take photo: takePhoto

      ...
      imageCapture.takePhoto()
      .then(blob => blobToImage(blob))
    }
  

Record

Record stream

    stream => {
      ... // Create video and put stream to video
      let chunks = []
      const mediaRecorder = new MediaRecorder(stream)
      ...
  

Record stream

      ...
      $startButton.onclick = () => mediaRecorder.start()
      $stopButton.onclick = () => mediaRecorder.stop()
      ...
  

Record stream

      ...
      mediaRecorder.ondataavailable = (e) => chunks.push(e.data)
      
      mediaRecorder.onstop = () =>
        blobToVideo(new Blob(chunks, {type: 'video/webm'}))
    }
  

Let's have fun

Filters

Filters

Just use CSS Filters for video

    video[data-filter=grayscale] {
      filter: grayscale(100%);
    }
  

Filters

To take photo with filter use drawImage way with tip:

    context.filter = window
      .getComputedStyle($video, null)
      .getPropertyValue('filter')
      
    context.drawImage($video, 0, 0)
  

ASCII Video Stream

Please allow this page to access your camera.

Face mask

Face mask

• Shape Detection API
• tracking.js

Augmented reality

Augmented reality

    <script src=".../aframe.min.js"></script>
    <script src=".../aframe-ar.js"></script>
  

Augmented reality

    <body style='margin : 0px; overflow: hidden;'>
    <a-scene embedded artoolkit='sourceType: webcam;'>
      <a-box position='0 0 0.5' material='opacity: 0.7;'></a-box>
      <a-marker-camera preset='hiro'></a-marker-camera>
    </a-scene>
    </body>
  

What's next?

You can do

1. Conference with WebRTC
2. PWA like native photo app
3. VR/AR

See more on GitHub

Thank you!