# A-Frame: "Look around" handsfree

Try it!

  • Turn head around to turn the camera
  • Move your head to move the camera
  • Try it on CodePen
Look around the A-Frame Handsfree

# The basic approach

First we start by instantiating Handsfree. We'll use the Weboji model which gives us head pose and translation:

handsfree = new Handsfree({weboji: true})

Next we create a plugin to match the A-Frame camera's pose with your head's:

$rig = document.querySelector('#camera-rig')

// Create the plugin
handsfree.use('lookHandsfree', ({weboji}) => {
  // Bail if we don't have weboji data
  if (!weboji.degree) return
  
  // [yaw, pitch, roll]
  const rot = weboji.degree

  // Let's shift the yaw slightly
  // - this assumes webcam is slightly below eye level, like on a laptop
  rot[0] += 15
  
  // Calculate position
  // - Positions are normalized between [0, 1]
  // - 0 is all the way to the left of the canvas, 1 all the way to the right
  // - Change the multiplier to change the range you want to move by
  const pos = {
    // Subtract .5 to "center" the x value
    x: (weboji.translation[0] - .5) * 10,
    // Subtract .5 to "center" the y value
    y: (weboji.translation[1] - .5) * 5,
    // Let's position the camera 5 units back from the center of the room
    z: 5 - weboji.translation[2] * 30
  }
  
  // Now let's just tell A-Frame to update our camera rig
  // - We flip yaw/roll because we are "looking into" the aframe vs how the tracker is "looking at" us
  // - We multiply by 1.5 to make it "look more", so we don't have to physically move our head as much
  $rig.setAttribute('rotation', `${-rot[0] * 1.5} ${-rot[1] * 1.5} ${rot[2] * 1.5}`)
  $rig.setAttribute('position', `${pos} ${pos} ${pos}`)
})

// Start tracking
handsfree.start()

# Adding Tweening

Although the above will definitely work, you'll notice that it jerks around quite a bit:

Jerky

Smooth

This is due to slight errors between frames and the fact that it isn't running at a full 30 or 60FPS. To fix this, we can modify the above to use Tweening:

<!-- Add a tweening library -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/3.5.1/gsap.min.js"></script>
// This will hold our tween values
tween = {yaw: 0, pitch: 0, roll: 0, x: 0, y: 0, z: 0}

handsfree.use('lookHandsfree', ({weboji}) => {
  if (!weboji.degree) return
  
  // Calculate rotation
  const rot = weboji.degree
  rot[0] += 15
  
  // Calculate position
  const pos = {
    x: (weboji.translation[0] - .5) * 10,
    y: (weboji.translation[1] - .5) * 5,
    z: 5 - weboji.translation[2] * 30
  }

  // Tween this values
  window.handsfree.TweenMax.to(tween, 1, {
  yaw: -rot[0] * 1.5,
  pitch: -rot[1] * 1.5,
  roll: rot[2] * 1.5,
    x: pos.x,
    y: pos.y,
    z: pos.z
  })
  
  // Use the tweened values instead of the actual current values from webcam
  $rig.setAttribute('rotation', `${tween.yaw} ${tween.pitch} ${tween.roll}`)
  $rig.setAttribute('position', `${tween.x} ${tween.y} ${tween.z}`)
})

// Start tracking
handsfree.start()

# See Also

# Boilerplate

The following is the boilerplate located in the repo at /boilerplate/aframe/look-around-handsfree/index.html (opens new window). You can also play with this demo on CodePen (opens new window), or by copy/pasting the following into a local .html file without a server.

<html>
  <head>
    <title>A-Frame: Look around handsfree</title>
    <script src="https://aframe.io/releases/1.1.0/aframe.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/3.5.1/gsap.min.js" integrity="sha512-IQLehpLoVS4fNzl7IfH8Iowfm5+RiMGtHykgZJl9AWMgqx0AmJ6cRWcB+GaGVtIsnC4voMfm8f2vwtY+6oPjpQ==" crossorigin="anonymous"></script>
    <style>button {font-size: 2.5em; cursor: pointer;}</style>
    
    <!-- Include Handsfree.js in head -->
    <link rel="stylesheet" href="https://unpkg.com/handsfree@8.4.3/build/lib/assets/handsfree.css" />
    <script src="https://unpkg.com/handsfree@8.4.3/build/lib/handsfree.js"></script>
  </head>
  <body>










    <!-- Instantiate Handsfree.js in body -->
    <script>
      // Let's use weboji. See: https://handsfree.js.org/ref/model/weboji
      handsfree = new Handsfree({weboji: true})

      // Used to hold tween values (without this things will be jerky)
      tween = {yaw: 0, pitch: 0, roll: 0, x: 0, y: 0, z: 0}

      // Create a new "plugin" to hook into the main loop
      // @see https://handsfree.js.org/guide/the-loop
      handsfree.use('lookHandsfree', ({weboji}) => {
        if (!weboji.degree) return
  
        // Calculate rotation
        const rot = weboji.degree
        rot[0] += 15
        
        // Calculate position
        const pos = {
          x: (weboji.translation[0] - .5) * 10,
          y: (weboji.translation[1] - .5) * 5,
          z: 5 - weboji.translation[2] * 30
        }

        // Tween this values
        TweenMax.to(tween, 1, {
          yaw: -rot[0] * 1.5,
          pitch: -rot[1] * 1.5,
          roll: rot[2] * 1.5,
          x: pos.x,
          y: pos.y,
          z: pos.z
        })
        
        // Use the tweened values instead of the actual current values from webcam
        $rig.setAttribute('rotation', `${tween.yaw} ${tween.pitch} ${tween.roll}`)
        $rig.setAttribute('position', `${tween.x} ${tween.y} ${tween.z}`)
      })

      // Cache the camera rig into a variable
      window.onload = function () {
        $rig = document.querySelector('#rig')
      }
    </script>
    















    <!-- Button. Notice the helper classes :) -->
    <div style="position: absolute; top: 0; left: 50%; transform: translateX(-50%); z-index: 100">
      <button onclick="handsfree.start()" class="handsfree-show-when-stopped handsfree-hide-when-loading">Start webcam</button>
      <button class="handsfree-show-when-loading">Loading...</button>
      <button onclick="handsfree.stop()" class="handsfree-show-when-started">Stop webcam</button>
    </div>
    
    <!-- Aframe -->
    <a-scene fog="type: exponential; color: #000; far: 30; density: 0.075">
      <a-assets>
        <img src="https://cdn.glitch.com/a2469ad6-a9ce-4918-8347-7348024d9f06%2F25P1geh.png?1543692382066" id="grid" crossorigin="anonymous">
      </a-assets>
      
      <!-- Camera Rig: This is what we control handsfree -->
      <a-entity id="rig" position="0 1 -5">
        <a-camera></a-camera>
      </a-entity>

      <!-- Walls -->
      <a-entity id="wall-bottom" geometry="primitive: plane; width: 10000; height: 10000;" rotation="-90 0 0" position="0 -2 0" material="src: #grid; repeat: 10000 10000;"></a-entity>
      <a-entity id="wall-top" geometry="primitive: plane; width: 10000; height: 10000;" rotation="90 0 0" position="0 7 0" material="src: #grid; repeat: 10000 10000;"></a-entity>
      <a-entity id="wall-right" geometry="primitive: plane; width: 10000; height: 10000;" rotation="0 -90 0" position="8 0 0" material="src: #grid; repeat: 10000 10000;"></a-entity>
      <a-entity id="wall-left" geometry="primitive: plane; width: 10000; height: 10000;" rotation="0 90 0" position="-8 0 0" material="src: #grid; repeat: 10000 10000;"></a-entity>
      <a-entity id="wall-back" geometry="primitive: plane; width: 10000; height: 10000;" rotation="0 0 0" position="0 0 -15" material="src: #grid; repeat: 10000 10000;"></a-entity>

      <!-- Some lights -->
      <a-entity light="color: #000; intensity: 1; type: ambient;" visible=""></a-entity>
      <a-entity light="color: #000; intensity: 1.5" position="5 5 5"></a-entity>
      <a-entity light="color: #000; intensity: 0.5" position="-5 5 15"></a-entity>
      <a-entity light="color: #aaa; type: ambient;"></a-entity>
    </a-scene>
  </body>
</html>
Debugger