Using TUIO with javascript

I recently developed an application for a museum running on a TILE display from Displax.
In order to get this up and running I needed to create a node.js server and websocket connection. Recent examples of this are hard to find online, so I want to share my code for anyone in the same situation, hopefully this saves you some time :-)

First you must have installed a server and node.js

Then install the packages needed through terminal/command prompt:

npm install osc express socket.io bufferutil utf-8-validate --no-audit

In your html file include this:
https://cdn.socket.io/4.7.5/socket.io.min.js

Now you need to create a server file, let's call it "server.js":

const bufferUtil = require('bufferutil');//maybe not needed, but maybe it speeds things up!
var osc = require('osc');
const express = require('express');
const { createServer } = require('node:http');
const { Server } = require('socket.io');
const app = express();
const server = createServer(app);
var socket;

const io = require("socket.io")(server, {cors:{origin: "*",methods: ["GET", "POST"]}});

//Listen to the TUIO data
const udpPort = new osc.UDPPort({
  localAddress: "127.0.0.1",
  localPort: 3333,
  metadata: true
});

//Listen/send on port 3000 or 5000
server.listen(3000, () => {
  console.log('Server running at http://localhost:3000');
});

io.on('connection', function (_socket) {
  socket = _socket;
    socket.on('config', function (obj) {
      console.log('config', obj);
    });
});

// Listen for incoming OSC bundles.
udpPort.on("bundle", function (oscBundle){
  if(socket && oscBundle.packets.length > 2) socket.emit('message', oscBundle);//only send TUIO v1.1
});

// Open the socket.
udpPort.open();

Now for connecting in the front-end, we need this script:

function WSConnection(_address){
    const socket = io(_address);//, {withCredentials: true}
    socket.on('error', function(){
        console.log("Error connecting!");
    });
    socket.on('connect', function(){
        console.log("Server connected");
        socket.emit('config', {server:{port: 3333,host: '127.0.0.1'},client: {port: 3334,host: '127.0.0.1'}});
    });
    socket.on('message', function(oscBundle){
    });
}
//Init
WSConnection("http://127.0.0.1:3000");//or port 5000

To start the server open terminal and cd into the server.js folder and enter:
node server.js

Now your have a running node.js server and a front-end that listens to any TUIO objects being sent through the server.

In order to test you can download the "TUIOSimulator.app" from here:
https://github.com/gregharding/TUIOSimulator

When above works you are facing the next challenge. Depending on the hardware and software you are using, TUIO objects can often be a little unreliable. Sometimes events are fired too slowly, so you think an object is removed from the display. So I made some custom work arounds for these scenarios. In my example I'm using both touch and object recognition, so I have to use both "tuio/2Dcur" and "tuio/2Dobj".

This is my "message" function:

var _alive = [], _aliveTags = [], _aliveCursors = [], _cursors = [];
var _l = 0, _id = 0, _numAlive = 0, _cursorId = 0;
var _v = "";

socket.on('message', function(oscBundle){
    _l = oscBundle.packets.length;
    for(var i=0;i<_l;i++){
        if(oscBundle.packets[i].address == "/tuio/2Dcur"){
            _v = oscBundle.packets[i].args[0].value;
            if(_v == "alive"){
                //Find alive cursors
                _numAlive = oscBundle.packets[i].args.length - 1;
                _alive = [];
                for(var l=0;l<_numAlive;l++){
                    _cursorId = oscBundle.packets[i].args[l+1].value%100;
                    _alive.push(_cursorId);
                    if(_aliveCursors.indexOf(_cursorId) == -1){
                        console.log("Add cursor",_cursorId);
                        _aliveCursors.push(_cursorId);
                        _cursors[_cursorId] = new TuioObj(_cursorId,false);
                    }
                    else _cursors[_cursorId]._removed = false;//keep alive (if set to be removed on next render)
                }
                //Find old cursors not alive anymore
                _numAlive = _aliveCursors.length;
                for(var l=0;l<_numAlive;l++){
                    if(_alive.indexOf(_aliveCursors[l]) == -1){
                        _id = _aliveCursors[l];
                        if(_cursors[_id]._removed){
                            console.log("Destroy cursor", _id);
                            _cursors[_id].destroy();
                            delete _cursors[_id];
                            _aliveCursors.splice(l,1);
                            _numAlive--;
                            l = 0;
                        }
                        else{
                            //console.log("Remove cursor", _id);
                            _cursors[_id]._removed = true;
                        }
                    }
                }
            }
            else if(_v == "set"){
                _cursorId = oscBundle.packets[i].args[1].value%100;
                if(_aliveCursors.indexOf(_cursorId) != -1) _cursors[_cursorId].setXY(oscBundle.packets[i].args[2].value * _appW,oscBundle.packets[i].args[3].value * _appH);
                else console.log("Cursor not found!", _cursorId);
            }
        }
        else if(oscBundle.packets[i].address == "/tuio/2Dobj"){
            _v = oscBundle.packets[i].args[0].value;
            if(_v == "alive"){
                //Find alive cursors
                _numAlive = oscBundle.packets[i].args.length - 1;
                _alive = [];
                for(var l=0;l<_numAlive;l++){
                    _cursorId = oscBundle.packets[i].args[l+1].value%100;
                    _alive.push(_cursorId);
                    if(_aliveTags.indexOf(_cursorId) == -1){
                        console.log("Add tag",_cursorId);
                        _aliveTags.push(_cursorId);
                        _tags[_cursorId] = new TuioObj(_cursorId,true);
                    }
                    else _tags[_cursorId]._removed = false;//keep alive (if set to be removed on next render)
                }
                //Find old cursors not alive anymore
                _numAlive = _aliveTags.length;
                for(var l=0;l<_numAlive;l++){
                    if(_alive.indexOf(_aliveTags[l]) == -1){
                        _id = _aliveTags[l];
                        if(_tags[_id]._removed){
                            console.log("Destroy tag", _id);
                            _tags[_id].destroy();
                            delete _tags[_id];
                            _aliveTags.splice(l,1);
                            _numAlive--;
                            l = 0;
                        }
                        else{
                            //console.log("Remove tag", _id);
                            _tags[_id]._removed = true;
                        }
                    }
                }
            }
            else if(_v == "set"){
                //console.log("Tag static id:", oscBundle.packets[i].args[2].value)
                _cursorId = oscBundle.packets[i].args[1].value%100;
                if(_aliveTags.indexOf(_cursorId) != -1) _tags[_cursorId].setXYR(oscBundle.packets[i].args[3].value * _appW,oscBundle.packets[i].args[4].value * _appH,oscBundle.packets[i].args[5].value);
                else console.log("Tag not found!", _cursorId);
            }
        }
    }
});

For each TUIO object, either a Tag or Touch, I am using this TuioObj, here's a simplified version of mine:

function TuioObj(_id,_isTag){
    var _this = this;
    _this._removed = false;//TUIO "alive" events are not reliable and often remove an object only to add it instantly again!

    //Touch (x and y coordinate)
    _this.setXY = function(x,y){}

    //Tag (x and y coordinate and rotation value)
    _this.setXYR = function(x,y,r){}

    //Destroy (remove DOM elements, event listeners etc.)
    _this.destroy = function(){}
}

Of course many more features, like idle handling, smooth movement (check my post on lerp) and distance measurement (for click handling etc.) can be added, but now you should have a template to get you started.

Blog

Using TUIO with javascript

Michael Vestergaard

Join Our Newsletter. No Spam, Only the good stuff.

Related