Node.js: What techniques are there for writing clean, simple callback code?

node.js code is known for turning into callback spaghetti.

What are the best techniques for overcoming this problem and writing clean, uncomplex, easy to understand callback code in node.js?

Answers


Take a look at Promises: http://promises-aplus.github.io/promises-spec/

It is an open standard which intended to solve this issue.

I am using node module 'q', which implements this standard: https://github.com/kriskowal/q

Simple use case:

var Q = require('q');

For example we have method like:

var foo = function(id) {
  var qdef = Q.defer();

  Model.find(id).success(function(result) {
    qdef.resolve(result);
  });

  return (qdef.promise);
}

Then we can chain promises by method .then():

foo(<any-id>)
.then(function(result) {
  // another promise
})
.then(function() {
  // so on
});

It is also possible to creating promise from values like:

Q([]).then(function(val) { val.push('foo') });

And much more, see docs.

See also:


Several things can be done to avoid the 'matrioska-style'.

  • You can store callbacks to variables:

    var on_read = function (foo, bar) {
          // some logic 
        },
    
        on_insert = function (err, data) {
          someAsyncRead(data, on_read);
        };
    
    someAsyncInsert('foo', on_insert);
    
  • You can use some modules that help in those scenarios.

    // Example using funk
    var funk = require('funk');
    for(var i = 0; i < 10; i++) {
      asyncFunction(i, funk.add(function (data) {
        this[i] = data;
      }));
    }
    
    funk.parallel(function () {
      console.log(this);
    });
    

I'd suggest 1) using CoffeeScript and 2) using named callbacks and passing state between them in a hash, rather than either nesting callbacks or allowing argument lists to get very long. So instead of

var callback1 = function(foo) {
  var callback2 = function(bar) {
    var callback3 = function(baz) {
      doLastThing(foo, bar, baz);
    }
    doSomethingElse(bar, callback3);
  }
  doSomething(foo, callback2);
}
someAsync(callback1);

you can instead simply write

callback1 = (state) -> doSomething state.foo, callback2
callback2 = (state) -> doSomethingElse state.bar, callback3
callback3 = (state) -> doLastThing state
someAsync callback1

once your doSomething, doSomethingElse and doLastThing have been rewritten to use/extend a hash. (You may need to write extra wrappers around external functions.)

As you can see, the code in this approach reads neatly and linearly. And because all callbacks are exposed, unit testing becomes much easier.


Try node-line

https://github.com/kevin0571/node-line

Usage:

var line = require("line");
line(function(next) {
    obj.action1(param1, function(err, rs) {
        next({
            err: err,
            rs: rs
        });
    });
}, function(next, data) {
    if (data.err) {
        console.error(err);
        return;
    }
    obj.action2(param2, function(err, rs) {
        if (err) {
            console.error(err);
            return;
        }
        next(rs);
   });
}, function(rs) {
   obj.finish(rs);
});

For the most part, working Twitter OAuth2 application-only example, using Kris' Q promise library with https.request, Nodejs Express api route. First attempt user timeline GET. If 401 response, refreshing bearer-token then retry user timeline. I had to use Q.when to handle a promise that returns another promise (chaining) or a value.

 /**
 * Using Rails-like standard naming convention for endpoints.
 * GET     /things              ->  index
 * POST    /things              ->  create
 * GET     /things/:id          ->  show
 * PUT     /things/:id          ->  update
 * DELETE  /things/:id          ->  destroy
 */

'use strict';

// var _ = require('lodash');
var http = require('http');
var https = require('https');
var querystring = require('querystring');
var Q = require('q')

// Get list of twtimelines
exports.index = function(req, res) {
    var tid = req.query.tid
    if (tid) {
        Q.when(reqTimeline(tid, true, res), function(value) {
            // > value
            // 404
            // > body1
            // '{"errors":[{"code":34,"message":"Sorry, that page does not exist."}]}'
        })
    } else {
        res.json({
            errors: [{
                message: 'no tid specified in query'
            }]
        });
    }
};


function reqPromise(options, postData) {
    var deferred = Q.defer()

    var req = https.request(options, function(res) {
        // console.log("statusCode: ", res.statusCode);
        // console.log("headers: ", res.headers);
        var statusCode = res.statusCode
        deferred.notify(res)

        res.on('data', function(d) {
            //process.stdout.write(d);
            deferred.notify(d)
        }).on('end', function() {
            deferred.resolve(statusCode)
        });
    });

    req.on('error', function(e) {
        console.error(e);
        deferred.reject(e)
    });

    req.write(postData);
    req.end();
    return deferred.promise
} // deferRequest

function isIncomingMessage(ot) {
    return ot instanceof http.IncomingMessage
}

function isBuffer(ot) {
    return ot instanceof Buffer
}

function reqTimeline(screen_name, reqBearerTokenOn401, res) {
    var optionsUserTimeline = {
        hostname: 'api.twitter.com',
        path: '/1.1/statuses/user_timeline.json?' + querystring.stringify({
            count: '3',
            screen_name: screen_name
        }),
        method: 'GET',
        headers: {
            //'Authorization': 'Bearer ' + JSON.parse(body1).access_token
            'Authorization': 'Bearer ' + process.env.BEARER_TOKEN
        } // headers
    };
    console.log("optionsUserTimeline", optionsUserTimeline)

    var statusCode;
    var body1 = new Buffer(''); // default utf8 string buffer ?
    return reqPromise(optionsUserTimeline, '')
        .then(function(value) { // done
                if (reqBearerTokenOn401 && value === 401) {
                    console.log("reqTimeline - requesting bearer token")
                    return reqBearerToken(screen_name, res)
                }
                console.log("reqTimeline - done done:", value)
                res.end()
                return value
            },
            function(reason) { // error
                console.log("reqTimeline - error:", body1)
            },
            function(progress) {
                console.log("reqTimeline - progress:", body1)
                if (isIncomingMessage(progress)) {
                    body1 = body1.slice(0, 0) // re-set buffer
                    statusCode = progress.statusCode;
                    if (reqBearerTokenOn401 && statusCode === 401) {
                        // readyn for retry
                    } else {
                        res.writeHead(statusCode)
                    }
                } else if (isBuffer(progress)) {
                    if (reqBearerTokenOn401 && statusCode === 401) {
                        body1 += progress
                    } else {
                        res.write(progress)
                    }
                } else {
                    throw "reqTimeline - unexpected progress"
                }
            });
} // reqTimeline

function reqBearerToken(screen_name, res) {
    var postData = querystring.stringify({
        'grant_type': 'client_credentials'
    })
    var optionsBearerToken = {
            hostname: 'api.twitter.com',
            path: '/oauth2/token',
            method: 'POST',
            headers: {
                'Authorization': 'Basic ' + new Buffer(
                    process.env.CONSUMER_KEY + ":" + process.env.CONSUMER_SECRET
                ).toString('base64'),
                'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
                'Content-Length': postData.length
            } // headers
        }
        // console.log("key", process.env.CONSUMER_KEY)
        // console.log("secret", process.env.CONSUMER_SECRET)
        // console.log("buf", new Buffer(
        //  process.env.CONSUMER_KEY + ":" + process.env.CONSUMER_SECRET
        // ).toString())
        console.log("optionsBearerToken", optionsBearerToken)

    var body2 = new Buffer(''); // default utf8 string buffer ?
    return reqPromise(optionsBearerToken, postData)
        .then(function(value) { // done
            console.log("reqBearerToken - done:", body2)
            if (value === 200) {
                console.log("reqBearerToken - done done")
                process.env.BEARER_TOKEN = JSON.parse(body2).access_token;
                return reqTimeline(screen_name, false, res)
            }
            return value
        }, function(reason) {
            throw "reqBearerToken - " + reason
        }, function(progress) {
            if (isIncomingMessage(progress)) {
                body2 = body2.slice(0, 0) // reset buffer
            } else if (isBuffer) {
                body2 += progress
            } else {
                throw "reqBearerToken - unexpected progress"
            }
        });
} // reqBearerToken

Need Your Help

Formatting Decimal places in R

r formatting rounding

I have a number, for example 1.128347132904321674821 that I would like to show as only two decimal places when output to screen (or written to a file). How does one do that?

using swift mailer of symfony 2, got an email but getting unnecessary response in email

symfony twig swiftmailer

getting this response by email "HTTP/1.0 200 OK Cache-Control: no-cache Content-Type: text/html; charset=UTF-8 Date: Tue, 13 Nov 2012 04:56:14 GMT".