A hidden feature: Total.js Chunker

// U.chunker(name, [items_per_page])
var chunker = U.chunker('products', 50);
// Filenames will be in this form: /tmp/products1.json, /tmp/products100.json

// We have a big data stream
somestream.on('data', U.streamer('<product>', '</product>', function(value, index) {
    var product = value.parseXML();

    // Chunker creates multiple files with a specific number of the "chunks" (in our case: 50 products).
    // chunker.write(object);
    chunker.write(product);

}));

// No more data, let's process the files chunker created
somestream.on('end', function() {

    // Flushes last open file
    chunker.end();

    // Reads each file
    chunker.each(function(items, next) {
        // items === 50 products
        // here we can update DB
        // here we have a lot of time :)

        // ....
        // ....

        // Read next items
        next();

    }, function() {
        // This method is called when the chunker doesn't contain any more items (optional).
        // Cleans up hdd and memory
        chunker.destroy();
    });

    // Reads items by page number
    // chunker.read(page, function(err, items) {
    //     items === products (max 50)
    // });

    // Deletes created files
    // chunker.clear();

    // Calls chunker.clear(); internally and deletes in-memory data
    // chunker.destroy();

});