| return // dont run this test for now since tape is weird and broken on 0.10 |
| |
| var fs = require('fs') |
| var JSONStream = require('../') |
| var file = process.argv[2] || '/tmp/JSONStream-test-large.json' |
| var size = Number(process.argv[3] || 100000) |
| var tape = require('tape') |
| // if (process.title !== 'browser') { |
| tape('out of mem', function (t) { |
| t.plan(1) |
| ////////////////////////////////////////////////////// |
| // Produces a random number between arg1 and arg2 |
| ////////////////////////////////////////////////////// |
| var randomNumber = function (min, max) { |
| var number = Math.floor(Math.random() * (max - min + 1) + min); |
| return number; |
| }; |
| |
| ////////////////////////////////////////////////////// |
| // Produces a random string of a length between arg1 and arg2 |
| ////////////////////////////////////////////////////// |
| var randomString = function (min, max) { |
| |
| // add several spaces to increase chanses of creating 'words' |
| var chars = ' 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; |
| var result = ''; |
| |
| var randomLength = randomNumber(min, max); |
| |
| for (var i = randomLength; i > 0; --i) { |
| result += chars[Math.round(Math.random() * (chars.length - 1))]; |
| } |
| return result; |
| }; |
| |
| ////////////////////////////////////////////////////// |
| // Produces a random JSON document, as a string |
| ////////////////////////////////////////////////////// |
| var randomJsonDoc = function () { |
| |
| var doc = { |
| "CrashOccurenceID": randomNumber(10000, 50000), |
| "CrashID": randomNumber(1000, 10000), |
| "SiteName": randomString(10, 25), |
| "MachineName": randomString(10, 25), |
| "Date": randomString(26, 26), |
| "ProcessDuration": randomString(18, 18), |
| "ThreadIdentityName": null, |
| "WindowsIdentityName": randomString(15, 40), |
| "OperatingSystemName": randomString(35, 65), |
| "DetailedExceptionInformation": randomString(100, 800) |
| }; |
| |
| doc = JSON.stringify(doc); |
| doc = doc.replace(/\,/g, ',\n'); // add new lines after each attribute |
| return doc; |
| }; |
| |
| ////////////////////////////////////////////////////// |
| // generates test data |
| ////////////////////////////////////////////////////// |
| var generateTestData = function (cb) { |
| |
| console.log('generating large data file...'); |
| |
| var stream = fs.createWriteStream(file, { |
| encoding: 'utf8' |
| }); |
| |
| var i = 0; |
| var max = size; |
| var writing = false |
| var split = ',\n'; |
| var doc = randomJsonDoc(); |
| stream.write('['); |
| |
| function write () { |
| if(writing) return |
| writing = true |
| while(++i < max) { |
| if(Math.random() < 0.001) |
| console.log('generate..', i + ' / ' + size) |
| if(!stream.write(doc + split)) { |
| writing = false |
| return stream.once('drain', write) |
| } |
| } |
| stream.write(doc + ']') |
| stream.end(); |
| console.log('END') |
| } |
| write() |
| stream.on('close', cb) |
| }; |
| |
| ////////////////////////////////////////////////////// |
| // Shows that parsing 100000 instances using JSONStream fails |
| // |
| // After several seconds, you will get this crash |
| // FATAL ERROR: JS Allocation failed - process out of memory |
| ////////////////////////////////////////////////////// |
| var testJSONStreamParse_causesOutOfMem = function (done) { |
| var items = 0 |
| console.log('parsing data files using JSONStream...'); |
| |
| var parser = JSONStream.parse([true]); |
| var stream = fs.createReadStream(file); |
| stream.pipe(parser); |
| |
| parser.on('data', function (data) { |
| items++ |
| if(Math.random() < 0.01) console.log(items, '...') |
| }); |
| |
| parser.on('end', function () { |
| t.equal(items, size) |
| }); |
| |
| }; |
| |
| ////////////////////////////////////////////////////// |
| // main |
| ////////////////////////////////////////////////////// |
| |
| fs.stat(file, function (err, stat) { |
| console.log(stat) |
| if(err) |
| generateTestData(testJSONStreamParse_causesOutOfMem); |
| else |
| testJSONStreamParse_causesOutOfMem() |
| }) |
| |
| }) |
| |
| // } |