(#445) Implement Incremental MD5 calculation Replace jkmyers-md5 with Spark-MD5 in order to calculate MD5 hashes incrementally. The use-case for this change is to avoid blocking the UI thread when adding large attachments.
diff --git a/lib/adapters/idb.js b/lib/adapters/idb.js index 4b682b9..995b935 100644 --- a/lib/adapters/idb.js +++ b/lib/adapters/idb.js
@@ -296,22 +296,26 @@ "Attachments need to be base64 encoded"); return callback(err); } - att.digest = 'md5-' + utils.MD5(data); if (blobSupport) { var type = att.content_type; data = utils.fixBinary(data); att.data = utils.createBlob([data], {type: type}); } - return finish(); + utils.MD5(data).then(function (result) { + att.digest = 'md5-' + result; + finish(); + }); } var reader = new FileReader(); reader.onloadend = function (e) { var binary = utils.arrayBufferToBinaryString(this.result); - att.digest = 'md5-' + utils.MD5(binary); if (!blobSupport) { att.data = btoa(binary); } - finish(); + utils.MD5(binary).then(function (result) { + att.digest = 'md5-' + result; + finish(); + }); }; reader.readAsArrayBuffer(att.data); }
diff --git a/lib/adapters/leveldb.js b/lib/adapters/leveldb.js index 3b3d151..ff82821 100644 --- a/lib/adapters/leveldb.js +++ b/lib/adapters/leveldb.js
@@ -202,7 +202,7 @@ stores.docStore.get(id, function (err, metadata) { db.removeListener('pouchdb-id-' + id, didDocChange); - + if (err || !metadata) { return callback(errors.MISSING_DOC); } @@ -459,11 +459,20 @@ collectResults(err); } - function onLoadEnd(e) { - var myData = global.escape( - utils.arrayBufferToBinaryString(e.target.result)); - var myDigest = 'md5-' + utils.MD5(myData); - saveAttachment(doc, myDigest, key, myData, attachmentSaved); + function onMD5Load(doc, prefix, key, data, attachmentSaved) { + return function (result) { + saveAttachment(doc, prefix + result, key, data, attachmentSaved); + }; + } + + function onLoadEnd(doc, prefix, key, attachmentSaved) { + return function (e) { + var data = global.escape( + utils.arrayBufferToBinaryString(e.target.result)); + utils.MD5(data).then( + onMD5Load(doc, prefix, key, data, attachmentSaved) + ); + }; } for (var i = 0; i < attachments.length; i++) { @@ -475,8 +484,8 @@ continue; } var att = doc.data._attachments[key]; - var digest; var data; + var prefix; if (typeof att.data === 'string') { try { data = utils.atob(att.data); @@ -488,17 +497,19 @@ {reason: "Attachments need to be base64 encoded"})); return; } - digest = (process.browser ? 'md5-' : '') + utils.MD5(data || ''); + prefix = process.browser ? 'md5-' : ''; } else if (!process.browser) { data = att.data; - digest = utils.MD5(data || ''); + prefix = ''; } else { // browser var reader = new FileReader(); - reader.onloadend = onLoadEnd; + reader.onloadend = onLoadEnd(doc, 'md5-', key, attachmentSaved); reader.readAsArrayBuffer(att.data); return; } - saveAttachment(doc, digest, key, data, attachmentSaved); + utils.MD5(data).then( + onMD5Load(doc, prefix, key, data, attachmentSaved) + ); } function finish() { @@ -733,7 +744,7 @@ }); docstream.on('error', callback); - + docstream.pipe(throughStream); }); };
diff --git a/lib/adapters/websql.js b/lib/adapters/websql.js index 5959faf..c679587 100644 --- a/lib/adapters/websql.js +++ b/lib/adapters/websql.js
@@ -112,7 +112,7 @@ var idRequests = []; var docCount = -1; // cache sqlite count(*) for performance var encoding; - + var db = openDB(name, POUCH_VERSION, name, size); if (!db) { return callback(errors.UNKNOWN_ERROR); @@ -439,9 +439,11 @@ var reader = new FileReader(); reader.onloadend = function (e) { var binary = utils.arrayBufferToBinaryString(this.result); - att.data = binary; - att.digest = 'md5-' + utils.MD5(binary); - finish(); + att.data = btoa(binary); + utils.MD5(binary).then(function (result) { + att.digest = 'md5-' + result; + finish(); + }); }; reader.readAsArrayBuffer(att.data); }
diff --git a/lib/replicate.js b/lib/replicate.js index afdd482..7d2a18f 100644 --- a/lib/replicate.js +++ b/lib/replicate.js
@@ -55,7 +55,9 @@ return target.id().then(function (target_id) { var queryData = src_id + target_id + filterFun + JSON.stringify(opts.query_params) + opts.doc_ids; - return '_local/' + utils.MD5(queryData); + return utils.MD5(queryData).then(function (md5) { + return '_local/' + md5; + }); }); }); }
diff --git a/lib/utils.js b/lib/utils.js index 76f06e9..d237dc6 100644 --- a/lib/utils.js +++ b/lib/utils.js
@@ -1,7 +1,7 @@ /*jshint strict: false */ /*global chrome */ var crypto = require('crypto'); -var md5 = require('md5-jkmyers'); +var Md5 = require('spark-md5'); var merge = require('./merge'); exports.extend = require('pouchdb-extend'); exports.ajax = require('./deps/ajax'); @@ -42,7 +42,7 @@ }; exports.inherits = require('inherits'); // Determine id an ID is valid -// - invalid IDs begin with an underescore that does not begin '_design' or +// - invalid IDs begin with an underescore that does not begin '_design' or // '_local' // - any other string value is a valid id // Returns the specific error object for each case @@ -536,11 +536,33 @@ promise.emit = emitter.emit.bind(emitter); return promise; }; -exports.Crypto = {}; -exports.MD5 = exports.Crypto.MD5 = function (string) { - if (!process.browser) { - return crypto.createHash('md5').update(string).digest('hex'); - } else { - return md5(string); + +exports.MD5 = exports.toPromise( + function (data, callback) { + if (!process.browser) { + callback(null, crypto.createHash('md5').update(data).digest('hex')); + return; + } + var chunkSize = Math.min(524288, data.length); + var chunks = Math.ceil(data.length / chunkSize); + var currentChunk = 0; + var buffer = new Md5(); + function loadNextChunk() { + var start = currentChunk * chunkSize; + var end = start + chunkSize; + if ((start + chunkSize) >= data.size) { + end = data.size; + } + currentChunk++; + if (currentChunk < chunks) { + buffer.append(data.slice(start, end)); + setImmediate(loadNextChunk); + } else { + buffer.append(data.slice(start, end)); + callback(null, buffer.end()); + buffer.destroy(); + } + } + loadNextChunk(); } -}; +);
diff --git a/package.json b/package.json index 600f159..ff12cb8 100644 --- a/package.json +++ b/package.json
@@ -18,6 +18,7 @@ "dependencies": { "argsarray": "0.0.1", "bluebird": "^1.2.4", + "es3ify": "^0.1.3", "inherits": "~2.0.1", "level-js": "^2.1.3", "level-sublevel": "~5.2.0", @@ -25,13 +26,13 @@ "levelup": "~0.18.4", "lie": "^2.6.0", "localstorage-down": "^0.4.4", + "md5-jkmyers": "0.0.1", + "memdown": "^0.8.0", + "pouchdb-extend": "^0.1.0", "pouchdb-mapreduce": "~2.2.4", "request": "~2.28.0", - "md5-jkmyers": "0.0.1", - "through2": "^0.4.1", - "es3ify": "^0.1.3", - "memdown": "^0.8.0", - "pouchdb-extend": "^0.1.0" + "spark-md5": "0.0.5", + "through2": "^0.4.1" }, "devDependencies": { "rimraf": "2.2.8",
diff --git a/tests/test.changes.js b/tests/test.changes.js index e6178d8..fb46cda 100644 --- a/tests/test.changes.js +++ b/tests/test.changes.js
@@ -1179,45 +1179,45 @@ {_id: '2', integer: 11}, {_id: '3', integer: 12}, ]; - - new PouchDB(dbs.name, function (err, localdb) { + new PouchDB(dbs.name).then(function (localdb) { var remotedb = new PouchDB(dbs.remote); - localdb.bulkDocs({ docs: docs1 }, function (err, info) { + return localdb.bulkDocs({ docs: docs1 }).then(function (info) { docs2[0]._rev = info[2].rev; docs2[1]._rev = info[3].rev; - localdb.put(docs2[0], function (err, info) { - localdb.put(docs2[1], function (err, info) { + return localdb.put(docs2[0]).then(function (info) { + return localdb.put(docs2[1]).then(function (info) { var rev2 = info.rev; - PouchDB.replicate(localdb, remotedb, function (err, done) { + return PouchDB.replicate(localdb, remotedb).then(function (done) { // update remote once, local twice, then replicate from // remote to local so the remote losing conflict is later in // the tree - localdb.put({ + return localdb.put({ _id: '3', _rev: rev2, integer: 20 - }, function (err, resp) { + }).then(function (resp) { var rev3Doc = { _id: '3', _rev: resp.rev, integer: 30 }; - localdb.put(rev3Doc, function (err, resp) { + return localdb.put(rev3Doc).then(function (resp) { var rev4local = resp.rev; var rev4Doc = { _id: '3', _rev: rev2, integer: 100 }; - remotedb.put(rev4Doc, function (err, resp) { + return remotedb.put(rev4Doc).then(function (resp) { var remoterev = resp.rev; - PouchDB.replicate(remotedb, localdb, - function (err, done) { - localdb.changes({ + return PouchDB.replicate(remotedb, localdb).then( + function (done) { + return localdb.changes({ include_docs: true, style: 'all_docs', - conflicts: true, - complete: function (err, changes) { + conflicts: true + }).on('error', testDone) + .then(function (changes) { changes.results.length.should.equal(4); var ch = changes.results[3]; ch.id.should.equal('3'); @@ -1231,9 +1231,7 @@ ch.doc.should.have.property('_conflicts'); ch.doc._conflicts.length.should.equal(1); ch.doc._conflicts[0].should.equal(remoterev); - testDone(); - } - }); + }); }); }); }); @@ -1241,7 +1239,9 @@ }); }); }); - }); + }).then(function () { + testDone(); + }, testDone); }); });