Skip to content

jio
jio
Commit fa8ad6e0 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

ReplicateStorage: use a property as hash 

Speed up document replication by using a document property to check if there was a change.
As allDocs can return this property directly, it reduces the required calculation.
parent 40ef31b0
Show whitespace changes
Inline Side-by-side
Showing with 4008 additions and 27 deletions
src/jio.storage/replicatestorage.js
View file @ fa8ad6e0
...@@ -28,6 +28,15 @@ ...@@ -28,6 +28,15 @@
CONFLICT_KEEP_REMOTE = 2, CONFLICT_KEEP_REMOTE = 2,
CONFLICT_CONTINUE = 3; CONFLICT_CONTINUE = 3;
function SkipError(message) {
if ((message !== undefined) && (typeof message !== "string")) {
throw new TypeError('You must pass a string.');
}
this.message = message || "Skip some asynchronous code";
}
SkipError.prototype = new Error();
SkipError.prototype.constructor = SkipError;
/**************************************************** /****************************************************
Use a local jIO to read/write/search documents Use a local jIO to read/write/search documents
Synchronize in background those document with a remote jIO. Synchronize in background those document with a remote jIO.
...@@ -46,6 +55,10 @@ ...@@ -46,6 +55,10 @@
function ReplicateStorage(spec) { function ReplicateStorage(spec) {
this._query_options = spec.query || {}; this._query_options = spec.query || {};
if (spec.signature_hash_key !== undefined) {
this._query_options.select_list = [spec.signature_hash_key];
}
this._signature_hash_key = spec.signature_hash_key;
this._local_sub_storage = jIO.createJIO(spec.local_sub_storage); this._local_sub_storage = jIO.createJIO(spec.local_sub_storage);
this._remote_sub_storage = jIO.createJIO(spec.remote_sub_storage); this._remote_sub_storage = jIO.createJIO(spec.remote_sub_storage);
...@@ -239,6 +252,26 @@ ...@@ -239,6 +252,26 @@
return result_promise_list[0]; return result_promise_list[0];
} }
function callAllDocsOnStorage(context, storage, cache, cache_key) {
return new RSVP.Queue()
.push(function () {
if (!cache.hasOwnProperty(cache_key)) {
return storage.allDocs(context._query_options)
.push(function (result) {
var i,
cache_entry = {};
for (i = 0; i < result.data.total_rows; i += 1) {
cache_entry[result.data.rows[i].id] = result.data.rows[i].value;
}
cache[cache_key] = cache_entry;
});
}
})
.push(function () {
return cache[cache_key];
});
}
function propagateAttachmentDeletion(context, skip_attachment_dict, function propagateAttachmentDeletion(context, skip_attachment_dict,
destination, destination,
id, name) { id, name) {
...@@ -602,14 +635,32 @@ ...@@ -602,14 +635,32 @@
function propagateModification(context, source, destination, doc, hash, id, function propagateModification(context, source, destination, doc, hash, id,
skip_document_dict, skip_document_dict,
options) { options) {
var result, var result = new RSVP.Queue(),
post_id, post_id,
to_skip = true; to_skip = true;
if (options === undefined) { if (options === undefined) {
options = {}; options = {};
} }
if (doc === null) {
result
.push(function () {
return source.get(id);
})
.push(function (source_doc) {
doc = source_doc;
}, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
throw new SkipError(id);
}
throw error;
});
}
if (options.use_post) { if (options.use_post) {
result = destination.post(doc) result
.push(function () {
return destination.post(doc);
})
.push(function (new_id) { .push(function (new_id) {
to_skip = false; to_skip = false;
post_id = new_id; post_id = new_id;
...@@ -658,7 +709,10 @@ ...@@ -658,7 +709,10 @@
skip_document_dict[post_id] = null; skip_document_dict[post_id] = null;
}); });
} else { } else {
result = destination.put(id, doc) result
.push(function () {
return destination.put(id, doc);
})
.push(function () { .push(function () {
return context._signature_sub_storage.put(id, { return context._signature_sub_storage.put(id, {
"hash": hash "hash": hash
...@@ -670,6 +724,12 @@ ...@@ -670,6 +724,12 @@
if (to_skip) { if (to_skip) {
skip_document_dict[id] = null; skip_document_dict[id] = null;
} }
})
.push(undefined, function (error) {
if (error instanceof SkipError) {
return;
}
throw error;
}); });
} }
...@@ -702,11 +762,24 @@ ...@@ -702,11 +762,24 @@
} }
function checkAndPropagate(context, skip_document_dict, function checkAndPropagate(context, skip_document_dict,
cache, destination_key,
status_hash, local_hash, doc, status_hash, local_hash, doc,
source, destination, id, source, destination, id,
conflict_force, conflict_revert, conflict_force, conflict_revert,
conflict_ignore, conflict_ignore,
options) { options) {
return new RSVP.Queue()
.push(function () {
if (options.signature_hash_key !== undefined) {
return callAllDocsOnStorage(context, destination,
cache, destination_key)
.push(function (result) {
if (result.hasOwnProperty(id)) {
return [null, result[id][options.signature_hash_key]];
}
return [null, null];
});
}
return destination.get(id) return destination.get(id)
.push(function (remote_doc) { .push(function (remote_doc) {
return [remote_doc, generateHash(stringify(remote_doc))]; return [remote_doc, generateHash(stringify(remote_doc))];
...@@ -716,11 +789,12 @@ ...@@ -716,11 +789,12 @@
return [null, null]; return [null, null];
} }
throw error; throw error;
});
}) })
.push(function (remote_list) { .push(function (remote_list) {
var remote_doc = remote_list[0], var remote_doc = remote_list[0],
remote_hash = remote_list[1]; remote_hash = remote_list[1];
if (local_hash === remote_hash) { if (local_hash === remote_hash) {
// Same modifications on both side // Same modifications on both side
if (local_hash === null) { if (local_hash === null) {
...@@ -783,14 +857,17 @@ ...@@ -783,14 +857,17 @@
local_hash, id, skip_document_dict, local_hash, id, skip_document_dict,
{use_post: options.use_post}); {use_post: options.use_post});
} }
doc = doc || local_hash;
remote_doc = remote_doc || remote_hash;
throw new jIO.util.jIOError("Conflict on '" + id + "': " + throw new jIO.util.jIOError("Conflict on '" + id + "': " +
stringify(doc || '') + " !== " + stringify(doc) + " !== " +
stringify(remote_doc || ''), stringify(remote_doc),
409); 409);
}); });
} }
function checkLocalDeletion(queue, context, skip_document_dict, function checkLocalDeletion(queue, context, skip_document_dict,
cache, destination_key,
destination, id, source, destination, id, source,
conflict_force, conflict_revert, conflict_force, conflict_revert,
conflict_ignore, options) { conflict_ignore, options) {
...@@ -802,6 +879,7 @@ ...@@ -802,6 +879,7 @@
.push(function (result) { .push(function (result) {
status_hash = result.hash; status_hash = result.hash;
return checkAndPropagate(context, skip_document_dict, return checkAndPropagate(context, skip_document_dict,
cache, destination_key,
status_hash, null, null, status_hash, null, null,
source, destination, id, source, destination, id,
conflict_force, conflict_revert, conflict_force, conflict_revert,
...@@ -811,20 +889,29 @@ ...@@ -811,20 +889,29 @@
} }
function checkSignatureDifference(queue, context, skip_document_dict, function checkSignatureDifference(queue, context, skip_document_dict,
cache, destination_key,
source, destination, id, source, destination, id,
conflict_force, conflict_revert, conflict_force, conflict_revert,
conflict_ignore, conflict_ignore,
status_hash, local_hash, status_hash,
options) { options) {
queue queue
.push(function () { .push(function () {
if (local_hash === null) {
// Hash was not provided by the allDocs query
return source.get(id); return source.get(id);
}
return null;
}) })
.push(function (doc) { .push(function (doc) {
var local_hash = generateHash(stringify(doc)); if (local_hash === null) {
// Hash was not provided by the allDocs query
local_hash = generateHash(stringify(doc));
}
if (local_hash !== status_hash) { if (local_hash !== status_hash) {
return checkAndPropagate(context, skip_document_dict, return checkAndPropagate(context, skip_document_dict,
cache, destination_key,
status_hash, local_hash, doc, status_hash, local_hash, doc,
source, destination, id, source, destination, id,
conflict_force, conflict_revert, conflict_force, conflict_revert,
...@@ -835,6 +922,7 @@ ...@@ -835,6 +922,7 @@
} }
function pushStorage(context, skip_document_dict, function pushStorage(context, skip_document_dict,
cache, source_key, destination_key,
source, destination, signature_allDocs, options) { source, destination, signature_allDocs, options) {
var argument_list = [], var argument_list = [],
argument_list_deletion = []; argument_list_deletion = [];
...@@ -844,7 +932,7 @@ ...@@ -844,7 +932,7 @@
if (!options.hasOwnProperty("use_revert_post")) { if (!options.hasOwnProperty("use_revert_post")) {
options.use_revert_post = false; options.use_revert_post = false;
} }
return source.allDocs(context._query_options) return callAllDocsOnStorage(context, source, cache, source_key)
.push(function (source_allDocs) { .push(function (source_allDocs) {
var i, var i,
local_dict = {}, local_dict = {},
...@@ -852,15 +940,26 @@ ...@@ -852,15 +940,26 @@
is_modification, is_modification,
is_creation, is_creation,
status_hash, status_hash,
local_hash,
key, key,
queue = new RSVP.Queue(); queue = new RSVP.Queue();
for (key in source_allDocs) {
if (source_allDocs.hasOwnProperty(key)) {
if (!skip_document_dict.hasOwnProperty(key)) {
local_dict[key] = source_allDocs[key];
}
}
}
/*
for (i = 0; i < source_allDocs.data.total_rows; i += 1) { for (i = 0; i < source_allDocs.data.total_rows; i += 1) {
if (!skip_document_dict.hasOwnProperty( if (!skip_document_dict.hasOwnProperty(
source_allDocs.data.rows[i].id source_allDocs.data.rows[i].id
)) { )) {
local_dict[source_allDocs.data.rows[i].id] = i; local_dict[source_allDocs.data.rows[i].id] =
source_allDocs.data.rows[i].value;
} }
} }
*/
for (i = 0; i < signature_allDocs.data.total_rows; i += 1) { for (i = 0; i < signature_allDocs.data.total_rows; i += 1) {
if (!skip_document_dict.hasOwnProperty( if (!skip_document_dict.hasOwnProperty(
signature_allDocs.data.rows[i].id signature_allDocs.data.rows[i].id
...@@ -882,14 +981,28 @@ ...@@ -882,14 +981,28 @@
status_hash = signature_dict[key]; status_hash = signature_dict[key];
} }
local_hash = null;
if (options.signature_hash_key !== undefined) {
local_hash = local_dict[key][options.signature_hash_key];
if (is_modification === true) {
// Bypass fetching all documents and calculating the sha
// Compare the select list values returned by allDocs calls
is_modification = false;
if (local_hash !== status_hash) {
is_modification = true;
}
}
}
if (is_modification === true || is_creation === true) { if (is_modification === true || is_creation === true) {
argument_list.push([undefined, context, skip_document_dict, argument_list.push([undefined, context, skip_document_dict,
cache, destination_key,
source, destination, source, destination,
key, key,
options.conflict_force, options.conflict_force,
options.conflict_revert, options.conflict_revert,
options.conflict_ignore, options.conflict_ignore,
status_hash, local_hash, status_hash,
options]); options]);
} }
} }
...@@ -910,6 +1023,7 @@ ...@@ -910,6 +1023,7 @@
argument_list_deletion.push([undefined, argument_list_deletion.push([undefined,
context, context,
skip_document_dict, skip_document_dict,
cache, destination_key,
destination, key, destination, key,
source, source,
options.conflict_force, options.conflict_force,
...@@ -941,7 +1055,8 @@ ...@@ -941,7 +1055,8 @@
ReplicateStorage.prototype.repair = function () { ReplicateStorage.prototype.repair = function () {
var context = this, var context = this,
argument_list = arguments, argument_list = arguments,
skip_document_dict = {}; skip_document_dict = {},
cache = {};
return new RSVP.Queue() return new RSVP.Queue()
.push(function () { .push(function () {
...@@ -1004,6 +1119,7 @@ ...@@ -1004,6 +1119,7 @@
context._check_local_creation || context._check_local_creation ||
context._check_local_deletion) { context._check_local_deletion) {
return pushStorage(context, skip_document_dict, return pushStorage(context, skip_document_dict,
cache, 'local', 'remote',
context._local_sub_storage, context._local_sub_storage,
context._remote_sub_storage, context._remote_sub_storage,
signature_allDocs, signature_allDocs,
...@@ -1018,7 +1134,8 @@ ...@@ -1018,7 +1134,8 @@
check_modification: context._check_local_modification, check_modification: context._check_local_modification,
check_creation: context._check_local_creation, check_creation: context._check_local_creation,
check_deletion: context._check_local_deletion, check_deletion: context._check_local_deletion,
operation_amount: context._parallel_operation_amount operation_amount: context._parallel_operation_amount,
signature_hash_key: context._signature_hash_key
}) })
.push(function () { .push(function () {
return signature_allDocs; return signature_allDocs;
...@@ -1031,6 +1148,7 @@ ...@@ -1031,6 +1148,7 @@
context._check_remote_creation || context._check_remote_creation ||
context._check_remote_deletion) { context._check_remote_deletion) {
return pushStorage(context, skip_document_dict, return pushStorage(context, skip_document_dict,
cache, 'remote', 'local',
context._remote_sub_storage, context._remote_sub_storage,
context._local_sub_storage, context._local_sub_storage,
signature_allDocs, { signature_allDocs, {
...@@ -1044,7 +1162,8 @@ ...@@ -1044,7 +1162,8 @@
check_modification: context._check_remote_modification, check_modification: context._check_remote_modification,
check_creation: context._check_remote_creation, check_creation: context._check_remote_creation,
check_deletion: context._check_remote_deletion, check_deletion: context._check_remote_deletion,
operation_amount: context._parallel_operation_amount operation_amount: context._parallel_operation_amount,
signature_hash_key: context._signature_hash_key
}); });
} }
}) })
......
test/jio.storage/replicatestorage.tests.js
View file @ fa8ad6e0
...@@ -73,6 +73,7 @@ ...@@ -73,6 +73,7 @@
equal(jio.__storage._check_remote_attachment_creation, false); equal(jio.__storage._check_remote_attachment_creation, false);
equal(jio.__storage._check_remote_attachment_deletion, false); equal(jio.__storage._check_remote_attachment_deletion, false);
equal(jio.__storage._check_remote_attachment_modification, false); equal(jio.__storage._check_remote_attachment_modification, false);
equal(jio.__storage._signature_hash_key, undefined);
equal(jio.__storage._custom_signature_sub_storage, false); equal(jio.__storage._custom_signature_sub_storage, false);
equal(jio.__storage._signature_hash, equal(jio.__storage._signature_hash,
...@@ -127,12 +128,14 @@ ...@@ -127,12 +128,14 @@
check_local_attachment_modification: true, check_local_attachment_modification: true,
check_remote_attachment_creation: true, check_remote_attachment_creation: true,
check_remote_attachment_deletion: true, check_remote_attachment_deletion: true,
check_remote_attachment_modification: true check_remote_attachment_modification: true,
signature_hash_key: 'bar'
}); });
deepEqual( deepEqual(
jio.__storage._query_options, jio.__storage._query_options,
{query: 'portal_type: "Foo"', limit: [0, 1234567890]} {query: 'portal_type: "Foo"', limit: [0, 1234567890],
select_list: ['bar']}
); );
equal(jio.__storage._use_remote_post, true); equal(jio.__storage._use_remote_post, true);
equal(jio.__storage._conflict_handling, 3); equal(jio.__storage._conflict_handling, 3);
...@@ -150,6 +153,7 @@ ...@@ -150,6 +153,7 @@
equal(jio.__storage._check_remote_attachment_creation, true); equal(jio.__storage._check_remote_attachment_creation, true);
equal(jio.__storage._check_remote_attachment_deletion, true); equal(jio.__storage._check_remote_attachment_deletion, true);
equal(jio.__storage._check_remote_attachment_modification, true); equal(jio.__storage._check_remote_attachment_modification, true);
equal(jio.__storage._signature_hash_key, 'bar');
equal(jio.__storage._custom_signature_sub_storage, false); equal(jio.__storage._custom_signature_sub_storage, false);
ok(jio.__storage._signature_sub_storage instanceof jio.constructor); ok(jio.__storage._signature_sub_storage instanceof jio.constructor);
...@@ -175,7 +179,7 @@ ...@@ -175,7 +179,7 @@
"replicatestorage200"); "replicatestorage200");
equal(jio.__storage._signature_hash, equal(jio.__storage._signature_hash,
"_replicate_11881e431308c0ec8c0e6430be98db380e1b92f8"); "_replicate_291eaf37f6fa1ba6b6b115ab92b44cc88be0bb06");
}); });
test("reject unknow conflict resolution", function () { test("reject unknow conflict resolution", function () {
......
test/jio.storage/replicatestorage_fastrepair.tests.js 0 → 100644
View file @ fa8ad6e0
This source diff could not be displayed because it is too large. You can view the blob instead.
test/tests.html
View file @ fa8ad6e0
...@@ -42,6 +42,7 @@ ...@@ -42,6 +42,7 @@
<script src="jio.storage/replicatestorage.tests.js"></script> <script src="jio.storage/replicatestorage.tests.js"></script>
<script src="jio.storage/replicatestorage_repair.tests.js"></script> <script src="jio.storage/replicatestorage_repair.tests.js"></script>
<script src="jio.storage/replicatestorage_repairattachment.tests.js"></script> <script src="jio.storage/replicatestorage_repairattachment.tests.js"></script>
<script src="jio.storage/replicatestorage_fastrepair.tests.js"></script>
<script src="jio.storage/shastorage.tests.js"></script> <script src="jio.storage/shastorage.tests.js"></script>
<!--script src="jio.storage/qiniustorage.tests.js"></script--> <!--script src="jio.storage/qiniustorage.tests.js"></script-->
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7