Skip to content

jio_mebibou
jio_mebibou
Commit 450a8e3e authored by Tristan Cavelier's avatar Tristan Cavelier
Browse files
Options

replicatestorage improved + jio tests

parent 1edc8e48
Hide whitespace changes
Inline Side-by-side
Showing with 254 additions and 74 deletions
src/jio.storage/replicaterevisionstorage.js
View file @ 450a8e3e
......@@ -21,6 +21,7 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
priv.storage_list_key = "storage_list";
priv.storage_list = spec[priv.storage_list_key];
my.env = my.env || spec.env || {};
priv.emptyFunction = function () {};
that.specToStore = function () {
var o = {};
......@@ -52,13 +53,34 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
};
/**
* Generates a hash code of a string
* @method hashCode
* Create an array containing dictionnary keys
* @method dictKeys2Array
* @param {object} dict The object to convert
* @return {array} The array of keys
*/
priv.dictKeys2Array = function (dict) {
var k, newlist = [];
for (k in dict) {
if (dict.hasOwnProperty(k)) {
newlist.push(k);
}
}
return newlist;
};
/**
* Generates the next revision
* @method generateNextRevision
* @param {number|string} previous_revision The previous revision
* @param {string} docid The document id
* @return {string} The next revision
*/
priv.getNextRevision = function (docid) {
priv.generateNextRevision = function (previous_revision, docid) {
my.env[docid].id += 1;
return my.env[docid].id.toString();
if (typeof previous_revision === "string") {
previous_revision = parseInt(previous_revision.split("-")[0], 10);
}
return (previous_revision + 1) + "-" + my.env[docid].id.toString();
};
/**
......@@ -68,7 +90,7 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
* @return {boolean} True if ok, else false
*/
priv.checkRevisionFormat = function (revision) {
return (/^[0-9a-zA-Z_]+$/.test(revision));
return (/^[0-9]+-[0-9a-zA-Z_]+$/.test(revision));
};
/**
......@@ -100,10 +122,10 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
};
/**
* Clones an object
* @method cloneObject
* @param {object} object The object to clone
* @return {object} The cloned object
* Clones an object in deep (without functions)
* @method clone
* @param {any} object The object to clone
* @return {any} The cloned object
*/
priv.clone = function (object) {
var tmp = JSON.stringify(object);
......@@ -113,8 +135,22 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
return JSON.parse(tmp);
};
/**
* Like addJob but also return the method and the index of the storage
* @method send
* @param {string} method The request method
* @param {number} index The storage index
* @param {object} doc The document object
* @param {object} option The request object
* @param {function} callback The callback. Parameters:
* - {string} The request method
* - {number} The storage index
* - {object} The error object
* - {object} The response object
*/
priv.send = function (method, index, doc, option, callback) {
var wrapped_callback_success, wrapped_callback_error;
callback = callback || priv.emptyFunction;
wrapped_callback_success = function (response) {
callback(method, index, undefined, response);
};
......@@ -131,6 +167,19 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
);
};
/**
* Use "send" method to all sub storages.
* Calling "callback" for each storage response.
* @method sendToAll
* @param {string} method The request method
* @param {object} doc The document object
* @param {object} option The request option
* @param {function} callback The callback. Parameters:
* - {string} The request method
* - {number} The storage index
* - {object} The error object
* - {object} The response object
*/
priv.sendToAll = function (method, doc, option, callback) {
var i;
for (i = 0; i < priv.storage_list.length; i += 1) {
......@@ -170,30 +219,20 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
} else {
doc_env = priv.initEnv(doc._id);
}
if (!priv.post_allowed && !doc_env.my_revisions[doc._rev]) {
that.error({
"status": 409,
"statusText": "Conflict",
"error": "conflict",
"message": "Cannot update a document",
"reason": "Document update conflict"
});
return;
}
my_rev = priv.getNextRevision(doc._id);
my_rev = priv.generateNextRevision(doc._rev || 0, doc._id);
functions.sendDocument();
};
functions.sendDocument = function () {
var i;
var i, cloned_doc;
for (i = 0; i < priv.storage_list.length; i += 1) {
var cloned_doc = priv.clone(doc);
cloned_doc = priv.clone(doc);
if (typeof cloned_doc._rev === "string" &&
doc_env.my_revisions[cloned_doc._rev] !== undefined) {
cloned_doc._rev = doc_env.my_revisions[cloned_doc._rev][i];
}
priv.send(
doc_env.last_revisions[i] === "unique_" + i ||
cloned_doc._rev !== undefined ? "put" : "post",
priv.put_only ? "put" : "post",
i,
cloned_doc,
command.cloneOption(),
......@@ -213,7 +252,7 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
return;
}
}
priv.updateEnv(doc_env, my_rev, index, undefined);
priv.updateEnv(doc_env, my_rev, index, null);
functions.error(err);
return;
}
......@@ -229,62 +268,142 @@ jIO.addStorageType('replicaterevision', function (spec, my) {
functions.success = function (response) {
// can be called once
that.success(response);
functions.success = function () {};
functions.success = priv.emptyFunction;
};
functions.error_count = 0;
functions.error = function (err) {
functions.error_count += 1;
if (functions.error_count === priv.storage_list.length) {
that.error(err);
functions.error = function () {};
functions.error = priv.emptyFunction;
}
};
functions.begin();
};
/**
* Get the document metadata from all sub storages, get the fastest.
* Put the document metadata to all sub storages
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.put_only = true;
that.post(command);
};
/**
* Put an attachment to a document to all sub storages
* @method putAttachment
* @param {object} command The JIO command
*/
// that.putAttachment = function (command) {
// };
/**
* Get the document or attachment from all sub storages, get the fastest.
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
var functions = {}, doc_env, docid, my_rev, waiting_response;
var functions = {}, doc_env, docid, my_rev, revs_array = [];
functions.begin = function () {
docid = command.cloneDocId();
var i, option;
docid = command.getDocId();
doc_env = my.env[doc._id];
doc_env = my.env[docid];
if (!doc_env || !doc_env.id) {
// document environment is not set
doc_env = priv.initEnv(doc._id);
doc_env = priv.initEnv(docid);
}
// document environment is set now
revs_array.length = priv.storage_list.length;
option = command.cloneOption() || {};
my_rev = option.rev;
if (my_rev) {
functions.update_env = false;
}
for (i = 0; i < priv.storage_list.length; i += 1) {
// request all sub storages
if (doc_env.my_revisions[my_rev]) {
// if my_rev exist, convert it to distant revision
option.rev = doc_env.my_revisions[my_rev][i];
}
priv.send("get", i, docid, priv.clone(option), functions.callback);
}
my_rev = priv.getNextRevision(doc._id);
priv.sendToAll("get", docid, command.cloneOption(), functions.callback);
};
functions.update_env = true;
functions.callback = function (method, index, err, response) {
if (err) {
priv.updateEnv(doc_env, my_rev, index, undefined);
revs_array[index] = null;
functions.error(err);
return;
}
priv.updateEnv(
doc_env,
my_rev,
index,
response._rev || "unique_" + index
);
doc_env.last_revisions[index] = response._rev || "unique_" + index;
revs_array[index] = response._rev || "unique_" + index;
if (doc_env.distant_revisions[response._rev || "unique_" + index]) {
// the document revision is already known
if (functions.update_env === true) {
my_rev = doc_env.distant_revisions[response._rev ||
"unique_" + index];
}
} else {
// the document revision is unknown
if (functions.update_env === true) {
my_rev = priv.generateNextRevision(0, docid);
doc_env.my_revisions[my_rev] = revs_array;
doc_env.distant_revisions[response._rev || "unique_" + index] =
my_rev;
}
functions.update_env = false;
}
response._rev = my_rev;
functions.success(response);
};
functions.success = function (response) {
var i, start, tmp, tmp_object;
functions.success = priv.emptyFunction;
if (doc_env.my_revisions[my_rev]) {
// this was not a specific revision
// we can convert revisions recieved by the sub storage
if (response._conflicts) {
// convert conflicting revisions to replicate revisions
tmp_object = {};
for (i = 0; i < response._conflicts.length; i += 1) {
tmp_object[doc_env.distant_revisions[response._conflicts[i]] ||
response._conflicts[i]] = true;
}
response._conflicts = priv.dictKeys2Array(tmp_object);
}
if (response._revisions) {
// convert revisions history to replicate revisions
tmp_object = {};
start = response._revisions.start;
for (i = 0; i < response._revisions.ids.length; i += 1, start -= 1) {
tmp = doc_env.distant_revisions[start + "-" + response._revisions.ids[i]];
if (tmp) {
response._revisions.ids[i] = tmp.split("-").slice(1).join("-");
}
}
}
if (response._revs_info) {
// convert revs info to replicate revisions
for (i = 0; i < response._revs_info.length; i += 1) {
tmp = doc_env.distant_revisions[response._revs_info[i].rev];
if (tmp) {
response._revs_info[i].rev = tmp;
}
}
}
}
that.success(response);
functions.success = function () {};
};
functions.error_count = 0;
functions.error = function (err) {
functions.error_count += 1;
if (functions.error_count === priv.storage_list.length) {
that.error(err);
functions.error = function () {};
functions.error = priv.emptyFunction;
}
};
functions.begin();
......
test/jiotests.js
View file @ 450a8e3e
......@@ -2004,7 +2004,6 @@ module ("JIO Replicate Revision Storage");
// post a new document without id
o.doc = {"title": "post document without id"};
o.revision = {"start": 0, "ids": []};
o.spy(o, "status", undefined, "Post document (without id)");
o.jio.post(o.doc, function (err, response) {
o.f.apply(arguments);
......@@ -2021,7 +2020,8 @@ module ("JIO Replicate Revision Storage");
// check document
o.doc._id = o.uuid;
o.rev = "1";
o.revision = {"start": 0, "ids": []};
o.rev = "1-1";
o.local_rev = "1-" + generateRevisionHash(o.doc, o.revision);
o.leavesAction(function (storage_description, param) {
var suffix = "", doc = clone(o.doc);
......@@ -2037,16 +2037,33 @@ module ("JIO Replicate Revision Storage");
);
});
// get the post document without revision
o.spy(o, "value", {
"_id": o.uuid,
"title": "post document without id",
"_rev": "1-1",
"_revisions": {"start": 1, "ids": ["1"]},
"_revs_info": [{"rev": "1-1", "status": "available"}]
}, "Get the previous document (without revision)");
o.jio.get(o.uuid, {
"conflicts": true,
"revs": true,
"revs_info": true
}, o.f);
o.tick(o);
// post a new document with id
o.doc = {"_id": "post1", "title": "post new doc with id"};
o.rev = "1"
o.spy(o, "value", {"ok": true, "id": "post1", "rev": o.rev},
o.doc = {"_id": "doc1", "title": "post new doc with id"};
o.spy(o, "value", {"ok": true, "id": "doc1", "rev": o.rev},
"Post document (with id)");
o.jio.post(o.doc, o.f);
o.tick(o);
// check document
o.local_rev = "1-" + generateRevisionHash(o.doc, o.revision);
o.local_rev_hash = generateRevisionHash(o.doc, o.revision);
o.local_rev = "1-" + o.local_rev_hash;
o.specific_rev_hash = o.local_rev_hash;
o.specific_rev = o.local_rev;
o.leavesAction(function (storage_description, param) {
var suffix = "", doc = clone(o.doc);
if (param.revision) {
......@@ -2055,15 +2072,30 @@ module ("JIO Replicate Revision Storage");
}
deepEqual(
localstorage.getItem(generateLocalPath(storage_description) +
"/post1" + suffix),
"/doc1" + suffix),
doc, "Check document"
);
});
// get the post document without revision
o.spy(o, "value", {
"_id": "doc1",
"title": "post new doc with id",
"_rev": "1-1",
"_revisions": {"start": 1, "ids": ["1"]},
"_revs_info": [{"rev": "1-1", "status": "available"}]
}, "Get the previous document (without revision)");
o.jio.get("doc1", {
"conflicts": true,
"revs": true,
"revs_info": true
}, o.f);
o.tick(o);
// post same document without revision
o.doc = {"_id": "post1", "title": "post same document without revision"};
o.rev = "2";
o.spy(o, "value", {"ok": true, "id": "post1", "rev": o.rev},
o.doc = {"_id": "doc1", "title": "post same document without revision"};
o.rev = "1-2";
o.spy(o, "value", {"ok": true, "id": "doc1", "rev": o.rev},
"Post same document (without revision)");
o.jio.post(o.doc, o.f);
o.tick(o);
......@@ -2078,15 +2110,15 @@ module ("JIO Replicate Revision Storage");
}
deepEqual(
localstorage.getItem(generateLocalPath(storage_description) +
"/post1" + suffix),
"/doc1" + suffix),
doc, "Check document"
);
});
// post a new revision
o.doc = {"_id": "post1", "title": "post new revision", "_rev": o.rev};
o.rev = "3";
o.spy(o, "value", {"ok": true, "id": "post1", "rev": o.rev},
o.doc = {"_id": "doc1", "title": "post new revision", "_rev": o.rev};
o.rev = "2-3";
o.spy(o, "value", {"ok": true, "id": "doc1", "rev": o.rev},
"Post document (with revision)");
o.jio.post(o.doc, o.f);
o.tick(o);
......@@ -2096,6 +2128,7 @@ module ("JIO Replicate Revision Storage");
o.revision.ids.unshift(o.local_rev.split("-").slice(1).join("-"));
o.doc._rev = o.local_rev;
o.local_rev = "2-" + generateRevisionHash(o.doc, o.revision);
o.specific_rev_conflict = o.local_rev;
o.leavesAction(function (storage_description, param) {
var suffix = "", doc = clone(o.doc);
delete doc._rev;
......@@ -2105,25 +2138,50 @@ module ("JIO Replicate Revision Storage");
}
deepEqual(
localstorage.getItem(generateLocalPath(storage_description) +
"/post1" + suffix),
"/doc1" + suffix),
doc, "Check document"
);
});
// get the post document with revision
o.spy(o, "value", {
"_id": "doc1",
"title": "post same document without revision",
"_rev": "1-2",
"_revisions": {"start": 1, "ids": ["2"]},
"_revs_info": [{"rev": "1-2", "status": "available"}],
"_conflicts": ["1-1"]
}, "Get the previous document (with revision)");
o.jio.get("doc1", {
"conflicts": true,
"revs": true,
"revs_info": true,
"rev": "1-2"
}, o.f);
o.tick(o);
// get the post document with specific revision
console.log(o.specific_rev);
o.spy(o, "value", {
"_id": "doc1",
"title": "post new doc with id",
"_rev": o.specific_rev,
"_revisions": {"start": 1, "ids": [o.specific_rev_hash]},
"_revs_info": [{"rev": o.specific_rev, "status": "available"}],
"_conflicts": [o.specific_rev_conflict]
}, "Get a previous document (with local storage revision)");
o.jio.get("doc1", {
"conflicts": true,
"revs": true,
"revs_info": true,
"rev": o.specific_rev
}, o.f);
o.tick(o);
o.jio.stop();
};
test ("[Local Storage] Scenario", function () {
testReplicateRevisionStorageGenerator(this, {
"type": "replicaterevision",
"storage_list": [{
"type": "local",
"username": "ureploc",
"application_name": "areploc"
}]
});
});
test ("[Revision + Local Storage] Scenario", function () {
testReplicateRevisionStorageGenerator(this, {
"type": "replicaterevision",
......@@ -2137,20 +2195,23 @@ module ("JIO Replicate Revision Storage");
}]
});
});
test ("[Revision + Local Storage, Local Storage] Scenario", function () {
test ("2x [Revision + Local Storage] Scenario", function () {
testReplicateRevisionStorageGenerator(this, {
"type": "replicaterevision",
"storage_list": [{
"type": "revision",
"sub_storage": {
"type": "local",
"username": "ureprevlocloc",
"application_name": "areprevlocloc"
"username": "ureprevlocloc1",
"application_name": "areprevloc1"
}
}, {
"type": "revision",
"sub_storage": {
"type": "local",
"username": "ureprevlocloc2",
"application_name": "areprevloc2"
}
},{
"type": "local",
"username": "ureprevlocloc2",
"application_name": "areprevlocloc2"
}]
});
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7