Skip to content

jio_mebibou
jio_mebibou
Commit f89b3df8 authored by Sven Franck's avatar Sven Franck
Browse files
Options

indexStorage: POST new API and qunit tests

parent a9c06bba
Hide whitespace changes
Inline Side-by-side
Showing with 418 additions and 109 deletions
src/jio.storage/indexstorage.js
View file @ f89b3df8
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global jIO: true, localStorage: true, setTimeout: true */
/**
* JIO Index Storage.
* Manages indexes for specified storages.
* Description:
* {
* "type": "index",
* "indices": [
* {"indexA",["field_A"]},
* {"indexAB",["field_A","field_B"]}
* ],
* "storage": [
* <sub storage description>,
* ...
* ]
* }
* Index file will contain
* {
* "_id": "ipost_indices.json",
* "indexA": {
* "keyword_abc": ["some_id","some_other_id",...]
* },
* "indexAB": {
* "keyword_abc": ["some_id"],
* "keyword_def": ["some_id"]
* }
* }
*/
jIO.addStorageType('indexed', function (spec, my) {
"use strict";
var that, priv = {}, spec;
spec = spec || {};
var that = my.basicStorage(spec, my),
priv = {},
validatestate_sub_storage = spec.storage || false,
storage_object_name = 'jio/indexed_storage_object',
storage_file_object_name,
super_serialized = that.serialized;
that = my.basicStorage(spec, my);
priv.sub_storage_spec = spec.storage || {
type: 'base'
};
priv.sub_storage_string = JSON.stringify(priv.sub_storage_spec);
priv.indices = spec.indices;
priv.substorage_key = "sub_storage";
priv.substorage = spec[priv.substorage_key];
priv.index_indicator = spec.sub_storage.application_name || "index";
priv.index_suffix = priv.index_indicator + "_indices.json";
storage_file_object_name = 'jio/indexed_file_object/' +
priv.sub_storage_string;
my.env = my.env || spec.env || {};
that.serialized = function () {
var o = super_serialized();
o.storage = priv.sub_storage_spec;
that.specToStore = function () {
var o = {};
o[priv.substorage_key] = priv.substorage;
o.env = my.env;
return o;
};
that.validateState = function () {
if (!validatestate_sub_storage) {
return 'Need at least one parameter: "storage" ' +
'containing storage specifications.';
/**
* Generate a new uuid
* @method generateUuid
* @return {string} The new uuid
*/
priv.generateUuid = function () {
var S4 = function () {
var i, string = Math.floor(
Math.random() * 0x10000 /* 65536 */
).toString(16);
for (i = string.length; i < 4; i += 1) {
string = "0" + string;
}
return string;
};
return S4() + S4() + "-" +
S4() + "-" +
S4() + "-" +
S4() + "-" +
S4() + S4() + S4();
};
/**
* Escape string before storing
* @method sanitizeValue
* @param {string} s The string to be sanitized
* @return {string} The sanitized string
*/
priv.sanitizeValue = function (s) {
return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
};
/**
* Get number of elements in object
* @method getObjectSize
* @param {object} obj The object to check
* @return {number} size The amount of elements in the object
*/
priv.getObjectSize = function (obj) {
var size = 0, key;
for (key in obj) {
if (obj.hasOwnProperty(key)) {
size += 1;
}
}
return '';
return size;
};
priv.secureDocId = function (string) {
var split = string.split('/'),
i;
if (split[0] === '') {
split = split.slice(1);
/**
* Creates an empty indices array
* @method createEmptyIndexArray
* @param {array} indices An array of indices (optional)
* @return {object} The new index array
*/
priv.createEmptyIndexArray = function (indices) {
var obj, i, j = priv.indices.length,
new_index_object = {}, new_index_name;
if (indices === undefined) {
for (i = 0; i < j; i += 1) {
new_index_name = priv.indices[i]["name"];
new_index_object[new_index_name] = {};
}
}
for (i = 0; i < split.length; i += 1) {
if (split[i] === '') {
return '';
return new_index_object;
};
/**
* Find id in indices
* @method docidInIndex
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
* @return {boolean} true/false
*/
priv.docidInIndex = function (indices, doc) {
var i, j, l = priv.indices.length, elements_to_check,
index, index_name, index_length;
// loop indices
for (i = 0; i < l; i += 1) {
index = priv.indices[i];
index_name = index["name"];
index_length = index.fields.length;
elements_to_check = priv.getObjectSize(indices[index_name]);
if (elements_to_check > 0) {
for (var key in indices[index_name]) {
var obj = indices[index_name][key];
for (var prop in obj) {
if (obj[prop] === doc._id) {
return true;
}
}
}
}
}
return split.join('%2F');
return false;
}
/**
* Adds entries to indices
* @method createEmptyIndexArray
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
*/
priv.updateIndeces = function (indices, doc) {
var i, j, k, m, value,
index, index_name, index_length, index_field_array,
l = priv.indices.length,
docid = doc._id;
// loop indices
for (i = 0; i < l; i += 1) {
index = priv.indices[i];
index_name = index["name"];
index_length = index.fields.length;
index_field_array = [];
// loop index fields [keywords]
for (j = 0; j < index_length; j += 1) {
value = doc[index.fields[j]];
if (value !== undefined) {
index_field_array.push(value);
}
}
m = index_field_array.length;
if (m) {
for (k = 0; k < m; k += 1) {
if (indices[index_name] !== undefined) {
if (indices[index_name][index_field_array[k]] === undefined) {
indices[index_name][index_field_array[k]] = [];
}
indices[index_name][index_field_array[k]].push(docid);
}
}
}
}
return indices;
};
priv.indexStorage = function () {
var obj = localStorage.getItem(storage_object_name) || {};
obj[priv.sub_storage_spec] = new Date().getTime();
localStorage.setItem(storage_object_name, obj);
/**
* Post document to substorage and create/update index file(s)
* @method post
* @param {object} command The JIO command
*/
that.post = function (command) {
var f = {}, indices, doc, docid;
doc = command.cloneDoc();
docid = command.getDocId();
if (typeof docid !== "string") {
doc._id = priv.generateUuid();
docid = doc._id;
}
f.getIndices = function () {
var option = command.cloneOption();
if (option.max_retry === 0) {
option.max_retry = 3;
}
that.addJob(
"get",
priv.substorage,
priv.index_suffix,
option,
function (response) {
indices = response;
f.postDocument("put");
},
function (err) {
switch (err.status) {
case 404:
indices = priv.createEmptyIndexArray();
f.postDocument("post");
break;
default:
err.message = "Cannot retrieve index array";
that.error(err);
break;
}
}
);
};
f.postDocument = function (index_update_method) {
// if the index file already has an entry with this id,
// the document already exists
if (priv.docidInIndex(indices, doc)) {
// POST the document already exists
that.error({
"status": 409,
"statusText": "Conflicts",
"error": "conflicts",
"message": "Cannot create a new document",
"reason": "Document already exists"
});
return;
} else {
indices = priv.updateIndeces(indices, doc);
that.addJob(
"post",
priv.substorage,
doc,
command.cloneOption(),
function () {
f.sendIndices(index_update_method);
},
function (err) {
switch (err.status) {
case 409:
// file already exists
f.sendIndices(index_update_method);
break;
default:
err.message = "Cannot upload document";
that.error(err);
break;
}
}
);
}
};
f.sendIndices = function (method) {
indices._id = priv.index_suffix;
that.addJob(
method,
priv.substorage,
indices,
command.cloneOption(),
function () {
that.success({
"ok": true,
"id": docid
});
},
function (err) {
// xxx do we try to delete the posted document ?
err.message = "Cannot save index file";
that.error(err);
}
);
};
f.getIndices();
};
/*
/**
* @method formatToFileObject
* @param {} row
* @return {} obj
*//*
priv.formatToFileObject = function (row) {
var k, obj = {
_id: row.id
......@@ -62,7 +309,12 @@ jIO.addStorageType('indexed', function (spec, my) {
}
return obj;
};
*/
/**
* @method allDocs
* @param {} files_object
* @return {} obj
*//*
priv.allDocs = function (files_object) {
var k, obj = {
rows: []
......@@ -79,7 +331,11 @@ jIO.addStorageType('indexed', function (spec, my) {
obj.total_rows = obj.rows.length;
return obj;
};
*/
/**
* @method setFileArray
* @param {} file_array
*//*
priv.setFileArray = function (file_array) {
var i, obj = {};
for (i = 0; i < file_array.length; i += 1) {
......@@ -87,67 +343,84 @@ jIO.addStorageType('indexed', function (spec, my) {
}
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* @method getFileObject
* @param {} docid
* @return {} obj
*//*
priv.getFileObject = function (docid) {
var obj = localStorage.getItem(storage_file_object_name) || {};
return obj[docid];
};
*/
/**
* @method addFile
* @param {} file_obj
*//*
priv.addFile = function (file_obj) {
var obj = localStorage.getItem(storage_file_object_name) || {};
obj[file_obj._id] = file_obj;
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* @method removeFile
* @param {} docid
*//*
priv.removeFile = function (docid) {
var obj = localStorage.getItem(storage_file_object_name) || {};
delete obj[docid];
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* updates the storage.
* It will retreive all files from a storage. It is an asynchronous task
* so the update can be on going even if IndexedStorage has already
* returned the result.
* @method update
*/
*//*
priv.update = function () {
var success = function (val) {
priv.setFileArray(val.rows);
};
that.addJob('allDocs', priv.sub_storage_spec, null, {
max_retry: 3
}, success, function () {});
};
that.post = function (command) {
that.put(command);
that.addJob(
'allDocs',
priv.sub_storage_spec,
null,
{max_retry: 3},
function (response) {
priv.setFileArray(response.rows);
},
function () {}
);
};
*/
/**
* Saves a document.
* Add put job to substorage and create/update index file(s)
* @method put
*/
* @param {object} command The JIO command
*//*
that.put = function (command) {
var cloned_doc = command.cloneDoc(),
cloned_option = command.cloneOption(),
success = function (val) {
cloned_option = command.cloneOption();
// create/update indexStorage
// fwd job
that.addJob('put', priv.sub_storage_spec, cloned_doc,
cloned_option,
function (response) {
priv.update();
that.success(val);
that.success(response);
},
error = function (err) {
that.error(err);
};
priv.indexStorage();
that.addJob('put', priv.sub_storage_spec, cloned_doc,
cloned_option, success, error);
}; // end put
function (error) {
that.error(error);
}
);
};
/**
*//**
* Loads a document.
* @method get
*/
*//*
that.get = function (command) {
// jslint unused var file_array
var success = function (val) {
......@@ -177,11 +450,27 @@ jIO.addStorageType('indexed', function (spec, my) {
get();
}
}; // end get
/**
*//**
* Removes a document.
* @method remove
*//*
that.remove = function (command) {
var success = function (val) {
priv.removeFile(command.getDocId());
priv.update();
that.success(val);
},
error = function (err) {
that.error(err);
};
that.addJob('remove', priv.sub_storage_spec, command.cloneDoc(),
command.cloneOption(), success, error);
};
*//**
* Gets a document list.
* @method allDocs
*/
/*
that.allDocs = function (command) {
var obj = localStorage.getItem(storage_file_object_name),
success,
......@@ -204,23 +493,6 @@ jIO.addStorageType('indexed', function (spec, my) {
command.cloneOption(), success, error);
}
}; // end allDocs
/**
* Removes a document.
* @method remove
*/
that.remove = function (command) {
var success = function (val) {
priv.removeFile(command.getDocId());
priv.update();
that.success(val);
},
error = function (err) {
that.error(err);
};
that.addJob('remove', priv.sub_storage_spec, command.cloneDoc(),
command.cloneOption(), success, error);
}; // end remove
*/
return that;
});
test/jiotests.js
View file @ f89b3df8
......@@ -2619,9 +2619,64 @@ test ('Remove document', function () {
o.mytest('DummyStorageAllOK,3tries: remove document.',{ok:true,id:'file'});
o.jio.stop();
});
*/
module ("Jio IndexStorage");
test ("Post", function () {
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [{
"name":"indexA",
"fields":["findMeA"]
},{
"name":"indexAB",
"fields":["findMeA","findMeB"]
}],
"sub_storage": {
"type": "local",
"username": "ipost",
"application_name": "ipost"
}
});
// post without id
o.spy (o, "status", undefined, "Post without id");
o.jio.post({}, o.f);
o.tick(o);
// post non empty document
o.doc = {"_id": "some_id", "title": "myPost1",
"findMeA":"keyword_abc", "findMeB":"keyword_def"
};
o.spy (o, "value", {"ok": true, "id": "some_id"}, "Post document");
o.jio.post(o.doc, o.f);
o.tick(o);
module ('Jio IndexedStorage');
// check document
o.indexPost = {
"indexAB": {"keyword_abc":["some_id"], "keyword_def":["some_id"]},
"indexA": {"keyword_abc":["some_id"]}
};
deepEqual(
o.jio.get("ipost_indices.json"),
o.indexPost,
"Check index file"
);
// post and document already exists
o.doc = {"_id": "some_id", "title": "myPost2",
"findMeA":"keyword_ghi", "findMeB":"keyword_jkl"
}
o.spy (o, "status", 409, "Post and document already exists");
o.jio.post(o.doc, o.f);
o.tick(o);
o.jio.stop();
});
/*
test ('Document load', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
......@@ -2665,27 +2720,6 @@ test ('Document load', function () {
o.jio.stop();
});
test ('Document save', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
o.jio = JIO.newJio({type:'indexed',
storage:{type:'dummyall3tries',
username:'indexsave'}});
o.f = function (err,val) {
if (err) {
err = err.status;
}
deepEqual (err || val,{ok:true,id:'file'},'document save');
};
this.spy(o,'f');
o.jio.put({_id:'file',content:'content'},{max_retry:3},o.f);
o.clock.tick(2000);
if (!o.f.calledOnce){
ok (false, 'no response / too much results');
}
o.jio.stop();
});
test ('Get document list', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
......@@ -2742,7 +2776,8 @@ test ('Remove document', function () {
o.jio.stop();
});
*/
/*
module ('Jio CryptedStorage');
test ('Document save' , function () {
......
test/jiotests_withoutrequirejs.html
View file @ f89b3df8
......@@ -28,9 +28,11 @@
src="../src/jio.storage/replicaterevisionstorage.js">
</script>
<!-- webDav -->
<script type="text/javascript" src="../lib/jquery/jquery.min.js"></script>
<script type="text/javascript" src="../src/jio.storage/davstorage.js">
</script>
<script type="text/javascript" src="../src/jio.storage/indexstorage.js">
</script>
</script>
<script type="text/javascript" src="./jiotests.js"></script>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7