Commit 555c5b3d authored by Sven Franck's avatar Sven Franck

indexStorage: fixed ALLDOCS complex queries, added QUnit tests

parent d2c42f76
...@@ -10,6 +10,10 @@ ...@@ -10,6 +10,10 @@
* {"indexA",["field_A"]}, * {"indexA",["field_A"]},
* {"indexAB",["field_A","field_B"]} * {"indexAB",["field_A","field_B"]}
* ], * ],
* "field_types": {
* "field_A": "dateTime",
* "field_B": "string"
* },
* "storage": [ * "storage": [
* <sub storage description>, * <sub storage description>,
* ... * ...
...@@ -17,7 +21,7 @@ ...@@ -17,7 +21,7 @@
* } * }
* Index file will contain * Index file will contain
* { * {
* "_id": "ipost_indices.json", * "_id": "app-name_indices.json",
* "indexA": * "indexA":
* "fieldA": { * "fieldA": {
* "keyword_abc": ["some_id","some_other_id",...] * "keyword_abc": ["some_id","some_other_id",...]
...@@ -54,6 +58,7 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -54,6 +58,7 @@ jIO.addStorageType('indexed', function (spec, my) {
that = my.basicStorage(spec, my); that = my.basicStorage(spec, my);
priv.indices = spec.indices; priv.indices = spec.indices;
priv.field_types = spec.field_types;
priv.substorage_key = "sub_storage"; priv.substorage_key = "sub_storage";
priv.substorage = spec[priv.substorage_key]; priv.substorage = spec[priv.substorage_key];
priv.index_indicator = spec.sub_storage.application_name || "index"; priv.index_indicator = spec.sub_storage.application_name || "index";
...@@ -164,8 +169,8 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -164,8 +169,8 @@ jIO.addStorageType('indexed', function (spec, my) {
* @returns {number} i Position of element in array * @returns {number} i Position of element in array
*/ */
priv.getPositionInArray = function (element, array) { priv.getPositionInArray = function (element, array) {
var i; var i, l = array.length;
for (i = 0; i < array.length; i += 1) { for (i = 0; i < l; i += 1) {
if (array[i] === element) { if (array[i] === element) {
return i; return i;
} }
...@@ -291,36 +296,45 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -291,36 +296,45 @@ jIO.addStorageType('indexed', function (spec, my) {
}; };
/** /**
* Check available indices to find the best one. This index must have * Check available indices to find the best one.
* all "id" parameters of the query and if it also contains all values from * TODOS: NOT NICE, redo
* the select-list, it can be used to run the whole query plus return results
* @method findBestIndexForQuery * @method findBestIndexForQuery
* @param {object} indices The index file * @param {object} syntax of query
* @returns {object} response The query object constructed from Index file * @returns {object} response The query object constructed from Index file
*/ */
priv.findBestIndexForQuery = function (indices, syntax) { priv.findBestIndexForQuery = function (syntax) {
var i, j, k, l, m, n, o, p, var i, j, k, l, n, p, o, element, key, block,
search_ids = [], search_ids, use_index = [], select_ids = {}, index, query_param,
select_ids = syntax.filter.select_list, // need to parse into object
index, query_param, search_param, use_index = []; current_query = jIO.ComplexQueries.parse(syntax.query);
// array of necessary query ids
if (syntax.query.query_list === undefined) {
search_ids.push(syntax.query.id);
} else {
for (j = 0; j < syntax.query.query_list.length; j += 1) {
search_ids.push(syntax.query.query_list[j].id);
}
}
// loop indices // loop indices
for (i = 0; i < priv.indices.length; i += 1) { for (i = 0; i < priv.indices.length; i += 1) {
search_ids = [];
block = false;
index = {}; index = {};
index.reference = priv.indices[i]; index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length; index.reference_size = index.reference.fields.length;
o = search_ids.length;
for (k = 0; k < o; k += 1) { // rebuild search_ids for iteration
// always check the first element in the array because it's spliced if (current_query.query_list === undefined) {
search_ids.push(current_query.id);
} else {
for (j = 0; j < current_query.query_list.length; j += 1) {
if (priv.getPositionInArray(current_query.query_list[j].id,
search_ids) === null) {
search_ids.push(current_query.query_list[j].id);
}
}
}
// rebuild select_ids
for (o = 0; o < syntax.filter.select_list.length; o += 1) {
element = syntax.filter.select_list[o];
select_ids[element] = true;
}
// loop search ids and find matches in index
for (k = 0; k < search_ids.length; k += 1) {
query_param = search_ids[0]; query_param = search_ids[0];
for (l = 0; l < index.reference_size; l += 1) { for (l = 0; l < index.reference_size; l += 1) {
if (query_param === index.reference.fields[l]) { if (query_param === index.reference.fields[l]) {
...@@ -332,40 +346,34 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -332,40 +346,34 @@ jIO.addStorageType('indexed', function (spec, my) {
} }
} }
// empty search_ids = index can be used, now check results // search_ids empty = all needed search fields found on index
if (search_ids.length === 0) { if (search_ids.length === 0) {
// can't return results if empty select_list (all fields) p = priv.getObjectSize(select_ids);
if (select_ids.length === 0) { if (p === 0) {
use_index.push({ use_index.push({
"name": index.reference.name, "name": index.reference.name,
"search": true, "search": true,
"results": false "results": false
}); });
} else { } else {
p = select_ids.length;
for (m = 0; m < p; m += 1) {
search_param = select_ids[0];
for (n = 0; n < index.reference_size; n += 1) { for (n = 0; n < index.reference_size; n += 1) {
if (search_param === index.reference.fields[n]) { delete select_ids[index.reference.fields[n]];
select_ids.splice(
priv.getPositionInArray(search_param, select_ids),
1
);
}
} }
} for (key in select_ids) {
// empty select_list = index can do do query and results! if (select_ids.hasOwnProperty(key)) {
if (select_ids.length === 0) {
use_index.push({ use_index.push({
"name": index.reference.name, "name": index.reference.name,
"search": true, "search": true,
"results": true "results": false
}); });
} else { block = true;
}
}
if (block === false) {
use_index.push({ use_index.push({
"name": index.reference.name, "name": index.reference.name,
"search": true, "search": true,
"results": false "results": true
}); });
} }
} }
...@@ -376,13 +384,78 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -376,13 +384,78 @@ jIO.addStorageType('indexed', function (spec, my) {
/** /**
* Converts the indices file into an object usable by complex queries * Converts the indices file into an object usable by complex queries
* @method convertIndicesToQueryObject * @method constructQueryObject
* @param {object} indices The index file * @param {object} indices The index file
* @returns {object} response The query object constructed from Index file * @returns {object} response The query object constructed from Index file
*/ */
priv.convertIndicesToQueryObject = function (indices, query_syntax) { priv.constructQueryObject = function (indices, query_syntax) {
var use_index = priv.findBestIndexForQuery(indices, query_syntax); var j, k, l, m, n, use_index, index,
return indices; index_name, field_names, field, key, element,
query_index, query_object = [], field_name,
entry;
// returns index-to-use|can-do-query|can-do-query-and-results
use_index = priv.findBestIndexForQuery(query_syntax);
if (use_index.length > 0) {
for (j = 0; j < use_index.length; j += 1) {
index = use_index[j];
// NOTED: the index could be used to:
// (a) get all document ids matching query
// (b) get all document ids and results (= run complex query on index)
// right now, only (b) is supported, because the complex query is
// a single step process. If it was possible to first get the
// relevant document ids, then get the results, the index could be
// used to do the first step plus use GET on the returned documents
if (index.search && index.results) {
index_name = use_index[j].name;
query_index = indices[index_name];
// get fieldnames from this index
for (k = 0; k < priv.indices.length; k += 1) {
if (priv.indices[k].name === use_index[j].name) {
field_names = priv.indices[k].fields;
}
}
for (l = 0; l < field_names.length; l += 1) {
field_name = field_names[l];
// loop entries for this field name
field = query_index[field_name];
for (key in field) {
if (field.hasOwnProperty(key)) {
element = field[key];
// key can be "string" or "number" right now
if (priv.field_types[field_name] === "number") {
key = +key;
}
for (m = 0; m < element.length; m += 1) {
if (priv.searchIndexByValue(
query_object,
element[m],
"bool"
)) {
// loop object
for (n = 0; n < query_object.length; n += 1) {
entry = query_object[n];
if (entry.id === element[m]) {
entry[field_name] = key;
}
}
} else {
entry = {};
entry.id = element[m];
entry[field_name] = key;
query_object.push(entry);
}
}
}
}
}
}
}
}
return query_object;
}; };
/** /**
* Build the alldocs response from the index file (overriding substorage) * Build the alldocs response from the index file (overriding substorage)
...@@ -490,7 +563,7 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -490,7 +563,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command * @param {object} command The JIO command
* @param {string} source The source of the function call * @param {string} source The source of the function call
*/ */
priv.postOrput = function (command, source) { priv.postOrPut = function (command, source) {
var f = {}, indices, doc, docid; var f = {}, indices, doc, docid;
doc = command.cloneDoc(); doc = command.cloneDoc();
docid = command.getDocId(); docid = command.getDocId();
...@@ -618,7 +691,7 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -618,7 +691,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.post = function (command) { that.post = function (command) {
priv.postOrput(command, 'POST'); priv.postOrPut(command, 'POST');
}; };
/** /**
...@@ -627,7 +700,7 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -627,7 +700,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.put = function (command) { that.put = function (command) {
priv.postOrput(command, 'PUT'); priv.postOrPut(command, 'PUT');
}; };
/** /**
...@@ -636,7 +709,7 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -636,7 +709,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.putAttachment = function (command) { that.putAttachment = function (command) {
priv.postOrput(command, 'PUTATTACHMENT'); priv.postOrPut(command, 'PUTATTACHMENT');
}; };
/** /**
...@@ -782,7 +855,8 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -782,7 +855,8 @@ jIO.addStorageType('indexed', function (spec, my) {
// ] // ]
//} //}
that.allDocs = function (command) { that.allDocs = function (command) {
var f = {}, option, all_docs_response, query_object, query_syntax; var f = {}, option, all_docs_response, query_object, query_syntax,
query_response;
option = command.cloneOption(); option = command.cloneOption();
if (option.max_retry === 0) { if (option.max_retry === 0) {
option.max_retry = 3; option.max_retry = 3;
...@@ -795,14 +869,61 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -795,14 +869,61 @@ jIO.addStorageType('indexed', function (spec, my) {
priv.index_suffix, priv.index_suffix,
option, option,
function (response) { function (response) {
// should include_docs be possible besides complex queries?
query_syntax = command.getOption('query'); query_syntax = command.getOption('query');
if (query_syntax !== undefined) { if (query_syntax !== undefined) {
// check to see if index can do the job
query_object = priv.convertIndicesToQueryObject( // build complex query object
response, query_object = priv.constructQueryObject(response, query_syntax);
query_syntax
if (query_object.length === 0) {
that.addJob(
"allDocs",
priv.substorage,
undefined,
option,
function (data) {
that.success(data);
},
function (err) {
switch (err.status) {
case 405:
that.error({
"status": 405,
"statusText": "Method Not Allowed",
"error": "method_not_allowed",
"message": "Method not allowed",
"reason": "Could not run AllDocs on this storage"
});
break;
default:
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Could not run allDocs command",
"reason": "There are no documents in the storage"
});
break;
}
}
); );
} else {
// we can use index, run query on index
query_response =
jIO.ComplexQueries.query({
query: query_syntax.query,
filter: {
sort_on: query_syntax.filter.sort_on,
limit: query_syntax.filter.limit,
select_list: query_syntax.filter.select_list
},
wildcard_character: query_syntax.wildcard_character
},
query_object
);
that.success(query_response);
}
} else if (command.getOption('include_docs')) { } else if (command.getOption('include_docs')) {
priv.allDocsResponseFromIndex(response, true, option); priv.allDocsResponseFromIndex(response, true, option);
} else { } else {
......
...@@ -244,7 +244,15 @@ generateTools = function (sinon) { ...@@ -244,7 +244,15 @@ generateTools = function (sinon) {
o.server.respondWith(method, url, o.server.respondWith(method, url,
[status, { "Content-Type": 'application/xml' }, response] [status, { "Content-Type": 'application/xml' }, response]
); );
};
o.sortArrayById = function(field, reverse, primer){
var key = function (x) {return primer ? primer(x[field]) : x[field]};
return function (a,b) {
var A = key(a), B = key(b);
return ( (A < B) ? -1 : ((A > B) ? 1 : 0) ) * [-1,1][+!!reverse];
} }
};
return o; return o;
}, },
...@@ -1097,7 +1105,7 @@ test ("AllDocs", function(){ ...@@ -1097,7 +1105,7 @@ test ("AllDocs", function(){
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson", "Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese" "Milos Forman", "Christopher Nolan", " Martin Scorsese"
] ]
// set documents
for (i = 0; i < m; i += 1) { for (i = 0; i < m; i += 1) {
o.fakeDoc = {}; o.fakeDoc = {};
o.fakeDoc._id = "doc_"+i; o.fakeDoc._id = "doc_"+i;
...@@ -2790,6 +2798,10 @@ test ("Post", function () { ...@@ -2790,6 +2798,10 @@ test ("Post", function () {
{"name":"indexA", "fields":["findMeA"]}, {"name":"indexA", "fields":["findMeA"]},
{"name":"indexAB", "fields":["findMeA","findMeB"]} {"name":"indexAB", "fields":["findMeA","findMeB"]}
], ],
"field_types": {
"findMeA": "string",
"findMeB": "string"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "ipost", "username": "ipost",
...@@ -2863,6 +2875,10 @@ test ("Put", function(){ ...@@ -2863,6 +2875,10 @@ test ("Put", function(){
{"name":"indexA", "fields":["author"]}, {"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]} {"name":"indexAB", "fields":["author","year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "iput", "username": "iput",
...@@ -3035,6 +3051,10 @@ test ("PutAttachment", function(){ ...@@ -3035,6 +3051,10 @@ test ("PutAttachment", function(){
{"name":"indexA", "fields":["author"]}, {"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]} {"name":"indexAB", "fields":["author","year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "iputatt", "username": "iputatt",
...@@ -3142,6 +3162,10 @@ test ("Get", function(){ ...@@ -3142,6 +3162,10 @@ test ("Get", function(){
{"name":"indexA", "fields":["author"]}, {"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]} {"name":"indexAB", "fields":["author","year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "iget", "username": "iget",
...@@ -3208,6 +3232,10 @@ test ("Remove", function(){ ...@@ -3208,6 +3232,10 @@ test ("Remove", function(){
{"name":"indexA", "fields":["author"]}, {"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]} {"name":"indexAB", "fields":["author","year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "irem", "username": "irem",
...@@ -3372,6 +3400,10 @@ test ("AllDocs", function () { ...@@ -3372,6 +3400,10 @@ test ("AllDocs", function () {
{"name":"indexA", "fields":["author"]}, {"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]} {"name":"indexAB", "fields":["author","year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "iall", "username": "iall",
...@@ -3461,24 +3493,157 @@ test ("AllDocs", function () { ...@@ -3461,24 +3493,157 @@ test ("AllDocs", function () {
o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)"); o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
o.jio.allDocs({"include_docs":true}, o.f); o.jio.allDocs({"include_docs":true}, o.f);
o.tick(o); o.tick(o);
/*
o.jio.stop();
});
test ("AllDocs Complex Queries", function () {
var o = generateTools(this), i, m = 15;
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["director"]},
{"name":"indexAB", "fields":["title","year"]}
//,
//{"name":"indexABC", "fields":["title","year","director"]}
],
"field_types": {
"director": "string",
"title": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "icomplex",
"application_name": "acomplex"
}
});
o.localpath = "jio/localstorage/icomplex/acomplex";
// sample data
o.titles = ["Shawshank Redemption", "Godfather", "Godfather 2",
"Pulp Fiction", "The Good, The Bad and The Ugly", "12 Angry Men",
"The Dark Knight", "Schindlers List",
"Lord of the Rings - Return of the King", "Fight Club",
"Star Wars Episode V", "Lord Of the Rings - Fellowship of the Ring",
"One flew over the Cuckoo's Nest", "Inception", "Godfellas"
];
o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
1975,2010,1990
];
o.director = ["Frank Darabont", "Francis Ford Coppola",
"Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
"Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese"
]
for (i = 0; i < m; i += 1) {
o.fakeDoc = {};
o.fakeDoc._id = ""+i;
o.fakeDoc.title = o.titles[i];
o.fakeDoc.year = o.years[i];
o.fakeDoc.director = o.director[i];
o.jio.put(o.fakeDoc);
o.clock.tick(1000);
}
// response
o.allDocsResponse = {};
o.allDocsResponse.rows = [];
o.allDocsResponse.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({
"id": ""+i,
"key": ""+i,
"value": {}
});
};
// alldocs
o.jio.allDocs(function (e, r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse,
"AllDocs response generated from index"
);
});
o.clock.tick(1000);
// include docs
o.allDocsResponse2 = {};
o.allDocsResponse2.rows = [];
o.allDocsResponse2.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse2.rows.push({
"id": ""+i,
"key": ""+i,
"value": {},
"doc": localstorage.getItem(o.localpath+"/"+i)
});
};
// alldocs
o.jio.allDocs({"include_docs":true}, function(e,r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse2,
"AllDocs response generated from index (include docs)"
);
});
o.clock.tick(1000);
// complex queries
o.thisShouldBeTheAnswer4 = [
{"title": "Inceptions", "year": 2010},
{"title": "The Dark Knight", "year": 2008},
{"title": "Lord of the Rings - Return of the King", "year": 2003},
{"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001},
{"title": "Fight Club", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer4,
"allDocs (complex queries year >= 1980, index used to do query)");
o.jio.allDocs({
"query":{
// "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": {
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['title','year']
},
"wildcard_character":'%'
}
}, o.f);
o.tick(o);
// complex queries // complex queries
o.thisShouldBeTheAnswer3 = {"nothing here":"yet"} o.thisShouldBeTheAnswer5 = [
o.spy(o, "value", o.thisShouldBeTheAnswer3, {"director": "Christopher Nolan", "year": 2010},
"allDocs (complex queries year >= 1985)"); {"director": "Christopher Nolan", "year": 2008},
{"director": "Peter Jackson", "year": 2003},
{"director": "Peter Jackson", "year": 2001},
{"director": "David Fincher", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer5,
"allDocs (complex queries year >= 1980, can't use index)");
o.jio.allDocs({ o.jio.allDocs({
"query":{ "query":{
"query":jIO.ComplexQueries.parse('(year: >= "1985" AND author:"D%")'), // "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": { "filter": {
"limit":[0,2], "limit":[0,5],
"sort_on":[['key','descending']], "sort_on":[['year','descending']],
"select_list":['author','year'] "select_list":['director','year']
}, },
"wildcard_character":'%' "wildcard_character":'%'
} }
}, o.f); }, o.f);
o.tick(o); o.tick(o);
*/
o.jio.stop(); o.jio.stop();
}); });
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment