Commit 15c81a44 authored by Tristan Cavelier's avatar Tristan Cavelier

indexstorage.js: repair added + tests

parent 2340f71c
...@@ -499,10 +499,10 @@ ...@@ -499,10 +499,10 @@
* @param {Function} callback The result callback(err, response) * @param {Function} callback The result callback(err, response)
*/ */
priv.storeIndexDatabaseList = function (database_list, option, callback) { priv.storeIndexDatabaseList = function (database_list, option, callback) {
var i, count = 0, onResponse, onError; var i, count = 0, count_max = 0, onResponse, onError;
onResponse = function (response) { onResponse = function (response) {
count += 1; count += 1;
if (count === priv.indices.length) { if (count === count_max) {
callback({"ok": true}); callback({"ok": true});
} }
}; };
...@@ -511,14 +511,17 @@ ...@@ -511,14 +511,17 @@
that.error(err); that.error(err);
}; };
for (i = 0; i < priv.indices.length; i += 1) { for (i = 0; i < priv.indices.length; i += 1) {
that.addJob( if (database_list[i] !== undefined) {
"put", count_max += 1;
priv.indices[i].sub_storage || priv.sub_storage, that.addJob(
database_list[i].serialized(), "put",
option, priv.indices[i].sub_storage || priv.sub_storage,
onResponse, database_list[i].serialized(),
onError option,
); onResponse,
onError
);
}
} }
}; };
...@@ -673,12 +676,112 @@ ...@@ -673,12 +676,112 @@
}); });
}; };
// that.repair = function (command) { that.check = function (command) {
// todo: repair todo
// easy but don't have time var database_index = -1, i;
// if _id is an index id, then repair the index by doing an for (i = 0; i < priv.indices.length; i += 1) {
// allDocs and recreating the database from scratch. end. if (priv.indices[i].id === command.getDocId()) {
// }; database_index = i;
break;
}
}
that.addJob(
"check",
priv.sub_storage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
if (database_index !== -1) {
// check index database
} else {
// regular document
}
},
function (err) {
err.message = "Could not repair sub storage";
that.error(err);
}
);
};
priv.repairIndexDatabase = function (command, index) {
var i;
that.addJob(
'allDocs',
priv.sub_storage,
{},
{'include_docs': true},
function (response) {
var db_list = [], db = new JSONIndex({
"_id": command.getDocId(),
"indexing": priv.indices[index].index
});
for (i = 0; i < response.rows.length; i += 1) {
db.put(response.rows[i].doc);
}
db_list[index] = db;
priv.storeIndexDatabaseList(db_list, {}, function () {
that.success({"ok": true, "_id": command.getDocId()});
});
},
function (err) {
err.message = "Unable to repair the index database";
that.error(err);
}
);
};
priv.repairDocument = function (command) {
var i, option = command.cloneOption();
that.addJob(
"get",
priv.sub_storage,
command.cloneDoc(),
{},
function (response) {
response._id = command.getDocId();
priv.getIndexDatabaseList(option, function (database_list) {
for (i = 0; i < database_list.length; i += 1) {
database_list[i].put(response);
}
priv.storeIndexDatabaseList(database_list, option, function () {
that.success({"ok": true, "id": command.getDocId()});
});
});
},
function (err) {
err.message = "Unable to repair document";
return that.error(err);
}
);
};
that.repair = function (command) {
var database_index = -1, i;
for (i = 0; i < priv.indices.length; i += 1) {
if (priv.indices[i].id === command.getDocId()) {
database_index = i;
break;
}
}
that.addJob(
"repair",
priv.sub_storage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
if (database_index !== -1) {
priv.repairIndexDatabase(command, database_index);
} else {
priv.repairDocument(command);
}
},
function (err) {
err.message = "Could not repair sub storage";
that.error(err);
}
);
};
return that; return that;
} }
...@@ -686,7 +789,7 @@ ...@@ -686,7 +789,7 @@
if (typeof exports === "object") { if (typeof exports === "object") {
// nodejs export module // nodejs export module
Object.defineProperty(exports, "indexStorage", { Object.defineProperty(exports, "jio_index_storage", {
configurable: false, configurable: false,
enumerable: true, enumerable: true,
writable: false, writable: false,
......
...@@ -4517,14 +4517,127 @@ test ("Put", function(){ ...@@ -4517,14 +4517,127 @@ test ("Put", function(){
o.jio.get({"_id": "A"}, o.f); o.jio.get({"_id": "A"}, o.f);
o.tick(o); o.tick(o);
ok(false,
"If we run several put at the same time, only one document is indexed.");
// there is several put job on the same index, so the job must wait for
// the previous one. But nothing change, event after thousands of seconds.
o.jio.stop(); o.jio.stop();
}); });
test("Repair", function () {
var o = generateTools(this), i;
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"id": "A", "index": ["director"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "indexstoragerepair"
}
});
o.fakeIndexA = {
"_id": "A",
"indexing": ["director"],
"free": [],
"database": []
};
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [],
"database": []
};
for (i = 0; i < 10; i += 1) {
o.jio.put({
"_id": "id" + i,
"director": "D" + i,
"year": i,
"title": "T" + i
});
o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
o.fakeIndexA.database[o.tmp] = {"_id": "id" + i, "director": "D" + i};
o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
o.fakeIndexB.database[o.tmp] = {"_id": "id" + i, "year": i};
}
o.clock.tick(5000);
o.spy(o, "value", {"_id": "A", "ok": true}, "Repair database");
o.jio.repair({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "value", {"_id": "B", "ok": true}, "Repair database");
o.jio.repair({"_id": "B"}, o.f);
o.tick(o);
// check index file
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.jio2 = JIO.newJio({"type": "local", "username": "indexstoragerepair"});
o.jio2.put({"_id": "blah", "title": "t", "year": "y", "director": "d"});
o.clock.tick(1000);
o.jio2.stop();
o.fakeIndexA.database.unshift({"_id": "blah", "director": "d"});
o.fakeIndexB.database.unshift({"_id": "blah", "year": "y"});
o.spy(o, "value", {"id": "blah", "ok": true}, "Repair Document");
o.jio.repair({"_id": "blah"}, o.f)
o.tick(o);
// check index file
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.jio.stop();
});
test ("PutAttachment", function(){ test ("PutAttachment", function(){
// not sure these need to be run, because the index does not change // not sure these need to be run, because the index does not change
...@@ -4953,8 +5066,6 @@ test ("AllDocs Complex Queries", function () { ...@@ -4953,8 +5066,6 @@ test ("AllDocs Complex Queries", function () {
"indices": [ "indices": [
{"id":"A", "index": ["director"]}, {"id":"A", "index": ["director"]},
{"id":"B", "index": ["title", "year"]} {"id":"B", "index": ["title", "year"]}
//,
//{"name":"indexABC", "fields":["title","year","director"]}
], ],
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
...@@ -4973,8 +5084,7 @@ test ("AllDocs Complex Queries", function () { ...@@ -4973,8 +5084,7 @@ test ("AllDocs Complex Queries", function () {
"One flew over the Cuckoo's Nest", "Inception", "Godfellas" "One flew over the Cuckoo's Nest", "Inception", "Godfellas"
]; ];
o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001, o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
1975,2010,1990 1975,2010,1990];
];
o.director = ["Frank Darabont", "Francis Ford Coppola", o.director = ["Frank Darabont", "Francis Ford Coppola",
"Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone", "Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
"Sidney Lumet", "Christopher Nolan", "Steven Spielberg", "Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
...@@ -4982,21 +5092,59 @@ test ("AllDocs Complex Queries", function () { ...@@ -4982,21 +5092,59 @@ test ("AllDocs Complex Queries", function () {
"Milos Forman", "Christopher Nolan", " Martin Scorsese" "Milos Forman", "Christopher Nolan", " Martin Scorsese"
]; ];
o.fakeIndexA = {
"_id": "A",
"indexing": ["director"],
"free": [],
"location": {},
"database": []
};
o.fakeIndexB = {
"_id": "B",
"indexing": ["title", "year"],
"free": [],
"location": {},
"database": []
};
for (i = 0; i < m; i += 1) { for (i = 0; i < m; i += 1) {
o.fakeDoc = {}; o.jio.put({
o.fakeDoc._id = ""+i; "_id": "" + i,
o.fakeDoc.title = o.titles[i]; "director": o.director[i],
o.fakeDoc.year = o.years[i]; "year": o.years[i],
o.fakeDoc.director = o.director[i]; "title": o.titles[i]
o.jio.put(o.fakeDoc); });
o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
o.fakeIndexA.database[o.tmp] = {"_id": "" + i, "director": o.director[i]};
o.fakeIndexA.location["" + i] = o.tmp;
o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
o.fakeIndexB.database[o.tmp] = {
"_id": "" + i,
"year": o.years[i],
"title": o.titles[i]
};
o.fakeIndexB.location["" + i] = o.tmp;
o.clock.tick(1000); o.clock.tick(1000);
} }
// o.clock.tick(1000); // o.clock.tick(1000);
// check index file
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// response // response
o.allDocsResponse = {}; o.allDocsResponse = {};
o.allDocsResponse.rows = []; o.allDocsResponse.rows = [];
o.allDocsResponse.total_rows = 15; o.allDocsResponse.total_rows = m;
for (i = 0; i < m; i += 1) { for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({ o.allDocsResponse.rows.push({
"id": ""+i, "id": ""+i,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment