indexstorage.js: repair added + tests

src/jio.storage/indexstorage.js

@@ -499,10 +499,10 @@
      * @param  {Function} callback The result callback(err, response)
      */
     priv.storeIndexDatabaseList = function (database_list, option, callback) {
-      var i, count = 0, onResponse, onError;
+      var i, count = 0, count_max = 0, onResponse, onError;
       onResponse = function (response) {
         count += 1;
-        if (count === priv.indices.length) {
+        if (count === count_max) {
           callback({"ok": true});
         }
       };
@@ -511,14 +511,17 @@
         that.error(err);
       };
       for (i = 0; i < priv.indices.length; i += 1) {
-        that.addJob(
-          "put",
-          priv.indices[i].sub_storage || priv.sub_storage,
-          database_list[i].serialized(),
-          option,
-          onResponse,
-          onError
-        );
+        if (database_list[i] !== undefined) {
+          count_max += 1;
+          that.addJob(
+            "put",
+            priv.indices[i].sub_storage || priv.sub_storage,
+            database_list[i].serialized(),
+            option,
+            onResponse,
+            onError
+          );
+        }
       }
     };
 
@@ -673,12 +676,112 @@
       });
     };
 
-    // that.repair = function (command) {
-    //   todo: repair
-    //   easy but don't have time
-    //   if _id is an index id, then repair the index by doing an
-    //   allDocs and recreating the database from scratch. end.
-    // };
+    that.check = function (command) {
+      todo
+      var database_index = -1, i;
+      for (i = 0; i < priv.indices.length; i += 1) {
+        if (priv.indices[i].id === command.getDocId()) {
+          database_index = i;
+          break;
+        }
+      }
+      that.addJob(
+        "check",
+        priv.sub_storage,
+        command.cloneDoc(),
+        command.cloneOption(),
+        function (response) {
+          if (database_index !== -1) {
+            // check index database
+          } else {
+            // regular document
+          }
+        },
+        function (err) {
+          err.message = "Could not repair sub storage";
+          that.error(err);
+        }
+      );
+    };
+
+    priv.repairIndexDatabase = function (command, index) {
+      var i;
+      that.addJob(
+        'allDocs',
+        priv.sub_storage,
+        {},
+        {'include_docs': true},
+        function (response) {
+          var db_list = [], db = new JSONIndex({
+            "_id": command.getDocId(),
+            "indexing": priv.indices[index].index
+          });
+          for (i = 0; i < response.rows.length; i += 1) {
+            db.put(response.rows[i].doc);
+          }
+          db_list[index] = db;
+          priv.storeIndexDatabaseList(db_list, {}, function () {
+            that.success({"ok": true, "_id": command.getDocId()});
+          });
+        },
+        function (err) {
+          err.message = "Unable to repair the index database";
+          that.error(err);
+        }
+      );
+    };
+
+    priv.repairDocument = function (command) {
+      var i, option = command.cloneOption();
+      that.addJob(
+        "get",
+        priv.sub_storage,
+        command.cloneDoc(),
+        {},
+        function (response) {
+          response._id = command.getDocId();
+          priv.getIndexDatabaseList(option, function (database_list) {
+            for (i = 0; i < database_list.length; i += 1) {
+              database_list[i].put(response);
+            }
+            priv.storeIndexDatabaseList(database_list, option, function () {
+              that.success({"ok": true, "id": command.getDocId()});
+            });
+          });
+        },
+        function (err) {
+          err.message = "Unable to repair document";
+          return that.error(err);
+        }
+      );
+    };
+
+    that.repair = function (command) {
+      var database_index = -1, i;
+      for (i = 0; i < priv.indices.length; i += 1) {
+        if (priv.indices[i].id === command.getDocId()) {
+          database_index = i;
+          break;
+        }
+      }
+      that.addJob(
+        "repair",
+        priv.sub_storage,
+        command.cloneDoc(),
+        command.cloneOption(),
+        function (response) {
+          if (database_index !== -1) {
+            priv.repairIndexDatabase(command, database_index);
+          } else {
+            priv.repairDocument(command);
+          }
+        },
+        function (err) {
+          err.message = "Could not repair sub storage";
+          that.error(err);
+        }
+      );
+    };
 
     return that;
   }
@@ -686,7 +789,7 @@
 
   if (typeof exports === "object") {
     // nodejs export module
-    Object.defineProperty(exports, "indexStorage", {
+    Object.defineProperty(exports, "jio_index_storage", {
       configurable: false,
       enumerable: true,
       writable: false,

test/jiotests.js

@@ -4517,14 +4517,127 @@ test ("Put", function(){
     o.jio.get({"_id": "A"}, o.f);
     o.tick(o);
 
-    ok(false,
-       "If we run several put at the same time, only one document is indexed.");
-    // there is several put job on the same index, so the job must wait for
-    // the previous one. But nothing change, event after thousands of seconds.
-
     o.jio.stop();
 });
 
+test("Repair", function () {
+  var o = generateTools(this), i;
+
+  o.jio = JIO.newJio({
+    "type": "indexed",
+    "indices": [
+      {"id": "A", "index": ["director"]},
+      {"id": "B", "index": ["year"]}
+    ],
+    "sub_storage": {
+      "type": "local",
+      "username": "indexstoragerepair"
+    }
+  });
+
+  o.fakeIndexA = {
+    "_id": "A",
+    "indexing": ["director"],
+    "free": [],
+    "database": []
+  };
+  o.fakeIndexB = {
+    "_id": "B",
+    "indexing": ["year"],
+    "free": [],
+    "database": []
+  };
+
+  for (i = 0; i < 10; i += 1) {
+    o.jio.put({
+      "_id": "id" + i,
+      "director": "D" + i,
+      "year": i,
+      "title": "T" + i
+    });
+
+    o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
+    o.fakeIndexA.database[o.tmp] = {"_id": "id" + i, "director": "D" + i};
+
+    o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
+    o.fakeIndexB.database[o.tmp] = {"_id": "id" + i, "year": i};
+  }
+  o.clock.tick(5000);
+
+  o.spy(o, "value", {"_id": "A", "ok": true}, "Repair database");
+  o.jio.repair({"_id": "A"}, o.f);
+  o.tick(o);
+
+  o.spy(o, "value", {"_id": "B", "ok": true}, "Repair database");
+  o.jio.repair({"_id": "B"}, o.f);
+  o.tick(o);
+
+  // check index file
+  o.spy(o, "value", o.fakeIndexA, "Check index file");
+  o.jio.get({"_id": "A"}, function (err, response) {
+    if (response) {
+      delete response.location;
+      response.database.sort(function (a, b) {
+        return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
+      });
+    }
+    o.f(err, response);
+  });
+  o.tick(o);
+
+  o.spy(o, "value", o.fakeIndexB, "Check index file");
+  o.jio.get({"_id": "B"}, function (err, response) {
+    if (response) {
+      delete response.location;
+      response.database.sort(function (a, b) {
+        return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
+      });
+    }
+    o.f(err, response);
+  });
+  o.tick(o);
+
+  o.jio2 = JIO.newJio({"type": "local", "username": "indexstoragerepair"});
+
+  o.jio2.put({"_id": "blah", "title": "t", "year": "y", "director": "d"});
+  o.clock.tick(1000);
+  o.jio2.stop();
+
+  o.fakeIndexA.database.unshift({"_id": "blah", "director": "d"});
+  o.fakeIndexB.database.unshift({"_id": "blah", "year": "y"});
+
+  o.spy(o, "value", {"id": "blah", "ok": true}, "Repair Document");
+  o.jio.repair({"_id": "blah"}, o.f)
+  o.tick(o);
+
+  // check index file
+  o.spy(o, "value", o.fakeIndexA, "Check index file");
+  o.jio.get({"_id": "A"}, function (err, response) {
+    if (response) {
+      delete response.location;
+      response.database.sort(function (a, b) {
+        return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
+      });
+    }
+    o.f(err, response);
+  });
+  o.tick(o);
+
+  o.spy(o, "value", o.fakeIndexB, "Check index file");
+  o.jio.get({"_id": "B"}, function (err, response) {
+    if (response) {
+      delete response.location;
+      response.database.sort(function (a, b) {
+        return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
+      });
+    }
+    o.f(err, response);
+  });
+  o.tick(o);
+
+  o.jio.stop();
+});
+
 test ("PutAttachment", function(){
 
     // not sure these need to be run, because the index does not change
@@ -4953,8 +5066,6 @@ test ("AllDocs Complex Queries", function () {
       "indices": [
         {"id":"A", "index": ["director"]},
         {"id":"B", "index": ["title", "year"]}
-          //,
-          //{"name":"indexABC", "fields":["title","year","director"]}
       ],
       "sub_storage": {
         "type": "local",
@@ -4973,8 +5084,7 @@ test ("AllDocs Complex Queries", function () {
       "One flew over the Cuckoo's Nest", "Inception", "Godfellas"
     ];
     o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
-      1975,2010,1990
-    ];
+               1975,2010,1990];
     o.director = ["Frank Darabont", "Francis Ford Coppola",
       "Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
       "Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
@@ -4982,21 +5092,59 @@ test ("AllDocs Complex Queries", function () {
       "Milos Forman", "Christopher Nolan", " Martin Scorsese"
     ];
 
+    o.fakeIndexA = {
+      "_id": "A",
+      "indexing": ["director"],
+      "free": [],
+      "location": {},
+      "database": []
+    };
+
+    o.fakeIndexB = {
+      "_id": "B",
+      "indexing": ["title", "year"],
+      "free": [],
+      "location": {},
+      "database": []
+    };
+
     for (i = 0; i < m; i += 1) {
-      o.fakeDoc = {};
-      o.fakeDoc._id = ""+i;
-      o.fakeDoc.title = o.titles[i];
-      o.fakeDoc.year = o.years[i];
-      o.fakeDoc.director = o.director[i];
-      o.jio.put(o.fakeDoc);
+      o.jio.put({
+        "_id": "" + i,
+        "director": o.director[i],
+        "year": o.years[i],
+        "title": o.titles[i]
+      });
+
+      o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
+      o.fakeIndexA.database[o.tmp] = {"_id": "" + i, "director": o.director[i]};
+      o.fakeIndexA.location["" + i] = o.tmp;
+
+      o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
+      o.fakeIndexB.database[o.tmp] = {
+        "_id": "" + i,
+        "year": o.years[i],
+        "title": o.titles[i]
+      };
+      o.fakeIndexB.location["" + i] = o.tmp;
+
       o.clock.tick(1000);
     }
     // o.clock.tick(1000);
 
+    // check index file
+    o.spy(o, "value", o.fakeIndexA, "Check index file");
+    o.jio.get({"_id": "A"}, o.f);
+    o.tick(o);
+
+    o.spy(o, "value", o.fakeIndexB, "Check index file");
+    o.jio.get({"_id": "B"}, o.f);
+    o.tick(o);
+
     // response
     o.allDocsResponse = {};
     o.allDocsResponse.rows = [];
-    o.allDocsResponse.total_rows = 15;
+    o.allDocsResponse.total_rows = m;
     for (i = 0; i < m; i += 1) {
       o.allDocsResponse.rows.push({
         "id": ""+i,