indexStorage: ALLDOCS new API and qunit

src/jio.storage/davstorage.js

@@ -790,6 +790,8 @@ jIO.addStorageType('dav', function (spec, my) {
 
   /**
    * Gets a document list from a distant dav storage
+   * Options:
+   * - {boolean} include_docs Also retrieve the actual document content.
    * @method allDocs
    * @param  {object} command The JIO command
    */

src/jio.storage/indexstorage.js

@@ -72,16 +72,6 @@ jIO.addStorageType('indexed', function (spec, my) {
       S4() + S4() + S4();
   };
 
-  /**
-  * Escape string before storing
-  * @method sanitizeValue
-  * @param  {string} s The string to be sanitized
-  * @return {string} The sanitized string
-  */
-  priv.sanitizeValue = function (s) {
-    return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
-  };
-
   /**
   * Get number of elements in object
   * @method getObjectSize
@@ -249,6 +239,91 @@ jIO.addStorageType('indexed', function (spec, my) {
     return indices;
   };
 
+  /**
+   * Build the alldocs response from the index file (overriding substorage)
+   * @method allDocsResponseFromIndex
+   * @param  {object} command The JIO command
+   * @param  {boolean} include_docs Whether to also supply the document
+   * @param  {object} option The options set for this method
+   * @returns {object} response The allDocs response
+   */
+  priv.allDocsResponseFromIndex = function (indices, include_docs, option) {
+     var i, j, k, m, n = 0, l = priv.indices.length,
+      index, key, obj, prop, found, file,
+      unique_count = 0, unique_docids = [], all_doc_response = {};
+
+    // loop indices
+    for (i = 0; i < l; i += 1) {
+      index = {};
+      index.reference = priv.indices[i];
+      index.name = index.reference["name"];
+      index.current = indices[index.name];
+      index.current_size = priv.getObjectSize(index.current);
+
+      // a lot of loops, not sure this is the fastest way
+      for (j = 0; j < index.current_size; j += 1) {
+        for (key in index.current) {
+          obj = index.current[key];
+          for (prop in obj) {
+            for ( k = 0; k < unique_docids.length; k++ ) {
+              if ( obj[prop] === unique_docids[k] ) {
+                found = true;
+                break;
+              }
+            }
+            if (!found) {
+              unique_docids.push( obj[prop] );
+              unique_count += 1;
+            }
+          }
+        }
+      }
+    }
+    // construct allDocs response
+    all_doc_response.total_rows = unique_count;
+    all_doc_response.rows = [];
+    for (m = 0; m < unique_count; m += 1) {
+      // include_docs
+      if (include_docs) {
+        that.addJob(
+          "get",
+          priv.substorage,
+          unique_docids[m],
+          option,
+          function (content) {
+            file = { value: {} };
+            file.id = unique_docids[n];
+            file.key = unique_docids[n];
+            file.doc = content;
+            all_doc_response.rows.push(file);
+            // async counter, must be in callback
+            n += 1;
+            if (n === (unique_count)) {
+              that.success(all_doc_response);
+            }
+          },
+          function (error) {
+            that.error({
+              "status": 404,
+              "statusText": "Not Found",
+              "error": "not_found",
+              "message": "Cannot find the document",
+              "reason": "Cannot get a document from substorage"
+            });
+          }
+        );
+      } else {
+        file = { value: {} };
+        file.id = unique_docids[m];
+        file.key = unique_docids[m];
+        all_doc_response.rows.push(file);
+        if (m === (unique_count-1)) {
+          return all_doc_response;
+        }
+      }
+    }
+  };
+
   /**
    * Post document to substorage and create/update index file(s)
    * @method post
@@ -507,7 +582,6 @@ jIO.addStorageType('indexed', function (spec, my) {
                 f.removeDocument('doc');
               },
               function (err) {
-                // xxx do we try to delete the posted document ?
                 err.message = "Cannot save index file";
                 that.error(err);
               }
@@ -529,33 +603,60 @@ jIO.addStorageType('indexed', function (spec, my) {
     f.getIndices();
   };
 
- /**
-   * Gets a document list.
+  /**
+   * Gets a document list from the substorage
+   * Options:
+   * - {boolean} include_docs Also retrieve the actual document content.
    * @method allDocs
+   * @param  {object} command The JIO command
    */
-  /*
+  //{
+  // "total_rows": 4,
+  // "rows": [
+  //    {
+  //    "id": "otherdoc",
+  //    "key": "otherdoc",
+  //    "value": {
+  //      "rev": "1-3753476B70A49EA4D8C9039E7B04254C"
+  //    }
+  //  },{...}
+  // ]
+  //}
   that.allDocs = function (command) {
-    var obj = localStorage.getItem(storage_file_object_name),
-      success,
-      error;
-
-    if (obj) {
-      priv.update();
-      setTimeout(function () {
-        that.success(priv.allDocs(obj));
-      });
-    } else {
-      success = function (val) {
-        priv.setFileArray(val.rows);
-        that.success(val);
-      };
-      error = function (err) {
-        that.error(err);
-      };
-      that.addJob('allDocs', priv.sub_storage_spec, null,
-        command.cloneOption(), success, error);
+    var f = {}, indices, option, include_docs, all_docs_response;
+    option = command.cloneOption();
+    if (option.max_retry === 0) {
+      option.max_retry = 3;
     }
-  }; // end allDocs
-  */
+
+    f.getIndices = function () {
+      that.addJob(
+        "get",
+        priv.substorage,
+        priv.index_suffix,
+        option,
+        function (response) {
+          if (command.getOption('include_docs')) {
+            priv.allDocsResponseFromIndex(response, true, option);
+          } else {
+            all_docs_response =
+              priv.allDocsResponseFromIndex(response, false, option);
+              that.success(all_docs_response);
+          }
+        },
+        function (err) {
+          that.error({
+            "status": 404,
+            "statusText": "Not Found",
+            "error": "not_found",
+            "message": "Document index not found",
+            "reason": "There are no documents in the storage"
+          });
+        return;
+        }
+      );
+    };
+    f.getIndices();
+  };
   return that;
 });

test/jiotests.js

@@ -2745,8 +2745,7 @@ test ("Put", function(){
 
     // modify document - modify keyword on index!
     o.doc = {"_id": "put1", "title": "myPuttter1", "author":"Jane Doe"};
-    o.spy (o, "value", {"ok": true, "id": "put1"},
-      "Modify document, update keyword on index");
+    o.spy (o, "value", {"ok": true, "id": "put1"}, "Modify existing document");
     o.jio.put(o.doc, o.f);
     o.tick(o);
 
@@ -2996,39 +2995,6 @@ test ("Get", function(){
     o.jio.stop();
 });
 
-
-/*
-test ('Get document list', function () {
-    var o = {}; o.clock = this.sandbox.useFakeTimers();
-    o.clock.tick(base_tick);
-    o.jio = JIO.newJio({type:'indexed',
-                        storage:{type:'dummyall3tries',
-                                 username:'indexgetlist'}});
-    o.doc1 = {id:'file',key:'file',value:{
-        _last_modified:15000,_creation_date:10000}};
-    o.doc2 = {id:'memo',key:'memo',value:{
-        _last_modified:25000,_creation_date:20000}};
-    // getting list must take long time with dummyall3tries
-    o.f = this.spy();
-    o.jio.allDocs({max_retry:3},o.f);
-    o.clock.tick(1000);
-    ok(!o.f.called,'Callback must not be called');
-    // wail long time too retreive list
-    o.clock.tick(1000);
-    // now we can test if the document list is loaded faster
-    o.f2 = function (err,val) {
-        deepEqual (err || objectifyDocumentArray(val.rows),
-                   objectifyDocumentArray([o.doc1,o.doc2]),'get document list');
-    };
-    this.spy(o,'f2');
-    o.jio.allDocs({max_retry:3},o.f2);
-    o.clock.tick(1000)
-    if (!o.f2.calledOnce) {
-        ok (false, 'no response / too much results');
-    }
-});
-*/
-
 test ("Remove", function(){
 
     // not sure these need to be run, because the index does not change
@@ -3037,17 +3003,17 @@ test ("Remove", function(){
     var o = generateTools(this);
 
     o.jio = JIO.newJio({
-          "type": "indexed",
-          "indices": [
-              {"name":"indexA", "fields":["author"]},
-              {"name":"indexAB", "fields":["author","year"]}
-          ],
-          "sub_storage": {
-            "type": "local",
-            "username": "irem",
-            "application_name": "irem"
-          }
-      });
+      "type": "indexed",
+      "indices": [
+          {"name":"indexA", "fields":["author"]},
+          {"name":"indexAB", "fields":["author","year"]}
+      ],
+      "sub_storage": {
+        "type": "local",
+        "username": "irem",
+        "application_name": "irem"
+      }
+    });
 
     // remove inexistent document
     o.spy(o, "status", 404, "Remove inexistent document");
@@ -3173,6 +3139,102 @@ test ("Remove", function(){
     o.jio.stop();
 });
 
+test ("AllDocs", function () {
+
+  var o = generateTools(this);
+
+    o.jio = JIO.newJio({
+      "type": "indexed",
+      "indices": [
+          {"name":"indexA", "fields":["author"]},
+          {"name":"indexAB", "fields":["author","year"]}
+      ],
+      "sub_storage": {
+        "type": "local",
+        "username": "iall",
+        "application_name": "iall"
+      }
+    });
+
+  // adding documents
+  o.all1 = { "_id": "dragon.doc",
+    "title": "some title", "author": "Dr. No", "year": "1968"
+  };
+  o.spy (o, "value", {"ok": true, "id": "dragon.doc"}, "Put 1");
+  o.jio.put(o.all1, o.f);
+  o.tick(o);
+  o.all2 = {"_id": "timemachine",
+    "title": "hello world", "author": "Dr. Who", "year": "1968"
+  }
+  o.spy (o, "value", {"ok": true, "id": "timemachine"}, "Put 2");
+  o.jio.put(o.all2, o.f);
+  o.tick(o);
+  o.all3 = {"_id": "rocket.ppt",
+    "title": "sunshine.", "author": "Dr. Snuggles", "year": "1985"
+  }
+  o.spy (o, "value", {"ok": true, "id": "rocket.ppt"}, "Put 3");
+  o.jio.put(o.all3, o.f);
+  o.tick(o);
+  o.all4 = {"_id": "stick.jpg",
+    "title": "clouds", "author": "Dr. House", "year": "2005"
+  }
+  o.spy (o, "value", {"ok": true, "id": "stick.jpg"}, "Put 4");
+  o.jio.put(o.all4, o.f);
+  o.tick(o);
+
+  // check index
+  o.fakeIndex = {
+    "_id": "iall_indices.json",
+    "indexA": {
+      "Dr. No": ["dragon.doc"],
+      "Dr. Who": ["timemachine"],
+      "Dr. Snuggles": ["rocket.ppt"],
+      "Dr. House":["stick.jpg"]
+    },
+    "indexAB": {
+      "Dr. No": ["dragon.doc"],
+      "Dr. Who": ["timemachine"],
+      "Dr. Snuggles": ["rocket.ppt"],
+      "Dr. House":["stick.jpg"],
+      "1968": ["dragon.doc", "timemachine"],
+      "1985": ["rocket.ppt"],
+      "2005":["stick.jpg"]
+    }
+  };
+  o.jio.get("iall_indices.json",function(err, response){
+      o.actualIndex = response;
+      deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
+  });
+  o.tick(o);
+
+  o.thisShouldBeTheAnswer = {
+    "rows": [
+      {"id": "dragon.doc", "key": "dragon.doc", "value": {} },
+      {"id": "timemachine", "key": "timemachine", "value": {} },
+      {"id": "rocket.ppt", "key": "rocket.ppt", "value": {} },
+      {"id": "stick.jpg", "key": "stick.jpg", "value": {} }
+    ],
+    "total_rows": 4
+  }
+  o.spy(o, "value", o.thisShouldBeTheAnswer, "allDocs (served by index)");
+  o.jio.allDocs(o.f);
+  o.tick(o);
+
+  o.thisShouldBeTheAnswer2 = {
+    "rows": [
+      {"id": "dragon.doc", "key": "dragon.doc", "value": {}, "doc": o.all1 },
+      {"id": "timemachine", "key": "timemachine", "value": {}, "doc": o.all2 },
+      {"id": "rocket.ppt", "key": "rocket.ppt", "value": {}, "doc": o.all3 },
+      {"id": "stick.jpg", "key": "stick.jpg", "value": {}, "doc": o.all4 }
+    ],
+    "total_rows": 4
+  }
+  o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
+  o.jio.allDocs({"include_docs":true}, o.f);
+  o.tick(o);
+
+  o.jio.stop();
+});
 /*
 module ('Jio CryptedStorage');