Merge branch 'master' of https://lab.nexedi.com/Junming/jio into ljm_qiniu_branch

Conflicts:
	examples/scenario.js

Gruntfile.js

@@ -180,6 +180,7 @@ module.exports = function (grunt) {
           'src/jio.storage/memorystorage.js',
           'src/jio.storage/localstorage.js',
           'src/jio.storage/zipstorage.js',
+          'src/jio.storage/dropboxstorage.js',
           'src/jio.storage/davstorage.js',
           'src/jio.storage/unionstorage.js',
           'src/jio.storage/erp5storage.js',

Makefile

@@ -36,6 +36,7 @@ zip:
 	@cp lib/require/require.js              $(TMPDIR)/jio/
 	@cp src/jio.storage/localstorage.js     $(TMPDIR)/jio/storage/
 	@cp src/jio.storage/davstorage.js       $(TMPDIR)/jio/storage/
+	@cp src/jio.storage/dropboxstorage.js   $(TMPDIR)/jio/storage/
 	@cp src/jio.storage/erp5storage.js      $(TMPDIR)/jio/storage/
 	@cp src/jio.storage/indexstorage.js     $(TMPDIR)/jio/storage/
 	@cp src/jio.storage/gidstorage.js       $(TMPDIR)/jio/storage/
@@ -68,6 +69,7 @@ zip:
 	@$(UGLIFY) lib/require/require.js              >$(TMPDIR)/jio/require.min.js 2>/dev/null
 	@$(UGLIFY) src/jio.storage/localstorage.js     >$(TMPDIR)/jio/storage/localstorage.min.js 2>/dev/null
 	@$(UGLIFY) src/jio.storage/davstorage.js       >$(TMPDIR)/jio/storage/davstorage.min.js 2>/dev/null
+	@$(UGLIFY) src/jio.storage/dropboxstorage.js   >$(TMPDIR)/jio/storage/dropboxstorage.min.js 2>/dev/null
 	@$(UGLIFY) src/jio.storage/erp5storage.js      >$(TMPDIR)/jio/storage/erp5storage.min.js 2>/dev/null
 	@$(UGLIFY) src/jio.storage/indexstorage.js     >$(TMPDIR)/jio/storage/indexstorage.min.js 2>/dev/null
 	@$(UGLIFY) src/jio.storage/gidstorage.js       >$(TMPDIR)/jio/storage/gidstorage.min.js 2>/dev/null

dist/jio-latest.js

@@ -7863,6 +7863,274 @@ Query.searchTextToRegExp = searchTextToRegExp;
 
   jIO.addStorage('zip', ZipStorage);
 }(RSVP, Blob, LZString, DOMException));
+;/*
+ * Copyright 2013, Nexedi SA
+ * Released under the LGPL license.
+ * http://www.gnu.org/licenses/lgpl.html
+ */
+/**
+ * JIO Dropbox Storage. Type = "dropbox".
+ * Dropbox "database" storage.
+ */
+/*global Blob, jIO, RSVP, UriTemplate*/
+/*jslint nomen: true*/
+
+(function (jIO, RSVP, Blob, UriTemplate) {
+  "use strict";
+  var UPLOAD_URL = "https://content.dropboxapi.com/1/files_put/" +
+      "{+root}{+id}{+name}{?access_token}",
+    upload_template = UriTemplate.parse(UPLOAD_URL),
+    CREATE_DIR_URL = "https://api.dropboxapi.com/1/fileops/create_folder" +
+      "{?access_token,root,path}",
+    create_dir_template = UriTemplate.parse(CREATE_DIR_URL),
+    REMOVE_URL = "https://api.dropboxapi.com/1/fileops/delete/" +
+      "{?access_token,root,path}",
+    remote_template = UriTemplate.parse(REMOVE_URL),
+    GET_URL = "https://content.dropboxapi.com/1/files" +
+      "{/root,id}{+name}{?access_token}",
+    get_template = UriTemplate.parse(GET_URL),
+    //LIST_URL = 'https://api.dropboxapi.com/1/metadata/sandbox/';
+    METADATA_URL = "https://api.dropboxapi.com/1/metadata" +
+      "{/root}{+id}{?access_token}",
+    metadata_template = UriTemplate.parse(METADATA_URL);
+
+  function restrictDocumentId(id) {
+    if (id.indexOf("/") !== 0) {
+      throw new jIO.util.jIOError("id " + id + " is forbidden (no begin /)",
+                                  400);
+    }
+    if (id.lastIndexOf("/") !== (id.length - 1)) {
+      throw new jIO.util.jIOError("id " + id + " is forbidden (no end /)",
+                                  400);
+    }
+    return id;
+  }
+
+  function restrictAttachmentId(id) {
+    if (id.indexOf("/") !== -1) {
+      throw new jIO.util.jIOError("attachment " + id + " is forbidden",
+                                  400);
+    }
+  }
+
+  /**
+   * The JIO Dropbox Storage extension
+   *
+   * @class DropboxStorage
+   * @constructor
+   */
+  function DropboxStorage(spec) {
+    if (typeof spec.access_token !== 'string' || !spec.access_token) {
+      throw new TypeError("Access Token' must be a string " +
+                          "which contains more than one character.");
+    }
+    if (typeof spec.root !== 'string' || !spec.root ||
+        (spec.root !== "dropbox" && spec.root !== "sandbox")) {
+      throw new TypeError("root must be 'dropbox' or 'sandbox'");
+    }
+    this._access_token = spec.access_token;
+    this._root = spec.root;
+  }
+
+  DropboxStorage.prototype.put = function (id, param) {
+    var that = this;
+    id = restrictDocumentId(id);
+    if (Object.getOwnPropertyNames(param).length > 0) {
+      // Reject if param has some properties
+      throw new jIO.util.jIOError("Can not store properties: " +
+                                  Object.getOwnPropertyNames(param), 400);
+    }
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "POST",
+          url: create_dir_template.expand({
+            access_token: that._access_token,
+            root: that._root,
+            path: id
+          })
+        });
+      })
+      .push(undefined, function (err) {
+        if ((err.target !== undefined) &&
+            (err.target.status === 405)) {
+          // Directory already exists, no need to fail
+          return;
+        }
+        throw err;
+      });
+  };
+
+  DropboxStorage.prototype.remove = function (id) {
+    id = restrictDocumentId(id);
+    return jIO.util.ajax({
+      type: "POST",
+      url: remote_template.expand({
+        access_token: this._access_token,
+        root: this._root,
+        path: id
+      })
+    });
+  };
+
+  DropboxStorage.prototype.get = function (id) {
+    var that = this;
+
+    if (id === "/") {
+      return {};
+    }
+    id = restrictDocumentId(id);
+
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: metadata_template.expand({
+            access_token: that._access_token,
+            root: that._root,
+            id: id
+          })
+        });
+      })
+      .push(function (evt) {
+        var obj = JSON.parse(evt.target.response ||
+                             evt.target.responseText);
+        if (obj.is_dir) {
+          return {};
+        }
+        throw new jIO.util.jIOError("Not a directory: " + id, 404);
+      }, function (error) {
+        if (error.target !== undefined && error.target.status === 404) {
+          throw new jIO.util.jIOError("Cannot find document: " + id, 404);
+        }
+        throw error;
+      });
+  };
+
+  DropboxStorage.prototype.allAttachments = function (id) {
+
+    var that = this;
+    id = restrictDocumentId(id);
+
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: metadata_template.expand({
+            access_token: that._access_token,
+            root: that._root,
+            id: id
+          })
+        });
+      })
+      .push(function (evt) {
+        var obj = JSON.parse(evt.target.response || evt.target.responseText),
+          i,
+          result = {};
+        if (!obj.is_dir) {
+          throw new jIO.util.jIOError("Not a directory: " + id, 404);
+        }
+        for (i = 0; i < obj.contents.length; i += 1) {
+          if (!obj.contents[i].is_dir) {
+            result[obj.contents[i].path.split("/").pop()] = {};
+          }
+        }
+        return result;
+      }, function (error) {
+        if (error.target !== undefined && error.target.status === 404) {
+          throw new jIO.util.jIOError("Cannot find document: " + id, 404);
+        }
+        throw error;
+      });
+  };
+
+  //currently, putAttachment will fail with files larger than 150MB,
+  //due to the Dropbox API. the API provides the "chunked_upload" method
+  //to pass this limit, but upload process becomes more complex to implement.
+  //
+  //putAttachment will also create a folder if you try to put an attachment
+  //to an inexisting foler.
+
+  DropboxStorage.prototype.putAttachment = function (id, name, blob) {
+    id = restrictDocumentId(id);
+    restrictAttachmentId(name);
+
+    return jIO.util.ajax({
+      type: "PUT",
+      url: upload_template.expand({
+        root: this._root,
+        id: id,
+        name: name,
+        access_token: this._access_token
+      }),
+      dataType: blob.type,
+      data: blob
+    });
+  };
+
+  DropboxStorage.prototype.getAttachment = function (id, name) {
+    var that = this;
+
+    id = restrictDocumentId(id);
+    restrictAttachmentId(name);
+
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          dataType: "blob",
+          url: get_template.expand({
+            root: that._root,
+            id: id,
+            name: name,
+            access_token: that._access_token
+          })
+        });
+      })
+      .push(function (evt) {
+        return new Blob(
+          [evt.target.response || evt.target.responseText],
+          {"type": evt.target.getResponseHeader('Content-Type') ||
+            "application/octet-stream"}
+        );
+      }, function (error) {
+        if (error.target !== undefined && error.target.status === 404) {
+          throw new jIO.util.jIOError("Cannot find attachment: " +
+                                      id + ", " + name, 404);
+        }
+        throw error;
+      });
+  };
+
+  //removeAttachment removes also directories.(due to Dropbox API)
+
+  DropboxStorage.prototype.removeAttachment = function (id, name) {
+    var that = this;
+    id = restrictDocumentId(id);
+    restrictAttachmentId(name);
+
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "POST",
+          url: remote_template.expand({
+            access_token: that._access_token,
+            root: that._root,
+            path: id + name
+          })
+        });
+      }).push(undefined, function (error) {
+        if (error.target !== undefined && error.target.status === 404) {
+          throw new jIO.util.jIOError("Cannot find attachment: " +
+                                      id + ", " + name, 404);
+        }
+        throw error;
+      });
+  };
+
+  jIO.addStorage('dropbox', DropboxStorage);
+
+}(jIO, RSVP, Blob, UriTemplate));
 ;/*
  * Copyright 2013, Nexedi SA
  * Released under the LGPL license.
@@ -8492,7 +8760,8 @@ Query.searchTextToRegExp = searchTextToRegExp;
           },
           form_data_json = {},
           field,
-          key;
+          key,
+          prefix_length;
 
         form_data_json.form_id = {
           "key": [form.form_id.key],
@@ -8502,15 +8771,20 @@ Query.searchTextToRegExp = searchTextToRegExp;
         for (key in form) {
           if (form.hasOwnProperty(key)) {
             field = form[key];
-            if ((key.indexOf('my_') === 0) &&
-                (field.editable) &&
+            prefix_length = 0;
+            if (key.indexOf('my_') === 0 && field.editable) {
+              prefix_length = 3;
+            }
+            if (key.indexOf('your_') === 0) {
+              prefix_length = 5;
+            }
+            if ((prefix_length !== 0) &&
                 (allowed_field_dict.hasOwnProperty(field.type))) {
-
-              form_data_json[key.substring(3)] = {
+              form_data_json[key.substring(prefix_length)] = {
                 "default": field["default"],
                 "key": field.key
               };
-              converted_json[key.substring(3)] = field["default"];
+              converted_json[key.substring(prefix_length)] = field["default"];
             }
           }
         }

examples/scenario.js

@@ -37,18 +37,18 @@
       ///////////////////////////
       // Memory storage
       ///////////////////////////
-//      return g.run({
-//        type: "query",
-//        sub_storage: {
-//          type: "uuid",
-//          sub_storage: {
-//            type: "union",
-//            storage_list: [{
-//              type: "memory"
-//            }]
-//          }
-//        }
-//      });
+      return g.run({
+        type: "query",
+        sub_storage: {
+          type: "uuid",
+          sub_storage: {
+            type: "union",
+            storage_list: [{
+              type: "memory"
+            }]
+          }
+        }
+      });
 
       ///////////////////////////
       // IndexedDB storage
@@ -80,23 +80,41 @@
 //             }
 //           }
 //         }
+//       });
+
+      ///////////////////////////
+      // Dropbox storage
+      ///////////////////////////
+//       return g.run({
+//         type: "query",
+//         sub_storage: {
+//           type: "uuid",
+//           sub_storage: {
+//             type: "drivetojiomapping",
+//             sub_storage: {
+//               "type": "dropbox",
+//               "access_token" : "TOKEN",
+//               "root" : "dropbox"
+//             }
+//           }
+//         }
 //       });
 
       ///////////////////////////
       // Qiniu storage
       ///////////////////////////
-      return g.run({
-        type: "query",
-        sub_storage: {
-          type: "uuid",
-          sub_storage: {
-            "type": "qiniu",
-            "bucket": "7xn150.com1.z0.glb.clouddn.com",
-            "access_key": "s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ",
-            "secret_key": "hfqndzXIfqP6aMpTOdgT_UjiUjARkiFXz98Cthjx"
-          }
-        }
-      });
+//       return g.run({
+//         type: "query",
+//         sub_storage: {
+//           type: "uuid",
+//           sub_storage: {
+//             "type": "qiniu",
+//             "bucket": "BUCKET",
+//             "access_key": "ACCESSKEY",
+//             "secret_key": "SECRETKEY"
+//           }
+//         }
+//       });
 
       ///////////////////////////
       // Replicate storage
@@ -159,10 +177,7 @@
             deepEqual(doc, {"title": "I don't have ID éà&\n"},
                       "Document correctly fetched");
             // Remove the doc
-            return jio.remove(doc_id)
-              .fail(function (error) {
-                console.log("remove error", error);
-              });
+            return jio.remove(doc_id);
           })
           .then(function (doc_id) {
             ok(doc_id, "Document removed");
@@ -229,20 +244,20 @@
           })
 
           .then(function () {
-            return jio.put("test.txt", {});
+            return jio.put("foo❤/test.txt", {});
           })
 
           .then(function () {
             return jio.putAttachment(
-              "test.txt",
+              "foo❤/test.txt",
               "enclosure",
-              new Blob(["fooé\nbar"], {type: "text/plain"})
+              new Blob(["fooé\nbar测试四😈"], {type: "text/plain"})
             );
           })
 
           .then(function () {
             ok(true, "Attachment stored");
-            return jio.getAttachment("test.txt", "enclosure");
+            return jio.getAttachment("foo❤/test.txt", "enclosure");
           })
 
           .then(function (blob) {
@@ -250,13 +265,14 @@
           })
 
           .then(function (result) {
-            equal(result.target.result, "fooé\nbar", "Attachment correctly fetched");
-            return jio.get("test.txt");
+            equal(result.target.result, "fooé\nbar测试四😈", "Attachment correctly fetched");
+            return jio.get("foo❤/test.txt");
+
           })
           .then(function (doc) {
             deepEqual(doc, {}, "Document correctly fetched");
 
-            return jio.allAttachments("test.txt");
+            return jio.allAttachments("foo❤/test.txt");
           })
           .then(function (doc) {
             deepEqual(doc, {
@@ -264,7 +280,7 @@
             },
               "Attachment list correctly fetched");
 
-            return jio.removeAttachment("test.txt", "enclosure");
+            return jio.removeAttachment("foo❤/test.txt", "enclosure");
           })
 
           .then(function () {

package.json

 {
   "name": "jio",
-  "version": "v3.3.0",
+  "version": "v3.4.0",
   "license": "LGPLv3",
   "author": "Nexedi SA",
   "contributors": [

src/jio.storage/erp5storage.js

@@ -79,7 +79,8 @@
           },
           form_data_json = {},
           field,
-          key;
+          key,
+          prefix_length;
 
         form_data_json.form_id = {
           "key": [form.form_id.key],
@@ -89,15 +90,20 @@
         for (key in form) {
           if (form.hasOwnProperty(key)) {
             field = form[key];
-            if ((key.indexOf('my_') === 0) &&
-                (field.editable) &&
+            prefix_length = 0;
+            if (key.indexOf('my_') === 0 && field.editable) {
+              prefix_length = 3;
+            }
+            if (key.indexOf('your_') === 0) {
+              prefix_length = 5;
+            }
+            if ((prefix_length !== 0) &&
                 (allowed_field_dict.hasOwnProperty(field.type))) {
-
-              form_data_json[key.substring(3)] = {
+              form_data_json[key.substring(prefix_length)] = {
                 "default": field["default"],
                 "key": field.key
               };
-              converted_json[key.substring(3)] = field["default"];
+              converted_json[key.substring(prefix_length)] = field["default"];
             }
           }
         }

test/jio.storage/erp5storage.tests.js

@@ -212,11 +212,17 @@
               type: "DateTimeField"
             },
             your_reference: {
-              key: "field_your_title",
+              key: "field_your_reference",
               "default": "bar",
               editable: true,
               type: "StringField"
             },
+            your_reference_non_editable: {
+              key: "field_your_reference_non_editable",
+              "default": "bar",
+              editable: false,
+              type: "StringField"
+            },
             sort_index: {
               key: "field_sort_index",
               "default": "foobar",
@@ -247,6 +253,8 @@
       .then(function (result) {
         deepEqual(result, {
           portal_type: "Person",
+          reference: "bar",
+          reference_non_editable: "bar",
           title: "foo"
         }, "Check document");
         equal(server.requests.length, 2);
@@ -1150,11 +1158,17 @@
               type: "DateTimeField"
             },
             your_reference: {
-              key: "field_your_title",
+              key: "field_your_reference",
               "default": "bar",
               editable: true,
               type: "StringField"
             },
+            your_reference_non_editable: {
+              key: "field_your_reference_non_editable",
+              "default": "bar",
+              editable: false,
+              type: "StringField"
+            },
             sort_index: {
               key: "field_sort_index",
               "default": "foobar",
@@ -1182,9 +1196,9 @@
     }, ""]);
 
     stop();
-    expect(21);
+    expect(23);
 
-    this.jio.put(id, {title: "barè", id: "foo"})
+    this.jio.put(id, {title: "barè", id: "foo", reference: "bar2"})
       .then(function (result) {
         equal(result, id);
         equal(server.requests.length, 3);
@@ -1201,7 +1215,7 @@
         ok(server.requests[2].requestBody instanceof FormData);
         equal(server.requests[2].withCredentials, true);
 
-        equal(context.spy.callCount, 3, "FormData.append count");
+        equal(context.spy.callCount, 4, "FormData.append count");
         equal(context.spy.firstCall.args[0], "form_id", "First append call");
         equal(context.spy.firstCall.args[1], "Base_view", "First append call");
         equal(context.spy.secondCall.args[0], "field_my_title",
@@ -1210,6 +1224,9 @@
         equal(context.spy.thirdCall.args[0], "field_my_id",
               "Third append call");
         equal(context.spy.thirdCall.args[1], "foo", "Third append call");
+        equal(context.spy.getCall(3).args[0], "field_your_reference",
+              "Fourth append call");
+        equal(context.spy.getCall(3).args[1], "bar2", "Fourth append call");
       })
       .fail(function (error) {
         ok(false, error);
@@ -1265,11 +1282,17 @@
               type: "DateTimeField"
             },
             your_reference: {
-              key: "field_your_title",
+              key: "field_your_reference",
               "default": "bar",
               editable: true,
               type: "StringField"
             },
+            your_reference_non_editable: {
+              key: "field_your_reference_non_editable",
+              "default": "bar",
+              editable: false,
+              type: "StringField"
+            },
             sort_index: {
               key: "field_sort_index",
               "default": "foobar",
@@ -1389,7 +1412,7 @@
               type: "DateTimeField"
             },
             your_reference: {
-              key: "field_your_title",
+              key: "field_your_reference",
               "default": "bar",
               editable: true,
               type: "StringField"
@@ -1425,13 +1448,14 @@
     }, ""]);
 
     stop();
-    expect(33);
+    expect(35);
 
     this.jio.post({
       title: "barè",
       id: "foo",
       portal_type: "Foo",
-      parent_relative_url: "foo_module"
+      parent_relative_url: "foo_module",
+      reference: "bar2"
     })
       .then(function (result) {
         equal(result, id);
@@ -1462,7 +1486,7 @@
         ok(server.requests[4].requestBody instanceof FormData);
         equal(server.requests[4].withCredentials, true);
 
-        equal(context.spy.callCount, 5, "FormData.append count");
+        equal(context.spy.callCount, 6, "FormData.append count");
 
         equal(context.spy.firstCall.args[0], "portal_type",
               "First append call");
@@ -1480,6 +1504,9 @@
         equal(context.spy.getCall(4).args[0], "field_my_id",
               "Fifth append call");
         equal(context.spy.getCall(4).args[1], "foo", "Fifth append call");
+        equal(context.spy.getCall(5).args[0], "field_your_reference",
+              "Sixth append call");
+        equal(context.spy.getCall(5).args[1], "bar2", "Sixth append call");
       })
       .fail(function (error) {
         ok(false, error);
@@ -1560,11 +1587,17 @@
               type: "DateTimeField"
             },
             your_reference: {
-              key: "field_your_title",
+              key: "field_your_reference",
               "default": "bar",
               editable: true,
               type: "StringField"
             },
+            your_reference_non_editable: {
+              key: "field_your_reference_non_editable",
+              "default": "bar",
+              editable: false,
+              type: "StringField"
+            },
             sort_index: {
               key: "field_sort_index",
               "default": "foobar",
@@ -1661,6 +1694,8 @@
         equal(result_list.length, 2);
         deepEqual(result, {
           portal_type: "Person",
+          reference: "bar",
+          reference_non_editable: "bar",
           title: "foo"
         }, "Check document");
         deepEqual(result2, {

test/tests.html

@@ -42,7 +42,7 @@
   <script src="jio.storage/shastorage.tests.js"></script>
 
   <!--script src="jio.storage/indexstorage.tests.js"></script-->
-  <!--script src="jio.storage/dropboxstorage.tests.js"></script-->
+  <script src="jio.storage/dropboxstorage.tests.js"></script>
   <script src="jio.storage/zipstorage.tests.js"></script>
 
   <!--script src="../lib/jquery/jquery.min.js"></script>