Skip to content

J
jio
Commit 374c1b07 authored by Junming Liu's avatar Junming Liu
Browse files
Options

QiniuStorage : The latest version of QiniuStorage 

This is the lastest QiniuStorage file. Because of Qiniu not support resource manage operations, so it's only have putAttachment and getAttachment.
The method to upload or download file are using URiTemplate via Ajax. Besides, i use CryptoJS to do HMAC-SHA1 algorithm.
parent cfea757e
Show whitespace changes
Inline Side-by-side
Showing with 171 additions and 313 deletions
examples/scenario.js
View file @ 374c1b07
...@@ -37,18 +37,18 @@ ...@@ -37,18 +37,18 @@
/////////////////////////// ///////////////////////////
// Memory storage // Memory storage
/////////////////////////// ///////////////////////////
return g.run({ // return g.run({
type: "query", // type: "query",
sub_storage: { // sub_storage: {
type: "uuid", // type: "uuid",
sub_storage: { // sub_storage: {
type: "union", // type: "union",
storage_list: [{ // storage_list: [{
type: "memory" // type: "memory"
}] // }]
} // }
} // }
}); // });
/////////////////////////// ///////////////////////////
// IndexedDB storage // IndexedDB storage
...@@ -85,18 +85,18 @@ ...@@ -85,18 +85,18 @@
/////////////////////////// ///////////////////////////
// Qiniu storage // Qiniu storage
/////////////////////////// ///////////////////////////
// return g.run({ return g.run({
// type: "query", type: "query",
// sub_storage: { sub_storage: {
// type: "uuid", type: "uuid",
// sub_storage: { sub_storage: {
// "type": "qiniu", "type": "qiniu",
// "bucket": "BUCKET", "bucket": "7xn150.com1.z0.glb.clouddn.com",
// "access_key": "ACCESSKEY", "access_key": "s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ",
// "secret_key": "SECRETKEY" "secret_key": "hfqndzXIfqP6aMpTOdgT_UjiUjARkiFXz98Cthjx"
// } }
// } }
// }); });
/////////////////////////// ///////////////////////////
// Replicate storage // Replicate storage
...@@ -159,7 +159,10 @@ ...@@ -159,7 +159,10 @@
deepEqual(doc, {"title": "I don't have ID éà&\n"}, deepEqual(doc, {"title": "I don't have ID éà&\n"},
"Document correctly fetched"); "Document correctly fetched");
// Remove the doc // Remove the doc
return jio.remove(doc_id); return jio.remove(doc_id)
.fail(function (error) {
console.log("remove error", error);
});
}) })
.then(function (doc_id) { .then(function (doc_id) {
ok(doc_id, "Document removed"); ok(doc_id, "Document removed");
...@@ -249,7 +252,6 @@ ...@@ -249,7 +252,6 @@
.then(function (result) { .then(function (result) {
equal(result.target.result, "fooé\nbar", "Attachment correctly fetched"); equal(result.target.result, "fooé\nbar", "Attachment correctly fetched");
return jio.get("test.txt"); return jio.get("test.txt");
}) })
.then(function (doc) { .then(function (doc) {
deepEqual(doc, {}, "Document correctly fetched"); deepEqual(doc, {}, "Document correctly fetched");
......
src/jio.storage/qiniustorage.js
View file @ 374c1b07
/* /*
* Copyright 2013, Nexedi SA * Copyright 2015, Nexedi SA
* Released under the LGPL license. * Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html * http://www.gnu.org/licenses/lgpl.html
*/ */
...@@ -7,22 +7,20 @@ ...@@ -7,22 +7,20 @@
/** /**
* JIO Qiniu Storage. Type = "qiniu". * JIO Qiniu Storage. Type = "qiniu".
* Qiniu "database" storage. * Qiniu "database" storage.
* Qiniu storage not support resource manage operation,
* so remove,removeAttachment,allDocs,allAttachment isn't working
*/ */
/*global FormData, btoa, Blob, CryptoJS, define, jIO */
/*jslint indent: 2, maxlen: 80, nomen: true, unparam: true, bitwise: true */ /*global JSON, FormData, btoa, Blob, CryptoJS, define,
(function (dependencies, module) { jIO, RSVP, console, UriTemplate */
"use strict"; /*jslint indent: 2, maxlen: 80, nomen: true, bitwise: true */
if (typeof define === 'function' && define.amd) { (function (jIO, RSVP, Blob, UriTemplate) {
return define(dependencies, module);
}
// if (typeof exports === 'object') {
// return module(exports, require('jio'));
// }
module(jIO);
}([
'jio'
], function (jIO) {
"use strict"; "use strict";
var METADATA_URL = "http://{+bucket}/{+key}?e=" +
"{+DEADLINE}&token={+access_key}:{+token}",
metadata_template = UriTemplate.parse(METADATA_URL),
UPLOAD_URL = "http://up.qiniu.com/",
DEADLINE = 2451491200;
function urlsafe_base64_encode(string) { function urlsafe_base64_encode(string) {
return string return string
...@@ -55,8 +53,6 @@ ...@@ -55,8 +53,6 @@
return urlsafe_base64_encode(btoa(encodedString)); return urlsafe_base64_encode(btoa(encodedString));
} }
var UPLOAD_URL = "http://up.qiniu.com/",
DEADLINE = 2451491200;
/** /**
* The JIO QiniuStorage extension * The JIO QiniuStorage extension
...@@ -93,15 +89,16 @@ ...@@ -93,15 +89,16 @@
data = new FormData(); data = new FormData();
if (update === true) { if (update === true) {
put_policy = JSON.stringify({ put_policy = JSON.stringify({
"scope": this._bucket + ':' + key, "scope": "bucket" + ':' + key,
"deadline": DEADLINE "deadline": DEADLINE
}); });
} else { } else {
put_policy = JSON.stringify({ put_policy = JSON.stringify({
"scope": this._bucket, "scope": "bucket",
"deadline": DEADLINE "deadline": DEADLINE
}); });
} }
encoded = btoa(put_policy); encoded = btoa(put_policy);
encode_signed = b64_hmac_sha1(this._secret_key, encoded); encode_signed = b64_hmac_sha1(this._secret_key, encoded);
upload_token = this._access_key + ":" + encode_signed + ":" + encoded; upload_token = this._access_key + ":" + encode_signed + ":" + encoded;
...@@ -110,10 +107,7 @@ ...@@ -110,10 +107,7 @@
data.append("token", upload_token); data.append("token", upload_token);
data.append( data.append(
"file", "file",
// new Blob([JSON.stringify(doc)], {type: "application/json"}),
// new Blob([doc], {type: "application/json"}),
blob, blob,
// new Blob([], {type: "application/octet-stream"}),
key key
); );
...@@ -126,247 +120,101 @@ ...@@ -126,247 +120,101 @@
}; };
/** /**
* Create a document. * Add an attachment to a document
*
* @method post
* @param {Object} command The JIO command
* @param {Object} metadata The metadata to store
*/
QiniuStorage.prototype.post = function (command, metadata) {
var doc = jIO.util.deepClone(metadata),
doc_id = metadata._id;
if (!doc_id) {
doc_id = jIO.util.generateUuid();
doc._id = doc_id;
}
return this._put(
doc_id,
new Blob([JSON.stringify(doc)], {type: "application/json"})
).then(function (doc) {
if (doc !== null) {
command.success({"id": doc_id});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to post doc"
);
});
};
/**
* Update/create a document.
* *
* @method put * @method putAttachment
* @param {Object} command The JIO command * @id {Object} Document id
* @param {Object} metadata The metadata to store * @param {Object} param The given parameters
* @blob {Object} attachment packaged into a blob
*/ */
QiniuStorage.prototype.put = function (command, metadata) { QiniuStorage.prototype.putAttachment = function (id, param, blob) {
return this._put( var gadget = this;
metadata._id, return new RSVP.Queue()
new Blob([JSON.stringify(metadata)], {type: "application/json"}), .push(function () {
return gadget._put(
id + "/" + param,
blob,
true true
).then(function (doc) {
if (doc !== null) {
command.success({"data": doc});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to put doc"
); );
}); });
}; };
QiniuStorage.prototype._get = function (key) { QiniuStorage.prototype._get = function (key) {
var download_url = 'http://' + this._bucket + '.u.qiniudn.com/' + key var download_url = 'http://' + this._bucket + '/' + key
// var download_url = 'http://' + this._bucket + '.dn.qbox.me/' + key
+ '?e=' + DEADLINE, + '?e=' + DEADLINE,
token = b64_hmac_sha1(this._secret_key, download_url); token = b64_hmac_sha1(this._secret_key, download_url),
gadget = this;
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({ return jIO.util.ajax({
"type": "GET", type: "GET",
"url": download_url + "&token=" + this._access_key + ':' + token url: metadata_template.expand({
// "dataType": "blob" bucket: gadget._bucket,
key: key,
DEADLINE: DEADLINE,
access_key: gadget._access_key,
token: token
})
}); });
};
/**
* Get a document or attachment
* @method get
* @param {object} command The JIO command
**/
QiniuStorage.prototype.get = function (command, param) {
return this._get(param._id)
.then(function (doc) {
if (doc.target.responseText !== undefined) {
command.success({"data": JSON.parse(doc.target.responseText)});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to get doc"
);
}); });
}; };
/** /**
* Get an attachment * Get an attaURITemplatechment
* *
* @method getAttachment * @method getAttachment
* @param {Object} command The JIO command
* @param {Object} param The given parameters * @param {Object} param The given parameters
* @param {Object} options The command options * @param {Object} attachment attachment name
*/ */
QiniuStorage.prototype.getAttachment = function (command, param) { QiniuStorage.prototype.getAttachment = function (param, attachment) {
return this._get(param._id + "/" + param._attachment) var gadget = this;
.then(function (doc) {
if (doc.target.response) { return new RSVP.Queue()
command.success({"data": doc.target.response}); .push(function () {
} else { return gadget._get(param + "/" + attachment);
// XXX Handle error })
command.error( .push(function (doc) {
"not_found", if (doc.target.response !== undefined) {
"missing", return new Blob([doc.target.response]);
"Cannot find document" }
); if (doc.target !== undefined) {
} doc.id = param;
}).fail(function (event) { doc.target.attachment = attachment;
command.error( return new Blob([JSON.stringify(doc)], {type: "application/json"});
event.target.status, }
event.target.statusText,
"Unable to get attachment" }).push(undefined, function (error) {
); if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find attachment: "
+ param._id + " , " + param._attachment,
404);
}
throw error;
}); });
}; };
/** /**
* Add an attachment to a document * Add an attachment to a document
* *
* @method putAttachment * @method putAttachment
* @param {Object} command The JIO command * @id {Object} Document id
* @param {Object} param The given parameters * @param {Object} param The given parameters
* @param {Object} options The command options * @blob {Object} attachment packaged into a blob
*/ */
QiniuStorage.prototype.putAttachment = function (command, param) { QiniuStorage.prototype.putAttachment = function (id, param, blob) {
return this._put( var gadget = this;
param._id + "/" + param._attachment, return new RSVP.Queue()
param._blob, .push(function () {
return gadget._put(
id + "/" + param,
blob,
true true
).then(function (doc) {
if (doc !== null) {
command.success({"data": doc});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to put attachment"
); );
}); });
}; };
/**
* Remove a document
*
* @method remove
* @param {Object} command The JIO command
* @param {Object} param The given parameters
*/
QiniuStorage.prototype.remove = function (command, param) {
var DELETE_HOST = "http://rs.qiniu.com",
DELETE_PREFIX = "/delete/",
encoded_entry_uri = urlsafe_base64_encode(btoa(
this._bucket + ':' + param._id
)),
delete_url = DELETE_HOST + DELETE_PREFIX + encoded_entry_uri,
data = DELETE_PREFIX + encoded_entry_uri + '\n',
token = b64_hmac_sha1(this._secret_key, data);
jIO.util.ajax({
"type": "POST",
"url": delete_url,
"headers": {
Authorization: "QBox " + this._access_key + ':' + token,
"Content-Type": 'application/x-www-form-urlencoded'
}
}).then(
command.success
).fail(function (error) {
command.error(
"not_found",
"missing",
"Unable to delete doc"
);
});
};
QiniuStorage.prototype.allDocs = function (command, param, options) {
var LIST_HOST = "http://rsf.qiniu.com",
LIST_PREFIX = "/list?bucket=" + this._bucket,
list_url = LIST_HOST + LIST_PREFIX,
token = b64_hmac_sha1(this._secret_key, LIST_PREFIX + '\n');
jIO.util.ajax({
"type": "POST",
"url": list_url,
"headers": {
Authorization: "QBox " + this._access_key + ':' + token,
"Content-Type": 'application/x-www-form-urlencoded'
}
}).then(function (response) {
var data = JSON.parse(response.target.responseText),
count = data.items.length,
result = [],
item,
i;
for (i = 0; i < count; i += 1) {
item = data.items[i];
result.push({
id: item.key,
key: item.key,
doc: {},
value: {}
});
}
command.success({"data": {"rows": result, "total_rows": count}});
}).fail(function (error) {
command.error(
"error",
"did not work as expected",
"Unable to call allDocs"
);
});
};
jIO.addStorage('qiniu', QiniuStorage); jIO.addStorage('qiniu', QiniuStorage);
})); }(jIO, RSVP, Blob, UriTemplate));
test/jio.storage/qiniustorage.tests.js
View file @ 374c1b07
/*global define, module, RSVP, jIO, test, ok,
/*global define, module, test_util, RSVP, jIO, test, ok, deepEqual, sinon, expect, stop, start, Blob, equal, define, console */
deepEqual, sinon, expect, stop, start, Blob, equal, define */
/*jslint indent: 2 */ /*jslint indent: 2 */
(function (dependencies, module) { (function (dependencies, module) {
"use strict"; "use strict";
if (typeof define === 'function' && define.amd) { if (typeof define === 'function' && define.amd) {
return define(dependencies, module); return define(dependencies, module);
} }
module(test_util, RSVP, jIO); module(RSVP, jIO);
}([ }([
'test_util',
'rsvp', 'rsvp',
'jio', 'jio',
'hmacsha1', 'hmacsha1',
...@@ -21,12 +19,12 @@ ...@@ -21,12 +19,12 @@
var qiniu_spec = { var qiniu_spec = {
"type": "qiniu", "type": "qiniu",
"bucket": "uth6nied", "bucket": "7xn150.com1.z0.glb.clouddn.com",
"access_key": "Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA", "access_key": "s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ",
"secret_key": "vFkNUlI2U4B7G1sz8UL_Z25kYHozfz82z4vMWPgo" "secret_key": "hfqndzXIfqP6aMpTOdgT_UjiUjARkiFXz98Cthjx"
}; };
module("QiniuStorage", { module("QiniuStorage ", {
setup: function () { setup: function () {
this.server = sinon.fakeServer.create(); this.server = sinon.fakeServer.create();
...@@ -45,9 +43,12 @@ ...@@ -45,9 +43,12 @@
test('get', function () { test('get', function () {
var key = "foobar12345", var key = "foobar12345",
server = this.server, server = this.server,
download_url = 'http://uth6nied.u.qiniudn.com/foobar12345?' + /* download_url = 'http://uth6nied.u.qiniudn.com/foobar12345?' +
'e=2451491200&token=Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA:' + 'e=2451491200&token=Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA:' +
'hISFzrC4dQvdOR8A_MozNsB5cME=', 'hISFzrC4dQvdOR8A_MozNsB5cME=', */
download_url = 'http://7xn150.com1.z0.glb.clouddn.com/foobar12345?' +
'e=2451491200&token=s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ:' +
'lWIEfRXIf6tbwWjuX381DHTSxnU=',
data = { data = {
"_id": key, "_id": key,
"foo": "bar" "foo": "bar"
...@@ -58,15 +59,12 @@ ...@@ -58,15 +59,12 @@
}, JSON.stringify(data)]); }, JSON.stringify(data)]);
stop(); stop();
this.jio_storage.get({"_id": key}) this.jio_storage.get(key)
.then(function (result) { .then(function (result) {
console.log("result", result);
deepEqual(result, { deepEqual(result, {
"data": data, "_id": key,
"id": key, "foo": "bar"
"method": "get",
"result": "success",
"status": 200,
"statusText": "Ok"
}); });
}) })
.fail(function (error) { .fail(function (error) {
...@@ -79,11 +77,14 @@ ...@@ -79,11 +77,14 @@
test('getAttachment', function () { test('getAttachment', function () {
var key = "foobar12345", var key = "foobar12345",
object,
resultdata,
attachment = "barfoo54321", attachment = "barfoo54321",
server = this.server, server = this.server,
download_url = 'http://uth6nied.u.qiniudn.com/foobar12345/barfoo54321' + download_url = 'http://7xn150.com1.z0.glb.clouddn.com/foobar12345' +
'?e=2451491200&token=Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA:' + '/barfoo54321?e=2451491200&token=s90kGV3JYDDQPivaPVpwxrHMi9RCpncL' +
'L88mHkZkfjr11DqPUqb5gsDjHFY=', 'gLctGDJQ:l_yS8PFderOhMyqHN0FN41tdJOM=',
data = { data = {
"_id": key, "_id": key,
"foo": "bar", "foo": "bar",
...@@ -95,24 +96,33 @@ ...@@ -95,24 +96,33 @@
}, JSON.stringify(data)]); }, JSON.stringify(data)]);
stop(); stop();
this.jio_storage.getAttachment({"_id": key, "_attachment": attachment}) this.jio_storage.getAttachment(key, attachment)
.then(function (result) { .then(function (result) {
return jIO.util.readBlobAsText(result.data).then(function (e) { return jIO.util.readBlobAsText(result).then(function (e) {
object = JSON.parse(e.target.result);
return { return {
"result": result, "result": object,
"text": e.target.result "text": object.target.responseText
}; };
}); });
}).then(function (result) { }).then(function (result) {
console.log("result", result);
resultdata = {
"data": result.result.target.responseText,
"id": result.result.id,
"attachment": result.result.target.attachment,
"method": result.result.target.method,
"status": result.result.target.status,
"statusText": result.result.target.statusText
};
result.result.data = result.text; result.result.data = result.text;
deepEqual(result.result, { deepEqual(resultdata, {
"data": JSON.stringify(data), "data": JSON.stringify(data),
"id": key, "id": key,
"attachment": attachment, "attachment": attachment,
"method": "getAttachment", "method": "GET",
"result": "success",
"status": 200, "status": 200,
"statusText": "Ok" "statusText": "OK"
}); });
}) })
.fail(function (error) { .fail(function (error) {
...@@ -122,7 +132,7 @@ ...@@ -122,7 +132,7 @@
start(); start();
}); });
}); });
/*
test('post', function () { test('post', function () {
var key = "foobar12345", var key = "foobar12345",
server = this.server, server = this.server,
...@@ -155,10 +165,10 @@ ...@@ -155,10 +165,10 @@
start(); start();
}); });
}); });
*/
test('put', function () { test('QiniuStorage.put', function () {
var key = "foobar12345", var server = this.server,
server = this.server,
upload_url = 'http://up.qiniu.com/', upload_url = 'http://up.qiniu.com/',
data = {"ok": "excellent"}; data = {"ok": "excellent"};
...@@ -167,9 +177,9 @@ ...@@ -167,9 +177,9 @@
}, JSON.stringify(data)]); }, JSON.stringify(data)]);
stop(); stop();
this.jio_storage.put({"_id": key}) this.jio_storage.put("bar", {"title": "foo"})
.then(function () { .then(function (result) {
throw new Error("Not implemented"); equal(result, "bar");
}) })
.fail(function (error) { .fail(function (error) {
ok(false, error); ok(false, error);
...@@ -179,25 +189,23 @@ ...@@ -179,25 +189,23 @@
}); });
}); });
test('putAttachment', function () { test('QiniuStorage putAttachment', function () {
var key = "foobar12345", var server = this.server,
attachment = "barfoo54321", blob = new Blob(["foo"]),
server = this.server, upload_url = 'http://up.qiniu.com/';
upload_url = 'http://up.qiniu.com/',
data = {"ok": "excellent"};
server.respondWith("POST", upload_url, [200, { server.respondWith("POST", upload_url, [200, {
"Content-Type": "application/json" "Content-Type": "application/json"
}, JSON.stringify(data)]); }, "foo"]);
stop(); stop();
this.jio_storage.putAttachment({ this.jio_storage.putAttachment("bar", "foo", blob)
"_id": key,
"_attachment": attachment,
"_blob": "bar"
})
.then(function () { .then(function () {
throw new Error("Not implemented"); equal(server.requests.length, 1);
equal(server.requests[0].method, "POST");
equal(server.requests[0].url, upload_url);
equal(server.requests[0].status, "200");
equal(server.requests[0].responseText, "foo");
}) })
.fail(function (error) { .fail(function (error) {
ok(false, error); ok(false, error);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7