Skip to content

J
jio
Commit 96cbccb1 authored by Tristan Cavelier's avatar Tristan Cavelier
Browse files
Options

Merge remote-tracking branch 's3/master' 

Conflicts:
	src/jio.storage/s3storage.js
	test/jiotests.js
parents ac7b4ae5 89d30927
Hide whitespace changes
Inline Side-by-side
Showing with 597 additions and 55 deletions
src/jio.storage/s3storage.js
View file @ 96cbccb1
......@@ -15,8 +15,6 @@ jIO.addStorageType("s3", function (spec, my) {
priv.AWSIdentifier = spec.AWSIdentifier || '';
priv.password = spec.password || '';
priv.server = spec.server || ''; /*|| jiobucket ||*/
priv.url = spec.url || ''; /*||> https://s3-eu-west-1.amazonaws.com <||*/
priv.acl = spec.acl || '';
/*||> "private,
......@@ -194,7 +192,6 @@ jIO.addStorageType("s3", function (spec, my) {
return {
"username": priv.username,
"password": priv.password,
"url": priv.url,
"server": priv.server,
"acl": priv.acl
};
......@@ -210,9 +207,6 @@ jIO.addStorageType("s3", function (spec, my) {
if (typeof priv.password === "string" && priv.password === '') {
return 'Need at least one parameter "password".';
}
if (typeof priv.url === "string" && priv.url === '') {
return 'Need at least one parameter "url".';
}
if (typeof priv.server === "string" && priv.server === '') {
return 'Need at least one parameter "server".';
}
......@@ -415,9 +409,7 @@ jIO.addStorageType("s3", function (spec, my) {
doc = JSON.stringify(doc);
break;
case "update":
console.log(doc._attachments);
doc._attachments[attachid] = data;
console.log(doc._attachments);
//update happened in the put request
doc = JSON.stringify(doc);
break;
......@@ -529,7 +521,7 @@ jIO.addStorageType("s3", function (spec, my) {
**/
that.post = function (command) {
//as S3 encoding key are directly inserted within the FormData(),
//as S3 encoding key are directly inserted within the FormData(),
//use of XHRwrapper function ain't pertinent
var doc, doc_id, mime;
......@@ -539,7 +531,7 @@ jIO.addStorageType("s3", function (spec, my) {
function postDocument() {
var http_response, fd, Signature, xhr;
doc_id = priv.secureName(priv.idsToFileName(doc_id));
//Meant to deep-serialize in order to avoid
//Meant to deep-serialize in order to avoid
//conflicts due to the multipart enctype
doc = JSON.stringify(doc);
http_response = '';
......@@ -872,9 +864,12 @@ jIO.addStorageType("s3", function (spec, my) {
requestUTC,
parse,
checkCounter;
keys = $(mon_document).find('Key');
resultTable = [];
counter = 0;
keys.each(function (index) {
var that, filename, docId;
that = $(this);
......@@ -899,6 +894,7 @@ jIO.addStorageType("s3", function (spec, my) {
//needed to save the index within the $.ajax.success() callback
count = resultTable.length - 1;
countB = 0;
dealCallback = function (i, countB, allDoc) {
return function (doc, statustext, response) {
allDoc.rows[i].doc = response.responseText;
......@@ -925,13 +921,16 @@ jIO.addStorageType("s3", function (spec, my) {
};
i = resultTable.length - 1;
if (command.getOption("include_docs") === true) {
for (i; i >= 0; i -= 1) {
keyId = resultTable[i];
Signature = that.encodeAuthorization(keyId);
callURL = priv.url + keyId;
callURL = 'http://' + priv.server + '.s3.amazonaws.com/' + keyId;
requestUTC = new Date().toUTCString();
parse = true;
allDocResponse.rows[i] = {
"id": priv.fileNameToIds(keyId).join(),
"key": keyId,
......@@ -950,7 +949,6 @@ jIO.addStorageType("s3", function (spec, my) {
+ priv.AWSIdentifier
+ ":"
+ Signature,
//'Host' : priv.url,
'x-amz-date' : requestUTC,
'Content-Type' : 'application/json'
//'Content-MD5' : ''
......@@ -991,46 +989,3 @@ jIO.addStorageType("s3", function (spec, my) {
};
return that;
});
/*
// It is not possible to attach listeners to xhr level 2 events
// AND validate the Qunit tests through sinon.js
// therefore, below methods are deprecated
var S3specifics = {};
S3specifics.uploadProgress = function(evt){
if (evt.lengthComputable) {
var percentComplete = Math.round(evt.loaded * 100 / evt.total);
console.log(percentComplete.toString() + '%');
} else {
console.log('Unable to compute.');
}
};
S3specifics.uploadComplete = function(evt){
var evt_txt = evt.target.responseText;
var parser = new DOMParser();
var xmlDoc = parser.parseFromString(evt_txt, "text/xml");
var responseURL = $(xmlDoc.getElementsByTagName('Location'))[0].text();
console.log(responseURL);
};
S3specifics.uploadFailed = function(evt){
var evt_txt = evt.target.responseText;
console.log("Erreur lors de la tentative d'upload : " + evt_txt);
};
S3specifics.uploadCanceled = function(evt){
console.log("Upload annulé par l'utilisateur ou le navigateur.");
};
S3specifics.onReadyStateChange = function(req, those, that) {
if (req.readyState === 4 && those.status === 200){
that.success({
ok: true,
id: command.getDocId()
});
}
};
*/
test/jiotests.js
View file @ 96cbccb1
......@@ -6575,6 +6575,593 @@ test ("AllDocs", function () {
});
*/
module("Amazon S3 Storage");
/*
Post without id
Create = POST non empty document
Post but document already exists
*/
test ("Post", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com"
});
o.server = sinon.fakeServer.create();
//Post without ID (ID is generated by storage)
o.server.respondWith("GET",
"https://jiobucket.s3.amazonaws.com/",
[404, {}, "HTML Response"]
);
o.server.respondWith("POST",
"https://jiobucket.s3.amazonaws.com/",
[204, {}, "HTML Response"]
);
//o.spy (o, "status", 405, "Post without id");
o.spy(o, "jobstatus", "done", "Post without ID");
o.jio.post({}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
//Post with ID
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"https://jiobucket.s3.amazonaws.com/post1",
[404, {}, "HTML Response"]
);
o.server.respondWith("POST",
"https://jiobucket.s3.amazonaws.com/",
[204, {}, "HTML Response"]
);
o.spy(o, "value", {'ok':true,'id':'post1'}, "Post with ID");
o.jio.post({"_id": "post1", "title": "myPost1"}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
//Post but document already exists (update)
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/post2",
[200, {}, "HTML Response"]
);
o.spy (o, "status", 409, "Post but document already exists (update)");
//o.spy(o, "jobstatus", "done", "Post without ID");
o.jio.post({"_id": "post2", "title": "myPost2"}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
/*
Put without id
Create = PUT non empty document
Updated the document
*/
test("Put", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"http://s3.amazonaws.com/jiobucket/put1"
});
// put without id => id required
o.server = sinon.fakeServer.create();
o.spy (o, "status", 20, "Put without id");
o.jio.put({}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//Put non empty document
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[404, {}, "HTML Response"]
);
o.server.respondWith("PUT",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {}, "HTML Response"]
);
o.spy (o, "value", {"ok": true, "id": "http://100%.json"},
"PUT non empty document");
o.jio.put({"_id": "http://100%.json", "title": "myPut1"}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//Put an existing document (update)
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {}, "HTML Response"]
);
o.server.respondWith("PUT",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {}, "HTML Response"]
);
o.spy (o, "value", {"ok": true, "id": "http://100%.json"},
"PUT non empty document");
o.jio.put({"_id": "http://100%.json", "title": "myPut1"}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
test ("PutAttachment", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com"
});
//PutAttachment without document ID (document ID is required)
o.server = sinon.fakeServer.create();
//PutAttachment without attachment ID (attachment ID is required)
o.spy(o, "status", 20, "PutAttachment without doc id -> 20");
o.jio.putAttachment({"_attachment": "putattmt1"}, o.f);
o.tick(o);
// putAttachment without attachment id => 22 Attachment Id Required
o.spy(o, "status", 22, "PutAttachment without attachment id -> 22");
o.jio.putAttachment({"_id": "http://100%.json"}, o.f);
o.tick(o);
//PutAttachment without underlying document (document is required)
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[404, {}, "HTML Response"]
);
o.spy(o, "status", 404, "PutAttachment without document -> 404");
o.jio.putAttachment({
"_id": "http://100%.json",
"_attachment": "putattmt2"
}, {"max_retry": 1}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
//PutAttachment
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200,{"Content-Type": "text/plain"},'{"_id":"http://100%.json","title":"Hi There!"}']
);
o.server.respondWith("PUT",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {}, "HTML Response"]
);
o.server.respondWith("PUT",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[200, {}, "HTML Response"]
);
o.spy(o, "value", {
"ok": true,
"id": "http://100%.json",
"attachment": "body.html"
}, "PutAttachment");
o.jio.putAttachment({
"_id": "http://100%.json",
"_attachment": "body.html",
"_mimetype": "text/html",
"_data": "<h1>Hi There!!</h1><p>How are you?</p>"
}, {"max_retry": 1}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
/*
Get non existing document
Get non existing attachment
Get document
Get non existing attachment (doc exists)
Get attachment
*/
test("Get", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com"
});
//Get non existing document
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"https://jiobucket.s3.amazonaws.com/doc",
[404, {}, "HTML Response"]
);
o.spy(o, "status", 404, "Get non existing document" );
o.jio.get("doc", {"max_retry":1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//Get document
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200,{"Content-Type": "text/plain"},'{"_id":"http://100%.json","title":"Hi There!"}']
);
o.spy(o, "value", {"_id":"http://100%.json","title":"Hi There!"}, "Get document" )
o.jio.get({"_id":"http://100%.json"}, {"max_retry":1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
test("GetAttachment", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com"
});
//Get non existing attachment
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[404, {}, "HTML Response"]
);
o.spy(o, "status", 404, "Get non existing attachment" );
o.jio.getAttachment({
"_id": "http://100%.json",
"_attachment": "body.html"
}, {"max_retry": 1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//Get attachment
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[200, {"Content-Type": "text/plain"}, "My Attachment Content"]
);
o.spy(o, "value", "My Attachment Content", "Get attachment");
o.jio.getAttachment({
"_id": "http://100%.json",
"_attachment": "body.html"
}, {"max_retry": 1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
//begins the remove tests
/*
Remove inexistent document
Remove inexistent document/attachment
Remove document
Check index file
Check if document has been removed
Remove one of multiple attachment
Check index file
Remove one document and attachment together
Check index file
Check if attachment has been removed
Check if document has been removed
*/
test ("Remove", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com/"
});
//Remove non-existing document
o.server = sinon.fakeServer.create();
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[404, {}, "HTML RESPONSE"]
);
o.spy(o, "status", 404, "Remove non existing document");
o.jio.remove({"_id": "http://100%.json"}, o.f);
o.clock.tick(5000);
o.server.respond();
o.server.restore();
//Remove document
o.server = sinon.fakeServer.create();
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {"Content-Type": "text/plain"}, "My Attachment Content"]
);
o.spy(o, "value", {"ok":true, "id":"http://100%.json"}, "Remove document");
o.jio.remove({"_id": "http://100%.json"}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//Remove document with multiple attachments
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[
200,
{"Content-Type": "text/html"},
JSON.stringify({
"_attachments": {
"body.html": {
"length": 32,
"digest": "md5-dontcare",
"content_type": "text/html"
},
"other": {
"length": 3,
"digest": "md5-dontcare-again",
"content_type": "text/plain"
}
}
})
]
);
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200, {"Content-Type": "text/plain"}, "<h1>Deleted</h1>"]
);
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[200, {"Content-Type": "text/plain"}, "<h1>Deleted</h1>"]
);
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.other",
[200, {"Content-Type": "text/plain"}, "<h1>Deleted</h1>"]
);
o.spy(o, "value", {"ok": true, "id": "http://100%.json"},
"Remove document containing multiple attachments");
o.jio.remove({"_id": "http://100%.json"}, {"max_retry": 1}, o.f);
o.clock.tick(1000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
test ("RemoveAttachment", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com/"
});
//Remove non-existing attachment
o.server = sinon.fakeServer.create();
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[404, {}, "HTML RESPONSE"]
);
o.spy(o, "status", 404, "Remove non existing attachment");
o.jio.removeAttachment({
"_id": "http://100%.json",
"_attachment": "body.html"
}, {"max_retry": 1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.server.restore();
//Remove attachment
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200,{"Content-Type": "text/plain"},'{"_id":"http://100%.json","title":"Hi There!"}']
);
o.server.respondWith("PUT",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json",
[200,{"Content-Type": "text/plain"},'{"_id":"http://100%.json","title":"Hi There!"}']
);
o.server.respondWith("DELETE",
"http://s3.amazonaws.com/jiobucket/http:%252F%252F100%2525_.json.body_.html",
[200, {"Content-Type": "text/plain"}, "My Attachment Content"]
);
o.spy(o, "value", {
"ok": true,
"id": "http://100%.json",
"attachment": "body.html"
}, "Remove attachment");
o.jio.removeAttachment({
"_id": "http://100%.json",
"_attachment": "body.html"
}, {"max_retry": 1}, o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
test ("AllDocs", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type":"s3",
"AWSIdentifier":"dontcare",
"password":"dontcare",
"server":"jiobucket",
"url":"https://jiobucket.s3.amazonaws.com/"
});
//allDocs without option
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/",
[204, {}, '<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>jiobucket</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>documentONE</Key><LastModified>2013-05-03T15:32:01.000Z</LastModified><ETag>&quot;8a65389818768e1f5e6530a949233581&quot;</ETag><Size>163</Size><Owner><ID>5d09e586ab92acad85e9d053f769cce65f82178e218d9ac9b0473f3ce7f89606</ID><DisplayName>jonathan.rivalan</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>documentONE.1st_Attachment_manual</Key><LastModified>2013-05-03T15:32:02.000Z</LastModified><ETag>&quot;f391dec65366d2b470406bc7b9595dea&quot;</ETag><Size>35</Size><Owner><ID>5d09e586ab92acad85e9d053f769cce65f82178e218d9ac9b0473f3ce7f89606</ID><DisplayName>jonathan.rivalan</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>']
);
o.spy(o, "jobstatus", "done", "AllDocs without include docs");
o.jio.allDocs(o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
//allDocs with the include docs option
o.server = sinon.fakeServer.create();
o.server.respondWith("GET",
"http://s3.amazonaws.com/jiobucket/",
[204, {}, '<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>jiobucket</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>documentONE</Key><LastModified>2013-05-03T15:32:01.000Z</LastModified><ETag>&quot;8a65389818768e1f5e6530a949233581&quot;</ETag><Size>163</Size><Owner><ID>5d09e586ab92acad85e9d053f769cce65f82178e218d9ac9b0473f3ce7f89606</ID><DisplayName>jonathan.rivalan</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>documentONE.1st_Attachment_manual</Key><LastModified>2013-05-03T15:32:02.000Z</LastModified><ETag>&quot;f391dec65366d2b470406bc7b9595dea&quot;</ETag><Size>35</Size><Owner><ID>5d09e586ab92acad85e9d053f769cce65f82178e218d9ac9b0473f3ce7f89606</ID><DisplayName>jonathan.rivalan</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>']
);
o.server.respondWith("GET",
"http://jiobucket.s3.amazonaws.com/documentONE",
[
200,
{"Content-Type": "text/html"},
JSON.stringify({
"_attachments": {
"body.html": {
"length": 32,
"digest": "md5-dontcare",
"content_type": "text/html"
},
"other": {
"length": 3,
"digest": "md5-dontcare-again",
"content_type": "text/plain"
}
}
})
]
);
console.log(o);
o.spy(o, "jobstatus", "done", "AllDocs with include docs");
o.jio.allDocs({"include_docs": true},o.f);
console.log(o.f);
o.clock.tick(5000);
o.server.respond();
o.tick(o);
o.server.restore();
o.jio.stop();
});
var nThen = function(next) {
var funcs = [];
var calls = 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7