Skip to content

J
jio
Commit 879a2d3a authored by Jonathan Rivalan's avatar Jonathan Rivalan
Browse files
Options

correcting jsling s3storage.js

parent dc97f827
Hide whitespace changes
Inline Side-by-side
Showing with 82 additions and 197 deletions
src/jio.storage/multisplitstorage.js
View file @ 879a2d3a
......@@ -26,28 +26,6 @@
}(['jio'], function (jIO) {
"use strict";
/**
* Generate a new uuid
*
* @method generateUuid
* @private
* @return {String} The new uuid
*/
function generateUuid() {
function S4() {
/* 65536 */
var i, string = Math.floor(
Math.random() * 0x10000
).toString(16);
for (i = string.length; i < 4; i += 1) {
string = '0' + string;
}
return string;
}
return S4() + S4() + "-" + S4() + "-" + S4() + "-" + S4() + "-" + S4() +
S4() + S4();
}
/**
* Select a storage to put the document part
*
......@@ -225,9 +203,10 @@
* A split storage instance is able to i/o on several sub storages with
* split documents.
*
* @class SplitStorage
* @class MultiSplitStorage
*/
function SplitStorage(spec) {
function MultiSplitStorage(spec) {
console.info('mutli creation');
var that = this, priv = {};
/**
......@@ -261,7 +240,7 @@
* @param {Object} option The command option
* @param {Function} callback Called at the end
*/
priv.send = function (command, method, doc, option, callback) {
function send(command, method, doc, option, callback) {
var i, answer_list = [], failed = false, currentServer;
function onEnd() {
i += 1;
......@@ -343,11 +322,8 @@
* @param {String} method The command method ('post' or 'put')
*/
priv.postOrPut = function (command, doc, option, method) {
console.log("multi post or put");
var i, data, doc_list = [], doc_underscores = {};
if (!doc._id) {
doc._id = generateUuid(); // XXX should let gidstorage guess uid
// in the future, complete id with gidstorage
}
for (i in doc) {
if (doc.hasOwnProperty(i)) {
if (i[0] === "_") {
......@@ -377,12 +353,13 @@
);
}
priv.send(command, method, doc_list, option, function (err) {
send(command, method, doc_list, option, function (err) {
if (err) {
err.message = "Unable to " + method + " document";
delete err.index;
return command.error(err);
}
console.log("post or put succes", doc_underscores._id);
command.success({"id": doc_underscores._id});
});
};
......@@ -426,7 +403,7 @@
data.type
);
}
priv.send(
send(
command,
'putAttachment',
attachment_list,
......@@ -450,7 +427,8 @@
*/
that.get = function (command, param, option) {
var doc = param;
priv.send(command, 'get', doc, option, function (err, response) {
send(command, 'get', doc, option, function (err, response) {
var i, k;
if (err) {
err.message = "Unable to get document";
......@@ -510,7 +488,7 @@
* @param {Object} command The JIO command
*/
that.getAttachment = function (command, param, option) {
priv.send(command, 'getAttachment', param, option, function (
send(command, 'getAttachment', param, option, function (
err,
response
) {
......@@ -533,7 +511,7 @@
* @param {Object} command The JIO command
*/
that.remove = function (command, param, option) {
priv.send(
send(
command,
'remove',
param,
......@@ -556,7 +534,7 @@
* @param {Object} command The JIO command
*/
that.removeAttachment = function (command, param, option) {
priv.send(
send(
command,
'removeAttachment',
param,
......@@ -583,7 +561,7 @@
*/
that.allDocs = function (command, param, option) {
option = {"include_docs": option.include_docs};
priv.send(
send(
command,
'allDocs',
param,
......@@ -601,8 +579,8 @@
}
);
};
console.info('mutli created');
} // end of MultiplitStorage
} // end of splitStorage
jIO.addStorage('split', SplitStorage);
jIO.addStorage('multisplit', MultiSplitStorage);
}));
src/jio.storage/s3storage.js
View file @ 879a2d3a
......@@ -16,7 +16,10 @@
"use strict";
var b64_hmac_sha1 = sha1.b64_hmac_sha1;
jIO.addStorage("s3", function (spec) {
console.info('S3 creation');
var that, priv = {}, lastDigest, isDelete;
that = this;
......@@ -68,103 +71,14 @@
return split.join('%2F');
};
/**
* Replace substrings to another strings
* @method recursiveReplace
* @param {string} string The string to do replacement
* @param {array} list_of_replacement An array of couple
* ["substring to select", "selected substring replaced by this string"].
* @return {string} The replaced string
*/
//priv.recursiveReplace = function (string, list_of_replacement) {
//var i, split_string = string.split(list_of_replacement[0][0]);
//if (list_of_replacement[1]) {
//for (i = 0; i < split_string.length; i += 1) {
//split_string[i] = priv.recursiveReplace(
//split_string[i],
//list_of_replacement.slice(1)
//);
//}
//}
//return split_string.join(list_of_replacement[0][1]);
//};
/**
* Changes / to %2F, % to %25 and . to _.
* @method secureName
* @param {string} name The name to secure
* @return {string} The secured name
*/
//priv.secureName = function (name) {
//return priv.recursiveReplace(name, [["/", "%2F"], ["%", "%25"]]);
//};
/**
* Restores the original name from a secured name
* @method restoreName
* @param {string} secured_name The secured name to restore
* @return {string} The original name
*/
//priv.restoreName = function (secured_name) {
//return priv.recursiveReplace(secured_name, [["%2F", "/"], ["%25", "%"]]);
//};
/**
* Convert document id and attachment id to a file name
* @method idsToFileName
* @param {string} doc_id The document id
* @param {string} attachment_id The attachment id (optional)
* @return {string} The file name
*/
//priv.idsToFileName = function (doc_id, attachment_id) {
//doc_id = priv.secureName(doc_id).split(".").join("_.");
//if (typeof attachment_id === "string") {
//attachment_id = priv.secureName(attachment_id).split(".").join("_.");
//return doc_id + "." + attachment_id;
//}
//return doc_id;
//};
/**
* Convert a file name to a document id (and attachment id if there)
* @method fileNameToIds
* @param {string} file_name The file name to convert
* @return {array} ["document id", "attachment id"] or ["document id"]
*/
//priv.fileNameToIds = function (file_name) {
//var separator_index = -1, split = file_name.split(".");
//split.slice(0, -1).forEach(function (file_name_part, index) {
//if (file_name_part.slice(-1) !== "_") {
//separator_index = index;
//}
//});
//if (separator_index === -1) {
//return [priv.restoreName(priv.restoreName(
//file_name
//).split("_.").join("."))];
//}
//return [
//priv.restoreName(priv.restoreName(
//split.slice(0, separator_index + 1).join(".")
//).split("_.").join(".")),
//priv.restoreName(priv.restoreName(
//split.slice(separator_index + 1).join(".")
//).split("_.").join("."))
//];
//};
priv.fileNameToIds = function (resourcename) {
var split, el, id = "", attmt = "", last;
split = resourcename.split('.');
function replaceAndNotLast() {
last = false;
return '.';
}
/*jslint ass: true */
while ((el = split.shift()) !== undefined) {
/*jslint ass: false */
last = true;
el = el.replace(/__/g, '%2595');
el = el.replace(/_$/, replaceAndNotLast);
......@@ -173,12 +87,10 @@
break;
}
}
attmt = split.join('.');
return [id, attmt];
}
};
priv.idsToFileName = function (document_id, attachment_id) {
document_id = encodeURI(document_id).
replace(/\//g, "%2F").
......@@ -193,7 +105,7 @@
return document_id + "." + attachment_id;
}
return document_id;
}
};
/**
* Removes the last character if it is a "/". "/a/b/c/" become "/a/b/c"
......@@ -209,29 +121,27 @@
};
/**
* Generate a new uuid
*
* @method generateUuid
* @private
* @return {String} The new uuid
*/
function generateUuid() {
function S4() {
/* 65536 */
var i, string = Math.floor(
Math.random() * 0x10000
).toString(16);
for (i = string.length; i < 4; i += 1) {
string = '0' + string;
/**
* Generate a new uuid
*
* @method generateUuid
* @private
* @return {String} The new uuid
*/
function generateUuid() {
function S4() {
/* 65536 */
var i, string = Math.floor(
Math.random() * 0x10000
).toString(16);
for (i = string.length; i < 4; i += 1) {
string = '0' + string;
}
return string;
}
return string;
return S4() + S4() + "-" + S4() + "-" + S4() + "-" + S4() + "-" + S4() +
S4() + S4();
}
return S4() + S4() + "-" + S4() + "-" + S4() + "-" + S4() + "-" + S4() +
S4() + S4();
}
that.documentObjectUpdate = function (doc, new_doc) {
var k;
......@@ -527,16 +437,13 @@
callback) {
var docFile, requestUTC, StringToSign, url, Signature, xhr;
if (command.method === "alldocs"){
if (command.method === "alldocs") {
docFile = '';
} else {
} else {
docFile = priv.idsToFileName(docId,
attachId || undefined);
}
console.trace('ma trace');
console.info('docfile = '+docId, attachId, docFile);
requestUTC = new Date().toUTCString();
StringToSign = priv.buildStringToSign(
......@@ -592,10 +499,8 @@
**/
that.post = function (command, metadata) {
console.info('S3 post')
//as S3 encoding key are directly inserted within the FormData(),
//use of XHRwrapper function ain't pertinent
console.log(metadata)
var doc, doc_id, mime;
doc = metadata;
//doc_id = (!doc._id) ? generateUuid() : doc._id;
......@@ -642,7 +547,6 @@
}
if (doc_id === '' || doc_id === undefined) {
// doc_id = 'no_document_id_' + ((Math.random() * 10).toString().split('.'))[1];
doc._id = generateUuid();
}
......@@ -670,7 +574,6 @@
**/
that.get = function (command, metadata) {
console.info('S3 get')
var docId, isJIO, mime;
docId = metadata._id;
isJIO = true;
......@@ -723,7 +626,6 @@
**/
that.put = function (command, metadata) {
console.info('S3 put')
var doc, docId, mime;
doc = metadata;
docId = doc._id;
......@@ -845,7 +747,6 @@
*/
that.remove = function (command, param) {
console.info('S3 remove')
var docId, mime;
docId = param._id;
mime = 'text/plain; charset=UTF-8';
......@@ -966,14 +867,13 @@
**/
that.allDocs = function (command, param, options) {
console.info('S3 allDocs');
var _succ = command.success;
command.success = function(){
console.log.apply(console, arguments);
_succ.apply(this, arguments)
}
/*jslint unparam: true */
var my_document, mime;
var _succ, my_document, mime;
_succ = command.success;
command.success = function () {
console.log.apply(console, arguments);
_succ.apply(this, arguments);
};
my_document = null;
mime = 'text/plain; charset=UTF-8';
......@@ -991,16 +891,16 @@
Signature,
callURL,
requestUTC;
keys = $($.parseXML(my_document)).find('Key');
keys = $($.parseXML(my_document)).find('Key');
if (keys.length === 0) {
return command.success( {"data":
return command.success({"data":
{
"total_rows": 0,
"rows": []
}
});
});
}
resultTable = [];
......@@ -1038,9 +938,8 @@
return command.success({
"data": allDoc
});
} else {
count -= 1;
}
count -= 1;
};
};
......@@ -1049,17 +948,7 @@
jQxhr.status,
jQxhr.statusText,
"S3 Alldocs failed."
)
//if (obj.status === 404) {
//obj.error = "not_found";
//console.info(obj);
//command.error(obj.error);
//} else {
////return command.retry(err);
//console.info(obj);
//return command.error(obj);
//}
);
};
i = resultTable.length - 1;
......@@ -1067,7 +956,6 @@
if (options.include_docs) {
for (i; i >= 0; i -= 1) {
keyId = resultTable[i];
console.log(keyId);
Signature = that.encodeAuthorization(keyId);
callURL = 'http://' + priv.server + '.s3.amazonaws.com/' + keyId;
requestUTC = new Date().toUTCString();
......@@ -1125,5 +1013,7 @@
getXML();
//fin alldocs
};
console.info('S3 created');
});
}));
test/jio.storage/multi.split.s3storage.tests.js
View file @ 879a2d3a
......@@ -13,8 +13,10 @@
'jio',
'test_util',
'rsvp',
'gidstorage',
's3storage',
'multisplitstorage'
], function (jIO, util, RSVP) {
"use strict";
......@@ -55,19 +57,33 @@
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
jio = jIO.createJIO({
"type": "split",
var jio_gid_description = {}, jio_gid_description2 = {};
jio_gid_description.type = jio_gid_description2.type = "gid";
jio_gid_description.constraints = jio_gid_description2.constraints = {
"default": {
"title": "string"
}
};
jio_gid_description.sub_storage = shared.s3_storage_description1;
jio_gid_description2.sub_storage = shared.s3_storage_description2;
var jio_multisplit_description = {
"type": "multisplit",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
jio_gid_description,
jio_gid_description2
]
}, {"workspace": shared.workspace});
};
jio = jIO.createJIO(jio_multisplit_description, {"workspace": shared.workspace});
jio_s3_list[0] = jIO.createJIO(shared.s3_storage_description1, {
"workspace": shared.workspace
});
jio_s3_list[1] = jIO.createJIO(shared.s3_storage_description2, {
"workspace": shared.workspace
});
jio_s3_list.run = function (method, argument) {
var i, promises = [];
for (i = 0; i < this.length; i += 1) {
......@@ -82,8 +98,7 @@
stop();
// post without id
jio.post({
"_underscored_meta": "uvalue",
"meta": "data"
"title": "preut"
})
.then(function (answer) {
shared.uuid = answer.id;
......
test/jio.storage/s3.multi.split.storage.livetests.html
View file @ 879a2d3a
......@@ -12,6 +12,7 @@
<script src="../../jio.js"></script>
<script src="../../complex_queries.js"></script>
<script src="../../src/sha1.amd.js"></script>
<script src="../../src/jio.storage/gidstorage.js"></script>
<script src="../../src/jio.storage/s3storage.js"></script>
<script src="../../src/jio.storage/multisplitstorage.js"></script>
<script src="../jio/util.js"></script>
......
test/jio.storage/split.s3storage.tests.js
View file @ 879a2d3a
......@@ -109,6 +109,7 @@
jio_s3_list[0] = jIO.createJIO(shared.s3_storage_description1, {
"workspace": shared.workspace
});
console.log(jio_s3_list[0])
jio_s3_list[1] = jIO.createJIO(shared.s3_storage_description2, {
"workspace": shared.workspace
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7