Skip to content

jio
jio
Commit 03ff747a authored by Jonathan Rivalan's avatar Jonathan Rivalan
Browse files
Options

update on some libraries (storages and tests)

parent 9b6bc977
Hide whitespace changes
Inline Side-by-side
Showing with 1680 additions and 12 deletions
src/jio.storage/multisplitstorage.js 0 → 100644
View file @ 03ff747a
/*
* Copyright 2013, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint indent:2, maxlen: 80, nomen: true */
/*global jIO, define, Blob */
/**
* Provides a split storage for JIO. This storage splits data
* and store them in the sub storages defined on the description.
*
* {
* "type": "split",
* "storage_list": [<storage description>, ...]
* }
*/
// define([dependencies], module);
(function (dependencies, module) {
"use strict";
if (typeof define === 'function' && define.amd) {
return define(dependencies, module);
}
module(jIO);
}(['jio'], function (jIO) {
"use strict";
/**
* Generate a new uuid
*
* @method generateUuid
* @private
* @return {String} The new uuid
*/
function generateUuid() {
function S4() {
/* 65536 */
var i, string = Math.floor(
Math.random() * 0x10000
).toString(16);
for (i = string.length; i < 4; i += 1) {
string = '0' + string;
}
return string;
}
return S4() + S4() + "-" + S4() + "-" + S4() + "-" + S4() + "-" + S4() +
S4() + S4();
}
/**
* Select a storage to put the document part
*
* @method selectStorage
* @private
* @return {Object} The selected storage
*/
function selectStorage(arg, iteration) {
var step = iteration % arg.length;
return arg[step];
}
/**
* Class to merge allDocs responses from several sub storages.
*
* @class AllDocsResponseMerger
* @constructor
*/
function AllDocsResponseMerger() {
/**
* A list of allDocs response.
*
* @attribute response_list
* @type {Array} Contains allDocs responses
* @default []
*/
this.response_list = [];
}
AllDocsResponseMerger.prototype.constructor = AllDocsResponseMerger;
/**
* Add an allDocs response to the response list.
*
* @method addResponse
* @param {Object} response The allDocs response.
* @return {AllDocsResponseMerger} This
*/
AllDocsResponseMerger.prototype.addResponse = function (response) {
this.response_list.push(response);
return this;
};
/**
* Add several allDocs responses to the response list.
*
* @method addResponseList
* @param {Array} response_list An array of allDocs responses.
* @return {AllDocsResponseMerger} This
*/
AllDocsResponseMerger.prototype.addResponseList = function (response_list) {
var i;
for (i = 0; i < response_list.length; i += 1) {
this.response_list.push(response_list[i]);
}
return this;
};
/**
* Merge the response_list to one allDocs response.
*
* The merger will find rows with the same id in order to merge them, thanks
* to the onRowToMerge method. If no row correspond to an id, rows with the
* same id will be ignored.
*
* @method merge
* @param {Object} [option={}] The merge options
* @param {Boolean} [option.include_docs=false] Tell the merger to also
* merge metadata if true.
* @return {Object} The merged allDocs response.
*/
AllDocsResponseMerger.prototype.merge = function (option) {
var result = [], row, to_merge = [], tmp, i;
if (this.response_list.length === 0) {
return [];
}
/*jslint ass: true */
while ((row = this.response_list[0].data.rows.shift()) !== undefined) {
to_merge[0] = row;
for (i = 1; i < this.response_list.length; i += 1) {
to_merge[i] = AllDocsResponseMerger.listPopFromRowId(
this.response_list[i].data.rows,
row.id
);
if (to_merge[i] === undefined) {
break;
}
}
tmp = this.onRowToMerge(to_merge, option || {});
if (tmp !== undefined) {
result[result.length] = tmp;
}
}
this.response_list = [];
return {"total_rows": result.length, "rows": result};
};
/**
* This method is called when the merger want to merge several rows with the
* same id.
*
* @method onRowToMerge
* @param {Array} row_list An array of rows.
* @param {Object} [option={}] The merge option.
* @param {Boolean} [option.include_docs=false] Also merge the metadata if
* true
* @return {Object} The merged row
*/
AllDocsResponseMerger.prototype.onRowToMerge = function (row_list, option) {
var i, k, new_row = {"value": {}}, data = "";
option = option || {};
for (i = 0; i < row_list.length; i += 1) {
new_row.id = row_list[i].id;
if (row_list[i].key) {
new_row.key = row_list[i].key;
}
if (option.include_docs) {
new_row.doc = new_row.doc || {};
for (k in row_list[i].doc) {
if (row_list[i].doc.hasOwnProperty(k)) {
if (k[0] === "_") {
new_row.doc[k] = row_list[i].doc[k];
}
}
}
data += row_list[i].doc.data;
}
}
if (option.include_docs) {
try {
data = JSON.parse(data);
} catch (e) { return undefined; }
for (k in data) {
if (data.hasOwnProperty(k)) {
new_row.doc[k] = data[k];
}
}
}
return new_row;
};
/**
* Search for a specific row and pop it. During the search operation, all
* parsed rows are stored on a dictionnary in order to be found instantly
* later.
*
* @method listPopFromRowId
* @param {Array} rows The row list
* @param {String} doc_id The document/row id
* @return {Object/undefined} The poped row
*/
AllDocsResponseMerger.listPopFromRowId = function (rows, doc_id) {
var row;
if (!rows.dict) {
rows.dict = {};
}
if (rows.dict[doc_id]) {
row = rows.dict[doc_id];
delete rows.dict[doc_id];
return row;
}
/*jslint ass: true*/
while ((row = rows.shift()) !== undefined) {
if (row.id === doc_id) {
return row;
}
rows.dict[row.id] = row;
}
};
/**
* The split storage class used by JIO.
*
* A split storage instance is able to i/o on several sub storages with
* split documents.
*
* @class SplitStorage
*/
function SplitStorage(spec) {
var that = this, priv = {};
/**
* The list of sub storages we want to use to store part of documents.
*
* @attribute storage_list
* @private
* @type {Array} Array of storage descriptions
*/
priv.storage_list = spec.storage_list;
//////////////////////////////////////////////////////////////////////
// Tools
/**
* Send a command to all sub storages. All the response are returned
* in a list. The index of the response correspond to the storage_list
* index. If an error occurs during operation, the callback is called with
* `callback(err, undefined)`. The response is given with
* `callback(undefined, response_list)`.
*
* `doc` is the document informations but can also be a list of dedicated
* document informations. In this case, each document is associated to one
* sub storage.
*
* @method send
* @private
* @param {String} method The command method
* @param {Object,Array} doc The document information to send to each sub
* storages or a list of dedicated document
* @param {Object} option The command option
* @param {Function} callback Called at the end
*/
priv.send = function (command, method, doc, option, callback) {
var i, answer_list = [], failed = false;
function onEnd() {
i += 1;
if (i === priv.storage_list.length) {
callback(undefined, answer_list);
}
}
function onSuccess(i) {
return function (response) {
if (!failed) {
answer_list[i] = response;
}
onEnd();
};
}
function onError(i) {
return function (err) {
if (!failed) {
failed = true;
err.index = i;
callback(err, undefined);
}
};
}
//dans quel cas le doc peut-il etre un array ?
//doc_list LENGTH != priv.storage_list LENGTH
if (!Array.isArray(doc)) {
for (i = 0; i < doc.length; i += 1) {
//fonction pour définir le storage en cours
var currentServer = selectStorage(priv.storage_list, i, doc.length);
console.log('current server :'+currentServer);
if (method === 'allDocs') {
command.storage(currentServer)[method](option).
then(onSuccess(i), onError(i));
} else {
command.storage(currentServer)[method](doc, option).
then(onSuccess(i), onError(i));
}
}
} else {
for (i = 0; i < doc.length; i += 1) {
var currentServer = selectStorage(priv.storage_list, i, doc.length);
console.log('current server :'+currentServer);
// we assume that alldocs is not called if the there is several docs
command.storage(priv.storage_list[i])[method](doc[i], option).
then(onSuccess(i), onError(i));
}
}
//default splitstorage method
/*
if (!Array.isArray(doc)) {
for (i = 0; i < priv.storage_list.length; i += 1) {
if (method === 'allDocs') {
command.storage(priv.storage_list[i])[method](option).
then(onSuccess(i), onError(i));
} else {
command.storage(priv.storage_list[i])[method](doc, option).
then(onSuccess(i), onError(i));
}
}
} else {
for (i = 0; i < priv.storage_list.length; i += 1) {
// we assume that alldocs is not called if the there is several docs
command.storage(priv.storage_list[i])[method](doc[i], option).
then(onSuccess(i), onError(i));
}
}
*/
//re-init
i = 0;
};
/**
* Split document metadata then store them to the sub storages.
*
* @method postOrPut
* @private
* @param {Object} doc A serialized document object
* @param {Object} option Command option properties
* @param {String} method The command method ('post' or 'put')
*/
priv.postOrPut = function (command, doc, option, method) {
var i, data, doc_list = [], doc_underscores = {};
if (!doc._id) {
doc._id = generateUuid(); // XXX should let gidstorage guess uid
//a posteriori, completer l'id avec l'index de découpage
}
for (i in doc) {
//si le doc a la clé
if (doc.hasOwnProperty(i)) {
//si l'index 0 de i est un undersore
console.log('i')
console.log(i);
if (i[0] === "_") {
doc_underscores[i] = doc[i];
delete doc[i];
}
}
}
data = JSON.stringify(doc);
//decoupage plus fin ?
//option ? !base 100
//for (i = 0; i < priv.storage_list.length; i += 1) {
//doc_list[i] = JSON.parse(JSON.stringify(doc_underscores));
//doc_list[i].data = data.slice(
//(data.length / priv.storage_list.length) * i,
//(data.length / priv.storage_list.length) * (i + 1)
//);
//console.info('doc_list[i].data');
//console.log(doc_list[i].data);
//}
for (i = 0; i < 100; i += 1) {
doc_list[i] = JSON.parse(JSON.stringify(doc_underscores));
doc_list[i]._id = doc_list[i]._id +'_'+i;
doc_list[i].data = data.slice(
(data.length / 100) * i,
(data.length / 100) * (i + 1)
);
}
priv.send(command, method, doc_list, option, function (err) {
if (err) {
err.message = "Unable to " + method + " document";
delete err.index;
return command.error(err);
}
command.success({"id": doc_underscores._id});
});
};
//////////////////////////////////////////////////////////////////////
// JIO commands
/**
* Split document metadata then store them to the sub storages.
*
* @method post
* @param {Object} command The JIO command
*/
that.post = function (command, metadata, option) {
priv.postOrPut(command, metadata, option, 'post');
};
/**
* Split document metadata then store them to the sub storages.
*
* @method put
* @param {Object} command The JIO command
*/
that.put = function (command, metadata, option) {
priv.postOrPut(command, metadata, option, 'put');
};
/**
* Puts an attachment to the sub storages.
*
* @method putAttachment
* @param {Object} command The JIO command
*/
that.putAttachment = function (command, param, option) {
var i, attachment_list = [], data = param._blob;
for (i = 0; i < priv.storage_list.length; i += 1) {
attachment_list[i] = jIO.util.deepClone(param);
attachment_list[i]._blob = data.slice(
data.size * i / priv.storage_list.length,
data.size * (i + 1) / priv.storage_list.length,
data.type
);
}
priv.send(
command,
'putAttachment',
attachment_list,
option,
function (err) {
if (err) {
err.message = "Unable to put attachment";
delete err.index;
return command.error(err);
}
command.success();
}
);
};
/**
* Gets splited document metadata then returns real document.
*
* @method get
* @param {Object} command The JIO command
*/
that.get = function (command, param, option) {
var doc = param;
priv.send(command, 'get', doc, option, function (err, response) {
var i, k;
if (err) {
err.message = "Unable to get document";
delete err.index;
return command.error(err);
}
doc = '';
for (i = 0; i < response.length; i += 1) {
response[i] = response[i].data;
doc += response[i].data;
}
doc = JSON.parse(doc);
for (i = 0; i < response.length; i += 1) {
for (k in response[i]) {
if (response[i].hasOwnProperty(k)) {
if (k[0] === "_") {
doc[k] = response[i][k];
}
}
}
}
delete doc._attachments;
for (i = 0; i < response.length; i += 1) {
if (response[i]._attachments) {
for (k in response[i]._attachments) {
if (response[i]._attachments.hasOwnProperty(k)) {
doc._attachments = doc._attachments || {};
doc._attachments[k] = doc._attachments[k] || {
"length": 0,
"content_type": ""
};
doc._attachments[k].length += response[i]._attachments[k].
length;
// if (response[i]._attachments[k].digest) {
// if (doc._attachments[k].digest) {
// doc._attachments[k].digest += " " + response[i].
// _attachments[k].digest;
// } else {
// doc._attachments[k].digest = response[i].
// _attachments[k].digest;
// }
// }
doc._attachments[k].content_type = response[i]._attachments[k].
content_type;
}
}
}
}
command.success({"data": doc});
});
};
/**
* Gets splited document attachment then returns real attachment data.
*
* @method getAttachment
* @param {Object} command The JIO command
*/
that.getAttachment = function (command, param, option) {
priv.send(command, 'getAttachment', param, option, function (
err,
response
) {
if (err) {
err.message = "Unable to get attachment";
delete err.index;
return command.error(err);
}
command.success({"data": new Blob(response.map(function (answer) {
return answer.data;
}), {"type": response[0].data.type})});
});
};
/**
* Removes a document from the sub storages.
*
* @method remove
* @param {Object} command The JIO command
*/
that.remove = function (command, param, option) {
priv.send(
command,
'remove',
param,
option,
function (err) {
if (err) {
err.message = "Unable to remove document";
delete err.index;
return command.error(err);
}
command.success();
}
);
};
/**
* Removes an attachment from the sub storages.
*
* @method removeAttachment
* @param {Object} command The JIO command
*/
that.removeAttachment = function (command, param, option) {
priv.send(
command,
'removeAttachment',
param,
option,
function (err) {
if (err) {
err.message = "Unable to remove attachment";
delete err.index;
return command.error(err);
}
command.success();
}
);
};
/**
* Retreive a list of all document in the sub storages.
*
* If include_docs option is false, then it returns the document list from
* the first sub storage. Else, it will merge results and return.
*
* @method allDocs
* @param {Object} command The JIO command
*/
that.allDocs = function (command, param, option) {
option = {"include_docs": option.include_docs};
priv.send(
command,
'allDocs',
param,
option,
function (err, response_list) {
var all_docs_merger;
if (err) {
err.message = "Unable to retrieve document list";
delete err.index;
return command.error(err);
}
all_docs_merger = new AllDocsResponseMerger();
all_docs_merger.addResponseList(response_list);
return command.success({"data": all_docs_merger.merge(option)});
}
);
};
} // end of splitStorage
jIO.addStorage('split', SplitStorage);
}));
src/jio.storage/s3storage.js 100644 → 100755
View file @ 03ff747a
...@@ -979,15 +979,14 @@ ...@@ -979,15 +979,14 @@
//allDocs = function (command, param, options) { //allDocs = function (command, param, options) {
//if (options.include_docs) { //if (options.include_docs) {
if (options.include_docs) { if (options.include_docs) {
for (i; i >= 0; i -= 1) { for (i; i >= 0; i -= 1) {
keyId = resultTable[i]; keyId = resultTable[i];
Signature = that.encodeAuthorization(keyId); Signature = that.encodeAuthorization(keyId);
callURL = 'http://' + priv.server + '.s3.amazonaws.com/' + keyId; callURL = 'http://' + priv.server + '.s3.amazonaws.com/' + keyId;
requestUTC = new Date().toUTCString(); requestUTC = new Date().toUTCString();
parse = true; parse = true;
allDocResponse.rows[i] = { allDocResponse.rows[i] = {
"id": priv.fileNameToIds(keyId).join(), "id": priv.fileNameToIds(keyId).join(),
"key": keyId,
"value": {} "value": {}
}; };
checkCounter = i; checkCounter = i;
...@@ -1020,7 +1019,6 @@ ...@@ -1020,7 +1019,6 @@
keyId = resultTable[i]; keyId = resultTable[i];
allDocResponse.rows[i] = { allDocResponse.rows[i] = {
"id": priv.fileNameToIds(keyId).join(), "id": priv.fileNameToIds(keyId).join(),
"key": priv.fileNameToIds(keyId).join(),
"value": {} "value": {}
}; };
} }
......
test/jio.storage/multi.split.s3storage.tests.js 0 → 100755
View file @ 03ff747a
/*jslint indent: 2, maxlen: 80, nomen: true */
/*global define, jIO, test_util, RSVP, test, ok, deepEqual, module, stop,
start, hex_sha256 */
// define([module_name], [dependencies], module);
(function (dependencies, module) {
"use strict";
if (typeof define === 'function' && define.amd) {
return define(dependencies, module);
}
module(jIO, test_util, RSVP);
}([
'jio',
'test_util',
'rsvp',
's3storage',
'multisplitstorage'
], function (jIO, util, RSVP) {
"use strict";
var tool = {
"readBlobAsBinaryString": jIO.util.readBlobAsBinaryString
};
function reverse(promise) {
return new RSVP.Promise(function (resolve, reject, notify) {
promise.then(reject, resolve, notify);
}, function () {
promise.cancel();
});
}
/**
* sequence(thens): Promise
*
* Executes a sequence of *then* callbacks. It acts like
* `smth().then(callback).then(callback)...`. The first callback is called
* with no parameter.
*
* Elements of `thens` array can be a function or an array contaning at most
* three *then* callbacks: *onFulfilled*, *onRejected*, *onNotified*.
*
* When `cancel()` is executed, each then promises are cancelled at the same
* time.
*
* @param {Array} thens An array of *then* callbacks
* @return {Promise} A new promise
*/
function sequence(thens) {
var promises = [];
return new RSVP.Promise(function (resolve, reject, notify) {
var i;
promises[0] = new RSVP.Promise(function (resolve) {
resolve();
});
for (i = 0; i < thens.length; i += 1) {
if (Array.isArray(thens[i])) {
promises[i + 1] = promises[i].
then(thens[i][0], thens[i][1], thens[i][2]);
} else {
promises[i + 1] = promises[i].then(thens[i]);
}
}
promises[i].then(resolve, reject, notify);
}, function () {
var i;
for (i = 0; i < promises.length; i += 1) {
promises[i].cancel();
}
});
}
function unexpectedError(error) {
if (error instanceof Error) {
deepEqual([
error.name + ": " + error.message,
error
], "UNEXPECTED ERROR", "Unexpected error");
} else {
deepEqual(error, "UNEXPECTED ERROR", "Unexpected error");
}
}
module("SplitStorage + S3 Storage");
test("Post", function () {
var shared = {}, jio, jio_s3_list = [];
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
jio_s3_list[0] = jIO.createJIO(shared.s3_storage_description1, {
"workspace": shared.workspace
});
jio_s3_list[1] = jIO.createJIO(shared.s3_storage_description2, {
"workspace": shared.workspace
});
jio_s3_list.run = function (method, argument) {
var i, promises = [];
for (i = 0; i < this.length; i += 1) {
promises[i] = this[i][method].apply(this[i], argument);
}
return RSVP.all(promises);
};
jio_s3_list.get = function () {
return this.run("get", arguments);
};
stop();
// post without id
jio.post({
"_underscored_meta": "uvalue",
"meta": "data"
})
.then(function (answer) {
shared.uuid = answer.id;
answer.id = "<uuid>";
ok(util.isUuid(shared.uuid), "Uuid should look like " +
"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx : " + shared.uuid);
deepEqual(answer, {
"id": "<uuid>",
"method": "post",
"result": "success",
"status": 201,
"statusText": "Created"
}, "Post document without id");
// check uploaded documents
return jio_s3_list.get({"_id": shared.uuid});
})
.then(function (answers) {
var i;
for (i = 0; i < answers.length; i += 1) {
deepEqual(answers[i].data, {
"_id": shared.uuid,
"_underscored_meta": "uvalue",
"data": i === 0 ? "{\"meta\"" : ":\"data\"}"
}, "Check uploaded document in sub storage " + (i + 1));
}
// post with id
return jio.post({
"_id": "one",
"_underscored_meta": "uvalue",
"meta": "data",
"hello": "world"
});
})
.then(function (answer) {
deepEqual(answer, {
"id": "one",
"method": "post",
"result": "success",
"status": 201,
"statusText": "Created"
}, "Post document with id");
// check uploaded documents
return jio_s3_list.get({"_id": "one"});
})
.then(function (answers) {
deepEqual(answers[0].data, {
"_id": "one",
"_underscored_meta": "uvalue",
"data": "{\"meta\":\"data\","
}, "Check uploaded document in sub storage 1");
deepEqual(answers[1].data, {
"_id": "one",
"_underscored_meta": "uvalue",
"data": "\"hello\":\"world\"}"
}, "Check uploaded document in sub storage 2");
// post with id
return reverse(jio.post({
"_id": "one",
"_underscored_meta": "uvalue",
"meta": "data",
"hello": "world"
}));
})
.then(function (answer) {
deepEqual(answer, {
"error": "conflict",
"id": "one",
"message": "Unable to post document",
"method": "post",
"reason": "Document already exists",
"result": "error",
"status": 409,
"statusText": "Conflict"
}, "Post document with same id -> 409 Conflict");
})
.fail(unexpectedError).always(start);
});
/*
test("PutAttachment", function () {
var shared = {}, jio, jio_s3_list = [];
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "putAttachment1",
"mode": "memory"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "putAttachment2",
"mode": "memory"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
jio_s3_list[0] = jIO.createJIO(shared.s3_storage_description1, {
"workspace": shared.workspace
});
jio_s3_list[1] = jIO.createJIO(shared.s3_storage_description2, {
"workspace": shared.workspace
});
jio_s3_list.run = function (method, argument) {
var i, promises = [];
for (i = 0; i < this.length; i += 1) {
promises[i] = this[i][method].apply(this[i], argument);
}
return RSVP.all(promises);
};
jio_s3_list.get = function () {
return this.run("get", arguments);
};
jio_s3_list.getAttachmentAsBinaryString = function () {
return this.run("getAttachment", arguments).then(function (answers) {
var i, promises = [];
for (i = 0; i < answers.length; i += 1) {
promises[i] = tool.readBlobAsBinaryString(answers[i].data);
}
return RSVP.all(promises);
});
};
stop();
return reverse(jio.putAttachment({
"_id": "two",
"_attachment": "my_attachment",
"_data": "My Data",
"_content_type": "text/plain"
})).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "two",
"message": "Unable to put attachment",
"method": "putAttachment",
"reason": "Document does not exist",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Put attachment on a inexistent document -> 404 Not Found");
return jio.post({
"_id": "two",
"_underscored_meta": "uvalue",
"meta": "data"
});
}).then(function () {
return jio.putAttachment({
"_id": "two",
"_attachment": "my_attachment",
"_data": "My Data",
"_mimetype": "text/plain"
});
}).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"id": "two",
"method": "putAttachment",
"result": "success",
"status": 204,
"statusText": "No Content"
}, "Put attachment on a document");
// check uploaded documents
return jio_s3_list.get({"_id": "two"});
}).then(function (answers) {
deepEqual(answers[0].data, {
"_attachments": {
"my_attachment": {
"content_type": "text/plain",
"digest": "sha256-ebf2d770a6a2dfa135f6c81431f22fc3cbcde9ae" +
"e52759ca9e520d4d964c1322", // sha256("My ")
"length": 3
}
},
"_id": "two",
"_underscored_meta": "uvalue",
"data": "{\"meta\""
}, "Check uploaded document in sub storage 1");
deepEqual(answers[1].data, {
"_attachments": {
"my_attachment": {
"content_type": "text/plain",
"digest": "sha256-cec3a9b89b2e391393d0f68e4bc12a9fa6cf358b" +
"3cdf79496dc442d52b8dd528", // sha256("Data")
"length": 4
}
},
"_id": "two",
"_underscored_meta": "uvalue",
"data": ":\"data\"}"
}, "Check uploaded document in sub storage 2");
return jio_s3_list.getAttachmentAsBinaryString({
"_id": "two",
"_attachment": "my_attachment"
});
}).then(function (events) {
deepEqual(events[0].target.result, "My ",
"Check uploaded document in sub storage 1");
deepEqual(events[1].target.result, "Data",
"Check uploaded document in sub storage 1");
}).fail(unexpectedError).always(start);
});
*/
/*
test("Get", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "get1",
"mode": "memory"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "get2",
"mode": "memory"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
reverse(jio.get({"_id": "three"})).then(function (answer) {
deepEqual(answer, {
"error": "not_found",
"id": "three",
"message": "Unable to get document",
"method": "get",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Get missing document");
return jio.post({
"_id": "three",
"_underscored_meta": "uvalue",
"meta": "data"
});
}).then(function () {
return jio.get({"_id": "three"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "three",
"_underscored_meta": "uvalue",
"meta": "data"
}, "Get posted document");
return jio.putAttachment({
"_id": "three",
"_attachment": "my_attachment",
"_data": "My Data",
"_content_type": "text/plain"
});
}).then(function () {
return jio.get({"_id": "three"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "three",
"_underscored_meta": "uvalue",
"meta": "data",
"_attachments": {
"my_attachment": {
"length": 7,
"content_type": "text/plain"
}
}
}, "Get document with attachment informations");
}).fail(unexpectedError).always(start);
});
*/
/*
test("GetAttachment", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "getAttachment",
"mode": "memory"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"application_name": "getAttachment2",
"mode": "memory"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
reverse(jio.getAttachment({
"_id": "four",
"_attachment": "my_attachment"
}))
.then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "four",
"message": "Unable to get attachment",
"method": "getAttachment",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Get attachment from missing document -> 404 Not Found");
return jio.post({
"_id": "four",
"_underscored_meta": "uvalue",
"meta": "data"
});
})
.then(function () {
return reverse(jio.getAttachment({
"_id": "four",
"_attachment": "my_attachment"
}));
}).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "four",
"message": "Unable to get attachment",
"method": "getAttachment",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Get missing attachment from document");
return jio.putAttachment({
"_id": "four",
"_attachment": "my_attachment",
"_data": "My Data",
"_mimetype": "text/plain"
});
})
.then(function () {
return jio.getAttachment({
"_id": "four",
"_attachment": "my_attachment"
});
}).then(function (answer) {
return tool.readBlobAsBinaryString(answer.data);
}).then(function (event) {
deepEqual(event.target.result, "My Data", "Get attachment");
})
.fail(unexpectedError).always(start);
});
*/
/*
test("RemoveAttachment", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"mode": "memory"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y",
"mode": "memory"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
reverse(jio.removeAttachment({
"_id": "five",
"_attachment": "my_attachment"
})).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "five",
"message": "Unable to remove attachment",
"method": "removeAttachment",
"reason": "missing document",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Remove attachment from inexistent document -> 404 Not Found");
return jio.post({
"_id": "five",
"_underscored_meta": "uvalue",
"meta": "data"
});
}).then(function () {
return reverse(jio.removeAttachment({
"_id": "five",
"_attachment": "my_attachment"
}));
}).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "five",
"message": "Unable to remove attachment",
"method": "removeAttachment",
"reason": "missing attachment",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Remove inexistent attachment -> 404 Not Found");
return jio.putAttachment({
"_id": "five",
"_attachment": "my_attachment",
"_data": "My Data",
"_mimetype": "text/plain"
});
})
.then(function () {
return jio.removeAttachment({
"_id": "five",
"_attachment": "my_attachment"
});
})
.then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"id": "five",
"method": "removeAttachment",
"result": "success",
"status": 204,
"statusText": "No Content"
}, "Remove attachment");
return jio.get({"_id": "five"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "five",
"_underscored_meta": "uvalue",
"meta": "data"
}, "Check document");
return reverse(jio.getAttachment({
"_id": "five",
"_attachment": "my_attachment"
}));
}).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "five",
"message": "Unable to get attachment",
"method": "getAttachment",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Check attachment -> 404 Not Found");
})
.fail(unexpectedError).always(start);
});
*/
/*
test("Remove", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
reverse(jio.remove({"_id": "six"})).then(function (answer) {
deepEqual(answer, {
"error": "not_found",
"id": "six",
"message": "Unable to remove document",
"method": "remove",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Remove missing document -> 404 Not Found");
return jio.post({
"_id": "six",
"_underscored_meta": "uvalue",
"meta": "data"
});
}).then(function () {
return jio.putAttachment({
"_id": "six",
"_attachment": "my_attachment",
"_data": "My Data",
"_mimetype": "text/plain"
});
}).then(function () {
return jio.remove({"_id": "six"});
}).then(function (answer) {
deepEqual(answer, {
"id": "six",
"method": "remove",
"result": "success",
"status": 204,
"statusText": "No Content"
}, "Remove document");
return reverse(jio.getAttachment({
"_id": "six",
"_attachment": "my_attachment"
}));
}).then(function (answer) {
deepEqual(answer, {
"attachment": "my_attachment",
"error": "not_found",
"id": "six",
"message": "Unable to get attachment",
"method": "getAttachment",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Check attachment -> 404 Not Found");
return reverse(jio.get({"_id": "six"}));
}).then(function (answer) {
deepEqual(answer, {
"error": "not_found",
"id": "six",
"message": "Unable to get document",
"method": "get",
"reason": "Not Found",
"result": "error",
"status": 404,
"statusText": "Not Found"
}, "Check document -> 404 Not Found");
}).fail(unexpectedError).always(start);
});
*/
/*
test("Put", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucketsplit",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucketsplit_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
jio.put({
"_id": "seven",
"_underscored_meta": "uvalue",
"meta": "data"
}).then(function (answer) {
deepEqual(answer, {
"id": "seven",
"method": "put",
"result": "success",
"status": 204,
"statusText": "No Content"
}, "Put new document");
return jio.get({"_id": "seven"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "seven",
"_underscored_meta": "uvalue",
"meta": "data"
}, "Check document");
return jio.put({
"_id": "seven",
"_underscored_meta": "uvalue",
"meow": "dog"
});
}).then(function (answer) {
deepEqual(answer, {
"id": "seven",
"method": "put",
"result": "success",
"status": 204,
"statusText": "No Content"
}, "Put same document again");
return jio.get({"_id": "seven"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "seven",
"_underscored_meta": "uvalue",
"meow": "dog"
}, "Get document for check");
}).fail(unexpectedError).always(start);
});
*/
/*
test("AllDocs", function () {
var shared = {}, jio;
shared.workspace = {};
shared.s3_storage_description1 = {
"type": "s3",
"server": "jiobucket_alldocs",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
shared.s3_storage_description2 = {
"type": "s3",
"server": "jiobucket_alldocs_bis",
"AWSIdentifier": "AKIAJLNYGVLTV66RHPEQ",
"password": "/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"
};
jio = jIO.createJIO({
"type": "split",
"storage_list": [
shared.s3_storage_description1,
shared.s3_storage_description2
]
}, {"workspace": shared.workspace});
stop();
function prepareDatabase() {
var i, do_list = [];
function post(i) {
return function () {
return jio.post({
"_id": "doc" + i,
"_underscored_meta": "uvalue" + i,
"meta": "data" + i
});
};
}
function putAttachment(i) {
return function () {
return jio.putAttachment({
"_id": "doc" + i,
"_attachment": "my_attachment" + i,
"_data": "My Data" + i,
"_content_type": "text/plain"
});
};
}
for (i = 0; i < 5; i += 1) {
do_list.push(post(i));
}
for (i = 0; i < 2; i += 1) {
do_list.push(putAttachment(i));
}
return sequence(do_list);
}
prepareDatabase().then(function () {
return jio.get({"_id": "doc1"});
}).then(function (answer) {
deepEqual(answer.data, {
"_id": "doc1",
"_underscored_meta": "uvalue1",
"meta": "data1",
"_attachments": {
"my_attachment1": {
"length": 8,
"content_type": "text/plain"
}
}
}, "Check document");
return jio.allDocs();
}).then(function (answer) {
answer.data.rows.sort(function (a, b) {
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
});
deepEqual(answer.data, {
"total_rows": 5,
"rows": [
{
"id": "doc0",
"key": "doc0",
"value": {}
},
{
"id": "doc1",
"key": "doc1",
"value": {}
},
{
"id": "doc2",
"key": "doc2",
"value": {}
},
{
"id": "doc3",
"key": "doc3",
"value": {}
},
{
"id": "doc4",
"key": "doc4",
"value": {}
}]
}, "AllDocs with document ids only");
}).fail(unexpectedError).always(start);
});
*/
}));
test/jio.storage/s3.multi.split.storage.livetests.html 0 → 100644
View file @ 03ff747a
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>JIO S3 Storage Qunit Live Tests</title>
<link rel="stylesheet" href="../../lib/qunit/qunit.css" />
<script src="../../lib/qunit/qunit.js"></script>
<script src="../../lib/rsvp/rsvp-custom.js"></script>
<script src="../../lib/jquery/jquery.min.js"></script>
<script src="../../src/sha256.amd.js"></script>
<script src="../../jio.js"></script>
<script src="../../complex_queries.js"></script>
<script src="../../src/sha1.amd.js"></script>
<script src="../../src/jio.storage/s3storage.js"></script>
<script src="../../src/jio.storage/multisplitstorage.js"></script>
<script src="../jio/util.js"></script>
</head>
<body>
<script>
var s3storage_spec = {};
console.log(location.href);
location.href.split('?')[1].split('&').forEach(function (item) {
s3storage_spec[item.split('=')[0]] = decodeURI(item.split('=').slice(1).join('=')).
replace(/%3A/ig, ':').replace(/%2F/ig, '/');
});
</script>
<h3>JIO initialization</h3>
<form method="get" action="">
<input type="hidden" name="type" value="s3"/>
<input type="text" name="url" placeholder="URL" value="https://jiobucket.s3.amazonaws.com"/>
<input type="text" name="server" placeholder="Bucket" value="jiobucket"/>
<input type="text" name="AWSIdentifier" placeholder="AWSIdentifier" value="AKIAJLNYGVLTV66RHPEQ"/>
<input type="password" name="password" placeholder="Password" value="/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"/>
<input type="submit" value="Run Tests"/>
</form>
<br />
<div id="qunit"></div>
<!--<script src="s3storage.tests.js"></script>-->
<script src="multi.split.s3storage.tests.js"></script>
</body>
</html>
test/jio.storage/s3.split.storage.livetests.html 0 → 100644
View file @ 03ff747a
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>JIO S3 Storage Qunit Live Tests</title>
<link rel="stylesheet" href="../../lib/qunit/qunit.css" />
<script src="../../lib/qunit/qunit.js"></script>
<script src="../../lib/rsvp/rsvp-custom.js"></script>
<script src="../../lib/jquery/jquery.min.js"></script>
<script src="../../src/sha256.amd.js"></script>
<script src="../../jio.js"></script>
<script src="../../complex_queries.js"></script>
<script src="../../src/sha1.amd.js"></script>
<script src="../../src/jio.storage/s3storage.js"></script>
<script src="../../src/jio.storage/splitstorage.js"></script>
<script src="../jio/util.js"></script>
</head>
<body>
<script>
var s3storage_spec = {};
console.log(location.href);
location.href.split('?')[1].split('&').forEach(function (item) {
s3storage_spec[item.split('=')[0]] = decodeURI(item.split('=').slice(1).join('=')).
replace(/%3A/ig, ':').replace(/%2F/ig, '/');
});
</script>
<h3>JIO initialization</h3>
<form method="get" action="">
<input type="hidden" name="type" value="s3"/>
<input type="text" name="url" placeholder="URL" value="https://jiobucket.s3.amazonaws.com"/>
<input type="text" name="server" placeholder="Bucket" value="jiobucket"/>
<input type="text" name="AWSIdentifier" placeholder="AWSIdentifier" value="AKIAJLNYGVLTV66RHPEQ"/>
<input type="password" name="password" placeholder="Password" value="/YHoa5r2X6EUHfvP31jdYx6t75h81pAjIZ4Mt94y"/>
<input type="submit" value="Run Tests"/>
</form>
<br />
<div id="qunit"></div>
<!--<script src="s3storage.tests.js"></script>-->
<script src="split.s3storage.tests.js"></script>
</body>
</html>
test/jio.storage/s3storage.livetests.html 100644 → 100755
View file @ 03ff747a
...@@ -13,7 +13,6 @@ ...@@ -13,7 +13,6 @@
<script src="../../complex_queries.js"></script> <script src="../../complex_queries.js"></script>
<script src="../../src/sha1.amd.js"></script> <script src="../../src/sha1.amd.js"></script>
<script src="../../src/jio.storage/s3storage.js"></script> <script src="../../src/jio.storage/s3storage.js"></script>
<script src="../../src/jio.storage/splitstorage.js"></script>
<script src="../jio/util.js"></script> <script src="../jio/util.js"></script>
</head> </head>
<body> <body>
...@@ -36,7 +35,6 @@ location.href.split('?')[1].split('&').forEach(function (item) { ...@@ -36,7 +35,6 @@ location.href.split('?')[1].split('&').forEach(function (item) {
</form> </form>
<br /> <br />
<div id="qunit"></div> <div id="qunit"></div>
<!--<script src="s3storage.tests.js"></script>--> <script src="s3storage.tests.js"></script>
<script src="split.s3storage.tests.js"></script>
</body> </body>
</html> </html>
test/jio.storage/s3storage.tests.js 100644 → 100755
View file @ 03ff747a
File mode changed from 100644 to 100755
test/jio.storage/split.s3storage.tests.js 100644 → 100755
View file @ 03ff747a
...@@ -935,27 +935,22 @@ ...@@ -935,27 +935,22 @@
"rows": [ "rows": [
{ {
"id": "doc0", "id": "doc0",
"key": "doc0",
"value": {} "value": {}
}, },
{ {
"id": "doc1", "id": "doc1",
"key": "doc1",
"value": {} "value": {}
}, },
{ {
"id": "doc2", "id": "doc2",
"key": "doc2",
"value": {} "value": {}
}, },
{ {
"id": "doc3", "id": "doc3",
"key": "doc3",
"value": {} "value": {}
}, },
{ {
"id": "doc4", "id": "doc4",
"key": "doc4",
"value": {} "value": {}
}] }]
}, "AllDocs with document ids only"); }, "AllDocs with document ids only");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7