Skip to content

J
jio
Commit 30571aa6 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

Add ZipStorage. 

This storage will transparently compress getAttachment/putAttachment results.
Only text, json, xml mme type are compressed.

Thanks to Lucas Parsy.
parent f6786d3e
Show whitespace changes
Inline Side-by-side
Showing with 704 additions and 29 deletions
Gruntfile.js
View file @ 30571aa6
......@@ -146,6 +146,7 @@ module.exports = function (grunt) {
src: [
'lib/uri/URI.js',
'node_modules/uritemplate/bin/uritemplate.js',
'node_modules/lz-string/libs/lz-string.js',
// 'node_modules/moment/moment.js',
'lib/moment/moment-2.5.0.js',
// 'src/*.js',
......@@ -178,6 +179,7 @@ module.exports = function (grunt) {
'src/jio.storage/uuidstorage.js',
'src/jio.storage/memorystorage.js',
'src/jio.storage/localstorage.js',
'src/jio.storage/zipstorage.js',
'src/jio.storage/davstorage.js',
'src/jio.storage/unionstorage.js',
'src/jio.storage/erp5storage.js',
......
Makefile
View file @ 30571aa6
......@@ -43,6 +43,7 @@ zip:
@cp src/jio.storage/splitstorage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/cryptstorage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/revisionstorage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/zipstorage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/replicaterevisionstorage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/s3storage.js $(TMPDIR)/jio/storage/
@cp src/jio.storage/splitstorage.js $(TMPDIR)/jio/storage/
......@@ -75,6 +76,7 @@ zip:
@$(UGLIFY) src/jio.storage/cryptstorage.js >$(TMPDIR)/jio/storage/cryptstorage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/revisionstorage.js >$(TMPDIR)/jio/storage/revisionstorage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/replicaterevisionstorage.js >$(TMPDIR)/jio/storage/replicaterevisionstorage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/zipstorage.js >$(TMPDIR)/jio/storage/zipstorage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/s3storage.js >$(TMPDIR)/jio/storage/s3storage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/splitstorage.js >$(TMPDIR)/jio/storage/splitstorage.min.js 2>/dev/null
@$(UGLIFY) src/jio.storage/xwikistorage.js >$(TMPDIR)/jio/storage/xwikistorage.min.js 2>/dev/null
......
examples/scenario.js
View file @ 30571aa6
......@@ -25,10 +25,13 @@
// type: "document",
// document_id: "/",
// sub_storage: {
// type: "zip",
// sub_storage: {
// type: "local"
// }
// }
// }
// }
// });
///////////////////////////
......
src/jio.js
View file @ 30571aa6
/*global window, RSVP, Blob, XMLHttpRequest, QueryFactory, Query, FileReader */
(function (window, RSVP, Blob, QueryFactory, Query,
FileReader) {
/*global window, RSVP, Blob, XMLHttpRequest, QueryFactory, Query, atob,
FileReader, ArrayBuffer, Uint8Array */
(function (window, RSVP, Blob, QueryFactory, Query, atob,
FileReader, ArrayBuffer, Uint8Array) {
"use strict";
var util = {},
......@@ -173,6 +174,25 @@
}
util.readBlobAsDataURL = readBlobAsDataURL;
// https://gist.github.com/davoclavo/4424731
function dataURItoBlob(dataURI) {
// convert base64 to raw binary data held in a string
var byteString = atob(dataURI.split(',')[1]),
// separate out the mime component
mimeString = dataURI.split(',')[0].split(':')[1],
// write the bytes of the string to an ArrayBuffer
arrayBuffer = new ArrayBuffer(byteString.length),
_ia = new Uint8Array(arrayBuffer),
i;
mimeString = mimeString.slice(0, mimeString.length - ";base64".length);
for (i = 0; i < byteString.length; i += 1) {
_ia[i] = byteString.charCodeAt(i);
}
return new Blob([arrayBuffer], {type: mimeString});
}
util.dataURItoBlob = dataURItoBlob;
// tools
function checkId(argument_list, storage, method_name) {
if (typeof argument_list[0] !== 'string' || argument_list[0] === '') {
......@@ -515,4 +535,5 @@
jIO = new JioBuilder();
window.jIO = jIO;
}(window, RSVP, Blob, QueryFactory, Query, FileReader));
}(window, RSVP, Blob, QueryFactory, Query, atob,
FileReader, ArrayBuffer, Uint8Array));
src/jio.storage/localstorage.js
View file @ 30571aa6
......@@ -5,8 +5,7 @@
*/
/*jslint nomen: true*/
/*global jIO, sessionStorage, localStorage, Blob, RSVP, atob,
ArrayBuffer, Uint8Array*/
/*global jIO, sessionStorage, localStorage, RSVP */
/**
* JIO Local Storage. Type = 'local'.
......@@ -22,8 +21,7 @@
* @class LocalStorage
*/
(function (jIO, sessionStorage, localStorage, Blob, RSVP,
atob, ArrayBuffer, Uint8Array) {
(function (jIO, sessionStorage, localStorage, RSVP) {
"use strict";
function LocalStorage(spec) {
......@@ -60,23 +58,6 @@
return attachments;
};
// https://gist.github.com/davoclavo/4424731
function dataURItoBlob(dataURI) {
// convert base64 to raw binary data held in a string
var byteString = atob(dataURI.split(',')[1]),
// separate out the mime component
mimeString = dataURI.split(',')[0].split(':')[1],
// write the bytes of the string to an ArrayBuffer
arrayBuffer = new ArrayBuffer(byteString.length),
_ia = new Uint8Array(arrayBuffer),
i;
mimeString = mimeString.slice(0, mimeString.length - ";base64".length);
for (i = 0; i < byteString.length; i += 1) {
_ia[i] = byteString.charCodeAt(i);
}
return new Blob([arrayBuffer], {type: mimeString});
}
LocalStorage.prototype.getAttachment = function (id, name) {
restrictDocumentId(id);
......@@ -88,7 +69,7 @@
404
);
}
return dataURItoBlob(textstring);
return jIO.util.dataURItoBlob(textstring);
};
LocalStorage.prototype.putAttachment = function (id, name, blob) {
......@@ -125,5 +106,4 @@
jIO.addStorage('local', LocalStorage);
}(jIO, sessionStorage, localStorage, Blob, RSVP,
atob, ArrayBuffer, Uint8Array));
}(jIO, sessionStorage, localStorage, RSVP));
src/jio.storage/zipstorage.js 0 → 100644
View file @ 30571aa6
/*jslint nomen: true*/
/*global RSVP, Blob, LZString, DOMException*/
(function (RSVP, Blob, LZString, DOMException) {
"use strict";
/**
* The jIO ZipStorage extension
*
* @class ZipStorage
* @constructor
*/
var MIME_TYPE = "application/x-jio-utf16_lz_string";
function ZipStorage(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
ZipStorage.prototype.get = function () {
return this._sub_storage.get.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.post = function () {
return this._sub_storage.post.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.remove = function () {
return this._sub_storage.remove.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.hasCapacity = function () {
return this._sub_storage.hasCapacity.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.getAttachment = function (id, name) {
var that = this;
return that._sub_storage.getAttachment(id, name)
.push(function (blob) {
if (blob.type !== MIME_TYPE) {
return blob;
}
return new RSVP.Queue()
.push(function () {
return jIO.util.readBlobAsText(blob, 'utf16');
})
.push(function (evt) {
var result =
LZString.decompressFromUTF16(evt.target.result);
if (result === '') {
return blob;
}
try {
return jIO.util.dataURItoBlob(
result
);
} catch (error) {
if (error instanceof DOMException) {
return blob;
}
throw error;
}
});
});
};
function myEndsWith(str, query) {
return (str.indexOf(query) === str.length - query.length);
}
ZipStorage.prototype.putAttachment = function (id, name, blob) {
var that = this;
if ((blob.type.indexOf("text/") === 0) || myEndsWith(blob.type, "xml") ||
myEndsWith(blob.type, "json")) {
return new RSVP.Queue()
.push(function () {
return jIO.util.readBlobAsDataURL(blob);
})
.push(function (data) {
var result = LZString.compressToUTF16(data.target.result);
blob = new Blob([result],
{type: MIME_TYPE});
return that._sub_storage.putAttachment(id, name, blob);
});
}
return this._sub_storage.putAttachment.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.removeAttachment = function () {
return this._sub_storage.removeAttachment.apply(this._sub_storage,
arguments);
};
ZipStorage.prototype.allAttachments = function () {
return this._sub_storage.allAttachments.apply(this._sub_storage,
arguments);
};
jIO.addStorage('zip', ZipStorage);
}(RSVP, Blob, LZString, DOMException));
test/jio.storage/zipstorage.tests.js 0 → 100644
View file @ 30571aa6
/*jslint nomen: true*/
/*global Blob, LZString*/
(function (jIO, QUnit, Blob, LZString) {
"use strict";
var test = QUnit.test,
stop = QUnit.stop,
start = QUnit.start,
ok = QUnit.ok,
expect = QUnit.expect,
deepEqual = QUnit.deepEqual,
equal = QUnit.equal,
throws = QUnit.throws,
module = QUnit.module;
/////////////////////////////////////////////////////////////////
// Custom test substorage definition
/////////////////////////////////////////////////////////////////
function Storage200() {
return this;
}
jIO.addStorage('zipstorage200', Storage200);
/////////////////////////////////////////////////////////////////
// ZipStorage.constructor
/////////////////////////////////////////////////////////////////
module("ZipStorage.constructor");
test("create substorage", function () {
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
equal(jio.__type, "zip");
equal(jio.__storage._sub_storage.__type, "zipstorage200");
});
/////////////////////////////////////////////////////////////////
// ZipStorage.get
/////////////////////////////////////////////////////////////////
module("ZipStorage.get");
test("get called substorage get", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.get = function (id) {
equal(id, "bar", "get 200 called");
return {title: "foo"};
};
jio.get("bar")
.then(function (result) {
deepEqual(result, {
"title": "foo"
}, "Check document");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.post
/////////////////////////////////////////////////////////////////
module("ZipStorage.post");
test("post called substorage post", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.post = function (id) {
equal(id, "bar", "post 200 called");
return {title: "foo"};
};
jio.post("bar")
.then(function (result) {
deepEqual(result, {
"title": "foo"
}, "Check document");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.put
/////////////////////////////////////////////////////////////////
module("ZipStorage.put");
test("put called substorage put", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.put = function (id) {
equal(id, "bar", "put 200 called");
return {title: "foo"};
};
jio.put("bar")
.then(function (result) {
equal(result, "bar");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.remove
/////////////////////////////////////////////////////////////////
module("ZipStorage.remove");
test("remove called substorage remove", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.remove = function (id) {
equal(id, "bar", "remove 200 called");
return {title: "foo"};
};
jio.remove("bar")
.then(function (result) {
equal(result, "bar");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.hasCapacity
/////////////////////////////////////////////////////////////////
module("ZipStorage.hasCapacity");
test("hasCapacity return substorage value", function () {
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
delete Storage200.prototype.hasCapacity;
throws(
function () {
jio.hasCapacity("foo");
},
function (error) {
ok(error instanceof jIO.util.jIOError);
equal(error.status_code, 501);
equal(error.message,
"Capacity 'foo' is not implemented on 'zipstorage200'");
return true;
}
);
});
/////////////////////////////////////////////////////////////////
// ZipStorage.buildQuery
/////////////////////////////////////////////////////////////////
module("ZipStorage.buildQuery");
test("buildQuery called substorage buildQuery", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.buildQuery = function (id) {
equal(id, "bar", "buildQuery 200 called");
return {title: "foo"};
};
jio.buildQuery("bar")
.then(function (result) {
deepEqual(result, {
"title": "foo"
}, "Check document");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.removeAttachment
/////////////////////////////////////////////////////////////////
module("ZipStorage.removeAttachment");
test("removeAttachment called substorage removeAttachment", function () {
stop();
expect(3);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.removeAttachment = function (id, name) {
equal(id, "bar", "removeAttachment 200 called");
equal(name, "foo", "removeAttachment 200 called");
return {title: "foo"};
};
jio.removeAttachment("bar", "foo")
.then(function (result) {
deepEqual(result, {
"title": "foo"
}, "Check document");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.allAttachments
/////////////////////////////////////////////////////////////////
module("ZipStorage.allAttachments");
test("allAttachments called substorage allAttachments", function () {
stop();
expect(2);
var jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
Storage200.prototype.allAttachments = function (id) {
equal(id, "bar", "allAttachments 200 called");
return {title: "foo"};
};
jio.allAttachments("bar")
.then(function (result) {
deepEqual(result, {
"title": "foo"
}, "Check document");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.getAttachment
/////////////////////////////////////////////////////////////////
module("ZipStorage.getAttachment", {
setup: function () {
this.jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
}
});
test("return substorage getattachment", function () {
var id = "/",
attachment = "stringattachment",
blob = new Blob(['foo']);
Storage200.prototype.getAttachment = function (arg1, arg2) {
equal(arg1, id, "getAttachment 200 called");
equal(arg2, attachment, "getAttachment 200 called");
return blob;
};
stop();
expect(3);
this.jio.getAttachment(id, attachment)
.then(function (result) {
equal(result, blob, "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("return substorage getattachment if uncompress fails", function () {
var id = "/",
attachment = "stringattachment",
blob = new Blob(['foo'], {type: 'application/x-utf16_lz_string'});
Storage200.prototype.getAttachment = function (arg1, arg2) {
equal(arg1, id, "getAttachment 200 called");
equal(arg2, attachment, "getAttachment 200 called");
return blob;
};
stop();
expect(3);
this.jio.getAttachment(id, attachment)
.then(function (result) {
equal(result, blob, "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("return substorage getattachment if not data url", function () {
var id = "/",
attachment = "stringattachment",
blob = new Blob([LZString.compressToUTF16('foo')],
{type: 'application/x-utf16_lz_string'});
Storage200.prototype.getAttachment = function (arg1, arg2) {
equal(arg1, id, "getAttachment 200 called");
equal(arg2, attachment, "getAttachment 200 called");
return blob;
};
stop();
expect(3);
this.jio.getAttachment(id, attachment)
.then(function (result) {
equal(result, blob, "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("uncompress blob lz-string mime type", function () {
var id = "/",
attachment = "stringattachment",
value = "azertyuio\npàç_è-('é&",
tocheck = "data:text/plain;charset=utf-8;base64," +
"YXplcnR5dWlvCnDDoMOnX8OoLSgnw6km",
blob = new Blob([LZString.compressToUTF16(tocheck)],
{type: 'application/x-jio-utf16_lz_string'});
Storage200.prototype.getAttachment = function (arg1, arg2) {
equal(arg1, id, "getAttachment 200 called");
equal(arg2, attachment, "getAttachment 200 called");
return blob;
};
stop();
expect(6);
this.jio.getAttachment(id, attachment)
.then(function (result) {
ok(result !== blob, "Does not return substorage result");
ok(result instanceof Blob, "Data is Blob");
deepEqual(result.type, "text/plain;charset=utf-8",
"Check mimetype");
return jIO.util.readBlobAsText(result);
})
.then(function (result) {
equal(result.target.result, value, "Attachment correctly fetched");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// ZipStorage.putAttachment
/////////////////////////////////////////////////////////////////
module("ZipStorage.putAttachment", {
setup: function () {
this.jio = jIO.createJIO({
type: "zip",
sub_storage: {type : "zipstorage200"}
});
}
});
test("store directly unhandled mime types", function () {
var id = "/",
attachment = "stringattachment",
blob = new Blob(['foo']);
Storage200.prototype.putAttachment = function (arg1, arg2, arg3) {
equal(arg1, id, "putAttachment 200 called");
equal(arg2, attachment, "putAttachment 200 called");
equal(arg3, blob, "putAttachment 200 called");
return "ok";
};
stop();
expect(4);
this.jio.putAttachment(id, attachment, blob)
.then(function (result) {
equal(result, "ok", "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("compress text mime type", function () {
var id = "/",
attachment = "stringattachment",
value = "azertyuio\npàç_è-('é&",
blob = new Blob([value],
{type: 'text/foo'});
Storage200.prototype.putAttachment = function (arg1, arg2, arg3) {
equal(arg1, id, "putAttachment 200 called");
equal(arg2, attachment, "putAttachment 200 called");
ok(true, arg3 !== blob, "putAttachment 200 called");
ok(arg3 instanceof Blob, "Data is Blob");
equal(arg3.type, "application/x-jio-utf16_lz_string",
"Check mimetype");
return jIO.util.readBlobAsText(arg3, 'utf16')
.then(function (result) {
var dataurl = LZString.decompressFromUTF16(result.target.result);
equal(
dataurl,
"data:text/foo;base64,YXplcnR5dWlvCnDDoMOnX8OoLSgnw6km"
);
return jIO.util.readBlobAsDataURL(arg3);
})
.then(function (result) {
equal(
result.target.result,
"data:application/x-jio-utf16_lz_string;base64,06LgsKvkkKvkmK" +
"rjgK/hoK/igLnkhJblkLvkgYPgoKznhqDjmKDisKDmoKTmoKHlgKHmgK3kgo" +
"PClsK0y5M94o2g45yh5IisxLfnlaDgrYDgvYDmhaDOtOOUoOCyoOOKoOOmo8" +
"KXxKDmrKDlrYAgIA=="
);
return "ok";
});
};
stop();
expect(8);
this.jio.putAttachment(id, attachment, blob)
.then(function (result) {
equal(result, "ok", "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("compress json mime type", function () {
var id = "/",
attachment = "stringattachment",
value = "azertyuio\npàç_è-('é&",
blob = new Blob([value],
{type: 'application/foo+json'});
Storage200.prototype.putAttachment = function (arg1, arg2, arg3) {
equal(arg1, id, "putAttachment 200 called");
equal(arg2, attachment, "putAttachment 200 called");
ok(true, arg3 !== blob, "putAttachment 200 called");
ok(arg3 instanceof Blob, "Data is Blob");
equal(arg3.type, "application/x-jio-utf16_lz_string",
"Check mimetype");
return jIO.util.readBlobAsText(arg3, 'utf16')
.then(function (result) {
var dataurl = LZString.decompressFromUTF16(result.target.result);
equal(
dataurl,
"data:application/foo+json;base64,YXplcnR5dWlvCnDDoMOnX8OoLSgnw6km"
);
return jIO.util.readBlobAsDataURL(arg3);
})
.then(function (result) {
equal(
result.target.result,
"data:application/x-jio-utf16_lz_string;base64,06LgsKvkkKvkiKD" +
"nhqHloLLmgoPjpLDjtqDnmKflgKzmvJTNsMuQ4LSF5IKOZuGFqOCqgOC2oOCs" +
"oOGooeGooOOQtuKmoOKUoeKzoMe154Ch5bCj1ZDCqeKfgNaw3rTmhIDHoOW6o" +
"Nmg4aWg4bOzW+SGoOOWoOK2sCAg"
);
return "ok";
});
};
stop();
expect(8);
this.jio.putAttachment(id, attachment, blob)
.then(function (result) {
equal(result, "ok", "Return substorage result");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit, Blob, LZString));
test/tests.html
View file @ 30571aa6
......@@ -43,6 +43,7 @@
<!--script src="jio.storage/indexstorage.tests.js"></script-->
<!--script src="jio.storage/dropboxstorage.tests.js"></script-->
<script src="jio.storage/zipstorage.tests.js"></script>
<!--script src="../lib/jquery/jquery.min.js"></script>
<script src="../src/jio.storage/xwikistorage.js"></script>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7