Skip to content

jio
jio
Commit a5515a0a authored by Alain Takoudjou's avatar Alain Takoudjou
2 Browse files
Options

Add webhttp and replicatedopmltree storage 

webhttp storage can get json documents by calling http GET method. for jio allDocs,
the storage will get a file called _document_list which should contain the list
of document id that can be downloaded.

replicatedopmltree storage can convert a list of opml storage (each opml can contain
one or more sub_storage) into an indexeddb. All document in opml tree will have an
entry into the indexeddb storage, the full document is added as attachment in the
indexeddb.
parent 27bf78f6
Show whitespace changes
Inline Side-by-side
Showing with 1598 additions and 2 deletions
Gruntfile.js
View file @ a5515a0a
......@@ -182,7 +182,9 @@ module.exports = function (grunt) {
'src/jio.storage/cryptstorage.js',
'src/jio.storage/websqlstorage.js',
'src/jio.storage/rssfeedstorage.js',
'src/jio.storage/opmlstorage.js'
'src/jio.storage/opmlstorage.js',
'src/jio.storage/replicatedopmltreestorage.js',
'src/jio.storage/webhttpstorage.js'
],
dest: 'dist/<%= pkg.name %>-<%= pkg.version %>.js'
// dest: 'jio.js'
......
examples/scenario_monitor.html 0 → 100644
View file @ a5515a0a
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Monitor Coverage Scenario</title>
<script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
<script src="../dist/jio-latest.js"></script>
<link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
<script src="../node_modules/grunt-contrib-qunit/test/libs/qunit.js" type="text/javascript"></script>
<script src="scenario_monitor.js"></script>
</head>
<body>
<h1 id="qunit-header">Monitor Coverage Scenario</h1>
<h2 id="qunit-banner"></h2>
<div id="qunit-testrunner-toolbar"></div>
<h2 id="qunit-userAgent"></h2>
<ol id="qunit-tests"></ol>
<div id="qunit-fixture">test markup, will be hidden</div>
</body>
</html>
\ No newline at end of file
examples/scenario_monitor.js 0 → 100644
View file @ a5515a0a
/*global Blob, Rusha, console*/
/*jslint nomen: true, maxlen: 80*/
(function (QUnit, jIO, Blob, Rusha, console) {
"use strict";
var test = QUnit.test,
// equal = QUnit.equal,
expect = QUnit.expect,
ok = QUnit.ok,
stop = QUnit.stop,
start = QUnit.start,
deepEqual = QUnit.deepEqual,
module = QUnit.module,
rusha = new Rusha(),
i,
name_list = ['get', 'put', 'remove', 'buildQuery',
'putAttachment', 'getAttachment', 'allAttachments'];
///////////////////////////////////////////////////////
// Fake Storage
///////////////////////////////////////////////////////
function resetCount(count) {
for (i = 0; i < name_list.length; i += 1) {
count[name_list[i]] = 0;
}
}
function RSSMockStorage(spec) {
this._rss_storage = jIO.createJIO({
type: "rss",
url: "http://example.com/rss.xml"
});
this._sub_storage = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
});
this._options = spec.options;
resetCount(spec.options.count);
}
RSSMockStorage.prototype.hasCapacity = function (name) {
return this._rss_storage.hasCapacity(name);
};
function WEBMockStorage(spec) {
this._web_storage = jIO.createJIO({
type: "webhttp",
url: "http://example.com/private/"
});
this._sub_storage = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
});
this._options = spec.options;
resetCount(spec.options.count);
}
WEBMockStorage.prototype.hasCapacity = function (name) {
return this._web_storage.hasCapacity(name);
};
function OPMLMockStorage(spec) {
this._opml_storage = jIO.createJIO({
type: "opml",
url: "http://example.com/opml.xml"
});
this._sub_storage = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
});
this._options = spec.options;
resetCount(spec.options.count);
}
OPMLMockStorage.prototype.hasCapacity = function (name) {
return this._opml_storage.hasCapacity(name);
};
function mockFunction(name) {
WEBMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
RSSMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
OPMLMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
}
for (i = 0; i < name_list.length; i += 1) {
mockFunction(name_list[i]);
}
jIO.addStorage('opmlmock', OPMLMockStorage);
jIO.addStorage('rssmock', RSSMockStorage);
jIO.addStorage('webmock', WEBMockStorage);
///////////////////////////////////////////////////////
// Helpers
///////////////////////////////////////////////////////
function generateHash(str) {
return rusha.digestFromString(str);
}
function putFullDoc(storage, id, doc) {
return storage.put(id, doc);
}
function equalStorage(storage, doc_tuple_list) {
return storage.allDocs({include_docs: true})
.push(function (result) {
var i,
promise_list = [];
for (i = 0; i < result.data.rows.length; i += 1) {
promise_list.push(RSVP.all([
result.data.rows[i].id,
storage.get(result.data.rows[i].id),
storage.getAttachment(result.data.rows[i].id,
result.data.rows[i].doc.name)
]));
}
return RSVP.all(promise_list);
})
.push(function (result) {
deepEqual(result, doc_tuple_list, 'Storage content');
});
}
function isEmptyStorage(storage) {
return equalStorage(storage, []);
}
function equalsubStorageCallCount(mock_count, expected_count) {
for (i = 0; i < name_list.length; i += 1) {
if (!expected_count.hasOwnProperty(name_list[i])
&& mock_count.hasOwnProperty(name_list[i])) {
expected_count[name_list[i]] = 0;
}
}
deepEqual(mock_count, expected_count, 'Expected method call count');
}
function getOpmlElement(doc, doc_id, url) {
var element,
id,
parent_id;
parent_id = generateHash(url);
id = generateHash(parent_id + doc_id);
element = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: id,
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: url
};
return {
id: id,
doc: element
};
}
function getSubOpmlElement(doc, doc_id, opml_doc, url, type) {
var id = generateHash(opml_doc.reference + url + doc_id);
return {
id: id,
doc: {
name: doc_id,
opml_title: opml_doc.opml_title,
parent_title: opml_doc.title,
parent_id: opml_doc.reference,
reference: id,
title: doc.title,
type: doc.type || type + "-item",
url: url,
status: doc.status,
creation_date: doc.date
}
};
}
///////////////////////////////////////////////////////
// Module
///////////////////////////////////////////////////////
module("scenario_monitor", {
setup: function () {
this.rss_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removehas_include_docs: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.opml_mock_options = JSON.parse(
JSON.stringify(this.rss_mock_options)
);
this.web_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.sub_opml_storage = {
type: "opmlmock",
options: this.opml_mock_options,
url: "http://example.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example.com/rss.xml",
has_include_docs: true,
options: this.rss_mock_options
},
{
type: "webmock",
url: "http://example.com/data/",
has_include_docs: true,
options: this.web_mock_options
}
],
basic_login: "YWRtaW46endfEzrJUZGw="
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage
],
local_sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
});
}
});
///////////////////////////////////////////////////////
// Do nothing cases
///////////////////////////////////////////////////////
test("empty: nothing to do", function () {
expect(4);
stop();
var test = this;
this.jio.repair()
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// sync done (XXX - to finish)
///////////////////////////////////////////////////////
test("allready synced: nothing to do", function () {
expect(2);
stop();
var test = this,
key,
doc_id = 'opml_foo',
doc = {
title: "opml item foo",
url: "http://example.com/rss.xml",
modified_date: "aftttt",
created_date: "adddb",
opml_title: "opml foo"
},
blob = new Blob([JSON.stringify(doc)]);
// initialise this storage here so we can put data
key = generateHash("http://example.com/opml.xml");
this.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: this.opml_mock_options
});
putFullDoc(this.jio.__storage._remote_storage_dict[key], doc_id, doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
resetCount(test.opml_mock_options.count);
return test.jio.repair();
})
.then(function () {
var storage_doc = getOpmlElement(doc,
  • Romain Courteaud @romain ·
    Owner

    getOpmlElement function make difficult to see what is the expected result. Usually, I prefer to hardcode the expected result and keep the test code simple.

Please register or sign in to reply
doc_id,
test.sub_opml_storage.url);
return RSVP.all([
equalStorage(test.jio, [[storage_doc.id, storage_doc.doc, blob]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// complete sync - one opml, 2 sub storages
///////////////////////////////////////////////////////
test("complete storage sync", function () {
expect(4);
stop();
var test = this,
key,
doc_id = 'opml_foo',
doc = {
title: "opml item foo",
url: "http://example.com/rss.xml",
modified_date: "aftttt",
created_date: "adddb",
opml_title: "opml foo"
},
blob = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
date: "Tue, 29 Aug 2006 09:00:00 -0400",
description: "This is an example of an Item",
guid: "1102345",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
reference: "This is an example of an RSS feed",
siteTitle: "RSS Example",
title: "Item Example",
status: "OK"
},
rss_blob = new Blob([JSON.stringify(rss_doc)]),
json_id = "promise_runner.status",
json_doc = {
title: "promise fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "promise",
foo_p: "fooo parameter",
bar_p: "bar parameter"
},
json_blob = new Blob([JSON.stringify(json_doc)]),
opml_gen,
rss_gen,
json_gen;
opml_gen = getOpmlElement(doc, doc_id, test.sub_opml_storage.url);
rss_gen = getSubOpmlElement(
rss_doc,
rss_id,
opml_gen.doc,
test.sub_opml_storage.sub_storage_list[0].url,
"rssmock"
);
json_gen = getSubOpmlElement(
json_doc,
json_id,
opml_gen.doc,
test.sub_opml_storage.sub_storage_list[1].url,
"webmock"
);
// initialise this storage here so we can put data
key = generateHash(test.sub_opml_storage.url);
this.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: this.opml_mock_options
});
putFullDoc(this.jio.__storage._remote_storage_dict[key], doc_id, doc)
.then(function () {
// put rss doc
key = generateHash(opml_gen.doc.reference +
test.sub_opml_storage.sub_storage_list[0].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "rssmock",
options: test.rss_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc);
})
.then(function () {
// put json doc
key = generateHash(opml_gen.doc.reference +
test.sub_opml_storage.sub_storage_list[1].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "webmock",
options: test.web_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
json_id, json_doc);
})
.then(function () {
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_gen.id, opml_gen.doc, blob],
[rss_gen.id, rss_gen.doc, rss_blob],
[json_gen.id, json_gen.doc, json_blob]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// document update
///////////////////////////////////////////////////////
test("remote document modified", function () {
expect(4);
stop();
var test = this,
key,
doc_id = 'opml_foo',
doc = {
title: "opml item foo",
url: "http://example.com/rss.xml",
modified_date: "aftttt",
created_date: "adddb",
opml_title: "opml foo"
},
blob = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
date: "Tue, 29 Aug 2006 09:00:00 -0400",
description: "This is an example of an Item",
guid: "1102345",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
reference: "This is an example of an RSS feed",
siteTitle: "RSS Example",
title: "Item Example",
status: "OK"
},
opml_gen,
rss_gen,
rss_doc2 = JSON.parse(JSON.stringify(rss_doc)),
rss_blob2 = new Blob([JSON.stringify(rss_doc2)]);
opml_gen = getOpmlElement(doc, doc_id, test.sub_opml_storage.url);
rss_gen = getSubOpmlElement(
rss_doc2,
rss_id,
opml_gen.doc,
test.sub_opml_storage.sub_storage_list[0].url,
"rssmock"
);
// initialise this storage here so we can put data
  • Romain Courteaud @romain ·
    Owner

    Applications are not supposed to hack the storages. Please remove this manual operation.

Please register or sign in to reply
key = generateHash(test.sub_opml_storage.url);
this.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: this.opml_mock_options
});
putFullDoc(this.jio.__storage._remote_storage_dict[key], doc_id, doc)
.then(function () {
// put rss doc
key = generateHash(opml_gen.doc.reference +
test.sub_opml_storage.sub_storage_list[0].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "rssmock",
options: test.rss_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc);
})
.then(function () {
return test.jio.repair();
})
.then(function () {
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc2);
})
.then(function () {
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_gen.id, opml_gen.doc, blob],
[rss_gen.id, rss_gen.doc, rss_blob2]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// document remove
///////////////////////////////////////////////////////
test("remote document deleted", function () {
expect(5);
stop();
var test = this,
key,
doc_id = 'opml_foo',
doc = {
title: "opml item foo",
url: "http://example.com/rss.xml",
modified_date: "aftttt",
created_date: "adddb",
opml_title: "opml foo"
},
blob = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
date: "Tue, 29 Aug 2006 09:00:00 -0400",
description: "This is an example of an Item",
guid: "1102345",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
reference: "This is an example of an RSS feed",
siteTitle: "RSS Example",
title: "Item Example",
status: "OK"
},
rss_blob = new Blob([JSON.stringify(rss_doc)]),
opml_gen,
rss_gen;
opml_gen = getOpmlElement(doc, doc_id, test.sub_opml_storage.url);
rss_gen = getSubOpmlElement(
rss_doc,
rss_id,
opml_gen.doc,
test.sub_opml_storage.sub_storage_list[0].url,
"rssmock"
);
// initialise this storage here so we can put data
key = generateHash(test.sub_opml_storage.url);
this.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: this.opml_mock_options
});
putFullDoc(this.jio.__storage._remote_storage_dict[key], doc_id, doc)
.then(function () {
// put rss doc
key = generateHash(opml_gen.doc.reference +
test.sub_opml_storage.sub_storage_list[0].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "rssmock",
options: test.rss_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc);
})
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_gen.id, opml_gen.doc, blob],
[rss_gen.id, rss_gen.doc, rss_blob]])
]);
})
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
return [];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_gen.id, opml_gen.doc, blob]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// complete sync - many opml (2 opmls, 4 sub storages)
///////////////////////////////////////////////////////
test("multi opml storage sync", function () {
expect(7);
stop();
var test = this,
key,
doc_id = 'opml_foo',
doc = {
title: "opml item foo",
url: "http://example.com/rss.xml",
modified_date: "aftttt",
created_date: "adddb",
opml_title: "opml foo"
},
blob = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
date: "Tue, 29 Aug 2006 09:00:00 -0400",
description: "This is an example of an Item",
guid: "1102345",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
reference: "This is an example of an RSS feed",
siteTitle: "RSS Example",
title: "Item Example",
status: "OK"
},
rss_blob = new Blob([JSON.stringify(rss_doc)]),
json_id = "promise_runner.status",
json_doc = {
title: "promise fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "promise",
foo_p: "fooo parameter",
bar_p: "bar parameter"
},
json_blob = new Blob([JSON.stringify(json_doc)]),
gen_dict = {};
// update storage with 2 opmls
this.sub_opml_storage2 = {
type: "opmlmock",
options: JSON.parse(JSON.stringify(this.opml_mock_options)),
url: "http://example2.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example2.com/rss.xml",
has_include_docs: true,
options: JSON.parse(JSON.stringify(this.rss_mock_options))
},
{
type: "webmock",
url: "http://example2.com/data/",
has_include_docs: true,
options: JSON.parse(JSON.stringify(this.web_mock_options))
}
],
basic_login: "YWRtaW46endfEzrJUZGw="
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage,
this.sub_opml_storage2
],
local_sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
});
gen_dict.opml = getOpmlElement(doc, doc_id, test.sub_opml_storage.url);
gen_dict.opml2 = getOpmlElement(doc, doc_id, test.sub_opml_storage2.url);
gen_dict.rss = getSubOpmlElement(
rss_doc,
rss_id,
gen_dict.opml.doc,
test.sub_opml_storage.sub_storage_list[0].url,
"rssmock"
);
gen_dict.json = getSubOpmlElement(
json_doc,
json_id,
gen_dict.opml.doc,
test.sub_opml_storage.sub_storage_list[1].url,
"webmock"
);
gen_dict.rss2 = getSubOpmlElement(
rss_doc,
rss_id,
gen_dict.opml2.doc,
test.sub_opml_storage2.sub_storage_list[0].url,
"rssmock"
);
gen_dict.json2 = getSubOpmlElement(
json_doc,
json_id,
gen_dict.opml2.doc,
test.sub_opml_storage2.sub_storage_list[1].url,
"webmock"
);
// initialise this storage here so we can put data
key = generateHash(test.sub_opml_storage.url);
this.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: this.opml_mock_options
});
putFullDoc(this.jio.__storage._remote_storage_dict[key], doc_id, doc)
.then(function () {
// put second opml doc
key = generateHash(test.sub_opml_storage2.url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "opmlmock",
options: test.sub_opml_storage2.options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
doc_id, doc);
})
.then(function () {
// put rss doc1
key = generateHash(gen_dict.opml.doc.reference +
test.sub_opml_storage.sub_storage_list[0].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "rssmock",
options: test.rss_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc);
})
.then(function () {
// put json doc1
key = generateHash(gen_dict.opml.doc.reference +
test.sub_opml_storage.sub_storage_list[1].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "webmock",
options: test.web_mock_options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
json_id, json_doc);
})
.then(function () {
// put rss doc2
key = generateHash(gen_dict.opml2.doc.reference +
test.sub_opml_storage2.sub_storage_list[0].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "rssmock",
options: test.sub_opml_storage2.sub_storage_list[0].options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
rss_id, rss_doc);
})
.then(function () {
// put json doc2
key = generateHash(gen_dict.opml2.doc.reference +
test.sub_opml_storage2.sub_storage_list[1].url);
test.jio.__storage._remote_storage_dict[key] = jIO.createJIO({
type: "webmock",
options: test.sub_opml_storage2.sub_storage_list[1].options
});
return putFullDoc(test.jio.__storage._remote_storage_dict[key],
json_id, json_doc);
})
.then(function () {
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
resetCount(test.sub_opml_storage2.options.count);
resetCount(test.sub_opml_storage2.sub_storage_list[0].options.count);
resetCount(test.sub_opml_storage2.sub_storage_list[1].options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio,
[[gen_dict.opml.id, gen_dict.opml.doc, blob],
[gen_dict.opml2.id, gen_dict.opml2.doc, blob],
[gen_dict.rss.id, gen_dict.rss.doc, rss_blob],
[gen_dict.rss2.id, gen_dict.rss2.doc, rss_blob],
[gen_dict.json.id, gen_dict.json.doc, json_blob],
[gen_dict.json2.id, gen_dict.json2.doc, json_blob]]
),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.sub_storage_list[0].options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.sub_storage_list[1].options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
}(QUnit, jIO, Blob, Rusha, console));
\ No newline at end of file
src/jio.storage/opmlstorage.js
View file @ a5515a0a
......@@ -42,7 +42,7 @@
element = {
opml_title: getElementsByTagName('title'),
create_date: getElementsByTagName('dateCreated'),
created_date: getElementsByTagName('dateCreated'),
modified_date: getElementsByTagName('dateModified'),
owner_name: getElementsByTagName('ownerName'),
owner_email: getElementsByTagName('ownerEmail'),
......
src/jio.storage/replicatedopmltreestorage.js 0 → 100644
View file @ a5515a0a
/*
* Copyright 2016, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint nomen: true */
/*global jIO, RSVP, Rusha, Blob, console */
(function (jIO, RSVP, Rusha, Blob, console) {
"use strict";
/**
*
* Sample OPML Tree Replicated Storage spec
*
* {
* "type": "replicatedopml",
* "opml_storage_list": [
* {
* "type": "opml",
* "url": "http://some.url.com",
* "sub_storage_list": [
* {"type": "rss", "url_attribute": "xmlUrl"},
* {"type": "webdav": "url_path": "/"},
* {"type": "webdav", "url_path": "/data/"},
* {"type": "webhttp": "url_path": "/storage"}
* ],
* "basic_login": "XUSODISOIDOISUJDD=="
* }
* ],
* local_sub_storage: {
* type: "query",
* sub_storage: {
* type: "indexeddb",
* database: "monitoring_local.db"
* }
* }
* }
*
*/
var rusha = new Rusha(),
KNOWN_SUB_STORAGE_LIST = ['rss', 'dav', 'webhttp'];
function generateHash(str) {
return rusha.digestFromString(str);
}
function createStorage(context, storage_spec, key) {
if (!context._remote_storage_dict.hasOwnProperty(key)) {
context._remote_storage_dict[key] = jIO.createJIO(storage_spec);
}
return context._remote_storage_dict[key];
}
/**
* The JIO OPML Tree Replicated Storage extension for monitor
*
* @class ReplicatedOPMLStorage
* @constructor
*/
function ReplicatedOPMLStorage(spec) {
var i,
j;
function checkSubStorage(sub_storage_spec) {
if (typeof sub_storage_spec.url !== 'string' &&
typeof sub_storage_spec.url_path !== 'string' &&
typeof sub_storage_spec.url_attribute !== 'string') {
throw new TypeError("one or more OPML sub storage(s) has no 'url' set");
}
if (typeof sub_storage_spec.type !== 'string') {
throw new TypeError(
"one or more OPML sub storage(s) has no attribute 'type' set"
);
}
}
if (typeof spec.opml_storage_list !== 'object') {
throw new TypeError("ReplicatedOPMLStorage 'opml_storage_list' " +
"is not of type object");
}
if (spec.local_sub_storage === undefined) {
throw new TypeError("ReplicatedOPMLStorage 'local_sub_storage' " +
"is not defined");
}
this._local_sub_storage = jIO.createJIO(spec.local_sub_storage);
this._remote_storage_dict = {};
this._opml_storage_list = spec.opml_storage_list;
for (i = 0; i < this._opml_storage_list.length; i += 1) {
if (typeof this._opml_storage_list[i].url !== 'string') {
throw new TypeError("opml storage 'url' is not of type string");
}
if (this._opml_storage_list[i].sub_storage_list !== undefined) {
for (j = 0; j < this._opml_storage_list[i].sub_storage_list.length;
j += 1) {
checkSubStorage(this._opml_storage_list[i].sub_storage_list[j]);
}
}
/*if (spec.opml_storage_list[i].type !== 'opml') {
throw new TypeError("ReplicatedOPMLStorage 'type' should be 'opml'");
}*/
}
}
ReplicatedOPMLStorage.prototype.get = function () {
return this._local_sub_storage.get.apply(this._local_sub_storage,
arguments);
};
ReplicatedOPMLStorage.prototype.buildQuery = function () {
return this._local_sub_storage.buildQuery.apply(this._local_sub_storage,
arguments);
};
/*ReplicatedOPMLStorage.prototype.put = function () {
return this._local_sub_storage.put.apply(this._local_sub_storage,
arguments);
};*/
ReplicatedOPMLStorage.prototype.hasCapacity = function (capacity) {
return (capacity === "list") || (capacity === "include");
};
ReplicatedOPMLStorage.prototype.getAttachment = function () {
return this._local_sub_storage.getAttachment.apply(this._local_sub_storage,
arguments);
};
ReplicatedOPMLStorage.prototype.allAttachments = function () {
return this._local_sub_storage.allAttachments.apply(this._local_sub_storage,
arguments);
};
function getSubOpmlStorageDescription(spec, opml_doc, basic_login) {
var storage_spec;
storage_spec = JSON.parse(JSON.stringify(spec));
if (storage_spec.basic_login === undefined && basic_login !== undefined) {
storage_spec.basic_login = basic_login;
}
if (KNOWN_SUB_STORAGE_LIST.indexOf(storage_spec.type) !== -1) {
if (storage_spec.url_attribute !== undefined &&
opml_doc.hasOwnProperty(storage_spec.url_attribute)) {
storage_spec.url = opml_doc[storage_spec.url_attribute];
delete storage_spec.url_attribute;
} else if (storage_spec.url_path !== undefined) {
storage_spec.url_path = storage_spec.url_path.replace(
new RegExp("^[/]+"),
""
);
storage_spec.url = opml_doc.url.replace(
new RegExp("[/]+$"),
""
) + "/" + storage_spec.url_path;
delete storage_spec.url_path;
}
// XXX - for compatibility, remove url with jio_private path
storage_spec.url = storage_spec.url.replace("jio_private", "private");
if (storage_spec.type === "dav") {
storage_spec = {
type: "query",
sub_storage: {
type: "drivetojiomapping",
sub_storage: storage_spec
}
};
}
}
return storage_spec;
}
function getStorageUrl(storage_spec) {
var spec = storage_spec;
while (spec !== undefined) {
if (spec.url !== undefined) {
return spec.url;
}
spec = spec.sub_storage;
}
throw new Error("No url found on sub storage: " +
JSON.stringify(storage_spec));
}
function loadSubStorage(context, spec, parent_id, opml_doc, basic_login) {
var sub_storage,
storage_spec,
options = {},
result_dict,
storage_key,
url;
if (spec.has_include_docs === true) {
options = {include_docs: true};
}
try {
storage_spec = getSubOpmlStorageDescription(
spec,
opml_doc,
basic_login
);
url = getStorageUrl(storage_spec);
storage_key = generateHash(parent_id + url);
sub_storage = createStorage(context, storage_spec, storage_key);
} catch (error) {
console.log(error);
throw error;
}
result_dict = {
parent_id: parent_id,
parent_title: opml_doc.title,
opml_title: opml_doc.opml_title,
type: storage_spec.type,
result: {
data: {
total_rows: 0
}
},
url: url
};
return sub_storage.allDocs(options)
.push(function (result) {
result_dict.result = result;
return result_dict;
}, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
console.log(error);
return result_dict;
}
throw error;
});
}
function getOpmlTree(context, opml_url, opml_spec) {
var opml_storage,
opml_document_list = [],
document_attachment_dict = {},
id;
id = generateHash(opml_url);
opml_storage = createStorage(context, opml_spec, id);
return opml_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return [[], {}];
}
throw error;
})
.push(function (opml_result_list) {
var j,
i,
item,
id_hash,
result_list = [];
for (i = 0; i < opml_result_list.data.total_rows; i += 1) {
item = opml_result_list.data.rows[i];
id_hash = generateHash(id + item.id);
opml_document_list.push({
id: id_hash,
doc: {
type: "opml-item",
name: item.id,
reference: id_hash,
parent_id: id,
creation_date: item.doc.created_date,
url: opml_url,
title: item.doc.title,
parent_title: undefined,
opml_title: item.doc.opml_title,
status: undefined
}
});
document_attachment_dict[id_hash] = {
name: item.id,
doc: item.doc
};
for (j = 0; j < opml_spec.sub_storage_list.length; j += 1) {
result_list.push(loadSubStorage(
context,
opml_spec.sub_storage_list[j],
id_hash,
item.doc,
opml_spec.basic_login
));
}
}
return RSVP.all(result_list);
})
.push(function (result_list) {
var i,
j;
function applyItemToTree(item, item_result) {
var id_hash,
element;
id_hash = generateHash(item_result.parent_id +
item_result.url + item.id);
if (item.doc !== undefined) {
element = item.doc;
} else {
element = item.value;
}
opml_document_list.push({
id: id_hash,
doc: {
parent_id: item_result.parent_id,
name: item.id,
type: (element.type || item_result.type + "-item"),
reference: id_hash,
creation_date: element.date || element["start-date"],
url: item_result.url,
status: (element.status || element.category),
title: (element.source || element.title),
parent_title: item_result.parent_title,
opml_title: item_result.opml_title
}
});
document_attachment_dict[id_hash] = {
name: item.id,
doc: element
};
}
for (i = 0; i < result_list.length; i += 1) {
for (j = 0; j < result_list[i].result.data.total_rows; j += 1) {
applyItemToTree(
result_list[i].result.data.rows[j],
result_list[i]
);
}
}
return [opml_document_list, document_attachment_dict];
});
}
function syncOpmlStorage(context) {
var i,
promise_list = [];
for (i = 0; i < context._opml_storage_list.length; i += 1) {
promise_list.push(
getOpmlTree(context,
context._opml_storage_list[i].url,
context._opml_storage_list[i])
);
}
return RSVP.all(promise_list);
}
function repairLocalStorage(context, new_document_list, document_key_list) {
var j,
promise_list = [];
function pushDocumentToStorage(document_list, attachment_dict) {
var document_queue = new RSVP.Queue(),
doc_index,
i;
function pushDocument(id, element, attachment) {
document_queue
.push(function () {
return context._local_sub_storage.put(id, element);
})
.push(function () {
return context._local_sub_storage.putAttachment(
id,
attachment.name,
new Blob([JSON.stringify(attachment.doc)])
);
});
}
for (i = 0; i < document_list.length; i += 1) {
doc_index = document_key_list.indexOf(document_list[i].id);
if (doc_index !== -1) {
delete document_key_list[doc_index];
}
pushDocument(
document_list[i].id,
document_list[i].doc,
attachment_dict[document_list[i].id]
);
}
return document_queue
.push(function () {
var k,
remove_queue = new RSVP.Queue();
// remove all document which were not updated
function removeDocument(key) {
remove_queue
.push(function () {
return context._local_sub_storage.get(key);
})
.push(undefined, function (error) {
throw error;
})
.push(function (element) {
return context._local_sub_storage.removeAttachment(
key,
element.name
);
})
.push(function () {
return context._local_sub_storage.remove(key);
})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return {};
}
throw error;
});
}
for (k = 0; k < document_key_list.length; k += 1) {
if (document_key_list[k] !== undefined) {
removeDocument(document_key_list[k]);
}
}
return remove_queue;
});
}
for (j = 0; j < new_document_list.length; j += 1) {
promise_list.push(pushDocumentToStorage(
new_document_list[j][0],
new_document_list[j][1]
));
}
return RSVP.all(promise_list);
}
ReplicatedOPMLStorage.prototype.repair = function () {
var context = this,
argument_list = arguments,
local_storage_index_list = [];
return new RSVP.Queue()
.push(function () {
return context._local_sub_storage.repair.apply(
context._local_sub_storage,
argument_list
);
})
.push(function () {
return context._local_sub_storage.allDocs();
})
.push(function (document_index) {
var i;
for (i = 0; i < document_index.data.total_rows; i += 1) {
local_storage_index_list.push(document_index.data.rows[i].id);
}
return syncOpmlStorage(context);
})
.push(function (result_list) {
return repairLocalStorage(context,
result_list,
local_storage_index_list);
});
};
jIO.addStorage('replicatedopml', ReplicatedOPMLStorage);
}(jIO, RSVP, Rusha, Blob, console));
src/jio.storage/webhttpstorage.js 0 → 100644
View file @ a5515a0a
/*
* Copyright 2016, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint nomen: true*/
/*global jIO, RSVP */
(function (jIO, RSVP) {
"use strict";
function ajax(storage, options) {
if (options === undefined) {
options = {};
}
if (storage._authorization !== undefined) {
if (options.headers === undefined) {
options.headers = {};
}
options.headers.Authorization = storage._authorization;
}
if (storage._with_credentials !== undefined) {
if (options.xhrFields === undefined) {
options.xhrFields = {};
}
options.xhrFields.withCredentials = storage._with_credentials;
}
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax(options);
});
}
function restrictDocumentId(id) {
var slash_index = id.indexOf("/");
if (slash_index !== 0 && slash_index !== -1) {
throw new jIO.util.jIOError("id " + id + " is forbidden (no begin /)",
400);
}
if (id.lastIndexOf("/") !== (id.length - 1)) {
throw new jIO.util.jIOError("id " + id + " is forbidden (no end /)",
400);
}
return id;
}
function getJsonDocument(context, id) {
return new RSVP.Queue()
.push(function () {
return ajax(context, {
type: "GET",
url: context._url + "/" + id + ".json",
dataType: "text"
});
})
.push(function (response) {
return {id: id, doc: JSON.parse(response.target.responseText)};
}, function (error) {
if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find document '" + id + "'", 404);
}
throw error;
});
}
/**
* The JIO WEB HTTP Storage extension
*
* @class WEBHTTPStorage
* @constructor
*/
function WEBHTTPStorage(spec) {
if (typeof spec.url !== 'string') {
throw new TypeError("WEBHTTPStorage 'url' is not of type string");
}
this._url = spec.url.replace(new RegExp("[/]+$"), "");
if (typeof spec.basic_login === 'string') {
this._authorization = "Basic " + spec.basic_login;
}
this._with_credentials = spec.with_credentials;
}
WEBHTTPStorage.prototype.get = function (id) {
var context = this,
element;
id = restrictDocumentId(id);
element = getJsonDocument(context, id);
if (element !== undefined) {
return element.doc;
}
};
WEBHTTPStorage.prototype.hasCapacity = function (capacity) {
return (capacity === "list") || (capacity === "include");
};
WEBHTTPStorage.prototype.buildQuery = function (options) {
var context = this,
item_list = [],
push_item;
if (options.include_docs === true) {
push_item = function (id, item) {
item_list.push({
"id": id,
"value": {},
"doc": item
});
};
} else {
push_item = function (id) {
item_list.push({
"id": id,
"value": {}
});
};
}
return new RSVP.Queue()
.push(function () {
return ajax(context, {
type: "GET",
url: context._url + "/_document_list",
dataType: "text"
});
})
.push(function (response) {
var document_list = [],
promise_list = [],
i;
document_list = response.target.responseText.split('\n');
for (i = 0; i < document_list.length; i += 1) {
if (document_list[i]) {
promise_list.push(getJsonDocument(context, document_list[i]));
}
}
return RSVP.all(promise_list);
}, function (error) {
if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find document", 404);
}
throw error;
})
.push(function (result_list) {
var i;
for (i = 0; i < result_list.length; i += 1) {
push_item(result_list[i].id, result_list[i].doc);
}
return item_list;
});
};
jIO.addStorage('webhttp', WEBHTTPStorage);
}(jIO, RSVP));
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7