Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
J
jio
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Ivan Tyagov
jio
Commits
dc97f827
Commit
dc97f827
authored
Feb 28, 2014
by
Jonathan Rivalan
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
s3storage.js update (+tests), now compatible with taskman.js
parent
65e213a7
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
184 additions
and
74 deletions
+184
-74
src/jio.storage/s3storage.js
src/jio.storage/s3storage.js
+178
-70
test/jio.storage/s3storage.tests.js
test/jio.storage/s3storage.tests.js
+6
-4
No files found.
src/jio.storage/s3storage.js
View file @
dc97f827
...
@@ -76,18 +76,18 @@
...
@@ -76,18 +76,18 @@
* ["substring to select", "selected substring replaced by this string"].
* ["substring to select", "selected substring replaced by this string"].
* @return {string} The replaced string
* @return {string} The replaced string
*/
*/
priv
.
recursiveReplace
=
function
(
string
,
list_of_replacement
)
{
//
priv.recursiveReplace = function (string, list_of_replacement) {
var
i
,
split_string
=
string
.
split
(
list_of_replacement
[
0
][
0
]);
//
var i, split_string = string.split(list_of_replacement[0][0]);
if
(
list_of_replacement
[
1
])
{
//
if (list_of_replacement[1]) {
for
(
i
=
0
;
i
<
split_string
.
length
;
i
+=
1
)
{
//
for (i = 0; i < split_string.length; i += 1) {
split_string
[
i
]
=
priv
.
recursiveReplace
(
//
split_string[i] = priv.recursiveReplace(
split_string
[
i
],
//
split_string[i],
list_of_replacement
.
slice
(
1
)
//
list_of_replacement.slice(1)
);
//
);
}
//
}
}
//
}
return
split_string
.
join
(
list_of_replacement
[
0
][
1
]);
//
return split_string.join(list_of_replacement[0][1]);
};
//
};
/**
/**
* Changes / to %2F, % to %25 and . to _.
* Changes / to %2F, % to %25 and . to _.
...
@@ -95,9 +95,9 @@
...
@@ -95,9 +95,9 @@
* @param {string} name The name to secure
* @param {string} name The name to secure
* @return {string} The secured name
* @return {string} The secured name
*/
*/
priv
.
secureName
=
function
(
name
)
{
//
priv.secureName = function (name) {
return
priv
.
recursiveReplace
(
name
,
[[
"
/
"
,
"
%2F
"
],
[
"
%
"
,
"
%25
"
]]);
//
return priv.recursiveReplace(name, [["/", "%2F"], ["%", "%25"]]);
};
//
};
/**
/**
* Restores the original name from a secured name
* Restores the original name from a secured name
...
@@ -105,9 +105,9 @@
...
@@ -105,9 +105,9 @@
* @param {string} secured_name The secured name to restore
* @param {string} secured_name The secured name to restore
* @return {string} The original name
* @return {string} The original name
*/
*/
priv
.
restoreName
=
function
(
secured_name
)
{
//
priv.restoreName = function (secured_name) {
return
priv
.
recursiveReplace
(
secured_name
,
[[
"
%2F
"
,
"
/
"
],
[
"
%25
"
,
"
%
"
]]);
//
return priv.recursiveReplace(secured_name, [["%2F", "/"], ["%25", "%"]]);
};
//
};
/**
/**
* Convert document id and attachment id to a file name
* Convert document id and attachment id to a file name
...
@@ -116,14 +116,14 @@
...
@@ -116,14 +116,14 @@
* @param {string} attachment_id The attachment id (optional)
* @param {string} attachment_id The attachment id (optional)
* @return {string} The file name
* @return {string} The file name
*/
*/
priv
.
idsToFileName
=
function
(
doc_id
,
attachment_id
)
{
//
priv.idsToFileName = function (doc_id, attachment_id) {
doc_id
=
priv
.
secureName
(
doc_id
).
split
(
"
.
"
).
join
(
"
_.
"
);
//
doc_id = priv.secureName(doc_id).split(".").join("_.");
if
(
typeof
attachment_id
===
"
string
"
)
{
//
if (typeof attachment_id === "string") {
attachment_id
=
priv
.
secureName
(
attachment_id
).
split
(
"
.
"
).
join
(
"
_.
"
);
//
attachment_id = priv.secureName(attachment_id).split(".").join("_.");
return
doc_id
+
"
.
"
+
attachment_id
;
//
return doc_id + "." + attachment_id;
}
//
}
return
doc_id
;
//
return doc_id;
};
//
};
/**
/**
* Convert a file name to a document id (and attachment id if there)
* Convert a file name to a document id (and attachment id if there)
...
@@ -131,27 +131,69 @@
...
@@ -131,27 +131,69 @@
* @param {string} file_name The file name to convert
* @param {string} file_name The file name to convert
* @return {array} ["document id", "attachment id"] or ["document id"]
* @return {array} ["document id", "attachment id"] or ["document id"]
*/
*/
priv
.
fileNameToIds
=
function
(
file_name
)
{
//priv.fileNameToIds = function (file_name) {
var
separator_index
=
-
1
,
split
=
file_name
.
split
(
"
.
"
);
//var separator_index = -1, split = file_name.split(".");
split
.
slice
(
0
,
-
1
).
forEach
(
function
(
file_name_part
,
index
)
{
//split.slice(0, -1).forEach(function (file_name_part, index) {
if
(
file_name_part
.
slice
(
-
1
)
!==
"
_
"
)
{
//if (file_name_part.slice(-1) !== "_") {
separator_index
=
index
;
//separator_index = index;
//}
//});
//if (separator_index === -1) {
//return [priv.restoreName(priv.restoreName(
//file_name
//).split("_.").join("."))];
//}
//return [
//priv.restoreName(priv.restoreName(
//split.slice(0, separator_index + 1).join(".")
//).split("_.").join(".")),
//priv.restoreName(priv.restoreName(
//split.slice(separator_index + 1).join(".")
//).split("_.").join("."))
//];
//};
priv
.
fileNameToIds
=
function
(
resourcename
)
{
var
split
,
el
,
id
=
""
,
attmt
=
""
,
last
;
split
=
resourcename
.
split
(
'
.
'
);
function
replaceAndNotLast
()
{
last
=
false
;
return
'
.
'
;
}
/*jslint ass: true */
while
((
el
=
split
.
shift
())
!==
undefined
)
{
/*jslint ass: false */
last
=
true
;
el
=
el
.
replace
(
/__/g
,
'
%2595
'
);
el
=
el
.
replace
(
/_$/
,
replaceAndNotLast
);
id
+=
el
.
replace
(
/%2595/g
,
'
_
'
);
if
(
last
)
{
break
;
}
}
attmt
=
split
.
join
(
'
.
'
);
return
[
id
,
attmt
];
}
priv
.
idsToFileName
=
function
(
document_id
,
attachment_id
)
{
document_id
=
encodeURI
(
document_id
).
replace
(
/
\/
/g
,
"
%2F
"
).
replace
(
/
\?
/g
,
"
%3F
"
);
document_id
=
encodeURI
(
document_id
).
replace
(
/_/g
,
"
__
"
).
replace
(
/
\.
/g
,
"
_.
"
);
if
(
attachment_id
)
{
attachment_id
=
encodeURI
(
attachment_id
).
replace
(
/
\/
/g
,
"
%2F
"
).
replace
(
/
\?
/g
,
"
%3F
"
);
return
document_id
+
"
.
"
+
attachment_id
;
}
return
document_id
;
}
}
});
if
(
separator_index
===
-
1
)
{
return
[
priv
.
restoreName
(
priv
.
restoreName
(
file_name
).
split
(
"
_.
"
).
join
(
"
.
"
))];
}
return
[
priv
.
restoreName
(
priv
.
restoreName
(
split
.
slice
(
0
,
separator_index
+
1
).
join
(
"
.
"
)
).
split
(
"
_.
"
).
join
(
"
.
"
)),
priv
.
restoreName
(
priv
.
restoreName
(
split
.
slice
(
separator_index
+
1
).
join
(
"
.
"
)
).
split
(
"
_.
"
).
join
(
"
.
"
))
];
};
/**
/**
* Removes the last character if it is a "/". "/a/b/c/" become "/a/b/c"
* Removes the last character if it is a "/". "/a/b/c/" become "/a/b/c"
...
@@ -167,6 +209,29 @@
...
@@ -167,6 +209,29 @@
};
};
/**
* Generate a new uuid
*
* @method generateUuid
* @private
* @return {String} The new uuid
*/
function
generateUuid
()
{
function
S4
()
{
/* 65536 */
var
i
,
string
=
Math
.
floor
(
Math
.
random
()
*
0x10000
).
toString
(
16
);
for
(
i
=
string
.
length
;
i
<
4
;
i
+=
1
)
{
string
=
'
0
'
+
string
;
}
return
string
;
}
return
S4
()
+
S4
()
+
"
-
"
+
S4
()
+
"
-
"
+
S4
()
+
"
-
"
+
S4
()
+
"
-
"
+
S4
()
+
S4
()
+
S4
();
}
that
.
documentObjectUpdate
=
function
(
doc
,
new_doc
)
{
that
.
documentObjectUpdate
=
function
(
doc
,
new_doc
)
{
var
k
;
var
k
;
...
@@ -460,10 +525,17 @@
...
@@ -460,10 +525,17 @@
jio
,
jio
,
is_attachment
,
is_attachment
,
callback
)
{
callback
)
{
var
docFile
,
requestUTC
,
StringToSign
,
url
,
Signature
,
xhr
;
var
docFile
,
requestUTC
,
StringToSign
,
url
,
Signature
,
xhr
;
docFile
=
priv
.
secureName
(
priv
.
idsToFileName
(
docId
,
attachId
||
undefined
));
if
(
command
.
method
===
"
alldocs
"
){
docFile
=
''
;
}
else
{
docFile
=
priv
.
idsToFileName
(
docId
,
attachId
||
undefined
);
}
console
.
trace
(
'
ma trace
'
);
console
.
info
(
'
docfile =
'
+
docId
,
attachId
,
docFile
);
requestUTC
=
new
Date
().
toUTCString
();
requestUTC
=
new
Date
().
toUTCString
();
...
@@ -520,16 +592,20 @@
...
@@ -520,16 +592,20 @@
**/
**/
that
.
post
=
function
(
command
,
metadata
)
{
that
.
post
=
function
(
command
,
metadata
)
{
console
.
info
(
'
S3 post
'
)
//as S3 encoding key are directly inserted within the FormData(),
//as S3 encoding key are directly inserted within the FormData(),
//use of XHRwrapper function ain't pertinent
//use of XHRwrapper function ain't pertinent
console
.
log
(
metadata
)
var
doc
,
doc_id
,
mime
;
var
doc
,
doc_id
,
mime
;
doc
=
metadata
;
doc
=
metadata
;
//doc_id = (!doc._id) ? generateUuid() : doc._id;
doc
.
_id
=
doc
.
_id
||
generateUuid
();
doc_id
=
doc
.
_id
;
doc_id
=
doc
.
_id
;
function
postDocument
()
{
function
postDocument
()
{
var
fd
,
Signature
,
xhr
;
var
fd
,
Signature
,
xhr
;
doc_id
=
priv
.
secureName
(
priv
.
idsToFileName
(
doc_id
));
doc_id
=
priv
.
idsToFileName
(
doc_id
);
console
.
log
(
doc_id
);
//Meant to deep-serialize in order to avoid
//Meant to deep-serialize in order to avoid
//conflicts due to the multipart enctype
//conflicts due to the multipart enctype
doc
=
JSON
.
stringify
(
doc
);
doc
=
JSON
.
stringify
(
doc
);
...
@@ -566,9 +642,8 @@
...
@@ -566,9 +642,8 @@
}
}
if
(
doc_id
===
''
||
doc_id
===
undefined
)
{
if
(
doc_id
===
''
||
doc_id
===
undefined
)
{
doc_id
=
'
no_document_id_
'
// doc_id = 'no_document_id_' + ((Math.random() * 10).toString().split('.'))[1];
+
((
Math
.
random
()
*
10
).
toString
().
split
(
'
.
'
))[
1
];
doc
.
_id
=
generateUuid
();
doc
.
_id
=
doc_id
;
}
}
mime
=
'
text/plain; charset=UTF-8
'
;
mime
=
'
text/plain; charset=UTF-8
'
;
...
@@ -595,6 +670,7 @@
...
@@ -595,6 +670,7 @@
**/
**/
that
.
get
=
function
(
command
,
metadata
)
{
that
.
get
=
function
(
command
,
metadata
)
{
console
.
info
(
'
S3 get
'
)
var
docId
,
isJIO
,
mime
;
var
docId
,
isJIO
,
mime
;
docId
=
metadata
.
_id
;
docId
=
metadata
.
_id
;
isJIO
=
true
;
isJIO
=
true
;
...
@@ -647,6 +723,7 @@
...
@@ -647,6 +723,7 @@
**/
**/
that
.
put
=
function
(
command
,
metadata
)
{
that
.
put
=
function
(
command
,
metadata
)
{
console
.
info
(
'
S3 put
'
)
var
doc
,
docId
,
mime
;
var
doc
,
docId
,
mime
;
doc
=
metadata
;
doc
=
metadata
;
docId
=
doc
.
_id
;
docId
=
doc
.
_id
;
...
@@ -681,6 +758,7 @@
...
@@ -681,6 +758,7 @@
};
};
that
.
putAttachment
=
function
(
command
,
param
)
{
that
.
putAttachment
=
function
(
command
,
param
)
{
console
.
info
(
'
S3 putAttachment
'
);
var
my_document
,
var
my_document
,
docId
,
docId
,
attachId
,
attachId
,
...
@@ -767,6 +845,7 @@
...
@@ -767,6 +845,7 @@
*/
*/
that
.
remove
=
function
(
command
,
param
)
{
that
.
remove
=
function
(
command
,
param
)
{
console
.
info
(
'
S3 remove
'
)
var
docId
,
mime
;
var
docId
,
mime
;
docId
=
param
.
_id
;
docId
=
param
.
_id
;
mime
=
'
text/plain; charset=UTF-8
'
;
mime
=
'
text/plain; charset=UTF-8
'
;
...
@@ -887,6 +966,12 @@
...
@@ -887,6 +966,12 @@
**/
**/
that
.
allDocs
=
function
(
command
,
param
,
options
)
{
that
.
allDocs
=
function
(
command
,
param
,
options
)
{
console
.
info
(
'
S3 allDocs
'
);
var
_succ
=
command
.
success
;
command
.
success
=
function
(){
console
.
log
.
apply
(
console
,
arguments
);
_succ
.
apply
(
this
,
arguments
)
}
/*jslint unparam: true */
/*jslint unparam: true */
var
my_document
,
mime
;
var
my_document
,
mime
;
my_document
=
null
;
my_document
=
null
;
...
@@ -907,7 +992,16 @@
...
@@ -907,7 +992,16 @@
callURL
,
callURL
,
requestUTC
;
requestUTC
;
keys
=
$
(
my_document
).
find
(
'
Key
'
);
keys
=
$
(
$
.
parseXML
(
my_document
)).
find
(
'
Key
'
);
if
(
keys
.
length
===
0
)
{
return
command
.
success
(
{
"
data
"
:
{
"
total_rows
"
:
0
,
"
rows
"
:
[]
}
});
}
resultTable
=
[];
resultTable
=
[];
counter
=
0
;
counter
=
0
;
...
@@ -939,22 +1033,33 @@
...
@@ -939,22 +1033,33 @@
dealCallback
=
function
(
i
,
countB
,
allDoc
)
{
dealCallback
=
function
(
i
,
countB
,
allDoc
)
{
/*jslint unparam: true */
/*jslint unparam: true */
return
function
(
doc
,
statustext
,
response
)
{
return
function
(
doc
,
statustext
,
response
)
{
allDoc
.
rows
[
i
].
doc
=
response
.
responseText
;
allDoc
.
rows
[
i
].
doc
=
JSON
.
parse
(
response
.
responseText
)
;
if
(
count
===
0
)
{
if
(
count
===
0
)
{
command
.
success
(
allDoc
);
return
command
.
success
({
"
data
"
:
allDoc
});
}
else
{
}
else
{
count
-=
1
;
count
-=
1
;
}
}
};
};
};
};
errCallback
=
function
(
err
)
{
errCallback
=
function
(
jQxhr
)
{
if
(
err
.
status
===
404
)
{
command
.
error
(
err
.
error
=
"
not_found
"
;
jQxhr
.
status
,
command
.
error
(
err
);
jQxhr
.
statusText
,
}
else
{
"
S3 Alldocs failed.
"
return
that
.
retry
(
err
);
)
}
//if (obj.status === 404) {
//obj.error = "not_found";
//console.info(obj);
//command.error(obj.error);
//} else {
////return command.retry(err);
//console.info(obj);
//return command.error(obj);
//}
};
};
i
=
resultTable
.
length
-
1
;
i
=
resultTable
.
length
-
1
;
...
@@ -962,11 +1067,12 @@
...
@@ -962,11 +1067,12 @@
if
(
options
.
include_docs
)
{
if
(
options
.
include_docs
)
{
for
(
i
;
i
>=
0
;
i
-=
1
)
{
for
(
i
;
i
>=
0
;
i
-=
1
)
{
keyId
=
resultTable
[
i
];
keyId
=
resultTable
[
i
];
console
.
log
(
keyId
);
Signature
=
that
.
encodeAuthorization
(
keyId
);
Signature
=
that
.
encodeAuthorization
(
keyId
);
callURL
=
'
http://
'
+
priv
.
server
+
'
.s3.amazonaws.com/
'
+
keyId
;
callURL
=
'
http://
'
+
priv
.
server
+
'
.s3.amazonaws.com/
'
+
keyId
;
requestUTC
=
new
Date
().
toUTCString
();
requestUTC
=
new
Date
().
toUTCString
();
allDocResponse
.
rows
[
i
]
=
{
allDocResponse
.
rows
[
i
]
=
{
"
id
"
:
priv
.
fileNameToIds
(
keyId
)
.
join
()
,
"
id
"
:
priv
.
fileNameToIds
(
keyId
)
[
0
]
,
"
value
"
:
{}
"
value
"
:
{}
};
};
$
.
ajax
({
$
.
ajax
({
...
@@ -988,28 +1094,30 @@
...
@@ -988,28 +1094,30 @@
//'x-amz-security-token' : ,
//'x-amz-security-token' : ,
},
},
success
:
dealCallback
(
i
,
countB
,
allDocResponse
),
success
:
dealCallback
(
i
,
countB
,
allDocResponse
),
error
:
errCallback
(
command
.
error
)
error
:
errCallback
});
});
countB
+=
1
;
countB
+=
1
;
}
}
}
else
{
}
else
{
for
(
i
;
i
>=
0
;
i
-=
1
)
{
for
(
i
;
i
>=
0
;
i
-=
1
)
{
keyId
=
resultTable
[
i
];
keyId
=
resultTable
[
i
];
console
.
log
(
keyId
);
allDocResponse
.
rows
[
i
]
=
{
allDocResponse
.
rows
[
i
]
=
{
"
id
"
:
priv
.
fileNameToIds
(
keyId
)
.
join
()
,
"
id
"
:
priv
.
fileNameToIds
(
keyId
)
[
0
]
,
"
value
"
:
{}
"
value
"
:
{}
};
};
}
}
allDocResponse
=
{
"
data
"
:
allDocResponse
};
allDocResponse
=
{
"
data
"
:
allDocResponse
};
command
.
success
(
allDocResponse
);
command
.
success
(
allDocResponse
);
}
}
}
}
function
getXML
()
{
function
getXML
()
{
//XHRwrapper(command,'PUT','text/plain; charset=UTF-8',true)
;
command
.
method
=
'
alldocs
'
;
that
.
XHRwrapper
(
command
,
''
,
''
,
'
GET
'
,
mime
,
''
,
false
,
false
,
that
.
XHRwrapper
(
command
,
''
,
''
,
'
GET
'
,
mime
,
''
,
false
,
false
,
function
(
reponse
)
{
function
(
re
s
ponse
)
{
my_document
=
reponse
;
my_document
=
re
s
ponse
;
makeJSON
();
makeJSON
();
}
}
);
);
...
...
test/jio.storage/s3storage.tests.js
View file @
dc97f827
...
@@ -68,11 +68,13 @@
...
@@ -68,11 +68,13 @@
"
status
"
:
204
,
"
status
"
:
204
,
"
statusText
"
:
"
No Content
"
"
statusText
"
:
"
No Content
"
},
"
Post a new document
"
);
},
"
Post a new document
"
);
ok
(
/^no_document_id_
[
0-9
]
+$/
.
test
(
uuid
),
//ok(/^no_document_id_[0-9]+$/.test(uuid),
"
New document id should look like no_document_id_479658600408584 :
"
+
//"New document id should look like no_document_id_479658600408584 : " +
uuid
);
//uuid);
//shared.created_document_id = uuid;
ok
(
test_util
.
isUuid
(
uuid
),
"
New document id should look like
"
+
"
xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx :
"
+
uuid
);
shared
.
created_document_id
=
uuid
;
shared
.
created_document_id
=
uuid
;
}
}
function
getCreatedDocument
()
{
function
getCreatedDocument
()
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment