Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Iliya Manolov
neoppod
Commits
d250deca
Commit
d250deca
authored
Jan 08, 2014
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Drop support for Python < 2.7
parent
14fd9cd3
Changes
28
Show whitespace changes
Inline
Side-by-side
Showing
28 changed files
with
82 additions
and
118 deletions
+82
-118
MANIFEST.in
MANIFEST.in
+0
-1
README
README
+1
-1
neo/client/__init__.py
neo/client/__init__.py
+0
-2
neo/client/config.py
neo/client/config.py
+2
-2
neo/lib/protocol.py
neo/lib/protocol.py
+1
-1
neo/lib/pt.py
neo/lib/pt.py
+5
-5
neo/master/app.py
neo/master/app.py
+2
-2
neo/master/handlers/client.py
neo/master/handlers/client.py
+12
-16
neo/master/pt.py
neo/master/pt.py
+14
-15
neo/master/recovery.py
neo/master/recovery.py
+2
-2
neo/neoctl/app.py
neo/neoctl/app.py
+1
-1
neo/scripts/simple.py
neo/scripts/simple.py
+2
-2
neo/storage/app.py
neo/storage/app.py
+1
-1
neo/storage/database/mysqldb.py
neo/storage/database/mysqldb.py
+4
-4
neo/storage/database/sqlite.py
neo/storage/database/sqlite.py
+4
-4
neo/storage/handlers/master.py
neo/storage/handlers/master.py
+2
-3
neo/tests/__init__.py
neo/tests/__init__.py
+0
-25
neo/tests/client/testClientApp.py
neo/tests/client/testClientApp.py
+1
-1
neo/tests/client/testStorageHandler.py
neo/tests/client/testStorageHandler.py
+11
-12
neo/tests/functional/__init__.py
neo/tests/functional/__init__.py
+4
-4
neo/tests/master/testClientHandler.py
neo/tests/master/testClientHandler.py
+1
-1
neo/tests/master/testMasterPT.py
neo/tests/master/testMasterPT.py
+1
-1
neo/tests/master/testStorageHandler.py
neo/tests/master/testStorageHandler.py
+3
-3
neo/tests/storage/testStorageDBTests.py
neo/tests/storage/testStorageDBTests.py
+2
-2
neo/tests/testPT.py
neo/tests/testPT.py
+1
-1
neo/tests/threaded/__init__.py
neo/tests/threaded/__init__.py
+3
-3
neo/tests/threaded/testReplication.py
neo/tests/threaded/testReplication.py
+2
-2
setup.py
setup.py
+0
-1
No files found.
MANIFEST.in
View file @
d250deca
graft tools
include neo.conf CHANGES TODO TESTS.txt ZODB3.patch
include neo/client/component.xml # required for Python < 2.7
README
View file @
d250deca
...
...
@@ -54,7 +54,7 @@ Requirements
- Linux 2.6 or later
- Python 2.
6.x or 2.
7.x
- Python 2.7.x
- For storage nodes using MySQL backend:
...
...
neo/client/__init__.py
View file @
d250deca
...
...
@@ -27,7 +27,6 @@ if 1:
# to tpc_finish/tpc_abort).
_check
(
Connection
.
tpc_finish
,
'f50ed2e5a74f584fa1ecaf1cd70a6b15'
,
# Python 2.6
'ab9b1b8d82c40e5fffa84f7bc4ea3a8b'
,
# Python 2.7
)
...
...
@@ -72,7 +71,6 @@ if 1:
# know any legitimate use of DB access outside a transaction.
_check
(
Connection
.
afterCompletion
,
'70dfc3df8a455d9f663dec619c607eb5'
,
# Python 2.6
'cd3a080b80fd957190ff3bb867149448'
,
# Python 2.7
)
...
...
neo/client/config.py
View file @
d250deca
...
...
@@ -21,5 +21,5 @@ class NeoStorage(BaseConfig):
def
open
(
self
):
from
.Storage
import
Storage
config
=
self
.
config
return
Storage
(
**
dict
((
k
,
getattr
(
config
,
k
)
)
for
k
in
config
.
getSectionAttributes
())
)
return
Storage
(
**
{
k
:
getattr
(
config
,
k
)
for
k
in
config
.
getSectionAttributes
()}
)
neo/lib/protocol.py
View file @
d250deca
...
...
@@ -181,7 +181,7 @@ UUID_NAMESPACES = {
}
uuid_str
=
(
lambda
ns
:
lambda
uuid
:
ns
[
uuid
>>
24
]
+
str
(
uuid
&
0xffffff
)
if
uuid
else
str
(
uuid
)
)(
dict
((
v
,
str
(
k
)[
0
])
for
k
,
v
in
UUID_NAMESPACES
.
iteritems
())
)
)(
{
v
:
str
(
k
)[
0
]
for
k
,
v
in
UUID_NAMESPACES
.
iteritems
()}
)
class
ProtocolError
(
Exception
):
""" Base class for protocol errors, close the connection """
...
...
neo/lib/pt.py
View file @
d250deca
...
...
@@ -129,9 +129,9 @@ class PartitionTable(object):
def
getNodeSet
(
self
,
readable
=
False
):
if
readable
:
return
set
(
x
.
getNode
()
for
row
in
self
.
partition_list
for
x
in
row
if
x
.
isReadable
()
)
return
set
(
x
.
getNode
()
for
row
in
self
.
partition_list
for
x
in
row
)
return
{
x
.
getNode
()
for
row
in
self
.
partition_list
for
x
in
row
if
x
.
isReadable
()
}
return
{
x
.
getNode
()
for
row
in
self
.
partition_list
for
x
in
row
}
def
getConnectedNodeList
(
self
):
return
[
node
for
node
in
self
.
getNodeSet
()
if
node
.
isConnected
()]
...
...
@@ -276,8 +276,8 @@ class PartitionTable(object):
if
row
is
None
:
line
.
append
(
'X'
*
len
(
node_list
))
else
:
cell_dict
=
dict
((
x
.
getNode
(),
cell_state_dict
[
x
.
getState
()])
for
x
in
row
)
cell_dict
=
{
x
.
getNode
():
cell_state_dict
[
x
.
getState
()]
for
x
in
row
}
line
.
append
(
''
.
join
(
cell_dict
.
get
(
x
,
'.'
)
for
x
in
node_list
))
if
line
:
append
(
'pt: %0*u: %s'
%
(
prefix_len
,
prefix
,
'|'
.
join
(
line
)))
...
...
neo/master/app.py
View file @
d250deca
...
...
@@ -282,8 +282,8 @@ class Application(object):
if
e
.
args
[
0
]
!=
ClusterStates
.
STARTING_BACKUP
:
raise
self
.
backup_tid
=
tid
=
self
.
getLastTransaction
()
self
.
pt
.
setBackupTidDict
(
dict
((
node
.
getUUID
(),
tid
)
for
node
in
self
.
nm
.
getStorageList
(
only_identified
=
True
)
)
)
self
.
pt
.
setBackupTidDict
(
{
node
.
getUUID
():
tid
for
node
in
self
.
nm
.
getStorageList
(
only_identified
=
True
)
}
)
def
playPrimaryRole
(
self
):
logging
.
info
(
'play the primary role with %r'
,
self
.
listening_conn
)
...
...
neo/master/handlers/client.py
View file @
d250deca
...
...
@@ -56,25 +56,21 @@ class ClientServiceHandler(MasterHandler):
def
askFinishTransaction
(
self
,
conn
,
ttid
,
oid_list
):
app
=
self
.
app
pt
=
app
.
pt
# Collect partitions related to this transaction.
getPartition
=
app
.
pt
.
getPartition
partition_set
=
set
(
getPartition
(
oid
)
for
oid
in
oid_list
)
partition_set
.
add
(
getPartition
(
ttid
))
partition_set
=
set
(
map
(
pt
.
getPartition
,
oid_list
))
partition_set
.
add
(
pt
.
getPartition
(
ttid
))
# Collect the UUIDs of nodes related to this transaction.
uuid_set
=
set
()
isStorageReady
=
app
.
isStorageReady
for
part
in
partition_set
:
uuid_set
.
update
((
uuid
for
uuid
in
(
cell
.
getUUID
()
for
cell
in
app
.
pt
.
getCellList
(
part
)
if
cell
.
getNodeState
()
!=
NodeStates
.
HIDDEN
)
if
isStorageReady
(
uuid
)))
if
not
uuid_set
:
uuid_list
=
filter
(
app
.
isStorageReady
,
{
cell
.
getUUID
()
for
part
in
partition_set
for
cell
in
pt
.
getCellList
(
part
)
if
cell
.
getNodeState
()
!=
NodeStates
.
HIDDEN
})
if
not
uuid_list
:
raise
ProtocolError
(
'No storage node ready for transaction'
)
identified_node_list
=
app
.
nm
.
getIdentifiedList
(
pool_set
=
uuid_set
)
identified_node_list
=
app
.
nm
.
getIdentifiedList
(
pool_set
=
set
(
uuid_list
)
)
# Request locking data.
# build a new set as we may not send the message to all nodes as some
...
...
@@ -83,9 +79,9 @@ class ClientServiceHandler(MasterHandler):
ttid
,
app
.
tm
.
prepare
(
ttid
,
app
.
pt
.
getPartitions
(),
pt
.
getPartitions
(),
oid_list
,
set
(
x
.
getUUID
()
for
x
in
identified_node_list
)
,
{
x
.
getUUID
()
for
x
in
identified_node_list
}
,
conn
.
getPeerId
(),
),
oid_list
,
...
...
@@ -98,7 +94,7 @@ class ClientServiceHandler(MasterHandler):
if
app
.
packing
is
None
:
storage_list
=
app
.
nm
.
getStorageList
(
only_identified
=
True
)
app
.
packing
=
(
conn
,
conn
.
getPeerId
(),
set
(
x
.
getUUID
()
for
x
in
storage_list
)
)
{
x
.
getUUID
()
for
x
in
storage_list
}
)
p
=
Packets
.
AskPack
(
tid
)
for
storage
in
storage_list
:
storage
.
getConnection
().
ask
(
p
)
...
...
neo/master/pt.py
View file @
d250deca
...
...
@@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from
collections
import
defaultdict
import
neo.lib.pt
from
neo.lib.protocol
import
CellStates
,
ZERO_TID
...
...
@@ -188,17 +189,17 @@ class PartitionTable(neo.lib.pt.PartitionTable):
This is done by computing a minimal diff between current partition table
and what make() would do.
"""
assigned_dict
=
dict
((
x
,
{})
for
x
in
self
.
count_dict
)
readable_
dict
=
dict
((
i
,
set
())
for
i
in
xrange
(
self
.
np
))
assigned_dict
=
{
x
:
{}
for
x
in
self
.
count_dict
}
readable_
list
=
[
set
()
for
x
in
xrange
(
self
.
np
)]
for
offset
,
row
in
enumerate
(
self
.
partition_list
):
for
cell
in
row
:
if
cell
.
isReadable
():
readable_
dic
t
[
offset
].
add
(
cell
)
readable_
lis
t
[
offset
].
add
(
cell
)
assigned_dict
[
cell
.
getNode
()][
offset
]
=
cell
pt
=
PartitionTable
(
self
.
np
,
self
.
nr
)
drop_list
=
set
(
x
for
x
in
drop_list
if
x
in
assigned_dict
)
node_set
=
set
(
MappedNode
(
x
)
for
x
in
assigned_dict
if
x
not
in
drop_list
)
drop_list
=
set
(
drop_list
).
intersection
(
assigned_dict
)
node_set
=
{
MappedNode
(
x
)
for
x
in
assigned_dict
if
x
not
in
drop_list
}
pt
.
make
(
node_set
)
for
offset
,
row
in
enumerate
(
pt
.
partition_list
):
for
cell
in
row
:
...
...
@@ -210,8 +211,8 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if
node
in
drop_list
:
yield
node
,
frozenset
()
continue
readable
=
set
(
offset
for
offset
,
cell
in
assigned
.
iteritems
()
if
cell
.
isReadable
())
readable
=
{
offset
for
offset
,
cell
in
assigned
.
iteritems
()
if
cell
.
isReadable
()}
# the criterion on UUID is purely cosmetic
node_list
.
append
((
len
(
readable
),
len
(
assigned
),
-
node
.
getUUID
(),
readable
,
node
))
...
...
@@ -226,7 +227,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
assert
not
node_set
changed_list
=
[]
uptodate_set
=
set
()
remove_dict
=
d
ict
((
i
,
[])
for
i
in
xrange
(
self
.
np
)
)
remove_dict
=
d
efaultdict
(
list
)
for
node
,
mapped
in
map_nodes
():
uuid
=
node
.
getUUID
()
assigned
=
assigned_dict
[
node
]
...
...
@@ -234,7 +235,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if
offset
in
mapped
:
if
cell
.
isReadable
():
uptodate_set
.
add
(
offset
)
readable_
dic
t
[
offset
].
remove
(
cell
)
readable_
lis
t
[
offset
].
remove
(
cell
)
if
cell
.
isFeeding
():
self
.
count_dict
[
node
]
+=
1
state
=
CellStates
.
UP_TO_DATE
...
...
@@ -251,11 +252,9 @@ class PartitionTable(neo.lib.pt.PartitionTable):
changed_list
.
append
((
offset
,
uuid
,
state
))
count_dict
=
self
.
count_dict
.
copy
()
for
offset
,
cell_list
in
remove_dict
.
iteritems
():
if
not
cell_list
:
continue
row
=
self
.
partition_list
[
offset
]
feeding
=
None
if
offset
in
uptodate_set
else
min
(
readable_
dic
t
[
offset
],
key
=
lambda
x
:
count_dict
[
x
.
getNode
()])
readable_
lis
t
[
offset
],
key
=
lambda
x
:
count_dict
[
x
.
getNode
()])
for
cell
in
cell_list
:
if
cell
is
feeding
:
count_dict
[
cell
.
getNode
()]
+=
1
...
...
@@ -305,10 +304,10 @@ class PartitionTable(neo.lib.pt.PartitionTable):
Return a set of all nodes which are part of at least one UP TO DATE
partition.
"""
return
set
(
cell
.
getNode
()
return
{
cell
.
getNode
()
for
row
in
self
.
partition_list
for
cell
in
row
if
cell
.
isReadable
()
)
if
cell
.
isReadable
()
}
def
clearReplicating
(
self
):
for
row
in
self
.
partition_list
:
...
...
neo/master/recovery.py
View file @
d250deca
...
...
@@ -63,8 +63,8 @@ class RecoveryManager(MasterHandler):
# A partition table exists, we are starting an existing
# cluster.
partition_node_set
=
pt
.
getReadableCellNodeSet
()
pending_node_set
=
set
(
x
for
x
in
partition_node_set
if
x
.
isPending
()
)
pending_node_set
=
{
x
for
x
in
partition_node_set
if
x
.
isPending
()
}
if
app
.
_startup_allowed
or
\
partition_node_set
==
pending_node_set
:
allowed_node_set
=
pending_node_set
...
...
neo/neoctl/app.py
View file @
d250deca
...
...
@@ -41,7 +41,7 @@ action_dict = {
uuid_int
=
(
lambda
ns
:
lambda
uuid
:
(
ns
[
uuid
[
0
]]
<<
24
)
+
int
(
uuid
[
1
:])
)(
dict
((
str
(
k
)[
0
],
v
)
for
k
,
v
in
UUID_NAMESPACES
.
iteritems
())
)
)(
{
str
(
k
)[
0
]:
v
for
k
,
v
in
UUID_NAMESPACES
.
iteritems
()}
)
class
TerminalNeoCTL
(
object
):
def
__init__
(
self
,
address
):
...
...
neo/scripts/simple.py
View file @
d250deca
...
...
@@ -44,8 +44,8 @@ def main():
options
,
args
=
parser
.
parse_args
()
if
options
.
seed
:
functional
.
random
=
random
.
Random
(
options
.
seed
)
cluster
=
functional
.
NEOCluster
(
args
,
**
dict
((
x
,
getattr
(
options
,
x
)
)
for
x
,
_
in
option_list
)
)
cluster
=
functional
.
NEOCluster
(
args
,
**
{
x
:
getattr
(
options
,
x
)
for
x
,
_
in
option_list
}
)
try
:
cluster
.
run
()
logging
.
info
(
"Cluster running ..."
)
...
...
neo/storage/app.py
View file @
d250deca
...
...
@@ -200,7 +200,7 @@ class Application(object):
conn
.
close
()
# create/clear event queue
self
.
event_queue
=
deque
()
self
.
event_queue_dict
=
dict
()
self
.
event_queue_dict
=
{}
try
:
self
.
verifyData
()
self
.
initialize
()
...
...
neo/storage/database/mysqldb.py
View file @
d250deca
...
...
@@ -259,12 +259,12 @@ class MySQLDatabaseManager(DatabaseManager):
def
_getLastIDs
(
self
,
all
=
True
):
p64
=
util
.
p64
q
=
self
.
query
trans
=
dict
((
partition
,
p64
(
tid
)
)
trans
=
{
partition
:
p64
(
tid
)
for
partition
,
tid
in
q
(
"SELECT partition, MAX(tid)"
" FROM trans GROUP BY partition"
)
)
obj
=
dict
((
partition
,
p64
(
tid
)
)
" FROM trans GROUP BY partition"
)
}
obj
=
{
partition
:
p64
(
tid
)
for
partition
,
tid
in
q
(
"SELECT partition, MAX(tid)"
" FROM obj GROUP BY partition"
)
)
" FROM obj GROUP BY partition"
)
}
oid
=
q
(
"SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj"
" GROUP BY partition) as t"
)[
0
][
0
]
if
all
:
...
...
neo/storage/database/sqlite.py
View file @
d250deca
...
...
@@ -221,12 +221,12 @@ class SQLiteDatabaseManager(DatabaseManager):
def
_getLastIDs
(
self
,
all
=
True
):
p64
=
util
.
p64
q
=
self
.
query
trans
=
dict
((
partition
,
p64
(
tid
)
)
trans
=
{
partition
:
p64
(
tid
)
for
partition
,
tid
in
q
(
"SELECT partition, MAX(tid)"
" FROM trans GROUP BY partition"
)
)
obj
=
dict
((
partition
,
p64
(
tid
)
)
" FROM trans GROUP BY partition"
)
}
obj
=
{
partition
:
p64
(
tid
)
for
partition
,
tid
in
q
(
"SELECT partition, MAX(tid)"
" FROM obj GROUP BY partition"
)
)
" FROM obj GROUP BY partition"
)
}
oid
=
q
(
"SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj"
" GROUP BY partition) as t"
).
next
()[
0
]
if
all
:
...
...
neo/storage/handlers/master.py
View file @
d250deca
...
...
@@ -64,9 +64,8 @@ class MasterOperationHandler(BaseMasterHandler):
conn
.
answer
(
Packets
.
AnswerPack
(
True
))
def
replicate
(
self
,
conn
,
tid
,
upstream_name
,
source_dict
):
self
.
app
.
replicator
.
backup
(
tid
,
dict
((
p
,
a
and
(
a
,
upstream_name
))
for
p
,
a
in
source_dict
.
iteritems
()))
self
.
app
.
replicator
.
backup
(
tid
,
{
p
:
a
and
(
a
,
upstream_name
)
for
p
,
a
in
source_dict
.
iteritems
()})
def
truncate
(
self
,
conn
,
tid
):
self
.
app
.
replicator
.
cancel
()
...
...
neo/tests/__init__.py
View file @
d250deca
...
...
@@ -514,28 +514,3 @@ class DoNothingConnector(Mock):
__builtin__
.
pdb
=
lambda
depth
=
0
:
\
debug
.
getPdb
().
set_trace
(
sys
.
_getframe
(
depth
+
1
))
def
_fixMockForInspect
():
"""
inspect module change broke Mock, see http://bugs.python.org/issue1785
Monkey-patch Mock class if needed by replacing predicate parameter on 2nd
getmembers call with isroutine (was ismethod).
"""
import
inspect
class
A
(
object
):
def
f
(
self
):
pass
if
not
inspect
.
getmembers
(
A
,
inspect
.
ismethod
):
from
mock
import
MockCallable
# _setupSubclassMethodInterceptors is under the FreeBSD license.
# See pyMock module for the whole license.
def
_setupSubclassMethodInterceptors
(
self
):
methods
=
inspect
.
getmembers
(
self
.
__class__
,
inspect
.
isroutine
)
baseMethods
=
dict
(
inspect
.
getmembers
(
Mock
,
inspect
.
isroutine
))
for
m
in
methods
:
name
=
m
[
0
]
# Don't record calls to methods of Mock base class.
if
not
name
in
baseMethods
:
self
.
__dict__
[
name
]
=
MockCallable
(
name
,
self
,
handcrafted
=
True
)
Mock
.
_setupSubclassMethodInterceptors
=
_setupSubclassMethodInterceptors
_fixMockForInspect
()
neo/tests/client/testClientApp.py
View file @
d250deca
...
...
@@ -337,7 +337,7 @@ class ClientApplicationTests(NeoUnitTestBase):
txn_context
[
'queue'
].
put
((
conn
,
packet
,
{}))
app
.
waitStoreResponses
(
txn_context
,
resolving_tryToResolveConflict
)
self
.
assertEqual
(
txn_context
[
'object_stored_counter_dict'
][
oid
],
{
tid
:
set
([
uuid
])
})
{
tid
:
{
uuid
}
})
self
.
assertEqual
(
txn_context
[
'cache_dict'
][
oid
],
'DATA'
)
self
.
assertFalse
(
oid
in
txn_context
[
'data_dict'
])
self
.
assertFalse
(
oid
in
txn_context
[
'conflict_serial_dict'
])
...
...
neo/tests/client/testStorageHandler.py
View file @
d250deca
...
...
@@ -65,11 +65,11 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
self
.
_getAnswerStoreObjectHandler
(
object_stored_counter_dict
,
conflict_serial_dict
,
resolved_conflict_serial_dict
,
).
answerStoreObject
(
conn
,
1
,
oid
,
tid
)
self
.
assertEqual
(
conflict_serial_dict
[
oid
],
set
([
tid
,
])
)
self
.
assertEqual
(
conflict_serial_dict
[
oid
],
{
tid
}
)
self
.
assertEqual
(
object_stored_counter_dict
[
oid
],
{})
self
.
assertFalse
(
oid
in
resolved_conflict_serial_dict
)
# object was already accepted by another storage, raise
handler
=
self
.
_getAnswerStoreObjectHandler
({
oid
:
{
tid
:
set
([
1
])
}},
{},
{})
handler
=
self
.
_getAnswerStoreObjectHandler
({
oid
:
{
tid
:
{
1
}
}},
{},
{})
self
.
assertRaises
(
NEOStorageError
,
handler
.
answerStoreObject
,
conn
,
1
,
oid
,
tid
)
...
...
@@ -80,21 +80,21 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
tid_2
=
self
.
getNextTID
()
# resolution-pending conflict
object_stored_counter_dict
=
{
oid
:
{}}
conflict_serial_dict
=
{
oid
:
set
([
tid
,
])
}
conflict_serial_dict
=
{
oid
:
{
tid
}
}
resolved_conflict_serial_dict
=
{}
self
.
_getAnswerStoreObjectHandler
(
object_stored_counter_dict
,
conflict_serial_dict
,
resolved_conflict_serial_dict
,
).
answerStoreObject
(
conn
,
1
,
oid
,
tid
)
self
.
assertEqual
(
conflict_serial_dict
[
oid
],
set
([
tid
,
])
)
self
.
assertEqual
(
conflict_serial_dict
[
oid
],
{
tid
}
)
self
.
assertFalse
(
oid
in
resolved_conflict_serial_dict
)
self
.
assertEqual
(
object_stored_counter_dict
[
oid
],
{})
# object was already accepted by another storage, raise
handler
=
self
.
_getAnswerStoreObjectHandler
({
oid
:
{
tid
:
set
([
1
])
}},
{
oid
:
set
([
tid
,
])
},
{})
handler
=
self
.
_getAnswerStoreObjectHandler
({
oid
:
{
tid
:
{
1
}
}},
{
oid
:
{
tid
}
},
{})
self
.
assertRaises
(
NEOStorageError
,
handler
.
answerStoreObject
,
conn
,
1
,
oid
,
tid
)
# detected conflict is different, don't raise
self
.
_getAnswerStoreObjectHandler
({
oid
:
{}},
{
oid
:
set
([
tid
,
])
},
{},
self
.
_getAnswerStoreObjectHandler
({
oid
:
{}},
{
oid
:
{
tid
}
},
{},
).
answerStoreObject
(
conn
,
1
,
oid
,
tid_2
)
def
test_answerStoreObject_3
(
self
):
...
...
@@ -108,17 +108,16 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
# storage accepted the resolved object.
object_stored_counter_dict
=
{
oid
:
{
tid_2
:
1
}}
conflict_serial_dict
=
{}
resolved_conflict_serial_dict
=
{
oid
:
set
([
tid
,
])
}
resolved_conflict_serial_dict
=
{
oid
:
{
tid
}
}
self
.
_getAnswerStoreObjectHandler
(
object_stored_counter_dict
,
conflict_serial_dict
,
resolved_conflict_serial_dict
,
).
answerStoreObject
(
conn
,
1
,
oid
,
tid
)
self
.
assertFalse
(
oid
in
conflict_serial_dict
)
self
.
assertEqual
(
resolved_conflict_serial_dict
[
oid
],
set
([
tid
,
]))
self
.
assertEqual
(
resolved_conflict_serial_dict
[
oid
],
{
tid
})
self
.
assertEqual
(
object_stored_counter_dict
[
oid
],
{
tid_2
:
1
})
# detected conflict is different, don't raise
self
.
_getAnswerStoreObjectHandler
({
oid
:
{
tid
:
1
}},
{},
{
oid
:
set
([
tid
,
])
}).
answerStoreObject
(
conn
,
1
,
oid
,
tid_2
)
{
oid
:
{
tid
}
}).
answerStoreObject
(
conn
,
1
,
oid
,
tid_2
)
def
test_answerStoreObject_4
(
self
):
uuid
=
self
.
getStorageUUID
()
...
...
@@ -135,7 +134,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
h
.
answerStoreObject
(
conn
,
0
,
oid
,
tid
)
self
.
assertFalse
(
oid
in
conflict_serial_dict
)
self
.
assertFalse
(
oid
in
resolved_conflict_serial_dict
)
self
.
assertEqual
(
object_stored_counter_dict
[
oid
],
{
tid
:
set
([
uuid
])
})
self
.
assertEqual
(
object_stored_counter_dict
[
oid
],
{
tid
:
{
uuid
}
})
def
test_answerTransactionInformation
(
self
):
conn
=
self
.
getFakeConnection
()
...
...
neo/tests/functional/__init__.py
View file @
d250deca
...
...
@@ -67,7 +67,7 @@ class NotFound(Exception):
class
PortAllocator
(
object
):
lock
=
SocketLock
(
'neo.PortAllocator'
)
allocator_set
=
weakref
.
Weak
KeyDictionary
()
# BBB: use WeakSet instead
allocator_set
=
weakref
.
Weak
Set
()
def
__init__
(
self
):
self
.
socket_list
=
[]
...
...
@@ -81,7 +81,7 @@ class PortAllocator(object):
s
.
setsockopt
(
socket
.
SOL_SOCKET
,
socket
.
SO_REUSEADDR
,
1
)
if
not
self
.
lock
.
locked
():
self
.
lock
.
acquire
()
self
.
allocator_set
[
self
]
=
None
self
.
allocator_set
.
add
(
self
)
self
.
socket_list
.
append
(
s
)
sock_port_set
=
self
.
sock_port_set
while
True
:
...
...
@@ -109,7 +109,7 @@ class PortAllocator(object):
def
reset
(
self
):
if
self
.
lock
.
locked
():
self
.
allocator_set
.
pop
(
self
,
None
)
self
.
allocator_set
.
discard
(
self
)
if
not
self
.
allocator_set
:
self
.
lock
.
release
()
self
.
release
()
...
...
@@ -126,7 +126,7 @@ class NEOProcess(object):
except
ImportError
:
raise
NotFound
,
'%s not found'
%
(
command
)
self
.
command
=
command
self
.
arg_dict
=
dict
((
'--'
+
k
,
v
)
for
k
,
v
in
arg_dict
.
iteritems
())
self
.
arg_dict
=
{
'--'
+
k
:
v
for
k
,
v
in
arg_dict
.
iteritems
()}
self
.
with_uuid
=
True
self
.
setUUID
(
uuid
)
...
...
neo/tests/master/testClientHandler.py
View file @
d250deca
...
...
@@ -186,7 +186,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
self
.
assertEqual
(
ptid
,
tid
)
self
.
assertTrue
(
self
.
app
.
packing
[
0
]
is
conn
)
self
.
assertEqual
(
self
.
app
.
packing
[
1
],
peer_id
)
self
.
assertEqual
(
self
.
app
.
packing
[
2
],
set
([
storage_uuid
,
])
)
self
.
assertEqual
(
self
.
app
.
packing
[
2
],
{
storage_uuid
}
)
# Asking again to pack will cause an immediate error
storage_uuid
=
self
.
identifyToMasterNode
(
port
=
10022
)
storage_conn
=
self
.
getFakeConnection
(
storage_uuid
,
...
...
neo/tests/master/testMasterPT.py
View file @
d250deca
...
...
@@ -220,7 +220,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
if
cell
.
isOutOfDate
():
pt
.
setUpToDate
(
cell
.
getNode
(),
offset
)
else
:
node_dict
=
dict
((
x
.
getUUID
(),
x
)
for
x
in
pt
.
count_dict
)
node_dict
=
{
x
.
getUUID
():
x
for
x
in
pt
.
count_dict
}
for
offset
,
uuid
,
state
in
change_list
:
if
state
is
CellStates
.
OUT_OF_DATE
:
pt
.
setUpToDate
(
node_dict
[
uuid
],
offset
)
...
...
neo/tests/master/testStorageHandler.py
View file @
d250deca
...
...
@@ -242,11 +242,11 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
'getPeerId'
:
512
,
})
client_peer_id
=
42
self
.
app
.
packing
=
(
client_conn
,
client_peer_id
,
set
([
conn1
.
getUUID
(),
conn2
.
getUUID
()])
)
self
.
app
.
packing
=
(
client_conn
,
client_peer_id
,
{
conn1
.
getUUID
(),
conn2
.
getUUID
()}
)
self
.
service
.
answerPack
(
conn1
,
False
)
self
.
checkNoPacketSent
(
client_conn
)
self
.
assertEqual
(
self
.
app
.
packing
[
2
],
set
([
conn2
.
getUUID
(),
])
)
self
.
assertEqual
(
self
.
app
.
packing
[
2
],
{
conn2
.
getUUID
()}
)
self
.
service
.
answerPack
(
conn2
,
False
)
status
=
self
.
checkAnswerPacket
(
client_conn
,
Packets
.
AnswerPack
,
decode
=
True
)[
0
]
...
...
neo/tests/storage/testStorageDBTests.py
View file @
d250deca
...
...
@@ -99,7 +99,7 @@ class StorageDBTests(NeoUnitTestBase):
cell2
=
(
1
,
uuid1
,
CellStates
.
UP_TO_DATE
)
db
.
setPartitionTable
(
ptid
,
[
cell1
,
cell2
])
result
=
db
.
getPartitionTable
()
self
.
assertEqual
(
set
(
result
),
set
([
cell1
,
cell2
])
)
self
.
assertEqual
(
set
(
result
),
{
cell1
,
cell2
}
)
def
getOIDs
(
self
,
count
):
return
map
(
self
.
getOID
,
xrange
(
count
))
...
...
@@ -253,7 +253,7 @@ class StorageDBTests(NeoUnitTestBase):
# add more entries
db
.
changePartitionTable
(
ptid
,
[
cell2
])
result
=
db
.
getPartitionTable
()
self
.
assertEqual
(
set
(
result
),
set
([
cell1
,
cell2
])
)
self
.
assertEqual
(
set
(
result
),
{
cell1
,
cell2
}
)
# drop discarded cells
db
.
changePartitionTable
(
ptid
,
[
cell2
,
cell3
])
result
=
db
.
getPartitionTable
()
...
...
neo/tests/testPT.py
View file @
d250deca
...
...
@@ -263,7 +263,7 @@ class PartitionTableTests(NeoUnitTestBase):
pt
.
setCell
(
0
,
sn4
,
CellStates
.
DISCARDED
)
# won't be added
# must get only two node as feeding and discarded not taken
# into account
self
.
assertEqual
(
pt
.
getNodeSet
(
True
),
set
((
sn1
,
sn3
))
)
self
.
assertEqual
(
pt
.
getNodeSet
(
True
),
{
sn1
,
sn3
}
)
self
.
assertEqual
(
len
(
pt
.
getNodeSet
()),
3
)
def
test_08_filled
(
self
):
...
...
neo/tests/threaded/__init__.py
View file @
d250deca
...
...
@@ -166,9 +166,9 @@ class Node(object):
def
getConnectionList
(
self
,
*
peers
):
addr
=
lambda
c
:
c
and
(
c
.
accepted_from
or
c
.
getAddress
())
addr_set
=
set
(
addr
(
c
.
connector
)
for
peer
in
peers
addr_set
=
{
addr
(
c
.
connector
)
for
peer
in
peers
for
c
in
peer
.
em
.
connection_dict
.
itervalues
()
if
isinstance
(
c
,
Connection
)
)
if
isinstance
(
c
,
Connection
)
}
addr_set
.
discard
(
None
)
return
(
c
for
c
in
self
.
em
.
connection_dict
.
itervalues
()
if
isinstance
(
c
,
Connection
)
and
addr
(
c
.
connector
)
in
addr_set
)
...
...
@@ -311,7 +311,7 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
checksum_dict
=
dict
(
dm
.
query
(
"SELECT id, hash FROM data"
))
assert
set
(
dm
.
_uncommitted_data
).
issubset
(
checksum_dict
)
get
=
dm
.
_uncommitted_data
.
get
return
dict
((
str
(
v
),
get
(
k
,
0
))
for
k
,
v
in
checksum_dict
.
iteritems
())
return
{
str
(
v
):
get
(
k
,
0
)
for
k
,
v
in
checksum_dict
.
iteritems
()}
class
ClientApplication
(
Node
,
neo
.
client
.
app
.
Application
):
...
...
neo/tests/threaded/testReplication.py
View file @
d250deca
...
...
@@ -66,7 +66,7 @@ class ReplicationTests(NEOThreadedTest):
np
=
pt
.
getPartitions
()
self
.
assertEqual
(
np
,
upstream_pt
.
getPartitions
())
checked
=
0
source_dict
=
dict
((
x
.
uuid
,
x
)
for
x
in
cluster
.
upstream
.
storage_list
)
source_dict
=
{
x
.
uuid
:
x
for
x
in
cluster
.
upstream
.
storage_list
}
for
storage
in
cluster
.
storage_list
:
self
.
assertEqual
(
np
,
storage
.
pt
.
getPartitions
())
for
partition
in
pt
.
getAssignedPartitionList
(
storage
.
uuid
):
...
...
@@ -349,7 +349,7 @@ class ReplicationTests(NEOThreadedTest):
cluster
.
start
()
cluster
.
populate
([
range
(
np
*
2
)]
*
tid_count
)
cluster
.
client
.
setPoll
(
0
)
storage_dict
=
dict
((
x
.
uuid
,
x
)
for
x
in
cluster
.
storage_list
)
storage_dict
=
{
x
.
uuid
:
x
for
x
in
cluster
.
storage_list
}
cluster
.
neoctl
.
checkReplicas
(
check_dict
,
ZERO_TID
,
None
)
cluster
.
tic
()
check
(
ClusterStates
.
RUNNING
,
0
)
...
...
setup.py
View file @
d250deca
...
...
@@ -9,7 +9,6 @@ Framework :: ZODB
Intended Audience :: Developers
License :: OSI Approved :: GNU General Public License (GPL)
Operating System :: POSIX :: Linux
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Topic :: Database
Topic :: Software Development :: Libraries :: Python Modules
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment