Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
d3780906
Commit
d3780906
authored
7 years ago
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
client: use a class instead of a simple dict to hold transaction information
parent
97e57031
Changes
7
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
158 additions
and
145 deletions
+158
-145
neo/client/app.py
neo/client/app.py
+49
-91
neo/client/handlers/master.py
neo/client/handlers/master.py
+1
-1
neo/client/handlers/storage.py
neo/client/handlers/storage.py
+10
-13
neo/client/pool.py
neo/client/pool.py
+0
-37
neo/client/transactions.py
neo/client/transactions.py
+95
-0
neo/tests/client/testClientApp.py
neo/tests/client/testClientApp.py
+1
-1
neo/tests/threaded/test.py
neo/tests/threaded/test.py
+2
-2
No files found.
neo/client/app.py
View file @
d3780906
This diff is collapsed.
Click to expand it.
neo/client/handlers/master.py
View file @
d3780906
...
...
@@ -147,7 +147,7 @@ class PrimaryNotificationsHandler(MTEventHandler):
logging
.
critical
(
msg
)
app
.
master_conn
=
None
for
txn_context
in
app
.
txn_contexts
():
txn_context
[
'error'
]
=
msg
txn_context
.
error
=
msg
try
:
del
app
.
pt
except
AttributeError
:
...
...
This diff is collapsed.
Click to expand it.
neo/client/handlers/storage.py
View file @
d3780906
...
...
@@ -22,7 +22,7 @@ from neo.lib.util import dump
from
neo.lib.exception
import
NodeNotReady
from
neo.lib.handler
import
MTEventHandler
from
.
import
AnswerBaseHandler
from
..
pool
import
InvolvedNodeDict
from
..
transactions
import
Transaction
from
..exception
import
NEOStorageError
,
NEOStorageNotFoundError
from
..exception
import
NEOStorageDoesNotExistError
...
...
@@ -82,26 +82,26 @@ class StorageAnswersHandler(AnswerBaseHandler):
if
conflict
!=
MAX_TID
:
# If this conflict is not already resolved, mark it for
# resolution.
if
conflict
<=
txn_context
[
'resolved_dict'
]
.
get
(
oid
,
''
):
if
conflict
<=
txn_context
.
resolved_dict
.
get
(
oid
,
''
):
return
txn_context
[
'conflict_dict'
]
[
oid
]
=
serial
,
conflict
txn_context
.
conflict_dict
[
oid
]
=
serial
,
conflict
else
:
try
:
data
=
txn_context
[
'data_dict'
]
.
pop
(
oid
)
data
=
txn_context
.
data_dict
.
pop
(
oid
)
except
KeyError
:
# replica, or multiple undo
return
if
type
(
data
)
is
str
:
size
=
len
(
data
)
txn_context
[
'data_size'
]
-=
size
size
+=
txn_context
[
'cache_size'
]
txn_context
.
data_size
-=
size
size
+=
txn_context
.
cache_size
if
size
<
self
.
app
.
_cache
.
_max_size
:
txn_context
[
'cache_size'
]
=
size
txn_context
.
cache_size
=
size
else
:
# Do not cache data past cache max size, as it
# would just flush it on tpc_finish. This also
# prevents memory errors for big transactions.
data
=
None
txn_context
[
'cache_dict'
]
[
oid
]
=
data
txn_context
.
cache_dict
[
oid
]
=
data
answerCheckCurrentSerial
=
answerStoreObject
...
...
@@ -112,11 +112,8 @@ class StorageAnswersHandler(AnswerBaseHandler):
def
connectionClosed
(
self
,
conn
):
txn_context
=
self
.
app
.
getHandlerData
()
# XXX: A 'Transaction' class would be cleaner.
if
type
(
txn_context
)
is
dict
:
involved_nodes
=
txn_context
.
get
(
'involved_nodes'
)
if
type
(
involved_nodes
)
is
InvolvedNodeDict
:
involved_nodes
[
conn
.
getUUID
()]
=
2
if
type
(
txn_context
)
is
Transaction
:
txn_context
.
involved_nodes
[
conn
.
getUUID
()]
=
2
super
(
StorageAnswersHandler
,
self
).
connectionClosed
(
conn
)
def
answerTIDsFrom
(
self
,
conn
,
tid_list
):
...
...
This diff is collapsed.
Click to expand it.
neo/client/pool.py
View file @
d3780906
...
...
@@ -37,23 +37,6 @@ CELL_GOOD = 0
CELL_FAILED
=
1
class
InvolvedNodeDict
(
dict
):
# Keys are node ids instead of Node objects because a node may disappear
# from the cluster. In any case, we always have to check if the id is
# still known by the NodeManager.
def
ask
(
self
,
conn
):
def
ask
(
*
args
,
**
kw
):
try
:
conn
.
ask
(
*
args
,
**
kw
)
except
ConnectionClosed
:
self
[
conn
.
getUUID
()]
=
2
else
:
self
.
fail
=
0
return
conn
.
getUUID
()
return
ask
class
ConnectionPool
(
object
):
"""This class manages a pool of connections to storage nodes."""
...
...
@@ -135,26 +118,6 @@ class ConnectionPool(object):
if
self
.
app
.
master_conn
is
None
:
raise
NEOPrimaryMasterLost
def
iterateForWrite
(
self
,
object_id
,
involved
,
store
=
1
):
pt
=
self
.
app
.
pt
involved
.
fail
=
1
for
cell
in
pt
.
getCellList
(
pt
.
getPartition
(
object_id
)):
node
=
cell
.
getNode
()
uuid
=
node
.
getUUID
()
status
=
involved
.
setdefault
(
uuid
,
store
)
if
status
<
store
:
involved
[
uuid
]
=
store
elif
status
>
1
:
continue
conn
=
self
.
getConnForNode
(
node
)
if
conn
is
None
:
involved
[
uuid
]
=
2
else
:
yield
involved
.
ask
(
conn
)
if
involved
.
fail
:
raise
NEOStorageError
(
'no storage available for write to partition %s'
%
object_id
)
def
getConnForNode
(
self
,
node
):
"""Return a locked connection object to a given node
If no connection exists, create a new one"""
...
...
This diff is collapsed.
Click to expand it.
neo/client/transactions.py
0 → 100644
View file @
d3780906
#
# Copyright (C) 2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from
ZODB.POSException
import
StorageTransactionError
from
neo.lib.connection
import
ConnectionClosed
from
neo.lib.locking
import
SimpleQueue
from
.exception
import
NEOStorageError
class
Transaction
(
object
):
cache_size
=
0
# size of data in cache_dict
data_size
=
0
# size of data in data_dict
error
=
None
voted
=
False
ttid
=
None
# XXX: useless, except for testBackupReadOnlyAccess
def
__init__
(
self
,
txn
):
self
.
queue
=
SimpleQueue
()
self
.
txn
=
txn
# data being stored
self
.
data_dict
=
{}
# data stored: this will go to the cache on tpc_finish
self
.
cache_dict
=
{}
# conflicts to resolve
self
.
conflict_dict
=
{}
# {oid: (base_serial, serial)}
# resolved conflicts
self
.
resolved_dict
=
{}
# {oid: serial}
# Keys are node ids instead of Node objects because a node may
# disappear from the cluster. In any case, we always have to check
# if the id is still known by the NodeManager.
# status: 0 -> check only, 1 -> store, 2 -> failed
self
.
involved_nodes
=
{}
# {node_id: status}
def
write
(
self
,
app
,
packet
,
object_id
,
store
=
1
,
**
kw
):
uuid_list
=
[]
pt
=
app
.
pt
involved
=
self
.
involved_nodes
object_id
=
pt
.
getPartition
(
object_id
)
for
cell
in
pt
.
getCellList
(
object_id
):
node
=
cell
.
getNode
()
uuid
=
node
.
getUUID
()
status
=
involved
.
setdefault
(
uuid
,
store
)
if
status
<
store
:
involved
[
uuid
]
=
store
elif
status
>
1
:
continue
conn
=
app
.
cp
.
getConnForNode
(
node
)
if
conn
is
not
None
:
try
:
conn
.
ask
(
packet
,
queue
=
self
.
queue
,
**
kw
)
uuid_list
.
append
(
uuid
)
continue
except
ConnectionClosed
:
pass
involved
[
uuid
]
=
2
if
uuid_list
:
return
uuid_list
raise
NEOStorageError
(
'no storage available for write to partition %s'
%
object_id
)
class
TransactionContainer
(
dict
):
# IDEA: Drop this container and use the new set_data/data API on
# transactions (requires transaction >= 1.6).
def
pop
(
self
,
txn
):
return
dict
.
pop
(
self
,
id
(
txn
),
None
)
def
get
(
self
,
txn
):
try
:
return
self
[
id
(
txn
)]
except
KeyError
:
raise
StorageTransactionError
(
"unknown transaction %r"
%
txn
)
def
new
(
self
,
txn
):
key
=
id
(
txn
)
if
key
in
self
:
raise
StorageTransactionError
(
"commit of transaction %r"
" already started"
%
txn
)
context
=
self
[
key
]
=
Transaction
(
txn
)
return
context
This diff is collapsed.
Click to expand it.
neo/tests/client/testClientApp.py
View file @
d3780906
...
...
@@ -70,7 +70,7 @@ class ClientApplicationTests(NeoUnitTestBase):
def
_begin
(
self
,
app
,
txn
,
tid
):
txn_context
=
app
.
_txn_container
.
new
(
txn
)
txn_context
[
'ttid'
]
=
tid
txn_context
.
ttid
=
tid
return
txn_context
def
getApp
(
self
,
master_nodes
=
None
,
name
=
'test'
,
**
kw
):
...
...
This diff is collapsed.
Click to expand it.
neo/tests/threaded/test.py
View file @
d3780906
...
...
@@ -370,7 +370,7 @@ class Test(NEOThreadedTest):
resolved
=
[]
last
=
lambda
txn
:
txn
.
_extension
[
'last'
]
# BBB
def
_handleConflicts
(
orig
,
txn_context
):
resolved
.
append
(
last
(
txn_context
[
'txn'
]
))
resolved
.
append
(
last
(
txn_context
.
txn
))
orig
(
txn_context
)
def
tpc_vote
(
orig
,
transaction
):
(
l3
if
last
(
transaction
)
else
l2
)()
...
...
@@ -969,7 +969,7 @@ class Test(NEOThreadedTest):
txn
=
transaction
.
Transaction
()
client
.
tpc_begin
(
None
,
txn
)
txn_context
=
client
.
_txn_container
.
get
(
txn
)
txn_context
[
'ttid'
]
=
add64
(
txn_context
[
'ttid'
]
,
1
)
txn_context
.
ttid
=
add64
(
txn_context
.
ttid
,
1
)
self
.
assertRaises
(
POSException
.
StorageError
,
client
.
tpc_finish
,
txn
)
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment