Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
nexedi
cython
Commits
67b3ef81
Commit
67b3ef81
authored
7 years ago
by
Stefan Behnel
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Optimise list.extend([a,b,c]) into fast sequential calls to list.append().
parent
3700e7f6
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
266 additions
and
22 deletions
+266
-22
CHANGES.rst
CHANGES.rst
+3
-0
Cython/Compiler/Nodes.py
Cython/Compiler/Nodes.py
+1
-1
Cython/Compiler/Optimize.py
Cython/Compiler/Optimize.py
+80
-13
Cython/Compiler/Visitor.py
Cython/Compiler/Visitor.py
+11
-4
tests/run/list.pyx
tests/run/list.pyx
+145
-4
tests/run/tuple.pyx
tests/run/tuple.pyx
+26
-0
No files found.
CHANGES.rst
View file @
67b3ef81
...
@@ -36,6 +36,9 @@ Features added
...
@@ -36,6 +36,9 @@ Features added
* Iteration over sets and frozensets is optimised.
* Iteration over sets and frozensets is optimised.
(Github issue #2048)
(Github issue #2048)
* ``alist.extend([a,b,c])`` is optimised into sequential ``list.append()`` calls
for short literal sequences.
* Calls to builtin methods that are not specifically optimised into C-API calls
* Calls to builtin methods that are not specifically optimised into C-API calls
now use a cache that avoids repeated lookups of the underlying C function.
now use a cache that avoids repeated lookups of the underlying C function.
(Github issue #2054)
(Github issue #2054)
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/Nodes.py
View file @
67b3ef81
...
@@ -5055,7 +5055,7 @@ class ExprStatNode(StatNode):
...
@@ -5055,7 +5055,7 @@ class ExprStatNode(StatNode):
self
.
expr
.
result_is_used
=
False
# hint that .result() may safely be left empty
self
.
expr
.
result_is_used
=
False
# hint that .result() may safely be left empty
self
.
expr
.
generate_evaluation_code
(
code
)
self
.
expr
.
generate_evaluation_code
(
code
)
if
not
self
.
expr
.
is_temp
and
self
.
expr
.
result
():
if
not
self
.
expr
.
is_temp
and
self
.
expr
.
result
():
code
.
putln
(
"
%s
;"
%
self
.
expr
.
result
())
code
.
putln
(
"
(void)(%s)
;"
%
self
.
expr
.
result
())
self
.
expr
.
generate_disposal_code
(
code
)
self
.
expr
.
generate_disposal_code
(
code
)
self
.
expr
.
free_temps
(
code
)
self
.
expr
.
free_temps
(
code
)
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/Optimize.py
View file @
67b3ef81
...
@@ -2046,11 +2046,18 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
...
@@ -2046,11 +2046,18 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
def
visit_ExprStatNode
(
self
,
node
):
def
visit_ExprStatNode
(
self
,
node
):
"""
"""
Drop useless coercions.
Drop
dead code and
useless coercions.
"""
"""
self
.
visitchildren
(
node
)
self
.
visitchildren
(
node
)
if
isinstance
(
node
.
expr
,
ExprNodes
.
CoerceToPyTypeNode
):
if
isinstance
(
node
.
expr
,
ExprNodes
.
CoerceToPyTypeNode
):
node
.
expr
=
node
.
expr
.
arg
node
.
expr
=
node
.
expr
.
arg
expr
=
node
.
expr
if
expr
is
None
or
expr
.
is_none
or
expr
.
is_literal
:
# Expression was removed or is dead code => remove ExprStatNode as well.
return
None
if
expr
.
is_name
and
expr
.
entry
and
(
expr
.
entry
.
is_local
or
expr
.
entry
.
is_arg
):
# Ignore dead references to local variables etc.
return
None
return
node
return
node
def
visit_CoerceToBooleanNode
(
self
,
node
):
def
visit_CoerceToBooleanNode
(
self
,
node
):
...
@@ -2815,6 +2822,62 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
...
@@ -2815,6 +2822,62 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
utility_code
=
load_c_utility
(
'append'
)
utility_code
=
load_c_utility
(
'append'
)
)
)
def
_handle_simple_method_list_extend
(
self
,
node
,
function
,
args
,
is_unbound_method
):
"""Replace list.extend([...]) for short sequence literals values by sequential appends
to avoid creating an intermediate sequence argument.
"""
if
len
(
args
)
!=
2
:
return
node
obj
,
value
=
args
if
not
value
.
is_sequence_constructor
or
value
.
mult_factor
is
not
None
:
return
node
items
=
list
(
value
.
args
)
if
len
(
items
)
>
4
:
# Appending wins for short sequences.
# Ignorantly assume that this a good enough limit that avoids repeated resizing.
return
node
wrapped_obj
=
self
.
_wrap_self_arg
(
obj
,
function
,
is_unbound_method
,
'extend'
)
if
not
items
:
# Empty sequences are not likely to occur, but why waste a call to list.extend() for them?
wrapped_obj
.
result_is_used
=
node
.
result_is_used
return
wrapped_obj
cloned_obj
=
obj
=
wrapped_obj
if
len
(
items
)
>
1
and
not
obj
.
is_simple
():
cloned_obj
=
UtilNodes
.
LetRefNode
(
obj
)
# Use ListComp_Append() for all but the last item and finish with PyList_Append()
# to shrink the list storage size at the very end if necessary.
temps
=
[]
arg
=
items
[
-
1
]
if
not
arg
.
is_simple
():
arg
=
UtilNodes
.
LetRefNode
(
arg
)
temps
.
append
(
arg
)
new_node
=
ExprNodes
.
PythonCapiCallNode
(
node
.
pos
,
"__Pyx_PyList_Append"
,
self
.
PyObject_Append_func_type
,
args
=
[
cloned_obj
,
arg
],
is_temp
=
True
,
utility_code
=
load_c_utility
(
"ListAppend"
))
for
arg
in
items
[
-
2
::
-
1
]:
if
not
arg
.
is_simple
():
arg
=
UtilNodes
.
LetRefNode
(
arg
)
temps
.
append
(
arg
)
new_node
=
ExprNodes
.
binop_node
(
node
.
pos
,
'|'
,
ExprNodes
.
PythonCapiCallNode
(
node
.
pos
,
"__Pyx_ListComp_Append"
,
self
.
PyObject_Append_func_type
,
args
=
[
cloned_obj
,
arg
],
py_name
=
"extend"
,
is_temp
=
True
,
utility_code
=
load_c_utility
(
"ListCompAppend"
)),
new_node
,
type
=
PyrexTypes
.
c_returncode_type
,
)
new_node
.
result_is_used
=
node
.
result_is_used
if
cloned_obj
is
not
obj
:
temps
.
append
(
cloned_obj
)
for
temp
in
temps
:
new_node
=
UtilNodes
.
EvalWithTempExprNode
(
temp
,
new_node
)
new_node
.
result_is_used
=
node
.
result_is_used
return
new_node
PyByteArray_Append_func_type
=
PyrexTypes
.
CFuncType
(
PyByteArray_Append_func_type
=
PyrexTypes
.
CFuncType
(
PyrexTypes
.
c_returncode_type
,
[
PyrexTypes
.
c_returncode_type
,
[
PyrexTypes
.
CFuncTypeArg
(
"bytearray"
,
PyrexTypes
.
py_object_type
,
None
),
PyrexTypes
.
CFuncTypeArg
(
"bytearray"
,
PyrexTypes
.
py_object_type
,
None
),
...
@@ -3791,18 +3854,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
...
@@ -3791,18 +3854,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
may_return_none
=
ExprNodes
.
PythonCapiCallNode
.
may_return_none
,
may_return_none
=
ExprNodes
.
PythonCapiCallNode
.
may_return_none
,
with_none_check
=
True
):
with_none_check
=
True
):
args
=
list
(
args
)
args
=
list
(
args
)
if
with_none_check
and
args
and
not
args
[
0
].
is_literal
:
if
with_none_check
and
args
:
self_arg
=
args
[
0
]
args
[
0
]
=
self
.
_wrap_self_arg
(
args
[
0
],
function
,
is_unbound_method
,
attr_name
)
if
is_unbound_method
:
self_arg
=
self_arg
.
as_none_safe_node
(
"descriptor '%s' requires a '%s' object but received a 'NoneType'"
,
format_args
=
[
attr_name
,
function
.
obj
.
name
])
else
:
self_arg
=
self_arg
.
as_none_safe_node
(
"'NoneType' object has no attribute '%{0}s'"
.
format
(
'.30'
if
len
(
attr_name
)
<=
30
else
''
),
error
=
"PyExc_AttributeError"
,
format_args
=
[
attr_name
])
args
[
0
]
=
self_arg
if
is_temp
is
None
:
if
is_temp
is
None
:
is_temp
=
node
.
is_temp
is_temp
=
node
.
is_temp
return
ExprNodes
.
PythonCapiCallNode
(
return
ExprNodes
.
PythonCapiCallNode
(
...
@@ -3814,6 +3867,20 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
...
@@ -3814,6 +3867,20 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
result_is_used
=
node
.
result_is_used
,
result_is_used
=
node
.
result_is_used
,
)
)
def
_wrap_self_arg
(
self
,
self_arg
,
function
,
is_unbound_method
,
attr_name
):
if
self_arg
.
is_literal
:
return
self_arg
if
is_unbound_method
:
self_arg
=
self_arg
.
as_none_safe_node
(
"descriptor '%s' requires a '%s' object but received a 'NoneType'"
,
format_args
=
[
attr_name
,
self_arg
.
type
.
name
])
else
:
self_arg
=
self_arg
.
as_none_safe_node
(
"'NoneType' object has no attribute '%{0}s'"
.
format
(
'.30'
if
len
(
attr_name
)
<=
30
else
''
),
error
=
"PyExc_AttributeError"
,
format_args
=
[
attr_name
])
return
self_arg
def
_inject_int_default_argument
(
self
,
node
,
args
,
arg_index
,
type
,
default_value
):
def
_inject_int_default_argument
(
self
,
node
,
args
,
arg_index
,
type
,
default_value
):
assert
len
(
args
)
>=
arg_index
assert
len
(
args
)
>=
arg_index
if
len
(
args
)
==
arg_index
:
if
len
(
args
)
==
arg_index
:
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/Visitor.py
View file @
67b3ef81
...
@@ -581,15 +581,23 @@ class MethodDispatcherTransform(EnvTransform):
...
@@ -581,15 +581,23 @@ class MethodDispatcherTransform(EnvTransform):
# into a C function call (defined in the builtin scope)
# into a C function call (defined in the builtin scope)
if
not
function
.
entry
:
if
not
function
.
entry
:
return
node
return
node
entry
=
function
.
entry
is_builtin
=
(
is_builtin
=
(
function
.
entry
.
is_builtin
or
entry
.
is_builtin
or
function
.
entry
is
self
.
current_env
().
builtin_scope
().
lookup_here
(
function
.
name
))
entry
is
self
.
current_env
().
builtin_scope
().
lookup_here
(
function
.
name
))
if
not
is_builtin
:
if
not
is_builtin
:
if
function
.
cf_state
and
function
.
cf_state
.
is_single
:
if
function
.
cf_state
and
function
.
cf_state
.
is_single
:
# we know the value of the variable
# we know the value of the variable
# => see if it's usable instead
# => see if it's usable instead
return
self
.
_delegate_to_assigned_value
(
return
self
.
_delegate_to_assigned_value
(
node
,
function
,
arg_list
,
kwargs
)
node
,
function
,
arg_list
,
kwargs
)
if
arg_list
and
entry
.
is_cmethod
and
entry
.
scope
and
entry
.
scope
.
parent_type
.
is_builtin_type
:
if
entry
.
scope
.
parent_type
is
arg_list
[
0
].
type
:
# Optimised (unbound) method of a builtin type => try to "de-optimise".
return
self
.
_dispatch_to_method_handler
(
entry
.
name
,
self_arg
=
None
,
is_unbound_method
=
True
,
type_name
=
entry
.
scope
.
parent_type
.
name
,
node
=
node
,
function
=
function
,
arg_list
=
arg_list
,
kwargs
=
kwargs
)
return
node
return
node
function_handler
=
self
.
_find_handler
(
function_handler
=
self
.
_find_handler
(
"function_%s"
%
function
.
name
,
kwargs
)
"function_%s"
%
function
.
name
,
kwargs
)
...
@@ -615,8 +623,7 @@ class MethodDispatcherTransform(EnvTransform):
...
@@ -615,8 +623,7 @@ class MethodDispatcherTransform(EnvTransform):
obj_type
=
self_arg
.
type
obj_type
=
self_arg
.
type
is_unbound_method
=
False
is_unbound_method
=
False
if
obj_type
.
is_builtin_type
:
if
obj_type
.
is_builtin_type
:
if
(
obj_type
is
Builtin
.
type_type
and
self_arg
.
is_name
and
if
obj_type
is
Builtin
.
type_type
and
self_arg
.
is_name
and
arg_list
and
arg_list
[
0
].
type
.
is_pyobject
:
arg_list
and
arg_list
[
0
].
type
.
is_pyobject
):
# calling an unbound method like 'list.append(L,x)'
# calling an unbound method like 'list.append(L,x)'
# (ignoring 'type.mro()' here ...)
# (ignoring 'type.mro()' here ...)
type_name
=
self_arg
.
name
type_name
=
self_arg
.
name
...
...
This diff is collapsed.
Click to expand it.
tests/run/list.pyx
View file @
67b3ef81
...
@@ -79,6 +79,10 @@ def test_list_reverse():
...
@@ -79,6 +79,10 @@ def test_list_reverse():
l1
.
reverse
()
l1
.
reverse
()
return
l1
return
l1
@
cython
.
test_assert_path_exists
(
'//SimpleCallNode//AttributeNode[@entry.cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_append
():
def
test_list_append
():
"""
"""
>>> test_list_append()
>>> test_list_append()
...
@@ -89,6 +93,36 @@ def test_list_append():
...
@@ -89,6 +93,36 @@ def test_list_append():
l1
.
append
(
4
)
l1
.
append
(
4
)
return
l1
return
l1
@
cython
.
test_assert_path_exists
(
'//SimpleCallNode//NameNode[@entry.cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_append_unbound
():
"""
>>> test_list_append_unbound()
[1, 2, 3, 4]
"""
cdef
list
l1
=
[
1
,
2
]
list
.
append
(
l1
,
3
)
list
.
append
(
l1
,
4
)
return
l1
@
cython
.
test_assert_path_exists
(
'//SimpleCallNode//NameNode[@entry.cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_append_unbound_assigned
():
"""
>>> test_list_append_unbound_assigned()
[1, 2, 3, 4]
"""
append
=
list
.
append
cdef
list
l1
=
[
1
,
2
]
append
(
l1
,
3
)
append
(
l1
,
4
)
return
l1
def
test_list_append_insert
():
def
test_list_append_insert
():
"""
"""
>>> test_list_append_insert()
>>> test_list_append_insert()
...
@@ -138,20 +172,127 @@ def test_list_pop_all():
...
@@ -138,20 +172,127 @@ def test_list_pop_all():
return
i
==
2
return
i
==
2
return
False
return
False
def
test_list_extend
():
@
cython
.
test_assert_path_exists
(
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]'
,
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_extend
(
seq
=
None
,
x
=
4
):
"""
"""
>>> test_list_extend()
>>> test_list_extend()
[1, 2, 3, 4, 5, 6]
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
>>> test_list_extend([])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
>>> test_list_extend([1])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 1]
>>> test_list_extend([1, 2])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 1, 2]
"""
"""
cdef
list
l
=
[
1
,
2
,
3
]
cdef
list
l
=
[
1
,
2
,
3
]
l
.
extend
([])
l
.
extend
([])
l
.
extend
(())
l
.
extend
(())
l
.
extend
(
set
())
l
.
extend
(
set
())
# not currently optimised (not worth the trouble)
assert
l
==
[
1
,
2
,
3
]
assert
l
==
[
1
,
2
,
3
]
assert
len
(
l
)
==
3
assert
len
(
l
)
==
3
l
.
extend
([
4
,
5
,
6
])
l
.
extend
([
4
,
x
+
1
,
6
])
l
.
extend
([
7
,
8
,
9
,
10
,
11
,
12
,
13
,
14
])
if
seq
is
not
None
:
l
.
extend
(
seq
)
return
l
return
l
@
cython
.
test_assert_path_exists
(
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]'
,
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_extend_unbound
(
seq
=
None
,
x
=
4
):
"""
>>> test_list_extend_unbound()
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
>>> test_list_extend_unbound([])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
>>> test_list_extend_unbound([1])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 1]
>>> test_list_extend_unbound([1, 2])
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 1, 2]
"""
cdef
list
l
=
[
1
,
2
,
3
]
list
.
extend
(
l
,
[])
list
.
extend
(
l
,
())
try
:
list
.
extend
((),
())
except
TypeError
:
pass
else
:
assert
False
,
"TypeError not raised!"
list
.
extend
(
l
,
set
())
# not currently optimised (not worth the trouble)
assert
l
==
[
1
,
2
,
3
]
assert
len
(
l
)
==
3
list
.
extend
(
l
,
[
4
,
x
+
1
,
6
])
list
.
extend
(
l
,
[
7
,
8
,
9
,
10
,
11
,
12
,
13
,
14
])
if
seq
is
not
None
:
list
.
extend
(
l
,
seq
)
return
l
@
cython
.
test_assert_path_exists
(
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]'
,
'//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]'
,
)
def
test_list_extend_sideeffect
(
seq
=
None
,
exc
=
False
):
"""
>>> test_list_extend_sideeffect()
([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [4, 6, 7, 8])
>>> test_list_extend_sideeffect([])
([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [4, 6, 7, 8])
>>> test_list_extend_sideeffect([], exc=True)
([1, 2, 3, 10, 11, 12, 13, 14, 15, 16], [4, 7, 8])
>>> test_list_extend_sideeffect([1])
([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1], [4, 6, 7, 8])
>>> test_list_extend_sideeffect([1], exc=True)
([1, 2, 3, 10, 11, 12, 13, 14, 15, 16, 1], [4, 7, 8])
>>> test_list_extend_sideeffect([1, 2])
([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2], [4, 6, 7, 8])
"""
calls
=
[]
def
sideeffect
(
value
):
calls
.
append
(
value
)
return
value
def
fail
(
value
):
if
exc
:
raise
TypeError
(
"HUHU"
)
return
value
cdef
list
l
=
[
1
,
2
,
3
]
l
.
extend
([])
l
.
extend
(())
l
.
extend
(
set
())
# not currently optimised (not worth the trouble)
assert
l
==
[
1
,
2
,
3
]
assert
len
(
l
)
==
3
# Must first build all items, then append them in order.
# If building one value fails, none of them must be appended.
try
:
l
.
extend
([
sideeffect
(
4
),
fail
(
5
),
sideeffect
(
6
)])
except
TypeError
as
e
:
assert
exc
assert
"HUHU"
in
str
(
e
)
else
:
assert
not
exc
try
:
l
.
extend
([
sideeffect
(
7
),
sideeffect
(
8
),
fail
(
9
)])
except
TypeError
as
e
:
assert
exc
assert
"HUHU"
in
str
(
e
)
else
:
assert
not
exc
l
.
extend
([
10
,
11
,
12
,
13
,
14
,
15
,
16
])
if
seq
is
not
None
:
l
.
extend
(
seq
)
return
l
,
calls
def
test_none_list_extend
(
list
l
):
def
test_none_list_extend
(
list
l
):
"""
"""
>>> test_none_list_extend([])
>>> test_none_list_extend([])
...
...
This diff is collapsed.
Click to expand it.
tests/run/tuple.pyx
View file @
67b3ef81
...
@@ -139,3 +139,29 @@ def tuple_of_tuple_or_none(tuple x):
...
@@ -139,3 +139,29 @@ def tuple_of_tuple_or_none(tuple x):
TypeError: ...itera...
TypeError: ...itera...
"""
"""
return
tuple
(
tuple
(
tuple
(
x
)))
return
tuple
(
tuple
(
tuple
(
x
)))
@
cython
.
test_fail_if_path_exists
(
"//ExprStatNode//TupleNode"
,
"//ExprStatNode"
,
)
def
unused_literals
():
"""
>>> unused_literals()
"""
(
1
,
2
,
3
)
(
1
,
2
,
3
+
4
)
(
"abc"
,
'def'
)
#(int(), 2, 3)
@
cython
.
test_assert_path_exists
(
"//ExprStatNode"
,
"//ExprStatNode//TupleNode"
,
)
def
unused_non_literal
():
"""
>>> unused_non_literal()
"""
(
set
(),
None
)
(
range
(
10
),
None
)
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment