Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
P
pyodide
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Boxiang Sun
pyodide
Commits
d4c92a5d
Commit
d4c92a5d
authored
Dec 27, 2018
by
Michael Droettboom
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Convert Numpy arrays to TypedArrays when possible
parent
09d8afea
Changes
8
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
702 additions
and
327 deletions
+702
-327
Makefile
Makefile
+1
-1
src/hiwire.c
src/hiwire.c
+46
-0
src/hiwire.h
src/hiwire.h
+94
-0
src/python2js.c
src/python2js.c
+2
-312
src/python2js_buffer.c
src/python2js_buffer.c
+492
-0
src/python2js_buffer.h
src/python2js_buffer.h
+18
-0
test/packages/test_numpy.py
test/packages/test_numpy.py
+10
-5
test/test_python.py
test/test_python.py
+39
-9
No files found.
Makefile
View file @
d4c92a5d
...
@@ -63,7 +63,7 @@ all: build/pyodide.asm.js \
...
@@ -63,7 +63,7 @@ all: build/pyodide.asm.js \
build/pyodide.asm.js
:
src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc
\
build/pyodide.asm.js
:
src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc
\
src/pyimport.bc src/pyproxy.bc src/python2js.bc
\
src/pyimport.bc src/pyproxy.bc src/python2js.bc
src/python2js_buffer.bc
\
src/runpython.bc src/hiwire.bc
src/runpython.bc src/hiwire.bc
[
-d
build
]
||
mkdir
build
[
-d
build
]
||
mkdir
build
$(CXX)
-s
EXPORT_NAME
=
"'pyodide'"
-o
build/pyodide.asm.html
$(
filter
%.bc,
$^
)
\
$(CXX)
-s
EXPORT_NAME
=
"'pyodide'"
-o
build/pyodide.asm.html
$(
filter
%.bc,
$^
)
\
...
...
src/hiwire.c
View file @
d4c92a5d
...
@@ -96,6 +96,46 @@ EM_JS(int, hiwire_bytes, (int ptr, int len), {
...
@@ -96,6 +96,46 @@ EM_JS(int, hiwire_bytes, (int ptr, int len), {
return
Module
.
hiwire_new_value
(
bytes
);
return
Module
.
hiwire_new_value
(
bytes
);
});
});
EM_JS
(
int
,
hiwire_int8array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Int8Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_uint8array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Uint8Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_int16array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Int16Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_uint16array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Uint16Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_int32array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Int32Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_uint32array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Uint32Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_float32array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Float32Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
EM_JS
(
int
,
hiwire_float64array
,
(
int
ptr
,
int
len
),
{
var
array
=
new
Float64Array
(
Module
.
HEAPU8
.
buffer
,
ptr
,
len
);
return
Module
.
hiwire_new_value
(
array
);
})
int
int
hiwire_undefined
()
hiwire_undefined
()
{
{
...
@@ -345,3 +385,9 @@ EM_JS(int, hiwire_get_dtype, (int idobj), {
...
@@ -345,3 +385,9 @@ EM_JS(int, hiwire_get_dtype, (int idobj), {
}
}
return
dtype
;
return
dtype
;
});
});
EM_JS
(
int
,
hiwire_subarray
,
(
int
idarr
,
int
start
,
int
end
),
{
var
jsarr
=
Module
.
hiwire_get_value
(
idarr
);
var
jssub
=
jsarr
.
subarray
(
start
,
end
);
return
Module
.
hiwire_new_value
(
jssub
);
});
src/hiwire.h
View file @
d4c92a5d
...
@@ -109,6 +109,94 @@ hiwire_string_ascii(int ptr);
...
@@ -109,6 +109,94 @@ hiwire_string_ascii(int ptr);
int
int
hiwire_bytes
(
int
ptr
,
int
len
);
hiwire_bytes
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Int8Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_int8array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Uint8Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_uint8array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Int16Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_int16array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Uint16Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_uint16array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Int32Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_int32array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Uint32Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_uint32array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Float32Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_float32array
(
int
ptr
,
int
len
);
/**
* Create a new Javascript Float64Array, given a pointer to a buffer and a
* length, in bytes.
*
* The array's data is not copied.
*
* Returns: New reference
*/
int
hiwire_float64array
(
int
ptr
,
int
len
);
/**
/**
* Create a new Javascript undefined value.
* Create a new Javascript undefined value.
*
*
...
@@ -435,4 +523,10 @@ hiwire_copy_to_ptr(int idobj, int ptr);
...
@@ -435,4 +523,10 @@ hiwire_copy_to_ptr(int idobj, int ptr);
int
int
hiwire_get_dtype
(
int
idobj
);
hiwire_get_dtype
(
int
idobj
);
/**
* Get a subarray from a TypedArray
*/
int
hiwire_subarray
(
int
idarr
,
int
start
,
int
end
);
#endif
/* HIWIRE_H */
#endif
/* HIWIRE_H */
src/python2js.c
View file @
d4c92a5d
...
@@ -2,13 +2,12 @@
...
@@ -2,13 +2,12 @@
#include <emscripten.h>
#include <emscripten.h>
#include <endian.h>
#include <stdint.h>
#include "hiwire.h"
#include "hiwire.h"
#include "jsproxy.h"
#include "jsproxy.h"
#include "pyproxy.h"
#include "pyproxy.h"
#include "python2js_buffer.h"
static
PyObject
*
tbmod
=
NULL
;
static
PyObject
*
tbmod
=
NULL
;
static
int
static
int
...
@@ -234,315 +233,6 @@ _python2js_dict(PyObject* x, PyObject* map)
...
@@ -234,315 +233,6 @@ _python2js_dict(PyObject* x, PyObject* map)
return
jsdict
;
return
jsdict
;
}
}
typedef
int
(
scalar_converter
)(
char
*
);
static
int
_convert_bool
(
char
*
data
)
{
char
v
=
*
((
char
*
)
data
);
if
(
v
)
{
return
hiwire_true
();
}
else
{
return
hiwire_false
();
}
}
static
int
_convert_int8
(
char
*
data
)
{
int8_t
v
=
*
((
int8_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_uint8
(
char
*
data
)
{
uint8_t
v
=
*
((
uint8_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_int16
(
char
*
data
)
{
int16_t
v
=
*
((
int16_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_int16_swap
(
char
*
data
)
{
int16_t
v
=
*
((
int16_t
*
)
data
);
return
hiwire_int
(
be16toh
(
v
));
}
static
int
_convert_uint16
(
char
*
data
)
{
uint16_t
v
=
*
((
uint16_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_uint16_swap
(
char
*
data
)
{
uint16_t
v
=
*
((
uint16_t
*
)
data
);
return
hiwire_int
(
be16toh
(
v
));
}
static
int
_convert_int32
(
char
*
data
)
{
int32_t
v
=
*
((
int32_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_int32_swap
(
char
*
data
)
{
int32_t
v
=
*
((
int32_t
*
)
data
);
return
hiwire_int
(
be32toh
(
v
));
}
static
int
_convert_uint32
(
char
*
data
)
{
uint32_t
v
=
*
((
uint32_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_uint32_swap
(
char
*
data
)
{
uint32_t
v
=
*
((
uint32_t
*
)
data
);
return
hiwire_int
(
be32toh
(
v
));
}
static
int
_convert_int64
(
char
*
data
)
{
int64_t
v
=
*
((
int64_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_int64_swap
(
char
*
data
)
{
int64_t
v
=
*
((
int64_t
*
)
data
);
return
hiwire_int
(
be64toh
(
v
));
}
static
int
_convert_uint64
(
char
*
data
)
{
uint64_t
v
=
*
((
uint64_t
*
)
data
);
return
hiwire_int
(
v
);
}
static
int
_convert_uint64_swap
(
char
*
data
)
{
uint64_t
v
=
*
((
uint64_t
*
)
data
);
return
hiwire_int
(
be64toh
(
v
));
}
static
int
_convert_float32
(
char
*
data
)
{
float
v
=
*
((
float
*
)
data
);
return
hiwire_double
(
v
);
}
static
int
_convert_float32_swap
(
char
*
data
)
{
union
float32_t
{
uint32_t
i
;
float
f
;
}
v
;
v
.
f
=
*
((
float
*
)
data
);
v
.
i
=
be32toh
(
v
.
i
);
return
hiwire_double
(
v
.
f
);
}
static
int
_convert_float64
(
char
*
data
)
{
double
v
=
*
((
double
*
)
data
);
return
hiwire_double
(
v
);
}
static
int
_convert_float64_swap
(
char
*
data
)
{
union
float64_t
{
uint64_t
i
;
double
f
;
}
v
;
v
.
f
=
*
((
double
*
)
data
);
v
.
i
=
be64toh
(
v
.
i
);
return
hiwire_double
(
v
.
f
);
}
static
scalar_converter
*
_python2js_buffer_get_converter
(
Py_buffer
*
buff
)
{
// Uses Python's struct typecodes as defined here:
// https://docs.python.org/3.7/library/array.html
char
format
;
char
swap
;
if
(
buff
->
format
==
NULL
)
{
swap
=
0
;
format
=
'B'
;
}
else
{
switch
(
buff
->
format
[
0
])
{
case
'>'
:
case
'!'
:
swap
=
1
;
format
=
buff
->
format
[
1
];
break
;
case
'='
:
case
'<'
:
case
'@'
:
swap
=
0
;
format
=
buff
->
format
[
1
];
break
;
default:
swap
=
0
;
format
=
buff
->
format
[
0
];
}
}
switch
(
format
)
{
case
'c'
:
case
'b'
:
return
_convert_int8
;
case
'B'
:
return
_convert_uint8
;
case
'?'
:
return
_convert_bool
;
case
'h'
:
if
(
swap
)
{
return
_convert_int16_swap
;
}
else
{
return
_convert_int16
;
}
case
'H'
:
if
(
swap
)
{
return
_convert_uint16_swap
;
}
else
{
return
_convert_uint16
;
}
case
'i'
:
case
'l'
:
case
'n'
:
if
(
swap
)
{
return
_convert_int32_swap
;
}
else
{
return
_convert_int32
;
}
case
'I'
:
case
'L'
:
case
'N'
:
if
(
swap
)
{
return
_convert_uint32_swap
;
}
else
{
return
_convert_uint32
;
}
case
'q'
:
if
(
swap
)
{
return
_convert_int64_swap
;
}
else
{
return
_convert_int64
;
}
case
'Q'
:
if
(
swap
)
{
return
_convert_uint64_swap
;
}
else
{
return
_convert_uint64
;
}
case
'f'
:
if
(
swap
)
{
return
_convert_float32_swap
;
}
else
{
return
_convert_float32
;
}
case
'd'
:
if
(
swap
)
{
return
_convert_float64_swap
;
}
else
{
return
_convert_float64
;
}
default:
return
NULL
;
}
}
static
int
_python2js_buffer_recursive
(
Py_buffer
*
buff
,
char
*
ptr
,
int
dim
,
scalar_converter
*
convert
)
{
// This function is basically a manual conversion of `recursive_tolist` in
// Numpy to use the Python buffer interface and output Javascript.
Py_ssize_t
i
,
n
,
stride
;
int
jsarray
,
jsitem
;
if
(
dim
>=
buff
->
ndim
)
{
return
convert
(
ptr
);
}
n
=
buff
->
shape
[
dim
];
stride
=
buff
->
strides
[
dim
];
jsarray
=
hiwire_array
();
for
(
i
=
0
;
i
<
n
;
++
i
)
{
jsitem
=
_python2js_buffer_recursive
(
buff
,
ptr
,
dim
+
1
,
convert
);
if
(
jsitem
==
HW_ERROR
)
{
hiwire_decref
(
jsarray
);
return
HW_ERROR
;
}
hiwire_push_array
(
jsarray
,
jsitem
);
hiwire_decref
(
jsitem
);
ptr
+=
stride
;
}
return
jsarray
;
}
static
int
_python2js_buffer
(
PyObject
*
x
)
{
PyObject
*
memoryview
=
PyMemoryView_FromObject
(
x
);
if
(
memoryview
==
NULL
)
{
PyErr_Clear
();
return
HW_ERROR
;
}
Py_buffer
*
buff
;
buff
=
PyMemoryView_GET_BUFFER
(
memoryview
);
scalar_converter
*
convert
=
_python2js_buffer_get_converter
(
buff
);
if
(
convert
==
NULL
)
{
Py_DECREF
(
memoryview
);
return
HW_ERROR
;
}
int
result
=
_python2js_buffer_recursive
(
buff
,
buff
->
buf
,
0
,
convert
);
Py_DECREF
(
memoryview
);
return
result
;
}
static
int
static
int
_python2js
(
PyObject
*
x
,
PyObject
*
map
)
_python2js
(
PyObject
*
x
,
PyObject
*
map
)
{
{
...
...
src/python2js_buffer.c
0 → 100644
View file @
d4c92a5d
This diff is collapsed.
Click to expand it.
src/python2js_buffer.h
0 → 100644
View file @
d4c92a5d
#ifndef PYTHON2JS_BUFFER_H
#define PYTHON2JS_BUFFER_H
/** Utilities to convert Python buffer objects to Javascript.
*/
#include <Python.h>
/** Convert a Python buffer object to a Javascript object.
*
* \param The Python object
* \return The Javascript object -- might be an Error object in the case of an
* exception.
*/
int
_python2js_buffer
(
PyObject
*
x
);
#endif
/* PYTHON2JS_BUFFER_H */
test/packages/test_numpy.py
View file @
d4c92a5d
def
test_numpy
(
selenium
):
def
test_numpy
(
selenium
):
selenium
.
load_package
(
"numpy"
)
selenium
.
load_package
(
"numpy"
)
selenium
.
run
(
"import numpy"
)
selenium
.
run
(
"import numpy"
)
x
=
selenium
.
run
(
"numpy.zeros((32, 64))"
)
selenium
.
run
(
"x = numpy.zeros((32, 64))"
)
assert
len
(
x
)
==
32
assert
selenium
.
run_js
(
"return pyodide.pyimport('x').length == 32"
)
assert
all
(
len
(
y
)
==
64
for
y
in
x
)
for
i
in
range
(
32
):
for
y
in
x
:
assert
selenium
.
run_js
(
assert
all
(
z
==
0
for
z
in
y
)
f"return pyodide.pyimport('x')[
{
i
}
].length == 64"
)
for
j
in
range
(
64
):
assert
selenium
.
run_js
(
f"return pyodide.pyimport('x')[
{
i
}
][
{
j
}
] == 0"
)
def
test_typed_arrays
(
selenium
):
def
test_typed_arrays
(
selenium
):
...
...
test/test_python.py
View file @
d4c92a5d
...
@@ -76,6 +76,16 @@ def test_python2js_numpy_dtype(selenium_standalone):
...
@@ -76,6 +76,16 @@ def test_python2js_numpy_dtype(selenium_standalone):
expected_result
=
[[[
0
,
1
],
[
2
,
3
]],
expected_result
=
[[[
0
,
1
],
[
2
,
3
]],
[[
4
,
5
],
[
6
,
7
]]]
[[
4
,
5
],
[
6
,
7
]]]
def
assert_equal
():
# We have to do this an element at a time, since the Selenium driver
# for Firefox does not convert TypedArrays to Python correctly
for
i
in
range
(
2
):
for
j
in
range
(
2
):
for
k
in
range
(
2
):
assert
selenium
.
run_js
(
f"return pyodide.pyimport('x')[
{
i
}
][
{
j
}
][
{
k
}
]"
)
==
expected_result
[
i
][
j
][
k
]
for
order
in
(
'C'
,
'F'
):
for
order
in
(
'C'
,
'F'
):
for
dtype
in
(
for
dtype
in
(
'int8'
,
'int8'
,
...
@@ -89,19 +99,36 @@ def test_python2js_numpy_dtype(selenium_standalone):
...
@@ -89,19 +99,36 @@ def test_python2js_numpy_dtype(selenium_standalone):
'float32'
,
'float32'
,
'float64'
'float64'
):
):
assert
selenium
.
run
(
selenium
.
run
(
f"""
f"""
x = np.arange(8, dtype=np.
{
dtype
}
)
x = np.arange(8, dtype=np.
{
dtype
}
)
x = x.reshape((2, 2, 2))
x = x.reshape((2, 2, 2))
x = x.copy(
{
order
!
r
}
)
x = x.copy(
{
order
!
r
}
)
x
"""
"""
)
==
expected_result
)
assert
selenium
.
run
(
assert_equal
()
classname
=
selenium
.
run_js
(
"return pyodide.pyimport('x')[0][0].constructor.name"
)
if
order
==
'C'
and
dtype
not
in
(
'uint64'
,
'int64'
):
assert
classname
.
endswith
(
'Array'
)
assert
classname
!=
'Array'
else
:
assert
classname
==
'Array'
selenium
.
run
(
"""
"""
x.byteswap().newbyteorder()
x
= x
.byteswap().newbyteorder()
"""
"""
)
==
expected_result
)
assert_equal
()
classname
=
selenium
.
run_js
(
"return pyodide.pyimport('x')[0][0].constructor.name"
)
if
order
==
'C'
and
dtype
in
(
'int8'
,
'uint8'
):
assert
classname
.
endswith
(
'Array'
)
assert
classname
!=
'Array'
else
:
assert
classname
==
'Array'
assert
selenium
.
run
(
"np.array([True, False])"
)
==
[
True
,
False
]
assert
selenium
.
run
(
"np.array([True, False])"
)
==
[
True
,
False
]
...
@@ -532,13 +559,16 @@ def test_recursive_dict(selenium_standalone):
...
@@ -532,13 +559,16 @@ def test_recursive_dict(selenium_standalone):
def
test_runpythonasync
(
selenium_standalone
):
def
test_runpythonasync
(
selenium_standalone
):
output
=
selenium_standalone
.
run_async
(
selenium_standalone
.
run_async
(
"""
"""
import numpy as np
import numpy as np
np.zeros(5)
x =
np.zeros(5)
"""
"""
)
)
assert
list
(
output
)
==
[
0
,
0
,
0
,
0
,
0
]
for
i
in
range
(
5
):
assert
selenium_standalone
.
run_js
(
f"return pyodide.pyimport('x')[
{
i
}
] == 0"
)
def
test_runpythonasync_different_package_name
(
selenium_standalone
):
def
test_runpythonasync_different_package_name
(
selenium_standalone
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment