Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
O
olapy
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
nexedi
olapy
Commits
cd49e537
Commit
cd49e537
authored
Jul 17, 2017
by
mouadh
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
format
parent
09c6d728
Changes
16
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
16 changed files
with
265 additions
and
172 deletions
+265
-172
micro_bench/__init__.py
micro_bench/__init__.py
+98
-36
micro_bench/cube_generator.py
micro_bench/cube_generator.py
+22
-11
micro_bench/micro_bench.py
micro_bench/micro_bench.py
+2
-1
olapy/core/mdx/executor/execute.py
olapy/core/mdx/executor/execute.py
+21
-17
olapy/core/mdx/executor/execute_config_file.py
olapy/core/mdx/executor/execute_config_file.py
+24
-8
olapy/core/mdx/executor/execute_csv_files.py
olapy/core/mdx/executor/execute_csv_files.py
+1
-0
olapy/core/mdx/executor/execute_db.py
olapy/core/mdx/executor/execute_db.py
+9
-5
olapy/core/mdx/tools/config_file_parser.py
olapy/core/mdx/tools/config_file_parser.py
+8
-8
olapy/core/mdx/tools/connection.py
olapy/core/mdx/tools/connection.py
+10
-7
olapy/core/mdx/tools/olapy_config_file_parser.py
olapy/core/mdx/tools/olapy_config_file_parser.py
+7
-9
olapy/core/services/models.py
olapy/core/services/models.py
+2
-3
olapy/core/services/xmla.py
olapy/core/services/xmla.py
+13
-16
olapy/core/services/xmla_discover_tools.py
olapy/core/services/xmla_discover_tools.py
+11
-15
olapy/core/services/xmla_execute_tools.py
olapy/core/services/xmla_execute_tools.py
+29
-24
tests/test_execute_mdx.py
tests/test_execute_mdx.py
+2
-0
tests/test_xmla_notox.py
tests/test_xmla_notox.py
+6
-12
No files found.
micro_bench/__init__.py
View file @
cd49e537
This diff is collapsed.
Click to expand it.
micro_bench/cube_generator.py
View file @
cd49e537
...
@@ -7,7 +7,8 @@ import shutil
...
@@ -7,7 +7,8 @@ import shutil
from
olapy.core.mdx.executor.execute
import
MdxEngine
from
olapy.core.mdx.executor.execute
import
MdxEngine
CUBE_NAME
=
"temp_cube"
CUBE_NAME
=
"temp_cube"
class
CubeGen
:
class
CubeGen
:
"""
"""
...
@@ -21,7 +22,8 @@ class CubeGen:
...
@@ -21,7 +22,8 @@ class CubeGen:
# We have to generate DataFrames and save them to csv format because XmlaProviderService and
# We have to generate DataFrames and save them to csv format because XmlaProviderService and
# MdxEngine classes use those files
# MdxEngine classes use those files
def
__init__
(
self
,
number_dimensions
=
1
,
rows_length
=
1000
,
columns_length
=
2
):
def
__init__
(
self
,
number_dimensions
=
1
,
rows_length
=
1000
,
columns_length
=
2
):
self
.
number_dimensions
=
number_dimensions
self
.
number_dimensions
=
number_dimensions
self
.
rows_length
=
rows_length
self
.
rows_length
=
rows_length
self
.
columns_length
=
columns_length
self
.
columns_length
=
columns_length
...
@@ -38,14 +40,20 @@ class CubeGen:
...
@@ -38,14 +40,20 @@ class CubeGen:
facts
=
pd
.
DataFrame
()
facts
=
pd
.
DataFrame
()
for
idx
,
dim
in
enumerate
(
range
(
self
.
number_dimensions
)):
for
idx
,
dim
in
enumerate
(
range
(
self
.
number_dimensions
)):
table_name
=
'table'
+
str
(
idx
)
table_name
=
'table'
+
str
(
idx
)
table_values
=
pd
.
DataFrame
(
np
.
random
.
randint
(
min_val
,
max_val
,
size
=
(
self
.
rows_length
,
self
.
columns_length
)),
table_values
=
pd
.
DataFrame
(
np
.
random
.
randint
(
min_val
,
max_val
,
size
=
(
self
.
rows_length
,
self
.
columns_length
)),
columns
=
list
(
columns
=
list
(
table_name
+
col
for
col
in
string
.
ascii_uppercase
[:
self
.
columns_length
]))
table_name
+
col
for
col
in
string
.
ascii_uppercase
[:
self
.
columns_length
]))
table_values
.
index
.
name
=
table_name
+
"_id"
table_values
.
index
.
name
=
table_name
+
"_id"
tables
[
table_name
]
=
table_values
.
reset_index
()
tables
[
table_name
]
=
table_values
.
reset_index
()
facts
[
table_name
+
"_id"
]
=
tables
[
table_name
][
table_name
+
"_id"
]
facts
[
table_name
+
"_id"
]
=
tables
[
table_name
][
table_name
+
"_id"
]
facts
[
'Amount'
]
=
np
.
random
.
randint
(
300
,
1000
,
size
=
(
self
.
rows_length
,
1
))
facts
[
'Amount'
]
=
np
.
random
.
randint
(
300
,
1000
,
size
=
(
self
.
rows_length
,
1
))
tables
[
'Facts'
]
=
facts
tables
[
'Facts'
]
=
facts
return
tables
return
tables
...
@@ -57,19 +65,22 @@ class CubeGen:
...
@@ -57,19 +65,22 @@ class CubeGen:
:param tables: dict of DataFrames
:param tables: dict of DataFrames
"""
"""
cube_path
=
os
.
path
.
join
(
cube_path
=
os
.
path
.
join
(
os
.
path
.
abspath
(
os
.
path
.
abspath
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
".."
)),
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
".."
)),
MdxEngine
.
CUBE_FOLDER
)
MdxEngine
.
CUBE_FOLDER
)
if
not
os
.
path
.
isdir
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)):
if
not
os
.
path
.
isdir
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)):
os
.
makedirs
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
))
os
.
makedirs
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
))
cube_path
=
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)
cube_path
=
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)
for
(
table_name
,
table_value
)
in
tables
.
items
():
for
(
table_name
,
table_value
)
in
tables
.
items
():
table_value
.
to_csv
(
os
.
path
.
join
(
os
.
path
.
join
(
cube_path
,
table_name
+
'.csv'
)),
sep
=
";"
,
index
=
False
)
table_value
.
to_csv
(
os
.
path
.
join
(
os
.
path
.
join
(
cube_path
,
table_name
+
'.csv'
)),
sep
=
";"
,
index
=
False
)
@
staticmethod
@
staticmethod
def
remove_temp_cube
():
def
remove_temp_cube
():
"""Remove the temporary cube."""
"""Remove the temporary cube."""
cube_path
=
os
.
path
.
join
(
cube_path
=
os
.
path
.
join
(
os
.
path
.
abspath
(
os
.
path
.
abspath
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
".."
)),
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
".."
)),
MdxEngine
.
CUBE_FOLDER
)
MdxEngine
.
CUBE_FOLDER
)
if
os
.
path
.
isdir
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)):
if
os
.
path
.
isdir
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
)):
shutil
.
rmtree
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
))
shutil
.
rmtree
(
os
.
path
.
join
(
cube_path
,
CUBE_NAME
))
micro_bench/micro_bench.py
View file @
cd49e537
...
@@ -21,4 +21,5 @@ class MicBench:
...
@@ -21,4 +21,5 @@ class MicBench:
to one million
to one million
:return: float execution time in seconds
:return: float execution time in seconds
"""
"""
return
Timer
(
lambda
:
connection
.
Execute
(
query
,
Catalog
=
cube
)).
timeit
(
number
=
number
)
return
Timer
(
lambda
:
connection
.
Execute
(
query
,
Catalog
=
cube
)).
timeit
(
number
=
number
)
olapy/core/mdx/executor/execute.py
View file @
cd49e537
...
@@ -41,6 +41,7 @@ class MdxEngine:
...
@@ -41,6 +41,7 @@ class MdxEngine:
# (before instantiate MdxEngine I need to access cubes information)
# (before instantiate MdxEngine I need to access cubes information)
csv_files_cubes
=
[]
csv_files_cubes
=
[]
postgres_db_cubes
=
[]
postgres_db_cubes
=
[]
# to show just config file's dimensions
# to show just config file's dimensions
def
__init__
(
self
,
def
__init__
(
self
,
...
@@ -113,7 +114,8 @@ class MdxEngine:
...
@@ -113,7 +114,8 @@ class MdxEngine:
try
:
try
:
db
=
MyDB
(
db_config_file_path
=
olapy_data_location
)
db
=
MyDB
(
db_config_file_path
=
olapy_data_location
)
# TODO this work only with postgres
# TODO this work only with postgres
result
=
db
.
engine
.
execute
(
'SELECT datname FROM pg_database WHERE datistemplate = false;'
)
result
=
db
.
engine
.
execute
(
'SELECT datname FROM pg_database WHERE datistemplate = false;'
)
available_tables
=
result
.
fetchall
()
available_tables
=
result
.
fetchall
()
# cursor.execute("""SELECT datname FROM pg_database
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
# WHERE datistemplate = false;""")
...
@@ -164,9 +166,10 @@ class MdxEngine:
...
@@ -164,9 +166,10 @@ class MdxEngine:
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
tables
=
{}
tables
=
{}
if
self
.
client
==
'excel'
and
config_file_parser
.
config_file_exist
(
client_type
=
self
.
client
if
self
.
client
==
'excel'
and
config_file_parser
.
config_file_exist
(
)
and
self
.
cube
in
config_file_parser
.
get_cubes_names
(
client_type
=
self
.
client
client_type
=
self
.
):
client
)
and
self
.
cube
in
config_file_parser
.
get_cubes_names
(
client_type
=
self
.
client
):
# for web (config file) we need only star_schema_dataframes, not all tables
# for web (config file) we need only star_schema_dataframes, not all tables
for
cubes
in
config_file_parser
.
construct_cubes
():
for
cubes
in
config_file_parser
.
construct_cubes
():
...
@@ -188,7 +191,8 @@ class MdxEngine:
...
@@ -188,7 +191,8 @@ class MdxEngine:
# if web, get measures from config file
# if web, get measures from config file
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
if
self
.
client
==
'web'
and
config_file_parser
.
config_file_exist
(
'web'
):
if
self
.
client
==
'web'
and
config_file_parser
.
config_file_exist
(
'web'
):
for
cubes
in
config_file_parser
.
construct_cubes
(
self
.
client
):
for
cubes
in
config_file_parser
.
construct_cubes
(
self
.
client
):
# update facts name
# update facts name
...
@@ -214,19 +218,19 @@ class MdxEngine:
...
@@ -214,19 +218,19 @@ class MdxEngine:
fusion
=
None
fusion
=
None
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
config_file_parser
=
ConfigParser
(
self
.
cube_path
)
if
config_file_parser
.
config_file_exist
(
if
config_file_parser
.
config_file_exist
(
self
.
client
self
.
)
and
self
.
cube
in
config_file_parser
.
get_cubes_names
(
client
)
and
self
.
cube
in
config_file_parser
.
get_cubes_names
(
client_type
=
self
.
client
):
client_type
=
self
.
client
):
for
cubes
in
config_file_parser
.
construct_cubes
(
self
.
client
):
for
cubes
in
config_file_parser
.
construct_cubes
(
self
.
client
):
# TODO cubes.source == 'csv'
# TODO cubes.source == 'csv'
if
cubes
.
source
==
'postgres'
:
if
cubes
.
source
==
'postgres'
:
# TODO one config file (I will try to merge dimensions between them in web part)
# TODO one config file (I will try to merge dimensions between them in web part)
if
self
.
client
==
'web'
:
if
self
.
client
==
'web'
:
fusion
=
_construct_web_star_schema_config_file
(
fusion
=
_construct_web_star_schema_config_file
(
self
,
self
,
cubes
)
cubes
)
else
:
else
:
fusion
=
_construct_star_schema_config_file
(
fusion
=
_construct_star_schema_config_file
(
self
,
self
,
cubes
)
cubes
)
elif
self
.
cube
in
self
.
csv_files_cubes
:
elif
self
.
cube
in
self
.
csv_files_cubes
:
fusion
=
_construct_star_schema_csv_files
(
self
)
fusion
=
_construct_star_schema_csv_files
(
self
)
...
@@ -256,7 +260,8 @@ class MdxEngine:
...
@@ -256,7 +260,8 @@ class MdxEngine:
:return: path to the cube
:return: path to the cube
"""
"""
if
MdxEngine
.
DATA_FOLDER
is
not
None
:
if
MdxEngine
.
DATA_FOLDER
is
not
None
:
return
os
.
path
.
join
(
MdxEngine
.
DATA_FOLDER
,
MdxEngine
.
CUBE_FOLDER
,
self
.
cube
)
return
os
.
path
.
join
(
MdxEngine
.
DATA_FOLDER
,
MdxEngine
.
CUBE_FOLDER
,
self
.
cube
)
return
os
.
path
.
join
(
self
.
cube_path
,
self
.
cube
)
return
os
.
path
.
join
(
self
.
cube_path
,
self
.
cube
)
# TODO temporary function
# TODO temporary function
...
@@ -309,7 +314,7 @@ class MdxEngine:
...
@@ -309,7 +314,7 @@ class MdxEngine:
for tup_att in tup[0].replace('.Members', '').split('.') if tup_att
for tup_att in tup[0].replace('.Members', '').split('.') if tup_att
]
]
for tup in re.compile(regex).findall(
for tup in re.compile(regex).findall(
query.encode("
utf
-
8
",'replace')[start:stop])
query.encode("
utf
-
8
",
'replace')[start:stop])
if len(tup[0].split('.')) > 1]
if len(tup[0].split('.')) > 1]
# TODO temporary function
# TODO temporary function
...
@@ -603,9 +608,8 @@ class MdxEngine:
...
@@ -603,9 +608,8 @@ class MdxEngine:
:return: updated columns_to_keep
:return: updated columns_to_keep
"""
"""
if len(
if len(tuple_as_list) == 3 and tuple_as_list[-1] in self.tables_loaded[
tuple_as_list
tuple_as_list[0]].columns:
) == 3 and tuple_as_list[-1] in self.tables_loaded[tuple_as_list[0]].columns:
# in case of [Geography].[Geography].[Country]
# in case of [Geography].[Geography].[Country]
cols = [tuple_as_list[-1]]
cols = [tuple_as_list[-1]]
else:
else:
...
...
olapy/core/mdx/executor/execute_config_file.py
View file @
cd49e537
...
@@ -4,6 +4,7 @@ from ..tools.connection import MyDB
...
@@ -4,6 +4,7 @@ from ..tools.connection import MyDB
import
pandas.io.sql
as
psql
import
pandas.io.sql
as
psql
import
os
import
os
# split execution into three part (execute from config files,
# split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
# and execute data base tables if they respect olapy's start schema model)
...
@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj):
...
@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj):
# just one facts table right now
# just one facts table right now
executer_instance
.
facts
=
cube_obj
.
facts
[
0
].
table_name
executer_instance
.
facts
=
cube_obj
.
facts
[
0
].
table_name
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
for
dimension
in
cube_obj
.
dimensions
:
for
dimension
in
cube_obj
.
dimensions
:
...
@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj):
...
@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj):
table_name
=
dimension
.
name
table_name
=
dimension
.
name
# rename columns if value not None
# rename columns if value not None
df
.
rename
(
columns
=
(
dict
((
k
,
v
)
for
k
,
v
in
dimension
.
columns
.
items
()
if
v
)),
inplace
=
True
)
df
.
rename
(
columns
=
(
dict
((
k
,
v
)
for
k
,
v
in
dimension
.
columns
.
items
()
if
v
)),
inplace
=
True
)
tables
[
table_name
]
=
df
[[
tables
[
table_name
]
=
df
[[
col
for
col
in
df
.
columns
if
col
.
lower
()[
-
2
:]
!=
'id'
col
for
col
in
df
.
columns
if
col
.
lower
()[
-
2
:]
!=
'id'
...
@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj):
...
@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj):
return
tables
return
tables
# excel client
# excel client
def
_construct_star_schema_config_file
(
executer_instance
,
cubes_obj
):
def
_construct_star_schema_config_file
(
executer_instance
,
cubes_obj
):
"""
"""
...
@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
:return: star schema DataFrame
:return: star schema DataFrame
"""
"""
executer_instance
.
facts
=
cubes_obj
.
facts
[
0
].
table_name
executer_instance
.
facts
=
cubes_obj
.
facts
[
0
].
table_name
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
# load facts table
# load facts table
fusion
=
psql
.
read_sql_query
(
fusion
=
psql
.
read_sql_query
(
...
@@ -74,7 +82,10 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -74,7 +82,10 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
# TODO CHOSE BETWEEN THOSES DF
# TODO CHOSE BETWEEN THOSES DF
fusion
=
fusion
.
merge
(
fusion
=
fusion
.
merge
(
df
,
left_on
=
fact_key
,
right_on
=
dimension_and_key
.
split
(
'.'
)[
1
],
how
=
'left'
,
df
,
left_on
=
fact_key
,
right_on
=
dimension_and_key
.
split
(
'.'
)[
1
],
how
=
'left'
,
# remove suffixe from dimension and keep the same column name for facts
# remove suffixe from dimension and keep the same column name for facts
suffixes
=
(
''
,
'_y'
))
suffixes
=
(
''
,
'_y'
))
...
@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
return
fusion
return
fusion
# web client
# web client
def
_construct_web_star_schema_config_file
(
executer_instance
,
cubes_obj
):
def
_construct_web_star_schema_config_file
(
executer_instance
,
cubes_obj
):
"""
"""
...
@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns
=
[]
all_columns
=
[]
executer_instance
.
facts
=
cubes_obj
.
facts
[
0
].
table_name
executer_instance
.
facts
=
cubes_obj
.
facts
[
0
].
table_name
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
db
=
MyDB
(
db_config_file_path
=
os
.
path
.
dirname
(
executer_instance
.
cube_path
),
db
=
executer_instance
.
cube
)
# load facts table
# load facts table
if
cubes_obj
.
facts
[
0
].
columns
:
if
cubes_obj
.
facts
[
0
].
columns
:
...
@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
# TODO check merge (how)
# TODO check merge (how)
fusion
=
fusion
.
merge
(
fusion
=
fusion
.
merge
(
df
,
left_on
=
fact_key
,
right_on
=
dimension_and_key
.
split
(
'.'
)[
1
],
how
=
'left'
,
df
,
left_on
=
fact_key
,
right_on
=
dimension_and_key
.
split
(
'.'
)[
1
],
how
=
'left'
,
# remove suffixe from dimension and keep the same column name for facts
# remove suffixe from dimension and keep the same column name for facts
suffixes
=
(
''
,
'_y'
))
suffixes
=
(
''
,
'_y'
))
return
fusion
[[
column
for
column
in
all_columns
if
'id'
!=
column
[
-
2
:]]]
return
fusion
[[
column
for
column
in
all_columns
if
'id'
!=
column
[
-
2
:]]]
olapy/core/mdx/executor/execute_csv_files.py
View file @
cd49e537
...
@@ -8,6 +8,7 @@ import pandas as pd
...
@@ -8,6 +8,7 @@ import pandas as pd
# execute csv files if they respect olapy's start schema model,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
# and execute data base tables if they respect olapy's start schema model)
def
_load_tables_csv_files
(
executer_instance
):
def
_load_tables_csv_files
(
executer_instance
):
"""
"""
Load tables from csv files.
Load tables from csv files.
...
...
olapy/core/mdx/executor/execute_db.py
View file @
cd49e537
...
@@ -9,8 +9,6 @@ import pandas.io.sql as psql
...
@@ -9,8 +9,6 @@ import pandas.io.sql as psql
# execute csv files if they respect olapy's start schema model,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
# and execute data base tables if they respect olapy's start schema model)
# class StringFolder(object):
# class StringFolder(object):
# """
# """
# Class that will fold strings. See 'fold_string'.
# Class that will fold strings. See 'fold_string'.
...
@@ -68,6 +66,7 @@ import pandas.io.sql as psql
...
@@ -68,6 +66,7 @@ import pandas.io.sql as psql
# TODO try pandas.read_sql_table and pandas.read_sql
# TODO try pandas.read_sql_table and pandas.read_sql
def
_load_tables_db
(
executer_instance
):
def
_load_tables_db
(
executer_instance
):
"""
"""
Load tables from database.
Load tables from database.
...
@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance):
...
@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance):
:return: tables dict with table name as key and dataframe as value
:return: tables dict with table name as key and dataframe as value
"""
"""
tables
=
{}
tables
=
{}
db
=
MyDB
(
db_config_file_path
=
executer_instance
.
DATA_FOLDER
,
db
=
executer_instance
.
cube
)
db
=
MyDB
(
db_config_file_path
=
executer_instance
.
DATA_FOLDER
,
db
=
executer_instance
.
cube
)
inspector
=
inspect
(
db
.
engine
)
inspector
=
inspect
(
db
.
engine
)
for
table_name
in
inspector
.
get_table_names
():
for
table_name
in
inspector
.
get_table_names
():
...
@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance):
...
@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance):
# 'SELECT * FROM "{0}"'.format(table_name), db.engine)
# 'SELECT * FROM "{0}"'.format(table_name), db.engine)
# results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
# results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
results
=
db
.
engine
.
execution_options
(
stream_results
=
True
).
execute
(
'SELECT * FROM "{0}"'
.
format
(
table_name
))
results
=
db
.
engine
.
execution_options
(
stream_results
=
True
).
execute
(
'SELECT * FROM "{0}"'
.
format
(
table_name
))
# Fetch all the results of the query
# Fetch all the results of the query
value
=
pd
.
DataFrame
(
iter
(
results
),
columns
=
results
.
keys
())
# Pass results as an iterator
value
=
pd
.
DataFrame
(
iter
(
results
),
columns
=
results
.
keys
())
# Pass results as an iterator
# with string_folding_wrapper we loose response time
# with string_folding_wrapper we loose response time
# value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
# value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
tables
[
table_name
]
=
value
[[
tables
[
table_name
]
=
value
[[
...
...
olapy/core/mdx/tools/config_file_parser.py
View file @
cd49e537
...
@@ -246,7 +246,7 @@ class ConfigParser:
...
@@ -246,7 +246,7 @@ class ConfigParser:
"""
"""
def
__init__
(
self
,
def
__init__
(
self
,
cube_path
=
None
,
cube_path
=
None
,
file_name
=
'cubes-config.xml'
,
file_name
=
'cubes-config.xml'
,
web_config_file_name
=
'web_cube_config.xml'
):
web_config_file_name
=
'web_cube_config.xml'
):
"""
"""
...
@@ -262,7 +262,8 @@ class ConfigParser:
...
@@ -262,7 +262,8 @@ class ConfigParser:
home_directory
=
expanduser
(
"~"
)
home_directory
=
expanduser
(
"~"
)
if
cube_path
is
None
:
if
cube_path
is
None
:
self
.
cube_path
=
os
.
path
.
join
(
home_directory
,
'olapy-data'
,
'cubes'
)
self
.
cube_path
=
os
.
path
.
join
(
home_directory
,
'olapy-data'
,
'cubes'
)
else
:
else
:
self
.
cube_path
=
cube_path
self
.
cube_path
=
cube_path
...
@@ -303,7 +304,7 @@ class ConfigParser:
...
@@ -303,7 +304,7 @@ class ConfigParser:
else
:
else
:
return
False
return
False
def
get_cubes_names
(
self
,
client_type
):
def
get_cubes_names
(
self
,
client_type
):
"""
"""
Get all cubes names in the config file.
Get all cubes names in the config file.
...
@@ -359,10 +360,10 @@ class ConfigParser:
...
@@ -359,10 +360,10 @@ class ConfigParser:
# column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
# column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
displayName
=
xml_dimension
.
find
(
'displayName'
).
text
,
displayName
=
xml_dimension
.
find
(
'displayName'
).
text
,
columns
=
OrderedDict
(
columns
=
OrderedDict
(
(
column_name
.
text
,
None
if
not
column_name
.
attrib
else
column_name
.
attrib
[
'column_new_name'
])
(
column_name
.
text
,
None
if
not
column_name
.
attrib
else
column_name
.
attrib
[
'column_new_name'
])
for
column_name
in
xml_dimension
.
findall
(
for
column_name
in
xml_dimension
.
findall
(
'columns/name'
)
'columns/name'
)))
))
for
xml_dimension
in
tree
.
xpath
(
for
xml_dimension
in
tree
.
xpath
(
'/cubes/cube/dimensions/dimension'
)
'/cubes/cube/dimensions/dimension'
)
]
]
...
@@ -449,8 +450,7 @@ class ConfigParser:
...
@@ -449,8 +450,7 @@ class ConfigParser:
global_table
=
{
global_table
=
{
'columns'
:
'columns'
:
dashboard
.
find
(
'Global_table/columns'
).
text
.
split
(
','
),
dashboard
.
find
(
'Global_table/columns'
).
text
.
split
(
','
),
'rows'
:
'rows'
:
dashboard
.
find
(
'Global_table/rows'
).
text
.
split
(
','
)
dashboard
.
find
(
'Global_table/rows'
).
text
.
split
(
','
)
},
},
pie_charts
=
dashboard
.
find
(
'PieCharts'
).
text
.
split
(
','
),
pie_charts
=
dashboard
.
find
(
'PieCharts'
).
text
.
split
(
','
),
bar_charts
=
dashboard
.
find
(
'BarCharts'
).
text
.
split
(
','
),
bar_charts
=
dashboard
.
find
(
'BarCharts'
).
text
.
split
(
','
),
...
...
olapy/core/mdx/tools/connection.py
View file @
cd49e537
from
__future__
import
absolute_import
,
division
,
print_function
# import psycopg2 as pg
# import psycopg2 as pg
from
sqlalchemy
import
create_engine
from
sqlalchemy
import
create_engine
# postgres connection
# postgres connection
from
olapy_config_file_parser
import
DbConfigParser
from
.olapy_config_file_parser
import
DbConfigParser
class
MyDB
(
object
):
class
MyDB
(
object
):
"""Connect to sql database (postgres only right now)."""
"""Connect to sql database (postgres only right now)."""
def
__init__
(
self
,
db_config_file_path
=
None
,
db
=
None
):
def
__init__
(
self
,
db_config_file_path
=
None
,
db
=
None
):
# TODO temporary
# TODO temporary
db_config
=
DbConfigParser
(
config_path
=
db_config_file_path
)
db_config
=
DbConfigParser
(
config_path
=
db_config_file_path
)
...
@@ -21,18 +23,19 @@ class MyDB(object):
...
@@ -21,18 +23,19 @@ class MyDB(object):
# first i want to show all databases to user (in excel)
# first i want to show all databases to user (in excel)
# self.engine = pg.connect("user={0} password={1} host='{2}'".
# self.engine = pg.connect("user={0} password={1} host='{2}'".
# format(username, password, host))
# format(username, password, host))
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
username
,
password
,
'postgres'
,
host
,
port
))
username
,
password
,
'postgres'
,
host
,
port
))
else
:
else
:
# and then we connect to the user db
# and then we connect to the user db
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
username
,
password
,
db
,
host
,
port
))
username
,
password
,
db
,
host
,
port
))
# self.connection = pg.connect(
# self.connection = pg.connect(
# "user={0} password={1} dbname='{2}' host='{3}'".format(
# "user={0} password={1} dbname='{2}' host='{3}'".format(
# username, password, db, host))
# username, password, db, host))
def
__del__
(
self
):
def
__del__
(
self
):
if
hasattr
(
self
,
'connection'
):
if
hasattr
(
self
,
'connection'
):
self
.
engine
.
dispose
()
self
.
engine
.
dispose
()
olapy/core/mdx/tools/olapy_config_file_parser.py
View file @
cd49e537
...
@@ -4,12 +4,11 @@ import os
...
@@ -4,12 +4,11 @@ import os
from
lxml
import
etree
from
lxml
import
etree
class
DbConfigParser
:
class
DbConfigParser
:
# TODO one config file (I will try to merge dimensions between them in web part)
# TODO one config file (I will try to merge dimensions between them in web part)
def
__init__
(
self
,
def
__init__
(
self
,
config_path
=
None
,
file_name
=
'olapy-config.xml'
):
config_path
=
None
,
file_name
=
'olapy-config.xml'
):
"""
"""
:param cube_path: path to cube (csv folders)
:param cube_path: path to cube (csv folders)
...
@@ -50,6 +49,5 @@ class DbConfigParser:
...
@@ -50,6 +49,5 @@ class DbConfigParser:
'password'
:
db
.
find
(
'password'
).
text
,
'password'
:
db
.
find
(
'password'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
'port'
:
db
.
find
(
'port'
).
text
,
'port'
:
db
.
find
(
'port'
).
text
,
}
}
for
db
in
tree
.
xpath
(
'/olapy/database'
)
for
db
in
tree
.
xpath
(
'/olapy/database'
)
]
]
olapy/core/services/models.py
View file @
cd49e537
...
@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function
...
@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function
from
spyne
import
ComplexModel
,
Integer
,
Unicode
,
XmlAttribute
from
spyne
import
ComplexModel
,
Integer
,
Unicode
,
XmlAttribute
# NOTE : I didn't respect python naming convention here
# NOTE : I didn't respect python naming convention here
# because we need to create the xmla response (generated by spyne) with the same variable names,
# because we need to create the xmla response (generated by spyne) with the same variable names,
# thus xmla requests from excel can be reached
# thus xmla requests from excel can be reached
class
Tuple
(
object
):
class
Tuple
(
object
):
"""Tuple description (used by spyne)."""
"""Tuple description (used by spyne)."""
...
@@ -110,7 +109,7 @@ class Propertielist(ComplexModel):
...
@@ -110,7 +109,7 @@ class Propertielist(ComplexModel):
class
Command
(
ComplexModel
):
class
Command
(
ComplexModel
):
"""Command description (used by spyne)."""
"""Command description (used by spyne)."""
_type_info
=
{
'Statement'
:
Unicode
,}
_type_info
=
{
'Statement'
:
Unicode
,
}
class
ExecuteRequest
(
ComplexModel
):
class
ExecuteRequest
(
ComplexModel
):
...
...
olapy/core/services/xmla.py
View file @
cd49e537
...
@@ -54,8 +54,7 @@ class XmlaProviderService(ServiceBase):
...
@@ -54,8 +54,7 @@ class XmlaProviderService(ServiceBase):
discover_tools
=
XmlaDiscoverTools
()
discover_tools
=
XmlaDiscoverTools
()
sessio_id
=
discover_tools
.
session_id
sessio_id
=
discover_tools
.
session_id
@
rpc
(
@
rpc
(
DiscoverRequest
,
DiscoverRequest
,
_returns
=
AnyXml
,
_returns
=
AnyXml
,
_body_style
=
"bare"
,
_body_style
=
"bare"
,
_out_header
=
Session
,
_out_header
=
Session
,
...
@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase):
...
@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase):
ctx
.
out_header
=
Session
(
SessionId
=
str
(
XmlaProviderService
.
sessio_id
))
ctx
.
out_header
=
Session
(
SessionId
=
str
(
XmlaProviderService
.
sessio_id
))
config_parser
=
ConfigParser
(
discover_tools
.
executer
.
cube_path
)
config_parser
=
ConfigParser
(
discover_tools
.
executer
.
cube_path
)
if
config_parser
.
xmla_authentication
(
if
config_parser
.
xmla_authentication
(
)
and
ctx
.
transport
.
req_env
[
)
and
ctx
.
transport
.
req_env
[
'QUERY_STRING'
]
!=
'admin'
:
'QUERY_STRING'
]
!=
'admin'
:
raise
InvalidCredentialsError
(
raise
InvalidCredentialsError
(
fault_string
=
fault_string
=
'You do not have permission to access this resource'
,
'You do not have permission to access this resource'
,
fault_object
=
None
)
fault_object
=
None
)
# TODO call (labster) login function or create login with token (according to labster db)
# TODO call (labster) login function or create login with token (according to labster db)
...
@@ -143,8 +141,7 @@ class XmlaProviderService(ServiceBase):
...
@@ -143,8 +141,7 @@ class XmlaProviderService(ServiceBase):
# Execute function must take 2 argument ( JUST 2 ! ) Command and Properties
# Execute function must take 2 argument ( JUST 2 ! ) Command and Properties
# we encapsulate them in ExecuteRequest object
# we encapsulate them in ExecuteRequest object
@
rpc
(
@
rpc
(
ExecuteRequest
,
ExecuteRequest
,
_returns
=
AnyXml
,
_returns
=
AnyXml
,
_body_style
=
"bare"
,
_body_style
=
"bare"
,
_out_header
=
Session
)
_out_header
=
Session
)
...
@@ -235,7 +232,7 @@ application = Application(
...
@@ -235,7 +232,7 @@ application = Application(
wsgi_application
=
WsgiApplication
(
application
)
wsgi_application
=
WsgiApplication
(
application
)
def
start_server
(
host
=
'0.0.0.0'
,
port
=
8000
,
write_on_file
=
False
):
def
start_server
(
host
=
'0.0.0.0'
,
port
=
8000
,
write_on_file
=
False
):
"""
"""
Start the xmla server.
Start the xmla server.
...
...
olapy/core/services/xmla_discover_tools.py
View file @
cd49e537
...
@@ -139,7 +139,7 @@ class XmlaDiscoverTools():
...
@@ -139,7 +139,7 @@ class XmlaDiscoverTools():
{1}
{1}
</root>
</root>
</return>
</return>
"""
.
format
(
xsd
,
rows
))
"""
.
format
(
xsd
,
rows
))
if
request
.
Restrictions
.
RestrictionList
.
PropertyName
==
'Catalog'
:
if
request
.
Restrictions
.
RestrictionList
.
PropertyName
==
'Catalog'
:
if
request
.
Properties
.
PropertyList
.
Catalog
is
not
None
:
if
request
.
Properties
.
PropertyList
.
Catalog
is
not
None
:
...
@@ -195,7 +195,7 @@ class XmlaDiscoverTools():
...
@@ -195,7 +195,7 @@ class XmlaDiscoverTools():
'MdpropMdxNamedSets'
,
'int'
,
'Read'
,
'false'
,
'MdpropMdxNamedSets'
,
'int'
,
'Read'
,
'false'
,
'15'
)
'15'
)
return
get_props
(
discover_preperties_xsd
,
''
,
''
,
''
,
''
,
''
,
''
)
return
get_props
(
discover_preperties_xsd
,
''
,
''
,
''
,
''
,
''
,
''
)
def
discover_schema_rowsets_response
(
self
,
request
):
def
discover_schema_rowsets_response
(
self
,
request
):
if
request
.
Restrictions
.
RestrictionList
.
SchemaName
==
"MDSCHEMA_HIERARCHIES"
and
\
if
request
.
Restrictions
.
RestrictionList
.
SchemaName
==
"MDSCHEMA_HIERARCHIES"
and
\
...
@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools():
...
@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools():
# french caracteres
# french caracteres
# TODO encode dataframe
# TODO encode dataframe
if
type
(
df
.
iloc
[
0
][
0
])
==
unicode
:
if
type
(
df
.
iloc
[
0
][
0
])
==
unicode
:
column_attribut
=
df
.
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
column_attribut
=
df
.
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
else
:
else
:
column_attribut
=
df
.
iloc
[
0
][
0
]
column_attribut
=
df
.
iloc
[
0
][
0
]
rows
+=
"""
rows
+=
"""
<row>
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
<CATALOG_NAME>{0}</CATALOG_NAME>
...
@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools():
...
@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
</row>
"""
.
format
(
self
.
selected_catalogue
,
"""
.
format
(
self
.
selected_catalogue
,
table_name
,
table_name
,
df
.
columns
[
0
],
column_attribut
)
df
.
columns
[
0
],
column_attribut
)
rows
+=
"""
rows
+=
"""
<row>
<row>
...
@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools():
...
@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools():
# french caracteres
# french caracteres
# TODO encode dataframe
# TODO encode dataframe
if
type
(
df
.
iloc
[
0
][
0
])
==
unicode
:
if
type
(
df
.
iloc
[
0
][
0
])
==
unicode
:
column_attribut
=
df
.
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
column_attribut
=
df
.
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
else
:
else
:
column_attribut
=
df
.
iloc
[
0
][
0
]
column_attribut
=
df
.
iloc
[
0
][
0
]
...
@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools():
...
@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
</row>
"""
.
format
(
self
.
selected_catalogue
,
"""
.
format
(
self
.
selected_catalogue
,
table_name
,
table_name
,
df
.
columns
[
0
],
column_attribut
)
df
.
columns
[
0
],
column_attribut
)
rows
+=
"""
rows
+=
"""
<row>
<row>
...
...
olapy/core/services/xmla_execute_tools.py
View file @
cd49e537
...
@@ -104,11 +104,11 @@ class XmlaExecuteTools():
...
@@ -104,11 +104,11 @@ class XmlaExecuteTools():
first_att
=
3
first_att
=
3
# query with on columns and on rows (without measure)
# query with on columns and on rows (without measure)
elif
mdx_execution_result
[
'columns_desc'
][
'columns'
]
and
mdx_execution_result
[
'columns_desc'
][
'rows'
]:
elif
mdx_execution_result
[
'columns_desc'
][
'columns'
]
and
mdx_execution_result
[
'columns_desc'
][
'rows'
]:
# ['Geography','America']
# ['Geography','America']
tuples
=
[
tuples
=
[
zip
(
zip
(
*
[[[
key
]
+
list
(
row
)
*
[[[
key
]
+
list
(
row
)
for
row
in
splited_df
[
key
].
itertuples
(
index
=
False
)]
for
row
in
splited_df
[
key
].
itertuples
(
index
=
False
)]
for
key
in
splited_df
.
keys
()
for
key
in
splited_df
.
keys
()
if
key
is
not
self
.
executer
.
facts
])
if
key
is
not
self
.
executer
.
facts
])
...
@@ -120,8 +120,7 @@ class XmlaExecuteTools():
...
@@ -120,8 +120,7 @@ class XmlaExecuteTools():
else
:
else
:
# ['Geography','Amount','America']
# ['Geography','Amount','America']
tuples
=
[
tuples
=
[
zip
(
zip
(
*
[[[
key
]
+
[
mes
]
+
list
(
row
)
*
[[[
key
]
+
[
mes
]
+
list
(
row
)
for
row
in
splited_df
[
key
].
itertuples
(
index
=
False
)]
for
row
in
splited_df
[
key
].
itertuples
(
index
=
False
)]
for
key
in
splited_df
.
keys
()
for
key
in
splited_df
.
keys
()
if
key
is
not
self
.
executer
.
facts
])
if
key
is
not
self
.
executer
.
facts
])
...
@@ -155,7 +154,10 @@ class XmlaExecuteTools():
...
@@ -155,7 +154,10 @@ class XmlaExecuteTools():
# french caracteres
# french caracteres
# TODO encode dataframe
# TODO encode dataframe
if
type
(
tuple_without_minus_1
[
-
1
])
==
unicode
:
if
type
(
tuple_without_minus_1
[
-
1
])
==
unicode
:
tuple_without_minus_1
=
[
x
.
encode
(
'utf-8'
,
'replace'
)
for
x
in
tuple_without_minus_1
]
tuple_without_minus_1
=
[
x
.
encode
(
'utf-8'
,
'replace'
)
for
x
in
tuple_without_minus_1
]
axis0
+=
"""
axis0
+=
"""
<Member Hierarchy="[{0}].[{0}]">
<Member Hierarchy="[{0}].[{0}]">
...
@@ -254,7 +256,8 @@ class XmlaExecuteTools():
...
@@ -254,7 +256,8 @@ class XmlaExecuteTools():
# TODO must be OPTIMIZED every time!!!!!
# TODO must be OPTIMIZED every time!!!!!
dfs
=
self
.
split_dataframe
(
mdx_execution_result
)
dfs
=
self
.
split_dataframe
(
mdx_execution_result
)
if
mdx_execution_result
[
'columns_desc'
][
'rows'
]
and
mdx_execution_result
[
'columns_desc'
][
'columns'
]:
if
mdx_execution_result
[
'columns_desc'
][
'rows'
]
and
mdx_execution_result
[
'columns_desc'
][
'columns'
]:
return
"""
return
"""
{0}
{0}
...
@@ -306,12 +309,11 @@ class XmlaExecuteTools():
...
@@ -306,12 +309,11 @@ class XmlaExecuteTools():
"""
"""
columns_loop
=
[]
columns_loop
=
[]
if
(
if
((
len
(
mdx_execution_result
[
'columns_desc'
][
'columns'
].
keys
())
==
0
)
(
len
(
mdx_execution_result
[
'columns_desc'
][
'columns'
].
keys
())
==
0
)
^
^
(
len
(
mdx_execution_result
[
'columns_desc'
][
'rows'
].
keys
())
==
0
)
(
len
(
mdx_execution_result
[
'columns_desc'
][
'rows'
].
keys
())
==
0
)
and
self
.
executer
.
facts
in
mdx_execution_result
[
'columns_desc'
][
'all'
].
keys
(
))
and
self
.
executer
.
facts
in
mdx_execution_result
[
):
'columns_desc'
][
'all'
].
keys
(
):
# iterate DataFrame horizontally
# iterate DataFrame horizontally
columns_loop
=
itertools
.
chain
(
*
[
columns_loop
=
itertools
.
chain
(
*
[
...
@@ -332,7 +334,7 @@ class XmlaExecuteTools():
...
@@ -332,7 +334,7 @@ class XmlaExecuteTools():
cell_data
=
""
cell_data
=
""
index
=
0
index
=
0
for
value
in
columns_loop
:
for
value
in
columns_loop
:
if
np
.
isnan
(
value
)
:
if
np
.
isnan
(
value
):
value
=
''
value
=
''
cell_data
+=
"""
cell_data
+=
"""
<Cell CellOrdinal="{0}">
<Cell CellOrdinal="{0}">
...
@@ -392,9 +394,9 @@ class XmlaExecuteTools():
...
@@ -392,9 +394,9 @@ class XmlaExecuteTools():
to_write
=
"[{0}].[{0}]"
.
format
(
dim_diff
)
to_write
=
"[{0}].[{0}]"
.
format
(
dim_diff
)
if
dim_diff
==
'Measures'
:
if
dim_diff
==
'Measures'
:
# if measures > 1 we don't have to write measure
# if measures > 1 we don't have to write measure
if
self
.
executer
.
facts
in
mdx_execution_result
[
'columns_desc'
][
'all'
]
and
len
(
if
self
.
executer
.
facts
in
mdx_execution_result
[
'columns_desc'
][
mdx_execution_result
[
'columns_desc'
][
'all'
]
'all'
]
and
len
(
mdx_execution_result
[
'columns_desc'
][
[
self
.
executer
.
facts
])
>
1
:
'all'
]
[
self
.
executer
.
facts
])
>
1
:
continue
continue
else
:
else
:
to_write
=
"[Measures]"
to_write
=
"[Measures]"
...
@@ -452,9 +454,9 @@ class XmlaExecuteTools():
...
@@ -452,9 +454,9 @@ class XmlaExecuteTools():
"""
"""
hierarchy_info
=
""
hierarchy_info
=
""
# measure must be written at the top
# measure must be written at the top
if
self
.
executer
.
facts
in
mdx_execution_result
[
'columns_desc'
][
mdx_query_axis
].
keys
(
if
self
.
executer
.
facts
in
mdx_execution_result
[
'columns_desc'
][
)
and
len
(
mdx_execution_result
[
'columns_desc'
][
mdx_query_axis
]
mdx_query_axis
].
keys
()
and
len
(
mdx_execution_result
[
[
self
.
executer
.
facts
])
>
1
:
'columns_desc'
][
mdx_query_axis
]
[
self
.
executer
.
facts
])
>
1
:
hierarchy_info
+=
"""
hierarchy_info
+=
"""
<HierarchyInfo name="{0}">
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
...
@@ -556,15 +558,18 @@ class XmlaExecuteTools():
...
@@ -556,15 +558,18 @@ class XmlaExecuteTools():
for
dim_diff
in
list
(
for
dim_diff
in
list
(
set
(
self
.
executer
.
get_all_tables_names
(
ignore_fact
=
True
))
-
set
(
self
.
executer
.
get_all_tables_names
(
ignore_fact
=
True
))
-
set
(
table_name
set
(
table_name
for
table_name
in
mdx_execution_result
[
'columns_desc'
]
for
table_name
in
mdx_execution_result
[
'columns_desc'
]
[
[
'all'
])):
'all'
])):
# TODO encode dataframe
# TODO encode dataframe
# french caracteres
# french caracteres
if
type
(
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
0
])
==
unicode
:
if
type
(
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
column_attribut
=
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
0
])
==
unicode
:
column_attribut
=
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
0
].
encode
(
'utf-8'
,
'replace'
)
else
:
else
:
column_attribut
=
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
0
]
column_attribut
=
self
.
executer
.
tables_loaded
[
dim_diff
].
iloc
[
0
][
0
]
tuple
+=
"""
tuple
+=
"""
<Member Hierarchy="[{0}].[{0}]">
<Member Hierarchy="[{0}].[{0}]">
...
...
tests/test_execute_mdx.py
View file @
cd49e537
from
__future__
import
absolute_import
,
division
,
print_function
import
pandas
as
pd
import
pandas
as
pd
from
olapy.core.mdx.executor.execute
import
MdxEngine
from
olapy.core.mdx.executor.execute
import
MdxEngine
...
...
tests/test_xmla_notox.py
View file @
cd49e537
...
@@ -198,8 +198,7 @@ def test_query2(conn):
...
@@ -198,8 +198,7 @@ def test_query2(conn):
mems
.
append
(
mems
.
append
(
Member
(
Member
(
_Hierarchy
=
'[Geography].[Geography]'
,
_Hierarchy
=
'[Geography].[Geography]'
,
UName
=
UName
=
'[Geography].[Geography].[Country].[America].[United States]'
,
'[Geography].[Geography].[Country].[America].[United States]'
,
Caption
=
'United States'
,
Caption
=
'United States'
,
LName
=
'[Geography].[Geography].[Country]'
,
LName
=
'[Geography].[Geography].[Country]'
,
LNum
=
'1'
,
LNum
=
'1'
,
...
@@ -209,14 +208,12 @@ def test_query2(conn):
...
@@ -209,14 +208,12 @@ def test_query2(conn):
mems
.
append
(
mems
.
append
(
Member
(
Member
(
_Hierarchy
=
'[Geography].[Geography]'
,
_Hierarchy
=
'[Geography].[Geography]'
,
UName
=
UName
=
'[Geography].[Geography].[City].[America].[United States].[New York]'
,
'[Geography].[Geography].[City].[America].[United States].[New York]'
,
Caption
=
'New York'
,
Caption
=
'New York'
,
LName
=
'[Geography].[Geography].[City]'
,
LName
=
'[Geography].[Geography].[City]'
,
LNum
=
'2'
,
LNum
=
'2'
,
DisplayInfo
=
'131076'
,
DisplayInfo
=
'131076'
,
PARENT_UNIQUE_NAME
=
PARENT_UNIQUE_NAME
=
'[Geography].[Geography].[Continent].[America].[United States]'
,
'[Geography].[Geography].[Continent].[America].[United States]'
,
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
mems
.
append
(
mems
.
append
(
Member
(
Member
(
...
@@ -250,14 +247,12 @@ def test_query2(conn):
...
@@ -250,14 +247,12 @@ def test_query2(conn):
mems
.
append
(
mems
.
append
(
Member
(
Member
(
_Hierarchy
=
'[Geography].[Geography]'
,
_Hierarchy
=
'[Geography].[Geography]'
,
UName
=
UName
=
'[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]'
,
'[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]'
,
Caption
=
'Barcelona'
,
Caption
=
'Barcelona'
,
LName
=
'[Geography].[Geography].[City]'
,
LName
=
'[Geography].[Geography].[City]'
,
LNum
=
'2'
,
LNum
=
'2'
,
DisplayInfo
=
'131076'
,
DisplayInfo
=
'131076'
,
PARENT_UNIQUE_NAME
=
PARENT_UNIQUE_NAME
=
'[Geography].[Geography].[Continent].[Europe].[Spain]'
,
'[Geography].[Geography].[Continent].[Europe].[Spain]'
,
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
mems
.
append
(
mems
.
append
(
Member
(
Member
(
...
@@ -267,8 +262,7 @@ def test_query2(conn):
...
@@ -267,8 +262,7 @@ def test_query2(conn):
LName
=
'[Geography].[Geography].[City]'
,
LName
=
'[Geography].[Geography].[City]'
,
LNum
=
'2'
,
LNum
=
'2'
,
DisplayInfo
=
'131076'
,
DisplayInfo
=
'131076'
,
PARENT_UNIQUE_NAME
=
PARENT_UNIQUE_NAME
=
'[Geography].[Geography].[Continent].[Europe].[Spain]'
,
'[Geography].[Geography].[Continent].[Europe].[Spain]'
,
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
HIERARCHY_UNIQUE_NAME
=
'[Geography].[Geography]'
))
mems
.
append
(
mems
.
append
(
Member
(
Member
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment