Commit cd49e537 authored by mouadh's avatar mouadh

format

parent 09c6d728
This diff is collapsed.
...@@ -7,7 +7,8 @@ import shutil ...@@ -7,7 +7,8 @@ import shutil
from olapy.core.mdx.executor.execute import MdxEngine from olapy.core.mdx.executor.execute import MdxEngine
CUBE_NAME ="temp_cube" CUBE_NAME = "temp_cube"
class CubeGen: class CubeGen:
""" """
...@@ -21,7 +22,8 @@ class CubeGen: ...@@ -21,7 +22,8 @@ class CubeGen:
# We have to generate DataFrames and save them to csv format because XmlaProviderService and # We have to generate DataFrames and save them to csv format because XmlaProviderService and
# MdxEngine classes use those files # MdxEngine classes use those files
def __init__(self, number_dimensions=1, rows_length=1000, columns_length=2): def __init__(self, number_dimensions=1, rows_length=1000,
columns_length=2):
self.number_dimensions = number_dimensions self.number_dimensions = number_dimensions
self.rows_length = rows_length self.rows_length = rows_length
self.columns_length = columns_length self.columns_length = columns_length
...@@ -38,14 +40,20 @@ class CubeGen: ...@@ -38,14 +40,20 @@ class CubeGen:
facts = pd.DataFrame() facts = pd.DataFrame()
for idx, dim in enumerate(range(self.number_dimensions)): for idx, dim in enumerate(range(self.number_dimensions)):
table_name = 'table' + str(idx) table_name = 'table' + str(idx)
table_values = pd.DataFrame(np.random.randint(min_val, max_val, size=(self.rows_length, self.columns_length)), table_values = pd.DataFrame(
np.random.randint(
min_val,
max_val,
size=(self.rows_length, self.columns_length)),
columns=list( columns=list(
table_name + col for col in string.ascii_uppercase[:self.columns_length])) table_name + col
for col in string.ascii_uppercase[:self.columns_length]))
table_values.index.name = table_name + "_id" table_values.index.name = table_name + "_id"
tables[table_name] = table_values.reset_index() tables[table_name] = table_values.reset_index()
facts[table_name + "_id"] = tables[table_name][table_name + "_id"] facts[table_name + "_id"] = tables[table_name][table_name + "_id"]
facts['Amount'] = np.random.randint(300, 1000, size=(self.rows_length, 1)) facts['Amount'] = np.random.randint(
300, 1000, size=(self.rows_length, 1))
tables['Facts'] = facts tables['Facts'] = facts
return tables return tables
...@@ -57,19 +65,22 @@ class CubeGen: ...@@ -57,19 +65,22 @@ class CubeGen:
:param tables: dict of DataFrames :param tables: dict of DataFrames
""" """
cube_path = os.path.join( cube_path = os.path.join(
os.path.abspath( os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER) MdxEngine.CUBE_FOLDER)
if not os.path.isdir(os.path.join(cube_path, CUBE_NAME)): if not os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
os.makedirs(os.path.join(cube_path, CUBE_NAME)) os.makedirs(os.path.join(cube_path, CUBE_NAME))
cube_path = os.path.join(cube_path, CUBE_NAME) cube_path = os.path.join(cube_path, CUBE_NAME)
for (table_name, table_value) in tables.items(): for (table_name, table_value) in tables.items():
table_value.to_csv(os.path.join(os.path.join(cube_path, table_name + '.csv')), sep=";", index=False) table_value.to_csv(
os.path.join(os.path.join(cube_path, table_name + '.csv')),
sep=";",
index=False)
@staticmethod @staticmethod
def remove_temp_cube(): def remove_temp_cube():
"""Remove the temporary cube.""" """Remove the temporary cube."""
cube_path = os.path.join( cube_path = os.path.join(
os.path.abspath( os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER) MdxEngine.CUBE_FOLDER)
if os.path.isdir(os.path.join(cube_path, CUBE_NAME)): if os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
shutil.rmtree(os.path.join(cube_path, CUBE_NAME)) shutil.rmtree(os.path.join(cube_path, CUBE_NAME))
...@@ -21,4 +21,5 @@ class MicBench: ...@@ -21,4 +21,5 @@ class MicBench:
to one million to one million
:return: float execution time in seconds :return: float execution time in seconds
""" """
return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(number=number) return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(
number=number)
...@@ -41,6 +41,7 @@ class MdxEngine: ...@@ -41,6 +41,7 @@ class MdxEngine:
# (before instantiate MdxEngine I need to access cubes information) # (before instantiate MdxEngine I need to access cubes information)
csv_files_cubes = [] csv_files_cubes = []
postgres_db_cubes = [] postgres_db_cubes = []
# to show just config file's dimensions # to show just config file's dimensions
def __init__(self, def __init__(self,
...@@ -113,7 +114,8 @@ class MdxEngine: ...@@ -113,7 +114,8 @@ class MdxEngine:
try: try:
db = MyDB(db_config_file_path=olapy_data_location) db = MyDB(db_config_file_path=olapy_data_location)
# TODO this work only with postgres # TODO this work only with postgres
result = db.engine.execute('SELECT datname FROM pg_database WHERE datistemplate = false;') result = db.engine.execute(
'SELECT datname FROM pg_database WHERE datistemplate = false;')
available_tables = result.fetchall() available_tables = result.fetchall()
# cursor.execute("""SELECT datname FROM pg_database # cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""") # WHERE datistemplate = false;""")
...@@ -164,9 +166,10 @@ class MdxEngine: ...@@ -164,9 +166,10 @@ class MdxEngine:
config_file_parser = ConfigParser(self.cube_path) config_file_parser = ConfigParser(self.cube_path)
tables = {} tables = {}
if self.client == 'excel' and config_file_parser.config_file_exist(client_type=self.client if self.client == 'excel' and config_file_parser.config_file_exist(
) and self.cube in config_file_parser.get_cubes_names(client_type=self.client client_type=self.
): client) and self.cube in config_file_parser.get_cubes_names(
client_type=self.client):
# for web (config file) we need only star_schema_dataframes, not all tables # for web (config file) we need only star_schema_dataframes, not all tables
for cubes in config_file_parser.construct_cubes(): for cubes in config_file_parser.construct_cubes():
...@@ -188,7 +191,8 @@ class MdxEngine: ...@@ -188,7 +191,8 @@ class MdxEngine:
# if web, get measures from config file # if web, get measures from config file
config_file_parser = ConfigParser(self.cube_path) config_file_parser = ConfigParser(self.cube_path)
if self.client == 'web' and config_file_parser.config_file_exist('web'): if self.client == 'web' and config_file_parser.config_file_exist(
'web'):
for cubes in config_file_parser.construct_cubes(self.client): for cubes in config_file_parser.construct_cubes(self.client):
# update facts name # update facts name
...@@ -214,19 +218,19 @@ class MdxEngine: ...@@ -214,19 +218,19 @@ class MdxEngine:
fusion = None fusion = None
config_file_parser = ConfigParser(self.cube_path) config_file_parser = ConfigParser(self.cube_path)
if config_file_parser.config_file_exist( if config_file_parser.config_file_exist(
self.client self.
) and self.cube in config_file_parser.get_cubes_names( client) and self.cube in config_file_parser.get_cubes_names(
client_type=self.client): client_type=self.client):
for cubes in config_file_parser.construct_cubes(self.client): for cubes in config_file_parser.construct_cubes(self.client):
# TODO cubes.source == 'csv' # TODO cubes.source == 'csv'
if cubes.source == 'postgres': if cubes.source == 'postgres':
# TODO one config file (I will try to merge dimensions between them in web part) # TODO one config file (I will try to merge dimensions between them in web part)
if self.client == 'web': if self.client == 'web':
fusion = _construct_web_star_schema_config_file( fusion = _construct_web_star_schema_config_file(self,
self, cubes) cubes)
else: else:
fusion = _construct_star_schema_config_file( fusion = _construct_star_schema_config_file(self,
self, cubes) cubes)
elif self.cube in self.csv_files_cubes: elif self.cube in self.csv_files_cubes:
fusion = _construct_star_schema_csv_files(self) fusion = _construct_star_schema_csv_files(self)
...@@ -256,7 +260,8 @@ class MdxEngine: ...@@ -256,7 +260,8 @@ class MdxEngine:
:return: path to the cube :return: path to the cube
""" """
if MdxEngine.DATA_FOLDER is not None: if MdxEngine.DATA_FOLDER is not None:
return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER, self.cube) return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER,
self.cube)
return os.path.join(self.cube_path, self.cube) return os.path.join(self.cube_path, self.cube)
# TODO temporary function # TODO temporary function
...@@ -309,7 +314,7 @@ class MdxEngine: ...@@ -309,7 +314,7 @@ class MdxEngine:
for tup_att in tup[0].replace('.Members', '').split('.') if tup_att for tup_att in tup[0].replace('.Members', '').split('.') if tup_att
] ]
for tup in re.compile(regex).findall( for tup in re.compile(regex).findall(
query.encode("utf-8",'replace')[start:stop]) query.encode("utf-8", 'replace')[start:stop])
if len(tup[0].split('.')) > 1] if len(tup[0].split('.')) > 1]
# TODO temporary function # TODO temporary function
...@@ -603,9 +608,8 @@ class MdxEngine: ...@@ -603,9 +608,8 @@ class MdxEngine:
:return: updated columns_to_keep :return: updated columns_to_keep
""" """
if len( if len(tuple_as_list) == 3 and tuple_as_list[-1] in self.tables_loaded[
tuple_as_list tuple_as_list[0]].columns:
) == 3 and tuple_as_list[-1] in self.tables_loaded[tuple_as_list[0]].columns:
# in case of [Geography].[Geography].[Country] # in case of [Geography].[Geography].[Country]
cols = [tuple_as_list[-1]] cols = [tuple_as_list[-1]]
else: else:
......
...@@ -4,6 +4,7 @@ from ..tools.connection import MyDB ...@@ -4,6 +4,7 @@ from ..tools.connection import MyDB
import pandas.io.sql as psql import pandas.io.sql as psql
import os import os
# split execution into three part (execute from config files, # split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model, # execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model) # and execute data base tables if they respect olapy's start schema model)
...@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj): ...@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj):
# just one facts table right now # just one facts table right now
executer_instance.facts = cube_obj.facts[0].table_name executer_instance.facts = cube_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube) db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
for dimension in cube_obj.dimensions: for dimension in cube_obj.dimensions:
...@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj): ...@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj):
table_name = dimension.name table_name = dimension.name
# rename columns if value not None # rename columns if value not None
df.rename(columns=(dict((k, v) for k, v in dimension.columns.items() if v)), inplace=True) df.rename(
columns=(dict((k, v) for k, v in dimension.columns.items() if v)),
inplace=True)
tables[table_name] = df[[ tables[table_name] = df[[
col for col in df.columns if col.lower()[-2:] != 'id' col for col in df.columns if col.lower()[-2:] != 'id'
...@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj): ...@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj):
return tables return tables
# excel client # excel client
def _construct_star_schema_config_file(executer_instance, cubes_obj): def _construct_star_schema_config_file(executer_instance, cubes_obj):
""" """
...@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj): ...@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
:return: star schema DataFrame :return: star schema DataFrame
""" """
executer_instance.facts = cubes_obj.facts[0].table_name executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube) db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
# load facts table # load facts table
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
...@@ -74,7 +82,10 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj): ...@@ -74,7 +82,10 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
# TODO CHOSE BETWEEN THOSES DF # TODO CHOSE BETWEEN THOSES DF
fusion = fusion.merge( fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left', df,
left_on=fact_key,
right_on=dimension_and_key.split('.')[1],
how='left',
# remove suffixe from dimension and keep the same column name for facts # remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y')) suffixes=('', '_y'))
...@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj): ...@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
return fusion return fusion
# web client # web client
def _construct_web_star_schema_config_file(executer_instance, cubes_obj): def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
""" """
...@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns = [] all_columns = []
executer_instance.facts = cubes_obj.facts[0].table_name executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube) db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
# load facts table # load facts table
if cubes_obj.facts[0].columns: if cubes_obj.facts[0].columns:
...@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
# TODO check merge (how) # TODO check merge (how)
fusion = fusion.merge( fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left', df,
left_on=fact_key,
right_on=dimension_and_key.split('.')[1],
how='left',
# remove suffixe from dimension and keep the same column name for facts # remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y')) suffixes=('', '_y'))
return fusion[[column for column in all_columns if 'id' != column[-2:]]] return fusion[[column for column in all_columns if 'id' != column[-2:]]]
...@@ -8,6 +8,7 @@ import pandas as pd ...@@ -8,6 +8,7 @@ import pandas as pd
# execute csv files if they respect olapy's start schema model, # execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model) # and execute data base tables if they respect olapy's start schema model)
def _load_tables_csv_files(executer_instance): def _load_tables_csv_files(executer_instance):
""" """
Load tables from csv files. Load tables from csv files.
......
...@@ -9,8 +9,6 @@ import pandas.io.sql as psql ...@@ -9,8 +9,6 @@ import pandas.io.sql as psql
# execute csv files if they respect olapy's start schema model, # execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model) # and execute data base tables if they respect olapy's start schema model)
# class StringFolder(object): # class StringFolder(object):
# """ # """
# Class that will fold strings. See 'fold_string'. # Class that will fold strings. See 'fold_string'.
...@@ -68,6 +66,7 @@ import pandas.io.sql as psql ...@@ -68,6 +66,7 @@ import pandas.io.sql as psql
# TODO try pandas.read_sql_table and pandas.read_sql # TODO try pandas.read_sql_table and pandas.read_sql
def _load_tables_db(executer_instance): def _load_tables_db(executer_instance):
""" """
Load tables from database. Load tables from database.
...@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance): ...@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance):
:return: tables dict with table name as key and dataframe as value :return: tables dict with table name as key and dataframe as value
""" """
tables = {} tables = {}
db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube) db = MyDB(
db_config_file_path=executer_instance.DATA_FOLDER,
db=executer_instance.cube)
inspector = inspect(db.engine) inspector = inspect(db.engine)
for table_name in inspector.get_table_names(): for table_name in inspector.get_table_names():
...@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance): ...@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance):
# 'SELECT * FROM "{0}"'.format(table_name), db.engine) # 'SELECT * FROM "{0}"'.format(table_name), db.engine)
# results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name)) # results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
results = db.engine.execution_options(stream_results=True).execute('SELECT * FROM "{0}"'.format(table_name)) results = db.engine.execution_options(stream_results=True).execute(
'SELECT * FROM "{0}"'.format(table_name))
# Fetch all the results of the query # Fetch all the results of the query
value = pd.DataFrame(iter(results),columns=results.keys()) # Pass results as an iterator value = pd.DataFrame(
iter(results),
columns=results.keys()) # Pass results as an iterator
# with string_folding_wrapper we loose response time # with string_folding_wrapper we loose response time
# value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys()) # value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
tables[table_name] = value[[ tables[table_name] = value[[
......
...@@ -246,7 +246,7 @@ class ConfigParser: ...@@ -246,7 +246,7 @@ class ConfigParser:
""" """
def __init__(self, def __init__(self,
cube_path = None, cube_path=None,
file_name='cubes-config.xml', file_name='cubes-config.xml',
web_config_file_name='web_cube_config.xml'): web_config_file_name='web_cube_config.xml'):
""" """
...@@ -262,7 +262,8 @@ class ConfigParser: ...@@ -262,7 +262,8 @@ class ConfigParser:
home_directory = expanduser("~") home_directory = expanduser("~")
if cube_path is None: if cube_path is None:
self.cube_path = os.path.join(home_directory, 'olapy-data', 'cubes') self.cube_path = os.path.join(home_directory, 'olapy-data',
'cubes')
else: else:
self.cube_path = cube_path self.cube_path = cube_path
...@@ -303,7 +304,7 @@ class ConfigParser: ...@@ -303,7 +304,7 @@ class ConfigParser:
else: else:
return False return False
def get_cubes_names(self,client_type): def get_cubes_names(self, client_type):
""" """
Get all cubes names in the config file. Get all cubes names in the config file.
...@@ -359,10 +360,10 @@ class ConfigParser: ...@@ -359,10 +360,10 @@ class ConfigParser:
# column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')], # column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
displayName=xml_dimension.find('displayName').text, displayName=xml_dimension.find('displayName').text,
columns=OrderedDict( columns=OrderedDict(
(column_name.text , None if not column_name.attrib else column_name.attrib['column_new_name']) (column_name.text, None if not column_name.attrib
else column_name.attrib['column_new_name'])
for column_name in xml_dimension.findall( for column_name in xml_dimension.findall(
'columns/name') 'columns/name')))
))
for xml_dimension in tree.xpath( for xml_dimension in tree.xpath(
'/cubes/cube/dimensions/dimension') '/cubes/cube/dimensions/dimension')
] ]
...@@ -449,8 +450,7 @@ class ConfigParser: ...@@ -449,8 +450,7 @@ class ConfigParser:
global_table={ global_table={
'columns': 'columns':
dashboard.find('Global_table/columns').text.split(','), dashboard.find('Global_table/columns').text.split(','),
'rows': 'rows': dashboard.find('Global_table/rows').text.split(',')
dashboard.find('Global_table/rows').text.split(',')
}, },
pie_charts=dashboard.find('PieCharts').text.split(','), pie_charts=dashboard.find('PieCharts').text.split(','),
bar_charts=dashboard.find('BarCharts').text.split(','), bar_charts=dashboard.find('BarCharts').text.split(','),
......
from __future__ import absolute_import, division, print_function
# import psycopg2 as pg # import psycopg2 as pg
from sqlalchemy import create_engine from sqlalchemy import create_engine
# postgres connection # postgres connection
from olapy_config_file_parser import DbConfigParser from .olapy_config_file_parser import DbConfigParser
class MyDB(object): class MyDB(object):
"""Connect to sql database (postgres only right now).""" """Connect to sql database (postgres only right now)."""
def __init__(self,db_config_file_path=None,db=None): def __init__(self, db_config_file_path=None, db=None):
# TODO temporary # TODO temporary
db_config = DbConfigParser(config_path=db_config_file_path) db_config = DbConfigParser(config_path=db_config_file_path)
...@@ -21,18 +23,19 @@ class MyDB(object): ...@@ -21,18 +23,19 @@ class MyDB(object):
# first i want to show all databases to user (in excel) # first i want to show all databases to user (in excel)
# self.engine = pg.connect("user={0} password={1} host='{2}'". # self.engine = pg.connect("user={0} password={1} host='{2}'".
# format(username, password, host)) # format(username, password, host))
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format( self.engine = create_engine(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, 'postgres', host, port)) username, password, 'postgres', host, port))
else: else:
# and then we connect to the user db # and then we connect to the user db
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format( self.engine = create_engine(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, db, host, port)) username, password, db, host, port))
# self.connection = pg.connect( # self.connection = pg.connect(
# "user={0} password={1} dbname='{2}' host='{3}'".format( # "user={0} password={1} dbname='{2}' host='{3}'".format(
# username, password, db, host)) # username, password, db, host))
def __del__(self): def __del__(self):
if hasattr(self, 'connection'): if hasattr(self, 'connection'):
self.engine.dispose() self.engine.dispose()
...@@ -4,12 +4,11 @@ import os ...@@ -4,12 +4,11 @@ import os
from lxml import etree from lxml import etree
class DbConfigParser: class DbConfigParser:
# TODO one config file (I will try to merge dimensions between them in web part) # TODO one config file (I will try to merge dimensions between them in web part)
def __init__(self, def __init__(self, config_path=None, file_name='olapy-config.xml'):
config_path = None,
file_name='olapy-config.xml'):
""" """
:param cube_path: path to cube (csv folders) :param cube_path: path to cube (csv folders)
...@@ -50,6 +49,5 @@ class DbConfigParser: ...@@ -50,6 +49,5 @@ class DbConfigParser:
'password': db.find('password').text, 'password': db.find('password').text,
'host': db.find('host').text, 'host': db.find('host').text,
'port': db.find('port').text, 'port': db.find('port').text,
} } for db in tree.xpath('/olapy/database')
for db in tree.xpath('/olapy/database')
] ]
...@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function ...@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function
from spyne import ComplexModel, Integer, Unicode, XmlAttribute from spyne import ComplexModel, Integer, Unicode, XmlAttribute
# NOTE : I didn't respect python naming convention here # NOTE : I didn't respect python naming convention here
# because we need to create the xmla response (generated by spyne) with the same variable names, # because we need to create the xmla response (generated by spyne) with the same variable names,
# thus xmla requests from excel can be reached # thus xmla requests from excel can be reached
class Tuple(object): class Tuple(object):
"""Tuple description (used by spyne).""" """Tuple description (used by spyne)."""
...@@ -110,7 +109,7 @@ class Propertielist(ComplexModel): ...@@ -110,7 +109,7 @@ class Propertielist(ComplexModel):
class Command(ComplexModel): class Command(ComplexModel):
"""Command description (used by spyne).""" """Command description (used by spyne)."""
_type_info = {'Statement': Unicode,} _type_info = {'Statement': Unicode, }
class ExecuteRequest(ComplexModel): class ExecuteRequest(ComplexModel):
......
...@@ -54,8 +54,7 @@ class XmlaProviderService(ServiceBase): ...@@ -54,8 +54,7 @@ class XmlaProviderService(ServiceBase):
discover_tools = XmlaDiscoverTools() discover_tools = XmlaDiscoverTools()
sessio_id = discover_tools.session_id sessio_id = discover_tools.session_id
@rpc( @rpc(DiscoverRequest,
DiscoverRequest,
_returns=AnyXml, _returns=AnyXml,
_body_style="bare", _body_style="bare",
_out_header=Session, _out_header=Session,
...@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase): ...@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase):
ctx.out_header = Session(SessionId=str(XmlaProviderService.sessio_id)) ctx.out_header = Session(SessionId=str(XmlaProviderService.sessio_id))
config_parser = ConfigParser(discover_tools.executer.cube_path) config_parser = ConfigParser(discover_tools.executer.cube_path)
if config_parser.xmla_authentication( if config_parser.xmla_authentication() and ctx.transport.req_env[
) and ctx.transport.req_env['QUERY_STRING'] != 'admin': 'QUERY_STRING'] != 'admin':
raise InvalidCredentialsError( raise InvalidCredentialsError(
fault_string= fault_string='You do not have permission to access this resource',
'You do not have permission to access this resource',
fault_object=None) fault_object=None)
# TODO call (labster) login function or create login with token (according to labster db) # TODO call (labster) login function or create login with token (according to labster db)
...@@ -143,8 +141,7 @@ class XmlaProviderService(ServiceBase): ...@@ -143,8 +141,7 @@ class XmlaProviderService(ServiceBase):
# Execute function must take 2 argument ( JUST 2 ! ) Command and Properties # Execute function must take 2 argument ( JUST 2 ! ) Command and Properties
# we encapsulate them in ExecuteRequest object # we encapsulate them in ExecuteRequest object
@rpc( @rpc(ExecuteRequest,
ExecuteRequest,
_returns=AnyXml, _returns=AnyXml,
_body_style="bare", _body_style="bare",
_out_header=Session) _out_header=Session)
...@@ -235,7 +232,7 @@ application = Application( ...@@ -235,7 +232,7 @@ application = Application(
wsgi_application = WsgiApplication(application) wsgi_application = WsgiApplication(application)
def start_server(host='0.0.0.0',port=8000,write_on_file=False): def start_server(host='0.0.0.0', port=8000, write_on_file=False):
""" """
Start the xmla server. Start the xmla server.
......
...@@ -139,7 +139,7 @@ class XmlaDiscoverTools(): ...@@ -139,7 +139,7 @@ class XmlaDiscoverTools():
{1} {1}
</root> </root>
</return> </return>
""".format(xsd,rows)) """.format(xsd, rows))
if request.Restrictions.RestrictionList.PropertyName == 'Catalog': if request.Restrictions.RestrictionList.PropertyName == 'Catalog':
if request.Properties.PropertyList.Catalog is not None: if request.Properties.PropertyList.Catalog is not None:
...@@ -195,7 +195,7 @@ class XmlaDiscoverTools(): ...@@ -195,7 +195,7 @@ class XmlaDiscoverTools():
'MdpropMdxNamedSets', 'int', 'Read', 'false', 'MdpropMdxNamedSets', 'int', 'Read', 'false',
'15') '15')
return get_props(discover_preperties_xsd, '','', '', '', '','') return get_props(discover_preperties_xsd, '', '', '', '', '', '')
def discover_schema_rowsets_response(self, request): def discover_schema_rowsets_response(self, request):
if request.Restrictions.RestrictionList.SchemaName == "MDSCHEMA_HIERARCHIES" and \ if request.Restrictions.RestrictionList.SchemaName == "MDSCHEMA_HIERARCHIES" and \
...@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools(): ...@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools():
# french caracteres # french caracteres
# TODO encode dataframe # TODO encode dataframe
if type(df.iloc[0][0]) == unicode: if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8','replace') column_attribut = df.iloc[0][0].encode('utf-8',
'replace')
else: else:
column_attribut = df.iloc[0][0] column_attribut = df.iloc[0][0]
rows += """ rows += """
<row> <row>
<CATALOG_NAME>{0}</CATALOG_NAME> <CATALOG_NAME>{0}</CATALOG_NAME>
...@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools(): ...@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN> <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION> <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row> </row>
""".format(self.selected_catalogue, """.format(self.selected_catalogue, table_name,
table_name, df.columns[0], column_attribut)
df.columns[0],
column_attribut)
rows += """ rows += """
<row> <row>
...@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools(): ...@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools():
# french caracteres # french caracteres
# TODO encode dataframe # TODO encode dataframe
if type(df.iloc[0][0]) == unicode: if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8','replace') column_attribut = df.iloc[0][0].encode('utf-8',
'replace')
else: else:
column_attribut = df.iloc[0][0] column_attribut = df.iloc[0][0]
...@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools(): ...@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN> <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION> <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row> </row>
""".format(self.selected_catalogue, """.format(self.selected_catalogue, table_name,
table_name, df.columns[0], column_attribut)
df.columns[0],
column_attribut)
rows += """ rows += """
<row> <row>
......
...@@ -104,11 +104,11 @@ class XmlaExecuteTools(): ...@@ -104,11 +104,11 @@ class XmlaExecuteTools():
first_att = 3 first_att = 3
# query with on columns and on rows (without measure) # query with on columns and on rows (without measure)
elif mdx_execution_result['columns_desc']['columns'] and mdx_execution_result['columns_desc']['rows']: elif mdx_execution_result['columns_desc'][
'columns'] and mdx_execution_result['columns_desc']['rows']:
# ['Geography','America'] # ['Geography','America']
tuples = [ tuples = [
zip( zip(*[[[key] + list(row)
*[[[key] + list(row)
for row in splited_df[key].itertuples(index=False)] for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys() for key in splited_df.keys()
if key is not self.executer.facts]) if key is not self.executer.facts])
...@@ -120,8 +120,7 @@ class XmlaExecuteTools(): ...@@ -120,8 +120,7 @@ class XmlaExecuteTools():
else: else:
# ['Geography','Amount','America'] # ['Geography','Amount','America']
tuples = [ tuples = [
zip( zip(*[[[key] + [mes] + list(row)
*[[[key] + [mes] + list(row)
for row in splited_df[key].itertuples(index=False)] for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys() for key in splited_df.keys()
if key is not self.executer.facts]) if key is not self.executer.facts])
...@@ -155,7 +154,10 @@ class XmlaExecuteTools(): ...@@ -155,7 +154,10 @@ class XmlaExecuteTools():
# french caracteres # french caracteres
# TODO encode dataframe # TODO encode dataframe
if type(tuple_without_minus_1[-1]) == unicode: if type(tuple_without_minus_1[-1]) == unicode:
tuple_without_minus_1 = [x.encode('utf-8','replace') for x in tuple_without_minus_1] tuple_without_minus_1 = [
x.encode('utf-8', 'replace')
for x in tuple_without_minus_1
]
axis0 += """ axis0 += """
<Member Hierarchy="[{0}].[{0}]"> <Member Hierarchy="[{0}].[{0}]">
...@@ -254,7 +256,8 @@ class XmlaExecuteTools(): ...@@ -254,7 +256,8 @@ class XmlaExecuteTools():
# TODO must be OPTIMIZED every time!!!!! # TODO must be OPTIMIZED every time!!!!!
dfs = self.split_dataframe(mdx_execution_result) dfs = self.split_dataframe(mdx_execution_result)
if mdx_execution_result['columns_desc']['rows'] and mdx_execution_result['columns_desc']['columns']: if mdx_execution_result['columns_desc'][
'rows'] and mdx_execution_result['columns_desc']['columns']:
return """ return """
{0} {0}
...@@ -306,12 +309,11 @@ class XmlaExecuteTools(): ...@@ -306,12 +309,11 @@ class XmlaExecuteTools():
""" """
columns_loop = [] columns_loop = []
if ( if ((len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
(len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
^ ^
(len(mdx_execution_result['columns_desc']['rows'].keys()) == 0) (len(mdx_execution_result['columns_desc']['rows'].keys()) == 0
) and self.executer.facts in mdx_execution_result['columns_desc']['all'].keys( )) and self.executer.facts in mdx_execution_result[
): 'columns_desc']['all'].keys():
# iterate DataFrame horizontally # iterate DataFrame horizontally
columns_loop = itertools.chain(*[ columns_loop = itertools.chain(*[
...@@ -332,7 +334,7 @@ class XmlaExecuteTools(): ...@@ -332,7 +334,7 @@ class XmlaExecuteTools():
cell_data = "" cell_data = ""
index = 0 index = 0
for value in columns_loop: for value in columns_loop:
if np.isnan(value) : if np.isnan(value):
value = '' value = ''
cell_data += """ cell_data += """
<Cell CellOrdinal="{0}"> <Cell CellOrdinal="{0}">
...@@ -392,9 +394,9 @@ class XmlaExecuteTools(): ...@@ -392,9 +394,9 @@ class XmlaExecuteTools():
to_write = "[{0}].[{0}]".format(dim_diff) to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures': if dim_diff == 'Measures':
# if measures > 1 we don't have to write measure # if measures > 1 we don't have to write measure
if self.executer.facts in mdx_execution_result['columns_desc']['all'] and len( if self.executer.facts in mdx_execution_result['columns_desc'][
mdx_execution_result['columns_desc']['all'] 'all'] and len(mdx_execution_result['columns_desc'][
[self.executer.facts]) > 1: 'all'][self.executer.facts]) > 1:
continue continue
else: else:
to_write = "[Measures]" to_write = "[Measures]"
...@@ -452,9 +454,9 @@ class XmlaExecuteTools(): ...@@ -452,9 +454,9 @@ class XmlaExecuteTools():
""" """
hierarchy_info = "" hierarchy_info = ""
# measure must be written at the top # measure must be written at the top
if self.executer.facts in mdx_execution_result['columns_desc'][mdx_query_axis].keys( if self.executer.facts in mdx_execution_result['columns_desc'][
) and len(mdx_execution_result['columns_desc'][mdx_query_axis] mdx_query_axis].keys() and len(mdx_execution_result[
[self.executer.facts]) > 1: 'columns_desc'][mdx_query_axis][self.executer.facts]) > 1:
hierarchy_info += """ hierarchy_info += """
<HierarchyInfo name="{0}"> <HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/> <UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
...@@ -556,15 +558,18 @@ class XmlaExecuteTools(): ...@@ -556,15 +558,18 @@ class XmlaExecuteTools():
for dim_diff in list( for dim_diff in list(
set(self.executer.get_all_tables_names(ignore_fact=True)) - set(self.executer.get_all_tables_names(ignore_fact=True)) -
set(table_name set(table_name
for table_name in mdx_execution_result['columns_desc'] for table_name in mdx_execution_result['columns_desc'][
['all'])): 'all'])):
# TODO encode dataframe # TODO encode dataframe
# french caracteres # french caracteres
if type(self.executer.tables_loaded[dim_diff].iloc[0][0]) == unicode: if type(self.executer.tables_loaded[dim_diff].iloc[0][
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0].encode('utf-8','replace') 0]) == unicode:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0].encode('utf-8', 'replace')
else: else:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0] column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0]
tuple += """ tuple += """
<Member Hierarchy="[{0}].[{0}]"> <Member Hierarchy="[{0}].[{0}]">
......
from __future__ import absolute_import, division, print_function
import pandas as pd import pandas as pd
from olapy.core.mdx.executor.execute import MdxEngine from olapy.core.mdx.executor.execute import MdxEngine
......
...@@ -198,8 +198,7 @@ def test_query2(conn): ...@@ -198,8 +198,7 @@ def test_query2(conn):
mems.append( mems.append(
Member( Member(
_Hierarchy='[Geography].[Geography]', _Hierarchy='[Geography].[Geography]',
UName= UName='[Geography].[Geography].[Country].[America].[United States]',
'[Geography].[Geography].[Country].[America].[United States]',
Caption='United States', Caption='United States',
LName='[Geography].[Geography].[Country]', LName='[Geography].[Geography].[Country]',
LNum='1', LNum='1',
...@@ -209,14 +208,12 @@ def test_query2(conn): ...@@ -209,14 +208,12 @@ def test_query2(conn):
mems.append( mems.append(
Member( Member(
_Hierarchy='[Geography].[Geography]', _Hierarchy='[Geography].[Geography]',
UName= UName='[Geography].[Geography].[City].[America].[United States].[New York]',
'[Geography].[Geography].[City].[America].[United States].[New York]',
Caption='New York', Caption='New York',
LName='[Geography].[Geography].[City]', LName='[Geography].[Geography].[City]',
LNum='2', LNum='2',
DisplayInfo='131076', DisplayInfo='131076',
PARENT_UNIQUE_NAME= PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[America].[United States]',
'[Geography].[Geography].[Continent].[America].[United States]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]')) HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append( mems.append(
Member( Member(
...@@ -250,14 +247,12 @@ def test_query2(conn): ...@@ -250,14 +247,12 @@ def test_query2(conn):
mems.append( mems.append(
Member( Member(
_Hierarchy='[Geography].[Geography]', _Hierarchy='[Geography].[Geography]',
UName= UName='[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
'[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
Caption='Barcelona', Caption='Barcelona',
LName='[Geography].[Geography].[City]', LName='[Geography].[Geography].[City]',
LNum='2', LNum='2',
DisplayInfo='131076', DisplayInfo='131076',
PARENT_UNIQUE_NAME= PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
'[Geography].[Geography].[Continent].[Europe].[Spain]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]')) HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append( mems.append(
Member( Member(
...@@ -267,8 +262,7 @@ def test_query2(conn): ...@@ -267,8 +262,7 @@ def test_query2(conn):
LName='[Geography].[Geography].[City]', LName='[Geography].[Geography].[City]',
LNum='2', LNum='2',
DisplayInfo='131076', DisplayInfo='131076',
PARENT_UNIQUE_NAME= PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
'[Geography].[Geography].[Continent].[Europe].[Spain]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]')) HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append( mems.append(
Member( Member(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment