Skip to content

O
olapy
Commit cf4dde4d authored by mouadh's avatar mouadh
Browse files
Options

web config file

parent 682e4901
Show whitespace changes
Inline Side-by-side
Showing with 302 additions and 56 deletions
olapy/core/mdx/executor/execute.py
View file @ cf4dde4d
...@@ -96,6 +96,7 @@ class MdxEngine: ...@@ -96,6 +96,7 @@ class MdxEngine:
MdxEngine.postgres_db_cubes = [ MdxEngine.postgres_db_cubes = [
database[0] for database in cursor.fetchall() database[0] for database in cursor.fetchall()
] ]
except Exception: except Exception:
print('no database connexion') print('no database connexion')
pass pass
...@@ -258,6 +259,68 @@ class MdxEngine: ...@@ -258,6 +259,68 @@ class MdxEngine:
return fusion return fusion
def _construct_web_star_schema_config_file(self, cube_name, cubes_obj):
"""
Construct star schema DataFrame from configuration file.
:param cube_name: cube name (or database name)
:param cubes_obj: cubes object
:return: star schema DataFrame
"""
all_columns = []
self.facts = cubes_obj.facts[0].table_name
db = MyDB(db=cube_name)
# load facts table
# measures in config-file only
if cubes_obj.facts[0].measures:
self.measures = cubes_obj.facts[0].measures
all_columns += cubes_obj.facts[0].measures
fusion = psql.read_sql_query("SELECT * FROM {0}".format(self.facts),
db.connection)
tables = {}
for table in cubes_obj.tables:
tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
db.connection)
try:
if table.columns:
tab = tab[table.columns]
except:
print("table columns doesn't exist")
print('pass with all columns')
try:
if table.new_names:
tab = tab.rename(columns=table.new_names)
except:
print("verify your old and new columns names")
print('pass with no change')
all_columns += list(tab.columns)
tables.update({table.name: tab})
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
dimension_name = dimension_and_key.split('.')[0]
if dimension_name in tables.keys():
df = tables[dimension_name]
else:
df = psql.read_sql_query(
"SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
db.connection)
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
return fusion[[column for column in all_columns if 'id' != column[-2:]]]
def _construct_star_schema_csv_files(self, cube_name): def _construct_star_schema_csv_files(self, cube_name):
""" """
Construct star schema DataFrame from csv files. Construct star schema DataFrame from csv files.
...@@ -306,7 +369,7 @@ class MdxEngine: ...@@ -306,7 +369,7 @@ class MdxEngine:
return fusion return fusion
def get_star_schema_dataframe(self, cube_name): def get_star_schema_dataframe(self, cube_name, client_type='excel'):
""" """
Merge all DataFrames as star schema. Merge all DataFrames as star schema.
...@@ -316,11 +379,16 @@ class MdxEngine: ...@@ -316,11 +379,16 @@ class MdxEngine:
fusion = None fusion = None
config_file_parser = ConfigParser(self.cube_path) config_file_parser = ConfigParser(self.cube_path)
if config_file_parser.config_file_exist( if config_file_parser.config_file_exist(client_type
) and cube_name in config_file_parser.get_cubes_names(): ) and cube_name in config_file_parser.get_cubes_names():
for cubes in config_file_parser.construct_cubes(): for cubes in config_file_parser.construct_cubes(client_type):
# TODO cubes.source == 'csv' # TODO cubes.source == 'csv'
if cubes.source == 'postgres': if cubes.source == 'postgres':
# TODO one config file (I will try to merge dimensions between them in web part)
if client_type == 'web':
fusion = self._construct_web_star_schema_config_file(
cube_name, cubes)
else:
fusion = self._construct_star_schema_config_file( fusion = self._construct_star_schema_config_file(
cube_name, cubes) cube_name, cubes)
......
olapy/core/mdx/tools/config_file_parser.py
View file @ cf4dde4d
...@@ -4,7 +4,7 @@ import os ...@@ -4,7 +4,7 @@ import os
from lxml import etree from lxml import etree
from .models import Cube, Dimension, Facts from .models import Cube, Dimension, Facts, Table
class ConfigParser: class ConfigParser:
...@@ -15,7 +15,7 @@ class ConfigParser: ...@@ -15,7 +15,7 @@ class ConfigParser:
Config file should be under 'home-directory/olapy-data/cubes/cubes-config.xml' Config file should be under 'home-directory/olapy-data/cubes/cubes-config.xml'
Config file Structure:: Excel Config file Structure::
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
...@@ -113,8 +113,109 @@ class ConfigParser: ...@@ -113,8 +113,109 @@ class ConfigParser:
</cubes> </cubes>
WEB Config file Structure::
<?xml version="1.0" encoding="UTF-8"?>
<cubes>
<cube>
<!-- cube name => db name -->
<name>mpr</name>
<!-- source : postgres | csv -->
<source>postgres</source>
<!--
star building customized star schema
-->
<facts>
<!-- facts table name -->
<table_name>projet</table_name>
<keys>
<!-- ref = table_name.column -->
<column_name ref="vocabulary_crm_status.id">status_id</column_name>
<column_name ref="vocabulary_crm_pole_leader.id">pole_leader_id</column_name>
<column_name ref="contact.id">contact_id</column_name>
<column_name ref="compte.id">compte_porteur_id</column_name>
<column_name ref="vocabulary_crm_aap_type.id">aap_name_id</column_name>
</keys>
<!-- specify measures explicitly -->
<measures>
<!-- by default, all number type columns in facts table, or you can specify them here -->
<name>budget_total</name>
<name>subvention_totale</name>
<name>duree_projet</name>
</measures>
</facts>
<!--
end building customized star schema
-->
<!-- -------------------------------------------------------- -->
<!-- FOR WEB -->
<tables>
<!-- Table name -->
<table name="vocabulary_crm_status">
<!-- Columns to keep (exclude id)-->
<!-- They must be seperated with comma ',' -->
<columns>label,active</columns>
<!-- Change insignificant table columns names -->
<!-- {IMPORTANT} Renaming COMMUN columns between dimensions and other columns if you want, other than ids column -->
<new_name old_column_name="label">status_label</new_name>
</table>
<!-- *********************** -->
<!-- Table name -->
<table name="vocabulary_crm_pole_leader">
<!-- Columns to keep (exclude id)-->
<!-- They must be seperated with comma ',' -->
<columns>label</columns>
<!-- Change insignificant table columns names -->
<!-- {IMPORTANT} Renaming COMMUN columns between dimensions and other columns if you want, other than ids column -->
<new_name old_column_name="label">pole_leader_label</new_name>
</table>
<!-- *********************** -->
<!-- Table name -->
<table name="contact">
<!-- Columns to keep (exclude id)-->
<!-- They must be seperated with comma ',' -->
<columns>nom,prenom,fonction</columns>
</table>
</tables>
<!-- END FOR WEB -->
<!-- -------------------------------------------------------- -->
</cube>
</cubes>
""" """
def __init__(self, cube_path, file_name='cubes-config.xml'):
# TODO one config file (I will try to merge dimensions between them in web part)
def __init__(self, cube_path, file_name='cubes-config.xml', web_config_file_name='web_cube_config.xml'):
""" """
:param cube_path: path to cube (csv folders) :param cube_path: path to cube (csv folders)
...@@ -122,13 +223,16 @@ class ConfigParser: ...@@ -122,13 +223,16 @@ class ConfigParser:
""" """
self.cube_path = cube_path self.cube_path = cube_path
self.file_name = file_name self.file_name = file_name
self.web_config_file_name = web_config_file_name
def config_file_exist(self): def config_file_exist(self,client_type='excel'):
""" """
Check whether the config file exists or not. Check whether the config file exists or not.
:return: True | False :return: True | False
""" """
if client_type == 'web':
return os.path.isfile(os.path.join(self.cube_path, self.web_config_file_name))
return os.path.isfile(os.path.join(self.cube_path, self.file_name)) return os.path.isfile(os.path.join(self.cube_path, self.file_name))
def xmla_authentication(self): def xmla_authentication(self):
...@@ -167,13 +271,7 @@ class ConfigParser: ...@@ -167,13 +271,7 @@ class ConfigParser:
except: except:
raise ('missed name or source tags') raise ('missed name or source tags')
def construct_cubes(self): def _construct_cubes_excel(self):
"""
Construct cube (with it dimensions) and facts from the config file.
:return: list of Cubes instance
"""
if self.config_file_exist():
try: try:
with open(os.path.join(self.cube_path, with open(os.path.join(self.cube_path,
self.file_name)) as config_file: self.file_name)) as config_file:
...@@ -218,5 +316,69 @@ class ConfigParser: ...@@ -218,5 +316,69 @@ class ConfigParser:
] ]
except: except:
raise ('Bad configuration in the configuration file') raise ('Bad configuration in the configuration file')
def _construct_cubes_web(self):
# try:
with open(os.path.join(self.cube_path,
self.web_config_file_name)) as config_file:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
facts = [
Facts(
table_name=xml_facts.find('table_name').text,
keys={
key.text: key.attrib['ref']
for key in xml_facts.findall(
'keys/column_name')
},
measures=[
mes.text
for mes in xml_facts.findall('measures/name')
]) for xml_facts in tree.xpath('/cubes/cube/facts')
]
tables = [
Table(
name=xml_column.attrib['name'],
columns = xml_column.find('columns').text.split(','),
new_names={
new_col.attrib['old_column_name'] : new_col.text
for new_col in xml_column.findall('new_name')
}
)
for xml_column in tree.xpath(
'/cubes/cube/tables/table')
]
return [
Cube(
name=xml_cube.find('name').text,
source=xml_cube.find('source').text,
facts=facts,
tables = tables
)
for xml_cube in tree.xpath('/cubes/cube')
]
# except:
# raise ('Bad configuration in the configuration file')
def construct_cubes(self,client_type='excel'):
"""
Construct cube (with it dimensions) and facts from the config file.
:param client_type: excel | web
:return: list of Cubes instance
"""
if self.config_file_exist(client_type):
if client_type == 'excel':
return self._construct_cubes_excel()
elif client_type == 'web':
return self._construct_cubes_web()
else: else:
raise ("Config file don't exist") raise ("Config file don't exist")
olapy/core/mdx/tools/models.py
View file @ cf4dde4d
...@@ -51,3 +51,19 @@ class Cube: ...@@ -51,3 +51,19 @@ class Cube:
def __str__(self): def __str__(self):
return str(self.__dict__) return str(self.__dict__)
class Table:
"""Column class used to encapsulate config file attributes for web client."""
def __init__(self, **kwargs):
"""
:param kwargs: {
table_name : 'something',
old_column_name : 'something',
new_column_name : 'something'
}
"""
self.__dict__.update(kwargs)
def __str__(self):
return str(self.__dict__)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7