Skip to content

O
olapy
Commit f6879c40 authored by mouadh's avatar mouadh
Browse files
Options

fix config file bug

parent 82c27c2d
Hide whitespace changes
Inline Side-by-side
Showing with 136 additions and 128 deletions
olapy/core/mdx/executor/execute.py
View file @ f6879c40
......@@ -42,17 +42,18 @@ class MdxEngine:
def __init__(self,
cube_name,
client_type='excel',
cubes_path=None,
mdx_query=None,
cube_folder=CUBE_FOLDER,
sep=';',
fact_table_name="Facts"):
self.cube_folder = cube_folder
self.cube = cube_name
self.sep = sep
self.facts = fact_table_name
self.mdx_query = mdx_query
if cubes_path is None:
self.cube_path = self._get_default_cube_directory()
else:
......@@ -60,10 +61,11 @@ class MdxEngine:
# to get cubes in db
self._ = self.get_cubes_names()
self.client = client_type
self.tables_loaded = self.load_tables()
# all measures
self.measures = self.get_measures()
self.load_star_schema_dataframe = self.get_star_schema_dataframe()
self.measures = self.get_measures()
self.tables_names = self._get_tables_name()
# default measure is the first one
self.selected_measures = [self.measures[0]]
......@@ -128,13 +130,13 @@ class MdxEngine:
def load_tables(self):
"""
Load all tables { Table name : DataFrame } of the current cube instance.
:return: dict with key as table name and DataFrame as value
"""
config_file_parser = ConfigParser(self.cube_path)
tables = {}
if config_file_parser.config_file_exist(
) and self.cube in config_file_parser.get_cubes_names():
) and self.cube in config_file_parser.get_cubes_names() and self.client != 'web':
for cubes in config_file_parser.construct_cubes():
# TODO working with cubes.source == 'csv'
......@@ -158,7 +160,7 @@ class MdxEngine:
include=[np.number]).columns if col.lower()[-2:] != 'id'
]
def get_star_schema_dataframe(self, client_type='excel'):
def get_star_schema_dataframe(self):
"""
Merge all DataFrames as star schema.
......@@ -169,13 +171,13 @@ class MdxEngine:
config_file_parser = ConfigParser(self.cube_path)
if config_file_parser.config_file_exist(
client_type
) and self.cube in config_file_parser.get_cubes_names():
for cubes in config_file_parser.construct_cubes(client_type):
self.client
) and self.cube in config_file_parser.get_cubes_names(client_type='web'):
for cubes in config_file_parser.construct_cubes(self.client):
# TODO cubes.source == 'csv'
if cubes.source == 'postgres':
# TODO one config file (I will try to merge dimensions between them in web part)
if client_type == 'web':
if self.client == 'web':
fusion = _construct_web_star_schema_config_file(
self, cubes)
else:
......@@ -232,7 +234,7 @@ class MdxEngine:
FROM {sales}
it returns :
[
['Geography','Geography','Continent'],
['Geography','Geography','Continent','Europe'],
......@@ -260,9 +262,9 @@ class MdxEngine:
tup_att.replace('All ', '').replace('[', "").replace("]", "")
for tup_att in tup[0].replace('.Members', '').split('.')
]
for tup in re.compile(regex).findall(
query.encode("utf-8")[start:stop])
if len(tup[0].split('.')) > 1]
for tup in re.compile(regex).findall(
query.encode("utf-8")[start:stop])
if len(tup[0].split('.')) > 1]
# TODO temporary function
def decorticate_query(self, query):
......@@ -313,13 +315,13 @@ class MdxEngine:
def change_measures(tuples_on_mdx):
"""
Set measures to which exists in the query.
:param tuples_on_mdx: list of tuples:
example : [ '[Measures].[Amount]' , '[Geography].[Geography].[Continent]' ]
:return: measures column's names
"""
return [
......@@ -362,7 +364,7 @@ class MdxEngine:
else:
tables_columns.update({
tupl[0]:
self.tables_loaded[tupl[0]].columns[:len(tupl[2:])]
self.tables_loaded[tupl[0]].columns[:len(tupl[2:])]
})
axes.update({axis: tables_columns})
......@@ -374,12 +376,12 @@ class MdxEngine:
Filter a DataFrame (Dataframe_in) with one tuple.
Example ::
tuple = ['Geography','Geography','Continent','Europe','France','olapy']
Dataframe_in in :
+-------------+----------+---------+---------+---------+
| Continent | Country | Company | Article | Amount |
+=============+==========+=========+=========+=========+
......@@ -389,9 +391,9 @@ class MdxEngine:
+-------------+----------+---------+---------+---------+
| ..... | ..... | ...... | ..... | ..... |
+-------------+----------+---------+---------+---------+
out :
+-------------+----------+---------+---------+---------+
| Continent | Country | Company | Article | Amount |
+=============+==========+=========+=========+=========+
......@@ -513,47 +515,47 @@ class MdxEngine:
If we have multiple dimensions, with many columns like:
columns_to_keep :
Geo -> Continent,Country
Prod -> Company
Time -> Year,Month,Day
we have to use only dimension's columns of current dimension that exist in tuple_as_list and keep other dimensions columns
so if tuple_as_list = ['Geography','Geography','Continent']
columns_to_keep will be:
columns_to_keep :
Geo -> Continent
Prod -> Company
Time -> Year,Month,Day
we need columns_to_keep for grouping our columns in the DataFrame
:param tuple_as_list: example : ['Geography','Geography','Continent']
:param columns_to_keep:
example :
{
'Geography':
['Continent','Country'],
'Time':
['Year','Month','Day']
}
:return: updated columns_to_keep
"""
if len(
......
olapy/core/mdx/tools/config_file_parser.py
View file @ f6879c40
......@@ -10,180 +10,180 @@ from .models import Cube, Dimension, Facts, Table, Dashboard
class ConfigParser:
"""
Parse olapy config files.
Config file used if you want to show only some measures, dimensions, columns... in excel
Config file should be under 'home-directory/olapy-data/cubes/cubes-config.xml'
Excel Config file Structure::
<?xml version="1.0" encoding="UTF-8"?>
<cubes>
<!-- if you want to set an authentication mechanism in excel so to access cube,
user must set a token with login url like 'http://127.0.0.1/admin -->
<!-- default password = admin -->
<xmla_authentication>False</xmla_authentication>
<cube>
<!-- cube name => db name -->
<name>labster</name>
<!-- source : postgres | csv -->
<source>postgres</source>
<!-- star building customized star schema -->
<facts>
<!-- facts table name -->
<table_name>stats_line</table_name>
<keys>
<!-- ref = table_name.column -->
<column_name ref="orgunit.id">departement_id</column_name>
</keys>
<!-- specify measures explicitly -->
<measures>
<!-- by default, all number type columns in facts table, or you can specify them here -->
<name>montant</name>
<name>salaire_brut_mensuel</name>
<name>cout_total_mensuel</name>
</measures>
</facts>
<!-- end building customized star schema -->
<!-- star building customized dimensions display in excel from the star schema -->
<dimensions>
<dimension>
<!-- if you want to keep the same name for excel display, just use the same name in name and displayName -->
<name>stats_line</name>
<displayName>Demande</displayName>
<columns>
<!-- columns order matter -->
<name>type_demande</name>
<name>financeur</name>
<name>wf_state</name>
<name>type_recrutement</name>
</columns>
</dimension>
<dimension>
<!-- if you want to keep the same name for excel display, just use the same name in name and displayName -->
<name>orgunit</name>
<displayName>Organisation</displayName>
<columns>
<!-- columns order matter -->
<name>type</name>
<name>nom</name>
<name>sigle</name>
</columns>
</dimension>
</dimensions>
<!-- end building customized dimensions display in excel from the star schema -->
</cube>
</cubes>
WEB Config file Structure::
<cubes>
<cube>
<!-- cube name => db name -->
<name>mpr</name>
<!-- source : postgres | csv -->
<source>postgres</source>
<!-- star building customized star schema -->
<facts>
<!-- facts table name -->
<table_name>projet</table_name>
<keys>
<!-- ref = table_name.column -->
<column_name ref="vocabulary_crm_status.id">status_id</column_name>
<column_name ref="vocabulary_crm_pole_leader.id">pole_leader_id</column_name>
<column_name ref="contact.id">contact_id</column_name>
<column_name ref="compte.id">compte_porteur_id</column_name>
<column_name ref="vocabulary_crm_aap_type.id">aap_name_id</column_name>
</keys>
<!-- specify measures explicitly -->
<measures>
<!-- by default, all number type columns in facts table, or you can specify them here -->
<name>budget_total</name>
<name>subvention_totale</name>
<name>duree_projet</name>
</measures>
</facts>
<!-- end building customized star schema -->
<tables>
<!-- Table name -->
<table name="vocabulary_crm_status">
<!-- Columns to keep (INCLUDING id)-->
<!-- They must be seperated with comma ',' -->
<columns>id,label</columns>
<!-- Change insignificant table columns names -->
<!-- {IMPORTANT} Renaming COMMUN columns between dimensions and other columns if you want, other than ids column -->
<new_name old_column_name="label">Status</new_name>
</table>
<table name="contact">
<columns>id,nom,prenom,fonction</columns>
</table>
</tables>
</cube>
</cubes>
"""
......@@ -194,7 +194,7 @@ class ConfigParser:
file_name='cubes-config.xml',
web_config_file_name='web_cube_config.xml'):
"""
:param cube_path: path to cube (csv folders)
:param file_name: config file name (DEFAULT = cubes-config.xml)
"""
......@@ -205,7 +205,7 @@ class ConfigParser:
def config_file_exist(self, client_type='excel'):
"""
Check whether the config file exists or not.
:return: True | False
"""
if client_type == 'web':
......@@ -216,27 +216,35 @@ class ConfigParser:
def xmla_authentication(self):
"""
Check if excel need authentication to access cubes or not. (xmla_authentication tag in the config file).
:return: True | False
"""
with open(os.path.join(self.cube_path, self.file_name)) as config_file:
if self.config_file_exist():
with open(os.path.join(self.cube_path, self.file_name)) as config_file:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
try:
return tree.xpath('/cubes/xmla_authentication')[
0].text == 'True'
except:
return False
try:
return tree.xpath('/cubes/xmla_authentication')[
0].text == 'True'
except:
return False
else:
return False
def get_cubes_names(self):
def get_cubes_names(self, client_type='excel'):
"""
Get all cubes names in the config file.
:return: dict of cube name as key and cube source as value (csv or postgres) (right now only postgres is supported)
"""
with open(os.path.join(self.cube_path, self.file_name)) as config_file:
if client_type == 'excel':
file_name = self.file_name
elif client_type == 'web':
file_name = self.web_config_file_name
with open(os.path.join(self.cube_path, file_name)) as config_file:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
......@@ -315,7 +323,6 @@ class ConfigParser:
# try:
with open(os.path.join(self.cube_path,
self.web_config_file_name)) as config_file:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
......@@ -358,7 +365,6 @@ class ConfigParser:
# try:
with open(os.path.join(self.cube_path,
self.web_config_file_name)) as config_file:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
......@@ -370,11 +376,11 @@ class ConfigParser:
'rows': dashboard.find('Global_table/rows').text.split(',')
},
pie_charts=dashboard.find('PieCharts').text.split(','),
bar_chats=dashboard.find('BarCharts').text.split(','),
bar_charts=dashboard.find('BarCharts').text.split(','),
line_charts={
table.find('name').text:
(table.find('columns').text.split(',')
if table.find('columns') is not None else 'ALL')
(table.find('columns').text.split(',')
if table.find('columns') is not None else 'ALL')
for table in dashboard.findall('LineCharts/table')
})
for dashboard in tree.xpath('/cubes/cube/Dashboards/Dashboard')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7