Skip to content

O
olapy
Commit 8cd36207 authored by mouadh's avatar mouadh
Browse files
Options

coms

parent e7dc4314
Hide whitespace changes
Inline Side-by-side
Showing with 82 additions and 79 deletions
olapy/core/mdx/executor/execute_config_file.py
View file @ 8cd36207
......@@ -5,7 +5,9 @@ from ..tools.connection import MyDB
import pandas.io.sql as psql
import os
# split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
def _load_table_config_file(executer_instance, cube_obj):
"""
Load tables from config file.
......@@ -45,10 +47,10 @@ def _load_table_config_file(executer_instance, cube_obj):
return tables
# excel client
def _construct_star_schema_config_file(executer_instance, cubes_obj):
"""
Construct star schema DataFrame from configuration file.
Construct star schema DataFrame from configuration file for excel client.
:param cube_name: cube name (or database name)
:param cubes_obj: cubes object
......@@ -82,53 +84,6 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
suffixes=('', '_y'))
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_config_file")
# TODO CHOSE BETWEEN THOSES DF
# if separated dimensions
# fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
# TODO CHOSE BETWEEN THOSES DF
# if facts contains all dimensions
# fusion = facts
# measures in config-file only
if cubes_obj.facts[0].measures:
executer_instance.measures = cubes_obj.facts[0].measures
return fusion
def _construct_star_schema_config_file_OLD(executer_instance, cubes_obj):
"""
Construct star schema DataFrame from configuration file.
:param cube_name: cube name (or database name)
:param cubes_obj: cubes object
:return: star schema DataFrame
"""
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db=executer_instance.cube)
# load facts table
memory_usage("1 - before executing query //// _construct_star_schema_config_file")
fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.engine)
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
df = psql.read_sql_query(
"SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
db.engine)
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_config_file")
# TODO CHOSE BETWEEN THOSES DF
# if separated dimensions
# fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
# TODO CHOSE BETWEEN THOSES DF
# if facts contains all dimensions
# fusion = facts
# measures in config-file only
if cubes_obj.facts[0].measures:
......@@ -136,10 +91,10 @@ def _construct_star_schema_config_file_OLD(executer_instance, cubes_obj):
return fusion
# web client
def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
"""
Construct star schema DataFrame from configuration file.
Construct star schema DataFrame from configuration file for web client.
:param cube_name: cube name (or database name)
:param cubes_obj: cubes object
......@@ -167,6 +122,7 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
db.engine)
# todo verify this
try:
if table.columns:
tab = tab[table.columns]
......
olapy/core/mdx/executor/execute_csv_files.py
View file @ 8cd36207
......@@ -4,6 +4,9 @@ import os
import pandas as pd
# split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
def _load_tables_csv_files(executer_instance):
"""
......
olapy/core/mdx/executor/execute_db.py
View file @ 8cd36207
......@@ -6,6 +6,9 @@ from ..tools.mem_bench import memory_usage
from ..tools.connection import MyDB
import pandas.io.sql as psql
# split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
class StringFolder(object):
"""
......
olapy/core/mdx/tools/config_file_parser.py
View file @ 8cd36207
......@@ -71,6 +71,13 @@ class ConfigParser:
<!-- star building customized dimensions display in excel from the star schema -->
<dimensions>
<!-- ADD facts table name to the dimensions section like this (this is a little bug to be solved soon) -->
<dimension>
<name>stats_line</name>
<displayName>stats_line</displayName>
</dimension>
<dimension>
<!-- if you want to keep the same name for excel display, just use the same name in name and displayName -->
......@@ -81,10 +88,12 @@ class ConfigParser:
<columns>
<!-- columns order matter -->
<name>type_demande</name>
<name>financeur</name>
<name>wf_state</name>
<name>type_recrutement</name>
<!-- column_new_name if you want to change column display name in excel -->
<!-- if you don't want to change display name , rewrite it in column_new_name -->
<name column_new_name="Type">type_demande</name>
<name column_new_name="Financeur">financeur</name>
<name column_new_name="Etat">wf_state</name>
<name column_new_name="type_recrutement">type_recrutement</name>
</columns>
......@@ -98,9 +107,9 @@ class ConfigParser:
<columns>
<!-- columns order matter -->
<name>type</name>
<name>nom</name>
<name>sigle</name>
<name column_new_name="type">type</name>
<name column_new_name="Nom">nom</name>
<name column_new_name="SIGLE">sigle</name>
</columns>
</dimension>
......@@ -154,6 +163,9 @@ class ConfigParser:
<name>duree_projet</name>
</measures>
<!-- additional columns to keep other than measures and ids -->
<columns>etat,aap,axes_de_developpement</columns>
</facts>
<!-- end building customized star schema -->
......@@ -177,19 +189,62 @@ class ConfigParser:
<table name="contact">
<columns>id,nom,prenom,fonction</columns>
<new_name old_column_name="fonction">Contact Fonction</new_name>
</table>
</tables>
<!-- Dashboards -->
<Dashboards>
<Dashboard>
<Global_table>
<!-- IMPORTANT !! columns and rows names must be specified as above with their new names -->
<!-- EXAMPLE <new_name old_column_name="label">Pole leader</new_name>, you put Pole leader -->
<!-- marches,axes_de_developpement,statut_pour_book are columns from facts table -->
<columns>marches,axes_de_developpement</columns>
<rows>statut_pour_book</rows>
</Global_table>
<!-- Contact Fonction,Type Organisation columns name from different tables (with ther new names) -->
<PieCharts>Contact Fonction,Type Organisation</PieCharts>
<!-- TODO BarCharts with Stacked Bar Chart -->
<BarCharts>Avis</BarCharts>
<!-- Preferably with time/date (or sequenced) tables-->
<LineCharts>
<table>
<!-- date_debut_envisagee a column from facts table -->
<name>date_debut_envisagee</name>
<!-- if not specified, then all columns attributs -->
<!--<columns>1945,2000,2006,2015</columns> -->
</table>
</LineCharts>
</Dashboard>
</Dashboards>
<!-- END Dashboards -->
</cube>
</cubes>
"""
# TODO one config file (I will try to merge dimensions between them in web part)
def __init__(self,
cube_path = None,
file_name='cubes-config.xml',
......@@ -214,7 +269,7 @@ class ConfigParser:
self.file_name = file_name
self.web_config_file_name = web_config_file_name
def config_file_exist(self, client_type='excel'):
def config_file_exist(self, client_type):
"""
Check whether the config file exists or not.
......@@ -231,7 +286,9 @@ class ConfigParser:
:return: True | False
"""
if self.config_file_exist():
# xmla authentication only in excel
if self.config_file_exist(client_type='excel'):
with open(os.path.join(self.cube_path,
self.file_name)) as config_file:
......@@ -246,7 +303,7 @@ class ConfigParser:
else:
return False
def get_cubes_names(self, client_type='excel'):
def get_cubes_names(self,client_type):
"""
Get all cubes names in the config file.
......
olapy/core/mdx/tools/connection.py
View file @ 8cd36207
......@@ -7,16 +7,6 @@ from olapy_config_file_parser import DbConfigParser
class MyDB(object):
"""Connect to sql database (postgres only right now)."""
# @staticmethod
# def db_credentials():
# db_config = DbConfigParser()
# if db_config.config_file_exist():
# # many databases in the future maybe
# return db_config.get_db_credentials()[0]
# else:
# raise Exception('Missing database config file')
def __init__(self,db_config_file_path=None,db=None):
# TODO temporary
......
olapy/core/services/xmla_discover_tools.py
View file @ 8cd36207
......@@ -1869,12 +1869,6 @@ class XmlaDiscoverTools():
for tables in self.executer.get_all_tables_names(ignore_fact=True):
# TODO in another idea, change this
# TO CHANGE NAME DISPLAY THAT EXISTS IN CONFIG FILE
# if MdxEngine.dimension_display_name != [] and tables in MdxEngine.dimension_display_name:
# continue
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7