format

micro_bench/__init__.py

@@ -23,7 +23,8 @@ PROFILING_LINES = 15
 
 
 def main():
-    file = open('bench_result' + str(datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")), 'w')
+    file = open('bench_result' +
+                str(datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")), 'w')
     gen = CubeGen(number_dimensions=3, rows_length=1000, columns_length=5)
     gen.generate_csv(gen.generate_cube(3, 1000))
     XmlaProviderService.discover_tools.change_catalogue(CUBE_NAME)
@@ -52,14 +53,18 @@ def main():
             WHERE ([Measures].[Amount])
             CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
 
-    file.write("Query 1 :\n" + cmd + "\n----------------------------------------------------------\n\n")
+    file.write(
+        "Query 1 :\n" + cmd +
+        "\n----------------------------------------------------------\n\n")
 
     t.add_row(['Query 1', mbench.bench(conn, cmd, CUBE_NAME)])
 
     cmd = """SELECT
         NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
         [table0].[table0].[All table0A].Members}}}, {
-        [table0].[table0].[table0A].[""" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[1]) + """]})))
+        [table0].[table0].[table0A].[""" + str(
+        XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
+            1]) + """]})))
         DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME
         ON COLUMNS
         FROM [""" + CUBE_NAME + """]
@@ -67,12 +72,18 @@ def main():
         CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
         """
 
-    file.write("Query 2 :\n" + cmd + "\n----------------------------------------------------------\n\n")
+    file.write(
+        "Query 2 :\n" + cmd +
+        "\n----------------------------------------------------------\n\n")
     t.add_row(['Query 2', mbench.bench(conn, cmd, CUBE_NAME)])
 
-    tup = "[table0].[table0].[table0A].[" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[0]) + "]"
+    tup = "[table0].[table0].[table0A].[" + str(
+        XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
+            0]) + "]"
     for d in range(REFINEMENT_LVL):
-        tup += ",\n[table0].[table0].[table0A].[" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[d + 1]) + "]"
+        tup += ",\n[table0].[table0].[table0A].[" + str(
+            XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
+                d + 1]) + "]"
 
     cmd = """
         SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
@@ -86,15 +97,25 @@ def main():
         CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
         """
 
-    file.write("Query 3 :\n" + cmd + "\n----------------------------------------------------------\n\n")
+    file.write(
+        "Query 3 :\n" + cmd +
+        "\n----------------------------------------------------------\n\n")
     t.add_row(['Query 3', mbench.bench(conn, cmd, CUBE_NAME)])
     file.write(str(t) + "\n\n")
 
     try:
-        file.write('******************************************************************************\n')
-        file.write('* mondrian with "warehouse" Cube (note the same as olapy but resemble to it) *\n')
-        file.write('* (olapy warehouse"s cube has more rows)                                     *\n')
-        file.write('******************************************************************************\n\n')
+        file.write(
+            '******************************************************************************\n'
+        )
+        file.write(
+            '* mondrian with "warehouse" Cube (note the same as olapy but resemble to it) *\n'
+        )
+        file.write(
+            '* (olapy warehouse"s cube has more rows)                                     *\n'
+        )
+        file.write(
+            '******************************************************************************\n\n'
+        )
 
         t = PrettyTable(['Query', 'mondrian', 'olapy'])
         p2 = xmla.XMLAProvider()
@@ -111,9 +132,13 @@ def main():
                FROM
                [Warehouse]"""
 
-        file.write("Query 1 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 1', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
-
+        file.write(
+            "Query 1 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 1', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
+                conn, cmd, 'foodmart')
+        ])
 
         cmd = """SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
           [Product].[Product].[All brand_name].Members}}}, {
@@ -131,8 +156,13 @@ def main():
             DIMENSION PROPERTIES PARENT_UNIQUE_NAME ON 0
             FROM [Warehouse]"""
 
-        file.write("Query 2 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 2', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
+        file.write(
+            "Query 2 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 2', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
+                conn, cmd, 'foodmart')
+        ])
 
         cmd = """SELECT NON EMPTY CrossJoin(Hierarchize(AddCalculatedMembers({
           [Product].[Product].[All brand_name].Members})), Hierarchize(AddCalculatedMembers({
@@ -151,8 +181,13 @@ def main():
             DIMENSION PROPERTIES PARENT_UNIQUE_NAME ON 0
             FROM [Warehouse]"""
 
-        file.write("Query 3 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 3', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
+        file.write(
+            "Query 3 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 3', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
+                conn, cmd, 'foodmart')
+        ])
 
         file.write(str(t) + "\n\n")
 
@@ -167,16 +202,23 @@ def main():
 
         t = PrettyTable(['Query', 'olapy', 'icCube'])
         p2 = xmla.XMLAProvider()
-        c2 = p2.connect(location="http://localhost:8282/icCube/xmla", username="demo",
-                        password="demo")
+        c2 = p2.connect(
+            location="http://localhost:8282/icCube/xmla",
+            username="demo",
+            password="demo")
 
         cmd = """SELECT
           FROM [Sales]
           WHERE ([Measures].[Amount])
           CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
 
-        file.write("Query 1 :\n" + cmd + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 1', mbench.bench(conn, cmd, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
+        file.write(
+            "Query 1 :\n" + cmd +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 1', mbench.bench(conn, cmd, 'sales'), mbench.bench(
+                c2, cmd, 'Sales (Excel)')
+        ])
 
         cmd = """SELECT
                 NON EMPTY Hierarchize(AddCalculatedMembers({DrilldownLevel({
@@ -194,8 +236,13 @@ def main():
                   FROM [Sales]
                   WHERE ([Measures].[Amount])
                   CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
-        file.write("Query 2 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 2', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
+        file.write(
+            "Query 2 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 2', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
+                c2, cmd, 'Sales (Excel)')
+        ])
 
         cmd = """SELECT
                  NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{DrilldownLevel({
@@ -224,8 +271,13 @@ def main():
                   FROM [sales]
                   WHERE ([Measures].[Amount])
                   CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
-        file.write("Query 3 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 3', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
+        file.write(
+            "Query 3 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 3', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
+                c2, cmd, 'Sales (Excel)')
+        ])
 
         cmd = """SELECT
                  NON EMPTY CrossJoin(Hierarchize(AddCalculatedMembers(DrilldownMember
@@ -257,8 +309,13 @@ def main():
                   WHERE ([Measures].[Amount])
                   CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
 
-        file.write("Query 4 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 4', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
+        file.write(
+            "Query 4 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 4', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
+                c2, cmd, 'Sales (Excel)')
+        ])
 
         cmd = """SELECT
                  NON EMPTY CrossJoin(CrossJoin(Hierarchize(AddCalculatedMembers
@@ -307,8 +364,13 @@ def main():
                   WHERE ([Measures].[Amount])
                   CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
 
-        file.write("Query 5 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
-        t.add_row(['Query 5', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
+        file.write(
+            "Query 5 :\n" + cmd2 +
+            "\n----------------------------------------------------------\n\n")
+        t.add_row([
+            'Query 5', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
+                c2, cmd, 'Sales (Excel)')
+        ])
 
         file.write(str(t) + "\n\n")
 
@@ -316,8 +378,8 @@ def main():
         print('Make sure icCube is running and containing sales Excel cube')
         pass
 
-
-    file.write('---------------- Profiling olapy Query 5 ------------------ \n\n')
+    file.write(
+        '---------------- Profiling olapy Query 5 ------------------ \n\n')
     cProfile.run("""cmd = '''
             SELECT
             NON EMPTY CrossJoin(CrossJoin(Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{{
@@ -347,19 +409,19 @@ request.Command = Command(Statement = cmd)
 request.Properties = Propertielist(PropertyList = Property(Catalog='sales'))
 
 XmlaProviderService().Execute(XmlaProviderService(),request)""",
-                 "{}.profile".format(__file__)
-                 )
+                 "{}.profile".format(__file__))
 
     s = pstats.Stats("{}.profile".format(__file__), stream=file)
     s.strip_dirs()
     s.sort_stats("time").print_stats(PROFILING_LINES)
 
     try:
-        os.system('gprof2dot -f pstats __init__.py.profile | dot -Tpng -o profile.png')
+        os.system(
+            'gprof2dot -f pstats main.py.profile | dot -Tpng -o profile.png')
     except:
         print('make sure gprof2dot and graphviz are installed')
 
-    os.remove('__init__.py.profile ')
+    os.remove('main.py.profile ')
     gen.remove_temp_cube()
     file.close()
     server.stop()

micro_bench/cube_generator.py

@@ -7,7 +7,8 @@ import shutil
 
 from olapy.core.mdx.executor.execute import MdxEngine
 
-CUBE_NAME ="temp_cube"
+CUBE_NAME = "temp_cube"
+
 
 class CubeGen:
     """
@@ -21,7 +22,8 @@ class CubeGen:
     # We have to generate DataFrames and save them to csv format because XmlaProviderService and
     # MdxEngine classes use those files
 
-    def __init__(self, number_dimensions=1, rows_length=1000, columns_length=2):
+    def __init__(self, number_dimensions=1, rows_length=1000,
+                 columns_length=2):
         self.number_dimensions = number_dimensions
         self.rows_length = rows_length
         self.columns_length = columns_length
@@ -38,14 +40,20 @@ class CubeGen:
         facts = pd.DataFrame()
         for idx, dim in enumerate(range(self.number_dimensions)):
             table_name = 'table' + str(idx)
-            table_values = pd.DataFrame(np.random.randint(min_val, max_val, size=(self.rows_length, self.columns_length)),
-                                        columns=list(
-                                            table_name + col for col in string.ascii_uppercase[:self.columns_length]))
+            table_values = pd.DataFrame(
+                np.random.randint(
+                    min_val,
+                    max_val,
+                    size=(self.rows_length, self.columns_length)),
+                columns=list(
+                    table_name + col
+                    for col in string.ascii_uppercase[:self.columns_length]))
             table_values.index.name = table_name + "_id"
             tables[table_name] = table_values.reset_index()
             facts[table_name + "_id"] = tables[table_name][table_name + "_id"]
 
-        facts['Amount'] = np.random.randint(300, 1000, size=(self.rows_length, 1))
+        facts['Amount'] = np.random.randint(
+            300, 1000, size=(self.rows_length, 1))
         tables['Facts'] = facts
         return tables
 
@@ -57,19 +65,22 @@ class CubeGen:
         :param tables: dict of DataFrames
         """
         cube_path = os.path.join(
-            os.path.abspath(
-                os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER)
+            os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
+            MdxEngine.CUBE_FOLDER)
         if not os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
             os.makedirs(os.path.join(cube_path, CUBE_NAME))
         cube_path = os.path.join(cube_path, CUBE_NAME)
         for (table_name, table_value) in tables.items():
-            table_value.to_csv(os.path.join(os.path.join(cube_path, table_name + '.csv')), sep=";", index=False)
+            table_value.to_csv(
+                os.path.join(os.path.join(cube_path, table_name + '.csv')),
+                sep=";",
+                index=False)
 
     @staticmethod
     def remove_temp_cube():
         """Remove the temporary cube."""
         cube_path = os.path.join(
-            os.path.abspath(
-                os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER)
+            os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
+            MdxEngine.CUBE_FOLDER)
         if os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
             shutil.rmtree(os.path.join(cube_path, CUBE_NAME))

micro_bench/micro_bench.py

@@ -21,4 +21,5 @@ class MicBench:
         to one million
         :return: float execution time in seconds
         """
-        return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(number=number)
+        return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(
+            number=number)

olapy/core/mdx/executor/execute.py

@@ -41,6 +41,7 @@ class MdxEngine:
     # (before instantiate MdxEngine I need to access cubes information)
     csv_files_cubes = []
     postgres_db_cubes = []
+
     # to show just config file's dimensions
 
     def __init__(self,
@@ -113,7 +114,8 @@ class MdxEngine:
         try:
             db = MyDB(db_config_file_path=olapy_data_location)
             # TODO this work only with postgres
-            result = db.engine.execute('SELECT datname FROM pg_database WHERE datistemplate = false;')
+            result = db.engine.execute(
+                'SELECT datname FROM pg_database WHERE datistemplate = false;')
             available_tables = result.fetchall()
             # cursor.execute("""SELECT datname FROM pg_database
             # WHERE datistemplate = false;""")
@@ -164,9 +166,10 @@ class MdxEngine:
         config_file_parser = ConfigParser(self.cube_path)
         tables = {}
 
-        if self.client == 'excel' and config_file_parser.config_file_exist(client_type=self.client
-        ) and self.cube in config_file_parser.get_cubes_names(client_type=self.client
-        ):
+        if self.client == 'excel' and config_file_parser.config_file_exist(
+                client_type=self.
+                client) and self.cube in config_file_parser.get_cubes_names(
+                    client_type=self.client):
             # for web (config file) we need only star_schema_dataframes, not all tables
             for cubes in config_file_parser.construct_cubes():
 
@@ -188,7 +191,8 @@ class MdxEngine:
 
         # if web, get measures from config file
         config_file_parser = ConfigParser(self.cube_path)
-        if self.client == 'web' and config_file_parser.config_file_exist('web'):
+        if self.client == 'web' and config_file_parser.config_file_exist(
+                'web'):
             for cubes in config_file_parser.construct_cubes(self.client):
 
                 # update facts name
@@ -214,19 +218,19 @@ class MdxEngine:
         fusion = None
         config_file_parser = ConfigParser(self.cube_path)
         if config_file_parser.config_file_exist(
-                self.client
-        ) and self.cube in config_file_parser.get_cubes_names(
-                client_type=self.client):
+                self.
+                client) and self.cube in config_file_parser.get_cubes_names(
+                    client_type=self.client):
             for cubes in config_file_parser.construct_cubes(self.client):
                 # TODO cubes.source == 'csv'
                 if cubes.source == 'postgres':
                     # TODO one config file (I will try to merge dimensions between them in web part)
                     if self.client == 'web':
-                        fusion = _construct_web_star_schema_config_file(
-                            self, cubes)
+                        fusion = _construct_web_star_schema_config_file(self,
+                                                                        cubes)
                     else:
-                        fusion = _construct_star_schema_config_file(
-                            self, cubes)
+                        fusion = _construct_star_schema_config_file(self,
+                                                                    cubes)
 
         elif self.cube in self.csv_files_cubes:
             fusion = _construct_star_schema_csv_files(self)
@@ -256,7 +260,8 @@ class MdxEngine:
         :return: path to the cube
         """
         if MdxEngine.DATA_FOLDER is not None:
-            return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER, self.cube)
+            return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER,
+                                self.cube)
         return os.path.join(self.cube_path, self.cube)
 
     # TODO temporary function
@@ -309,7 +314,7 @@ class MdxEngine:
             for tup_att in tup[0].replace('.Members', '').split('.') if tup_att
         ]
                 for tup in re.compile(regex).findall(
-                    query.encode("utf-8",'replace')[start:stop])
+                    query.encode("utf-8", 'replace')[start:stop])
                 if len(tup[0].split('.')) > 1]
 
     # TODO temporary function
@@ -603,9 +608,8 @@ class MdxEngine:
 
         :return: updated columns_to_keep
         """
-        if len(
-                tuple_as_list
-        ) == 3 and tuple_as_list[-1] in self.tables_loaded[tuple_as_list[0]].columns:
+        if len(tuple_as_list) == 3 and tuple_as_list[-1] in self.tables_loaded[
+                tuple_as_list[0]].columns:
             # in case of [Geography].[Geography].[Country]
             cols = [tuple_as_list[-1]]
         else:

olapy/core/mdx/executor/execute_config_file.py

@@ -4,6 +4,7 @@ from ..tools.connection import MyDB
 import pandas.io.sql as psql
 import os
 
+
 # split execution into three part (execute from config files,
 # execute csv files if they respect olapy's start schema model,
 # and execute data base tables if they respect olapy's start schema model)
@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj):
     # just one facts table right now
     executer_instance.facts = cube_obj.facts[0].table_name
 
-    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
+    db = MyDB(
+        db_config_file_path=os.path.dirname(executer_instance.cube_path),
+        db=executer_instance.cube)
 
     for dimension in cube_obj.dimensions:
 
@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj):
             table_name = dimension.name
 
         # rename columns if value not None
-        df.rename(columns=(dict((k, v) for k, v in dimension.columns.items() if v)), inplace=True)
+        df.rename(
+            columns=(dict((k, v) for k, v in dimension.columns.items() if v)),
+            inplace=True)
 
         tables[table_name] = df[[
             col for col in df.columns if col.lower()[-2:] != 'id'
@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj):
 
     return tables
 
+
 # excel client
 def _construct_star_schema_config_file(executer_instance, cubes_obj):
     """
@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
     :return: star schema DataFrame
     """
     executer_instance.facts = cubes_obj.facts[0].table_name
-    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
+    db = MyDB(
+        db_config_file_path=os.path.dirname(executer_instance.cube_path),
+        db=executer_instance.cube)
     # load facts table
 
     fusion = psql.read_sql_query(
@@ -74,8 +82,11 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
 
         # TODO CHOSE BETWEEN THOSES DF
         fusion = fusion.merge(
-            df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
-        # remove suffixe from dimension and keep the same column name for facts
+            df,
+            left_on=fact_key,
+            right_on=dimension_and_key.split('.')[1],
+            how='left',
+            # remove suffixe from dimension and keep the same column name for facts
             suffixes=('', '_y'))
 
     # measures in config-file only
@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
 
     return fusion
 
+
 # web client
 def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
     """
@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
     all_columns = []
 
     executer_instance.facts = cubes_obj.facts[0].table_name
-    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
+    db = MyDB(
+        db_config_file_path=os.path.dirname(executer_instance.cube_path),
+        db=executer_instance.cube)
     # load facts table
 
     if cubes_obj.facts[0].columns:
@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
 
         # TODO check merge (how)
         fusion = fusion.merge(
-            df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
+            df,
+            left_on=fact_key,
+            right_on=dimension_and_key.split('.')[1],
+            how='left',
             # remove suffixe from dimension and keep the same column name for facts
             suffixes=('', '_y'))
 
     return fusion[[column for column in all_columns if 'id' != column[-2:]]]
-

olapy/core/mdx/executor/execute_csv_files.py

@@ -8,6 +8,7 @@ import pandas as pd
 # execute csv files if they respect olapy's start schema model,
 # and execute data base tables if they respect olapy's start schema model)
 
+
 def _load_tables_csv_files(executer_instance):
     """
     Load tables from csv files.

olapy/core/mdx/executor/execute_db.py

@@ -9,8 +9,6 @@ import pandas.io.sql as psql
 # execute csv files if they respect olapy's start schema model,
 # and execute data base tables if they respect olapy's start schema model)
 
-
-
 # class StringFolder(object):
 #     """
 #     Class that will fold strings. See 'fold_string'.
@@ -68,6 +66,7 @@ import pandas.io.sql as psql
 
 # TODO try pandas.read_sql_table and pandas.read_sql
 
+
 def _load_tables_db(executer_instance):
     """
     Load tables from database.
@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance):
     :return: tables dict with table name as key and dataframe as value
     """
     tables = {}
-    db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube)
+    db = MyDB(
+        db_config_file_path=executer_instance.DATA_FOLDER,
+        db=executer_instance.cube)
     inspector = inspect(db.engine)
 
     for table_name in inspector.get_table_names():
@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance):
         #     'SELECT * FROM "{0}"'.format(table_name), db.engine)
 
         # results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
-        results = db.engine.execution_options(stream_results=True).execute('SELECT * FROM "{0}"'.format(table_name))
+        results = db.engine.execution_options(stream_results=True).execute(
+            'SELECT * FROM "{0}"'.format(table_name))
         # Fetch all the results of the query
-        value = pd.DataFrame(iter(results),columns=results.keys())  # Pass results as an iterator
+        value = pd.DataFrame(
+            iter(results),
+            columns=results.keys())  # Pass results as an iterator
         # with string_folding_wrapper we loose response time
         # value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
         tables[table_name] = value[[

olapy/core/mdx/tools/config_file_parser.py

@@ -246,7 +246,7 @@ class ConfigParser:
     """
 
     def __init__(self,
-                 cube_path = None,
+                 cube_path=None,
                  file_name='cubes-config.xml',
                  web_config_file_name='web_cube_config.xml'):
         """
@@ -262,7 +262,8 @@ class ConfigParser:
             home_directory = expanduser("~")
 
         if cube_path is None:
-            self.cube_path = os.path.join(home_directory, 'olapy-data', 'cubes')
+            self.cube_path = os.path.join(home_directory, 'olapy-data',
+                                          'cubes')
         else:
             self.cube_path = cube_path
 
@@ -303,7 +304,7 @@ class ConfigParser:
         else:
             return False
 
-    def get_cubes_names(self,client_type):
+    def get_cubes_names(self, client_type):
         """
         Get all cubes names in the config file.
 
@@ -359,10 +360,10 @@ class ConfigParser:
                         # column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
                         displayName=xml_dimension.find('displayName').text,
                         columns=OrderedDict(
-                            (column_name.text , None if not column_name.attrib else column_name.attrib['column_new_name'])
+                            (column_name.text, None if not column_name.attrib
+                             else column_name.attrib['column_new_name'])
                             for column_name in xml_dimension.findall(
-                                'columns/name')
-                        ))
+                                'columns/name')))
                     for xml_dimension in tree.xpath(
                         '/cubes/cube/dimensions/dimension')
                 ]
@@ -449,8 +450,7 @@ class ConfigParser:
                 global_table={
                     'columns':
                     dashboard.find('Global_table/columns').text.split(','),
-                    'rows':
-                    dashboard.find('Global_table/rows').text.split(',')
+                    'rows': dashboard.find('Global_table/rows').text.split(',')
                 },
                 pie_charts=dashboard.find('PieCharts').text.split(','),
                 bar_charts=dashboard.find('BarCharts').text.split(','),

olapy/core/mdx/tools/connection.py

+from __future__ import absolute_import, division, print_function
 # import psycopg2 as pg
 from sqlalchemy import create_engine
 
 # postgres connection
-from olapy_config_file_parser import DbConfigParser
+from .olapy_config_file_parser import DbConfigParser
+
 
 class MyDB(object):
     """Connect to sql database (postgres only right now)."""
 
-    def __init__(self,db_config_file_path=None,db=None):
+    def __init__(self, db_config_file_path=None, db=None):
 
         # TODO temporary
         db_config = DbConfigParser(config_path=db_config_file_path)
@@ -21,18 +23,19 @@ class MyDB(object):
             # first i want to show all databases to user (in excel)
             # self.engine = pg.connect("user={0} password={1} host='{2}'".
             #                              format(username, password, host))
-            self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
-                username, password, 'postgres', host, port))
+            self.engine = create_engine(
+                'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
+                    username, password, 'postgres', host, port))
 
         else:
             # and then we connect to the user db
-            self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
-                username, password, db, host, port))
+            self.engine = create_engine(
+                'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
+                    username, password, db, host, port))
             # self.connection = pg.connect(
             #     "user={0} password={1} dbname='{2}' host='{3}'".format(
             #         username, password, db, host))
 
-
     def __del__(self):
         if hasattr(self, 'connection'):
             self.engine.dispose()

olapy/core/mdx/tools/olapy_config_file_parser.py

@@ -4,12 +4,11 @@ import os
 
 from lxml import etree
 
+
 class DbConfigParser:
 
     # TODO one config file (I will try to merge dimensions between them in web part)
-    def __init__(self,
-                 config_path = None,
-                 file_name='olapy-config.xml'):
+    def __init__(self, config_path=None, file_name='olapy-config.xml'):
         """
 
         :param cube_path: path to cube (csv folders)
@@ -46,10 +45,9 @@ class DbConfigParser:
             return [
                 {
                     # 'sgbd': db.find('sgbd').text,
-                 'user_name': db.find('user_name').text,
-                 'password': db.find('password').text,
-                 'host': db.find('host').text,
-                 'port': db.find('port').text,
-                 }
-                for db in tree.xpath('/olapy/database')
+                    'user_name': db.find('user_name').text,
+                    'password': db.find('password').text,
+                    'host': db.find('host').text,
+                    'port': db.find('port').text,
+                } for db in tree.xpath('/olapy/database')
             ]

olapy/core/services/models.py

@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function
 
 from spyne import ComplexModel, Integer, Unicode, XmlAttribute
 
-
-
 # NOTE : I didn't respect python naming convention here
 # because we need to create the xmla response (generated by spyne) with the same variable names,
 # thus xmla requests from excel can be reached
 
+
 class Tuple(object):
     """Tuple description (used by spyne)."""
 
@@ -110,7 +109,7 @@ class Propertielist(ComplexModel):
 class Command(ComplexModel):
     """Command description (used by spyne)."""
 
-    _type_info = {'Statement': Unicode,}
+    _type_info = {'Statement': Unicode, }
 
 
 class ExecuteRequest(ComplexModel):

olapy/core/services/xmla.py

@@ -54,12 +54,11 @@ class XmlaProviderService(ServiceBase):
     discover_tools = XmlaDiscoverTools()
     sessio_id = discover_tools.session_id
 
-    @rpc(
-        DiscoverRequest,
-        _returns=AnyXml,
-        _body_style="bare",
-        _out_header=Session,
-        _throws=InvalidCredentialsError)
+    @rpc(DiscoverRequest,
+         _returns=AnyXml,
+         _body_style="bare",
+         _out_header=Session,
+         _throws=InvalidCredentialsError)
     def Discover(ctx, request):
         """
         The first principle function of xmla protocol.
@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase):
         ctx.out_header = Session(SessionId=str(XmlaProviderService.sessio_id))
 
         config_parser = ConfigParser(discover_tools.executer.cube_path)
-        if config_parser.xmla_authentication(
-        ) and ctx.transport.req_env['QUERY_STRING'] != 'admin':
+        if config_parser.xmla_authentication() and ctx.transport.req_env[
+                'QUERY_STRING'] != 'admin':
 
             raise InvalidCredentialsError(
-                fault_string=
-                'You do not have permission to access this resource',
+                fault_string='You do not have permission to access this resource',
                 fault_object=None)
             # TODO call (labster) login function or create login with token (according to labster db)
 
@@ -143,11 +141,10 @@ class XmlaProviderService(ServiceBase):
 
     # Execute function must take 2 argument ( JUST 2 ! ) Command and Properties
     # we encapsulate them in ExecuteRequest object
-    @rpc(
-        ExecuteRequest,
-        _returns=AnyXml,
-        _body_style="bare",
-        _out_header=Session)
+    @rpc(ExecuteRequest,
+         _returns=AnyXml,
+         _body_style="bare",
+         _out_header=Session)
     def Execute(ctx, request):
         """
         The second principle function of xmla protocol.
@@ -235,7 +232,7 @@ application = Application(
 wsgi_application = WsgiApplication(application)
 
 
-def start_server(host='0.0.0.0',port=8000,write_on_file=False):
+def start_server(host='0.0.0.0', port=8000, write_on_file=False):
     """
     Start the xmla server.
 

olapy/core/services/xmla_discover_tools.py

@@ -85,7 +85,7 @@ class XmlaDiscoverTools():
                     <Value>{5}</Value>
                 </row>
                 """.format(PropertyName, PropertyDescription, PropertyType,
-                       PropertyAccessType, IsRequired, Value)
+                           PropertyAccessType, IsRequired, Value)
             else:
                 rows = """
                 <row>
@@ -139,7 +139,7 @@ class XmlaDiscoverTools():
                 {1}
                 </root>
             </return>
-            """.format(xsd,rows))
+            """.format(xsd, rows))
 
         if request.Restrictions.RestrictionList.PropertyName == 'Catalog':
             if request.Properties.PropertyList.Catalog is not None:
@@ -195,7 +195,7 @@ class XmlaDiscoverTools():
                                  'MdpropMdxNamedSets', 'int', 'Read', 'false',
                                  '15')
 
-        return get_props(discover_preperties_xsd, '','', '', '', '','')
+        return get_props(discover_preperties_xsd, '', '', '', '', '', '')
 
     def discover_schema_rowsets_response(self, request):
         if request.Restrictions.RestrictionList.SchemaName == "MDSCHEMA_HIERARCHIES" and \
@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools():
                     # french caracteres
                     # TODO encode dataframe
                     if type(df.iloc[0][0]) == unicode:
-                        column_attribut = df.iloc[0][0].encode('utf-8','replace')
+                        column_attribut = df.iloc[0][0].encode('utf-8',
+                                                               'replace')
                     else:
                         column_attribut = df.iloc[0][0]
 
-
                     rows += """
                     <row>
                         <CATALOG_NAME>{0}</CATALOG_NAME>
@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools():
                         <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
                         <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
                     </row>
-                    """.format(self.selected_catalogue,
-                               table_name,
-                               df.columns[0],
-                               column_attribut)
-
+                    """.format(self.selected_catalogue, table_name,
+                               df.columns[0], column_attribut)
 
                 rows += """
                 <row>
@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools():
                     # french caracteres
                     # TODO encode dataframe
                     if type(df.iloc[0][0]) == unicode:
-                        column_attribut = df.iloc[0][0].encode('utf-8','replace')
+                        column_attribut = df.iloc[0][0].encode('utf-8',
+                                                               'replace')
                     else:
                         column_attribut = df.iloc[0][0]
 
@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools():
                         <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
                         <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
                     </row>
-                        """.format(self.selected_catalogue,
-                                   table_name,
-                                   df.columns[0],
-                                   column_attribut)
+                        """.format(self.selected_catalogue, table_name,
+                                   df.columns[0], column_attribut)
 
                 rows += """
                 <row>

olapy/core/services/xmla_execute_tools.py

@@ -104,11 +104,11 @@ class XmlaExecuteTools():
                 first_att = 3
 
         # query with on columns and on rows (without measure)
-        elif mdx_execution_result['columns_desc']['columns'] and mdx_execution_result['columns_desc']['rows']:
+        elif mdx_execution_result['columns_desc'][
+                'columns'] and mdx_execution_result['columns_desc']['rows']:
             # ['Geography','America']
             tuples = [
-                zip(
-                    *[[[key] + list(row)
+                zip(*[[[key] + list(row)
                        for row in splited_df[key].itertuples(index=False)]
                       for key in splited_df.keys()
                       if key is not self.executer.facts])
@@ -120,8 +120,7 @@ class XmlaExecuteTools():
         else:
             # ['Geography','Amount','America']
             tuples = [
-                zip(
-                    *[[[key] + [mes] + list(row)
+                zip(*[[[key] + [mes] + list(row)
                        for row in splited_df[key].itertuples(index=False)]
                       for key in splited_df.keys()
                       if key is not self.executer.facts])
@@ -155,7 +154,10 @@ class XmlaExecuteTools():
                 # french caracteres
                 # TODO encode dataframe
                 if type(tuple_without_minus_1[-1]) == unicode:
-                    tuple_without_minus_1 = [x.encode('utf-8','replace') for x in tuple_without_minus_1]
+                    tuple_without_minus_1 = [
+                        x.encode('utf-8', 'replace')
+                        for x in tuple_without_minus_1
+                    ]
 
                 axis0 += """
                 <Member Hierarchy="[{0}].[{0}]">
@@ -254,7 +256,8 @@ class XmlaExecuteTools():
         # TODO must be OPTIMIZED every time!!!!!
 
         dfs = self.split_dataframe(mdx_execution_result)
-        if mdx_execution_result['columns_desc']['rows'] and mdx_execution_result['columns_desc']['columns']:
+        if mdx_execution_result['columns_desc'][
+                'rows'] and mdx_execution_result['columns_desc']['columns']:
 
             return """
             {0}
@@ -306,12 +309,11 @@ class XmlaExecuteTools():
         """
         columns_loop = []
 
-        if (
-            (len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
+        if ((len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
                 ^
-            (len(mdx_execution_result['columns_desc']['rows'].keys()) == 0)
-        ) and self.executer.facts in mdx_execution_result['columns_desc']['all'].keys(
-        ):
+            (len(mdx_execution_result['columns_desc']['rows'].keys()) == 0
+             )) and self.executer.facts in mdx_execution_result[
+                 'columns_desc']['all'].keys():
 
             # iterate DataFrame horizontally
             columns_loop = itertools.chain(*[
@@ -332,7 +334,7 @@ class XmlaExecuteTools():
         cell_data = ""
         index = 0
         for value in columns_loop:
-            if np.isnan(value) :
+            if np.isnan(value):
                 value = ''
             cell_data += """
             <Cell CellOrdinal="{0}">
@@ -392,9 +394,9 @@ class XmlaExecuteTools():
             to_write = "[{0}].[{0}]".format(dim_diff)
             if dim_diff == 'Measures':
                 # if measures > 1 we don't have to write measure
-                if self.executer.facts in mdx_execution_result['columns_desc']['all'] and len(
-                        mdx_execution_result['columns_desc']['all']
-                    [self.executer.facts]) > 1:
+                if self.executer.facts in mdx_execution_result['columns_desc'][
+                        'all'] and len(mdx_execution_result['columns_desc'][
+                            'all'][self.executer.facts]) > 1:
                     continue
                 else:
                     to_write = "[Measures]"
@@ -452,9 +454,9 @@ class XmlaExecuteTools():
         """
         hierarchy_info = ""
         # measure must be written at the top
-        if self.executer.facts in mdx_execution_result['columns_desc'][mdx_query_axis].keys(
-        ) and len(mdx_execution_result['columns_desc'][mdx_query_axis]
-                  [self.executer.facts]) > 1:
+        if self.executer.facts in mdx_execution_result['columns_desc'][
+                mdx_query_axis].keys() and len(mdx_execution_result[
+                    'columns_desc'][mdx_query_axis][self.executer.facts]) > 1:
             hierarchy_info += """
             <HierarchyInfo name="{0}">
                 <UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
@@ -556,15 +558,18 @@ class XmlaExecuteTools():
         for dim_diff in list(
                 set(self.executer.get_all_tables_names(ignore_fact=True)) -
                 set(table_name
-                    for table_name in mdx_execution_result['columns_desc']
-                    ['all'])):
+                    for table_name in mdx_execution_result['columns_desc'][
+                        'all'])):
 
             # TODO encode dataframe
             # french caracteres
-            if type(self.executer.tables_loaded[dim_diff].iloc[0][0]) == unicode:
-                column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0].encode('utf-8','replace')
+            if type(self.executer.tables_loaded[dim_diff].iloc[0][
+                    0]) == unicode:
+                column_attribut = self.executer.tables_loaded[dim_diff].iloc[
+                    0][0].encode('utf-8', 'replace')
             else:
-                column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0]
+                column_attribut = self.executer.tables_loaded[dim_diff].iloc[
+                    0][0]
 
             tuple += """
             <Member Hierarchy="[{0}].[{0}]">

tests/test_execute_mdx.py

+from __future__ import absolute_import, division, print_function
+
 import pandas as pd
 from olapy.core.mdx.executor.execute import MdxEngine
 

tests/test_xmla_notox.py

@@ -198,8 +198,7 @@ def test_query2(conn):
     mems.append(
         Member(
             _Hierarchy='[Geography].[Geography]',
-            UName=
-            '[Geography].[Geography].[Country].[America].[United States]',
+            UName='[Geography].[Geography].[Country].[America].[United States]',
             Caption='United States',
             LName='[Geography].[Geography].[Country]',
             LNum='1',
@@ -209,14 +208,12 @@ def test_query2(conn):
     mems.append(
         Member(
             _Hierarchy='[Geography].[Geography]',
-            UName=
-            '[Geography].[Geography].[City].[America].[United States].[New York]',
+            UName='[Geography].[Geography].[City].[America].[United States].[New York]',
             Caption='New York',
             LName='[Geography].[Geography].[City]',
             LNum='2',
             DisplayInfo='131076',
-            PARENT_UNIQUE_NAME=
-            '[Geography].[Geography].[Continent].[America].[United States]',
+            PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[America].[United States]',
             HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
     mems.append(
         Member(
@@ -250,14 +247,12 @@ def test_query2(conn):
     mems.append(
         Member(
             _Hierarchy='[Geography].[Geography]',
-            UName=
-            '[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
+            UName='[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
             Caption='Barcelona',
             LName='[Geography].[Geography].[City]',
             LNum='2',
             DisplayInfo='131076',
-            PARENT_UNIQUE_NAME=
-            '[Geography].[Geography].[Continent].[Europe].[Spain]',
+            PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
             HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
     mems.append(
         Member(
@@ -267,8 +262,7 @@ def test_query2(conn):
             LName='[Geography].[Geography].[City]',
             LNum='2',
             DisplayInfo='131076',
-            PARENT_UNIQUE_NAME=
-            '[Geography].[Geography].[Continent].[Europe].[Spain]',
+            PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
             HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
     mems.append(
         Member(