Merge branch 'update_excel_config_file'

# Conflicts:
#	requirements.txt

config/olapy-config.xml

 <?xml version="1.0" encoding="UTF-8"?>
-
+<!-- this config file will be deleted ASAP -->
 <olapy>
 
     <database>
@@ -8,6 +8,7 @@
         <user_name>postgres</user_name>
         <password>root</password>
         <host>localhost</host>
+        <port>5432</port>
 
     </database>
 

olapy/core/mdx/executor/execute.py

@@ -33,12 +33,14 @@ class MdxEngine:
     :param sep: separator in the csv files
     """
 
+    # DATA_FOLDER useful for olapy web (falsk instance_path)
+    # get olapy-data path with instance_path instead of 'expanduser'
+    DATA_FOLDER = None
     CUBE_FOLDER = "cubes"
     # (before instantiate MdxEngine I need to access cubes information)
     csv_files_cubes = []
     postgres_db_cubes = []
     # to show just config file's dimensions
-    dimension_display_name = []
 
     def __init__(self,
                  cube_name,
@@ -64,8 +66,8 @@ class MdxEngine:
         self.client = client_type
         self.tables_loaded = self.load_tables()
         # all measures
-        self.load_star_schema_dataframe = self.get_star_schema_dataframe()
         self.measures = self.get_measures()
+        self.load_star_schema_dataframe = self.get_star_schema_dataframe()
         self.tables_names = self._get_tables_name()
         # default measure is the first one
         self.selected_measures = [self.measures[0]]
@@ -75,32 +77,42 @@ class MdxEngine:
         """:return: list cubes name that exists in cubes folder (under ~/olapy-data/cubes) and postgres database (if connected)."""
         # get csv files folders (cubes)
         # toxworkdir does not expanduser properly under tox
-        if RUNNING_TOX:
+
+        # surrended with try, except and PASS so we continue getting cubes from different
+        # sources (db, csv...) without interruption
+        if cls.DATA_FOLDER is not None:
+            home_directory = os.path.dirname(cls.DATA_FOLDER)
+        elif RUNNING_TOX:
             home_directory = os.environ.get('HOME_DIR')
         else:
             home_directory = expanduser("~")
 
         location = os.path.join(home_directory, 'olapy-data', cls.CUBE_FOLDER)
 
+        # surrended with try, except and PASS so we continue getting cubes from different
+        # sources (db, csv...) without interruption
         try:
             MdxEngine.csv_files_cubes = [
                 file for file in os.listdir(location)
                 if os.path.isdir(os.path.join(location, file))
             ]
-
         except Exception:
             print('no csv folders')
             pass
 
         # get postgres databases
+        # surrended with try, except and PASS so we continue getting cubes from different
+        # sources (db, csv...) without interruption
         try:
-            db = MyDB()
-            cursor = db.connection.cursor()
-            cursor.execute("""SELECT datname FROM pg_database
-            WHERE datistemplate = false;""")
+            db = MyDB(db_config_file_path=cls.DATA_FOLDER)
+            # TODO this work only with postgres
+            result = db.engine.execute('SELECT datname FROM pg_database WHERE datistemplate = false;')
+            available_tables = result.fetchall()
+            # cursor.execute("""SELECT datname FROM pg_database
+            # WHERE datistemplate = false;""")
 
             MdxEngine.postgres_db_cubes = [
-                database[0] for database in cursor.fetchall()
+                database[0] for database in available_tables
             ]
 
         except Exception:
@@ -112,12 +124,19 @@ class MdxEngine:
     def _get_default_cube_directory(self):
 
         # toxworkdir does not expanduser properly under tox
-        if RUNNING_TOX:
+        if 'OLAPY_PATH' in os.environ:
+            home_directory = os.environ.get('OLAPY_PATH')
+        elif MdxEngine.DATA_FOLDER is not None:
+            home_directory = MdxEngine.DATA_FOLDER
+        elif RUNNING_TOX:
             home_directory = os.environ.get('HOME_DIR')
         else:
             home_directory = expanduser("~")
 
-        return os.path.join(home_directory, 'olapy-data', self.cube_folder)
+        if 'olapy-data' not in home_directory:
+            home_directory = os.path.join(home_directory, 'olapy-data')
+
+        return os.path.join(home_directory, self.cube_folder)
 
     def _get_tables_name(self):
         """
@@ -135,6 +154,7 @@ class MdxEngine:
         """
         config_file_parser = ConfigParser(self.cube_path)
         tables = {}
+
         if config_file_parser.config_file_exist(
         ) and self.cube in config_file_parser.get_cubes_names(
         ) and self.client != 'web':
@@ -155,6 +175,20 @@ class MdxEngine:
     def get_measures(self):
         """:return: all numerical columns in facts table."""
         # col.lower()[-2:] != 'id' to ignore any id column
+
+        # if web get measures from config file
+        config_file_parser = ConfigParser(self.cube_path)
+        if self.client == 'web' and config_file_parser.config_file_exist('web'):
+            for cubes in config_file_parser.construct_cubes(self.client):
+
+                # TODO temp
+                # update facts name
+                self.facts = cubes.facts[0].table_name
+
+
+                if cubes.facts[0].measures:
+                    return cubes.facts[0].measures
+
         return [
             col
             for col in self.tables_loaded[self.facts].select_dtypes(
@@ -169,7 +203,6 @@ class MdxEngine:
         :return: star schema DataFrame
         """
         fusion = None
-
         config_file_parser = ConfigParser(self.cube_path)
         if config_file_parser.config_file_exist(
                 self.client
@@ -213,6 +246,8 @@ class MdxEngine:
 
         :return: path to the cube
         """
+        if MdxEngine.DATA_FOLDER is not None:
+            return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER, self.cube)
         return os.path.join(self.cube_path, self.cube)
 
     # TODO temporary function

olapy/core/mdx/executor/execute_config_file.py

 from __future__ import absolute_import, division, print_function
 
+from ..tools.mem_bench import memory_usage
 from ..tools.connection import MyDB
 import pandas.io.sql as psql
+import os
 
 
 def _load_table_config_file(executer_instance, cube_obj):
@@ -15,22 +17,31 @@ def _load_table_config_file(executer_instance, cube_obj):
     # just one facts table right now
     executer_instance.facts = cube_obj.facts[0].table_name
 
-    db = MyDB(db=executer_instance.cube)
+    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
+
+    memory_usage("1 - before executing query //// _load_table_config_file")
+    for dimension in cube_obj.dimensions:
 
-    for table in cube_obj.dimensions:
-        value = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
-                                    db.connection)
+        df = psql.read_sql_query("SELECT * FROM {0}".format(dimension.name),
+                                 db.engine)
+        # only certain columns
+        if dimension.columns.keys():
+            df = df[dimension.columns.keys()]
 
-        tables[table.name] = value[[
-            col for col in value.columns if col.lower()[-3:] != '_id'
+        # change table display name
+        if dimension.displayName:
+            table_name = dimension.displayName
+        else:
+            table_name = dimension.name
+
+        # rename columns if value not None
+        df.rename(columns=(dict((k, v) for k, v in dimension.columns.items() if v)), inplace=True)
+
+        tables[table_name] = df[[
+            col for col in df.columns if col.lower()[-2:] != 'id'
         ]]
 
-    # update table display name
-    for dimension in cube_obj.dimensions:
-        if dimension.displayName and dimension.name and dimension.displayName != dimension.name:
-            tables[dimension.displayName] = tables[dimension.name][
-                dimension.columns]
-            executer_instance.dimension_display_name.append(dimension.name)
+    memory_usage("2 - after query, before fetchall  /////// _load_table_config_file")
 
     return tables
 
@@ -44,22 +55,76 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
     :return: star schema DataFrame
     """
     executer_instance.facts = cubes_obj.facts[0].table_name
-    db = MyDB(db=executer_instance.cube)
+    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
     # load facts table
+
+    memory_usage("1 - before executing query //// _construct_star_schema_config_file")
     fusion = psql.read_sql_query(
-        "SELECT * FROM {0}".format(executer_instance.facts), db.connection)
+        "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
 
     for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
         df = psql.read_sql_query(
             "SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
-            db.connection)
+            db.engine)
+
+        for dimension in cubes_obj.dimensions:
+            if dimension_and_key.split('.')[0] == dimension.name:
+                df.rename(columns=dimension.columns, inplace=True)
 
+        # todo test with this
         fusion = fusion.merge(
             df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
 
         # TODO CHOSE BETWEEN THOSES DF
-        # if separated dimensions
-        # fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
+        # fusion = fusion.merge(
+        #     df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
+        #     # remove suffixe from dimension and keep the same column name for facts
+        #     suffixes=('', '_y'))
+
+    memory_usage("2 - after query, before fetchall  /////// _construct_star_schema_config_file")
+    # TODO CHOSE BETWEEN THOSES DF
+    # if separated dimensions
+    # fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
+
+    # TODO CHOSE BETWEEN THOSES DF
+    # if facts contains all dimensions
+    # fusion = facts
+
+    # measures in config-file only
+    if cubes_obj.facts[0].measures:
+        executer_instance.measures = cubes_obj.facts[0].measures
+
+    return fusion
+
+
+def _construct_star_schema_config_file_OLD(executer_instance, cubes_obj):
+    """
+    Construct star schema DataFrame from configuration file.
+
+    :param cube_name:  cube name (or database name)
+    :param cubes_obj: cubes object
+    :return: star schema DataFrame
+    """
+    executer_instance.facts = cubes_obj.facts[0].table_name
+    db = MyDB(db=executer_instance.cube)
+    # load facts table
+
+    memory_usage("1 - before executing query //// _construct_star_schema_config_file")
+    fusion = psql.read_sql_query(
+        "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
+
+    for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
+        df = psql.read_sql_query(
+            "SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
+            db.engine)
+
+        fusion = fusion.merge(
+            df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
+
+    memory_usage("2 - after query, before fetchall  /////// _construct_star_schema_config_file")
+    # TODO CHOSE BETWEEN THOSES DF
+    # if separated dimensions
+    # fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
 
     # TODO CHOSE BETWEEN THOSES DF
     # if facts contains all dimensions
@@ -83,20 +148,24 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
     all_columns = []
 
     executer_instance.facts = cubes_obj.facts[0].table_name
-    db = MyDB(db=executer_instance.cube)
+    db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
     # load facts table
 
     if cubes_obj.facts[0].columns:
         all_columns += cubes_obj.facts[0].columns
 
+    memory_usage("1 - before executing query //// 1111 _construct_web_star_schema_config_file ")
     fusion = psql.read_sql_query(
-        "SELECT * FROM {0}".format(executer_instance.facts), db.connection)
+        "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
+
+    memory_usage("2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file")
 
     tables = {}
+    memory_usage("1 - before executing query //// 3333333333 _construct_web_star_schema_config_file ")
     for table in cubes_obj.tables:
 
         tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
-                                  db.connection)
+                                  db.engine)
 
         try:
             if table.columns:
@@ -117,11 +186,14 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
         all_columns += list(tab.columns)
         tables.update({table.name: tab})
 
+    memory_usage("2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file")
+
     # measures in config-file only
     if cubes_obj.facts[0].measures:
         executer_instance.measures = cubes_obj.facts[0].measures
         all_columns += cubes_obj.facts[0].measures
 
+    memory_usage("1 - before executing query //// 55555555 _construct_web_star_schema_config_file ")
     for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
         dimension_name = dimension_and_key.split('.')[0]
         if dimension_name in tables.keys():
@@ -129,13 +201,15 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
         else:
             df = psql.read_sql_query(
                 "SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
-                db.connection)
+                db.engine)
 
         # TODO check merge (how)
         fusion = fusion.merge(
             df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
-              # remove suffixe from dimension and keep the same column name for facts
-              suffixes=('', '_y'))
+            # remove suffixe from dimension and keep the same column name for facts
+            suffixes=('', '_y'))
+
+    memory_usage("2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file")
 
     return fusion[[column for column in all_columns if 'id' != column[-2:]]]
 

olapy/core/mdx/executor/execute_db.py

 from __future__ import absolute_import, division, print_function
 
+from sqlalchemy import inspect
+import pandas as pd
+from ..tools.mem_bench import memory_usage
 from ..tools.connection import MyDB
 import pandas.io.sql as psql
 
 
+class StringFolder(object):
+    """
+    Class that will fold strings. See 'fold_string'.
+    This object may be safely deleted or go out of scope when
+    strings have been folded.
+    """
+    def __init__(self):
+        self.unicode_map = {}
+
+    def fold_string(self, s):
+        """
+        Given a string (or unicode) parameter s, return a string object
+        that has the same value as s (and may be s). For all objects
+        with a given value, the same object will be returned. For unicode
+        objects that can be coerced to a string with the same value, a
+        string object will be returned.
+        If s is not a string or unicode object, it is returned unchanged.
+        :param s: a string or unicode object.
+        :return: a string or unicode object.
+        """
+        # If s is not a string or unicode object, return it unchanged
+        if not isinstance(s, basestring):
+            return s
+
+        # If s is already a string, then str() has no effect.
+        # If s is Unicode, try and encode as a string and use intern.
+        # If s is Unicode and can't be encoded as a string, this try
+        # will raise a UnicodeEncodeError.
+        try:
+            return intern(str(s))
+        except UnicodeEncodeError:
+            # Fall through and handle s as Unicode
+            pass
+
+        # Look up the unicode value in the map and return
+        # the object from the map. If there is no matching entry,
+        # store this unicode object in the map and return it.
+        return self.unicode_map.setdefault(s, s)
+
+
+def string_folding_wrapper(results):
+    """
+    This generator yields rows from the results as tuples,
+    with all string values folded.
+    """
+    # Get the list of keys so that we build tuples with all
+    # the values in key order.
+    keys = results.keys()
+    folder = StringFolder()
+    for row in results:
+        yield tuple(
+            folder.fold_string(row[key])
+            for key in keys
+        )
+
+# TODO try pandas.read_sql_table and pandas.read_sql
+
 def _load_tables_db(executer_instance):
     """
     Load tables from database.
@@ -11,18 +71,40 @@ def _load_tables_db(executer_instance):
     :return: tables dict with table name as key and dataframe as value
     """
     tables = {}
-    db = MyDB(db=executer_instance.cube)
-    cursor = db.connection.cursor()
-    cursor.execute("""SELECT table_name FROM information_schema.tables
-                      WHERE table_schema = 'public'""")
+    db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube)
+    inspector = inspect(db.engine)
+
+    memory_usage("1 - before executing query //// _load_tables_db")
+    for table_name in inspector.get_table_names():
 
-    for table_name in cursor.fetchall():
-        value = psql.read_sql_query(
-            'SELECT * FROM "{0}" '.format(table_name[0]), db.connection)
+        # value = psql.read_sql_query(
+        #     'SELECT * FROM "{0}"'.format(table_name), db.engine)
 
-        tables[table_name[0]] = value[[
+        # results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
+        results = db.engine.execution_options(stream_results=True).execute('SELECT * FROM "{0}"'.format(table_name))
+        # Fetch all the results of the query
+        value = pd.DataFrame(iter(results),columns=results.keys())  # Pass results as an iterator
+        # with string_folding_wrapper we loose response time
+        # value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
+        tables[table_name] = value[[
             col for col in value.columns if col.lower()[-3:] != '_id'
         ]]
+
+    # tables = {}
+    # db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER, db=executer_instance.cube)
+    # # inspector = inspect(db.engine)
+    # cursor = db.engine.cursor()
+    # cursor.execute("""SELECT table_name FROM information_schema.tables
+    #                           WHERE table_schema = 'public'""")
+    #
+    # for table_name in cursor.fetchall():
+    #     value = psql.read_sql_query(
+    #         'SELECT * FROM "{0}" '.format(table_name[0]), db.engine)
+    #
+    #     tables[table_name[0]] = value[[
+    #         col for col in value.columns if col.lower()[-3:] != '_id'
+    #     ]]
+    memory_usage("2 - after query, before fetchall  /////// _load_tables_db")
     return tables
 
 
@@ -34,21 +116,22 @@ def _construct_star_schema_db(executer_instance):
     :return: star schema DataFrame
     """
     db = MyDB(db=executer_instance.cube)
-
+    memory_usage("1 - before executing query //// _construct_star_schema_db")
     # load facts table
-    fusion = psql.read_sql_query(
-        'SELECT * FROM "{0}" '.format(executer_instance.facts), db.connection)
+    with db.engine as connection:
+        fusion = psql.read_sql_query(
+            'SELECT * FROM "{0}" '.format(executer_instance.facts), connection)
 
-    cursor = db.connection.cursor()
-    cursor.execute("""SELECT table_name FROM information_schema.tables
-                          WHERE table_schema = 'public'""")
-    for db_table_name in cursor.fetchall():
-        try:
-            fusion = fusion.merge(
-                psql.read_sql_query("SELECT * FROM {0}".format(
-                    db_table_name[0]), db.connection))
-        except:
-            print('No common column')
-            pass
+        inspector = inspect(connection)
+
+        for db_table_name in inspector.get_table_names():
+            try:
+                fusion = fusion.merge(
+                    psql.read_sql_query("SELECT * FROM {0}".format(
+                        db_table_name[0]), connection))
+            except:
+                print('No common column')
+                pass
 
+    memory_usage("2 - after query, before fetchall  /////// _construct_star_schema_db")
     return fusion

olapy/core/mdx/parser/gen_parser/models.py

-from __future__ import absolute_import, division, print_function
-
-
-class SelectStatement:
-
-    def __init__(self, select_statement):
-        self.select_statement = select_statement
-
-    def __str__(self):
-        return '{}'.format(self.select_statement)

olapy/core/mdx/parser/grammar/ebnf_mdx.ebnf

-(* The ebnf file is the translation and improvement of microsoft mdx's spec, from  https://msdn.microsoft.com/fr-fr/library/windows/desktop/ms717923%28v=vs.85%29.aspx *)
-(* The ebnf conversion was guided by http://stackoverflow.com/questions/14922242/how-to-convert-bnf-to-ebnf *)
-(* The generation of mdx_parser.py is as easy as: /path/to/grako bnf_mdx.ebnf -o mdx_parser.py *)
-
-
-mdx_statement =
-			select_statement;
-		
-select_statement::SelectStatement = name:'SELECT' [axis_specification_columns:axis_specification]
-						[',' axis_specification_rows:axis_specification]
-						'FROM' cube_specification:cube_specification
-						['WHERE' condition_specification:condition_specification]$
-                        ;
-
-axis_specification = [left_accolade] [fetch_form] @:dim_props_place [right_accolade] 'ON' axis_name; 
-
-dim_props_place = [left_parentheses] @:dim_props_type [right_parentheses] ;
-
-dim_props_type = [left_accolade] [fetch_type] @:dim_props_op_l1 [right_accolade] [{@:operator dim_props_type}*]; 
-
-dim_props_op_l1 = [left_parentheses] @:dim_props_op [right_parentheses];
-
-dim_props_op = [left_accolade] @:dim_props_ligne [right_accolade] [{@:comma dim_props_op}*];
-
-dim_props_ligne = [left_parentheses] @:dimension_place [right_parentheses] [{ (@:comma | @:dpoint ) @:dim_props_ligne}*] ;
-
-dimension_place = [left_accolade] @:dim_props [ point @:laste_node] [ @:comma @:dim_props [ point @:laste_node ]] [@:dpoint @:dim_props [ point @:laste_node ]] [right_accolade] | @:dimension_shortcut;
-
-dim_props =  {[point] [left_bracket] @:dimension [right_bracket]}* ;
-
-laste_node = "members" | "children" | "Members" | 'ALLMEMBERS' ;
-
-dimension = @:/[a-zA-Z0-9'_'' '',']*/ ;
-
-axis_name = '0' | '1' |'COLUMNS' | 'ROWS' | '_ROWS';
-
-cube_specification = [left_bracket] dimension [right_bracket];
-
-condition_specification =  [left_parentheses] {[point] [left_bracket] @:dimension [right_bracket]}* [right_parentheses] ;
-
-digit = 
-"0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" ;
-
-fetch_type = 'CROSSJOIN' | 'NONEMPTY' | 'union' | 'except' | 'extract' ;
-
-dimension_shortcut = 'all' | 'time' ;
-
-fetch_form = 'NONEMPTY' | 'non_empty' | 'non empty' ;
-	
-left_bracket = '[';
-
-right_bracket = ']';
-
-left_parentheses = '(';
-
-right_parentheses = ')';
-
-left_accolade = '{';
-
-right_accolade = '}';
-
-point = '.' ;
-
-dpoint = ':' ;
-
-comma = ',' ;
-
-operator = '+' | '-' | '/' | '*' ; 

olapy/core/mdx/parser/parse.py

-from __future__ import absolute_import, division, print_function
-
-from grako.model import ModelBuilderSemantics
-
-from .gen_parser.mdxparser import MdxParserGen
-from .gen_parser.models import SelectStatement
-
-
-class MdxParser:
-    """Parse the mdx query and split it into well-defined parts."""
-    START = 'MDX_statement'
-
-    @staticmethod
-    def parsing_mdx_query(axis, query):
-        """Split the query into axis.
-
-        **Example**::
-
-
-            SELECT
-
-            { [Geography].[Geo].[Country].[France],
-              [Geography].[Geo].[Country].[Spain] } ON COLUMNS,
-
-            { [Product].[Prod].[Company].[Crazy Development] } ON ROWS
-
-              FROM [Sales]
-
-              WHERE [Time].[Calendar].[Year].[2010]
-
-
-        +------------+------------------------------------------------+
-        |            | [Geography].[Geo].[Country].[France]           |
-        | column     |                                                |
-        |            | [Geography].[Geo].[Country].[Spain]            |
-        +------------+------------------------------------------------+
-        | row        | [Product].[Prod].[Company].[Crazy Development] |
-        +------------+------------------------------------------------+
-        | cube       | [Sales]                                        |
-        +------------+------------------------------------------------+
-        | condition  | [Time].[Calendar].[Year].[2010]                |
-        +------------+------------------------------------------------+
-
-        :param query: MDX Query
-        :param axis: column | row | cube | condition | all
-        :return: Tuples in the axis, from the MDX query
-        """
-        model = MdxParserGen(semantics=ModelBuilderSemantics(
-            types=[SelectStatement]))
-        ast = model.parse(query, rule_name=MdxParser.START, ignorecase=True)
-        if axis == "column":
-            if ast.select_statement.axis_specification_columns is not None and \
-                    u'' in ast.select_statement.axis_specification_columns:
-                ast.select_statement.axis_specification_columns.remove(u'')
-            return ast.select_statement.axis_specification_columns
-        elif axis == "row":
-            if ast.select_statement.axis_specification_rows is not None and \
-                    u'' in ast.select_statement.axis_specification_rows:
-                ast.select_statement.axis_specification_rows.remove(u'')
-            return ast.select_statement.axis_specification_rows
-        elif axis == "cube":
-            if ast.select_statement.cube_specification is not None and \
-                    u'' in ast.select_statement.cube_specification:
-                ast.select_statement.cube_specification.remove(u'')
-            return ast.select_statement.cube_specification[1] if \
-                isinstance(ast.select_statement.cube_specification, list) \
-                else ast.select_statement.cube_specification
-        elif axis == "condition":
-            if ast.select_statement.condition_specification is not None and \
-                    type(ast.select_statement.condition_specification) not in (
-                    unicode, str) and \
-                    u'' in ast.select_statement.condition_specification:
-                ast.select_statement.condition_specification.remove(u'')
-            return ast.select_statement.condition_specification
-        elif axis == "all":
-
-            return 'Operation = {} \n' \
-                   'Columns = {} \n' \
-                   'Rows = {} \n' \
-                   'From = {} \n' \
-                   'Where = {} \n'.format(ast.select_statement.name,
-                                          ast.select_statement.from_,
-                                          ast.select_statement.axis_specification_columns,
-                                          ast.select_statement.axis_specification_rows,
-                                          ast.select_statement.cube_specification,
-                                          ast.select_statement.condition_specification,
-                                          )

olapy/core/mdx/tools/bech_mem_opt2.txt

+Memory summary:1 - before executing query //// _load_tables_db
+
+VM: 700.28Mb
+
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+
+VM: 2781.79Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 2782.04Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 2782.29Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 2782.29Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 2784.29Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 2784.29Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 2784.29Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 2784.04Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 2783.79Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 2783.79Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 2784.29Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 2784.29Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 2784.29Mb
+
+
+Memory summary:1 - before executing query //// _load_tables_db
+  types |   # objects |   total size
+======= | =========== | ============
+   dict |        7429 |      8.72 MB
+    str |       58990 |      8.30 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |      2.01 GB
+                                 dict |        7802 |      8.88 MB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         592 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         179 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         180 |      2.01 GB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         180 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |      2.01 GB
+    <class 'pandas.core.series.Series |         618 |      2.01 GB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         630 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         185 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |      2.01 GB
+    <class 'pandas.core.series.Series |         604 |      2.01 GB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |      2.01 GB
+    <class 'pandas.core.series.Series |         592 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      2.01 GB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      2.01 GB
+    <class 'pandas.core.series.Series |         629 |      2.01 GB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         641 |      2.01 GB
+  <class 'pandas.core.frame.DataFrame |         186 |      2.01 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      2.01 GB
+    <class 'pandas.core.series.Series |         615 |      2.01 GB
+
+

olapy/core/mdx/tools/bech_mem_opt3.txt

+Memory summary:1 - before executing query //// _load_tables_db
+
+VM: 700.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+
+VM: 2719.36Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 2719.36Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 2719.86Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 2719.86Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 2721.61Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 2721.61Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 2721.61Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 2721.61Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 2721.36Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 2721.36Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 2721.86Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 2721.86Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 2721.86Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 2721.86Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 2721.36Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 2721.36Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 2721.86Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 2721.86Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 2721.86Mb
+
+****************************************
+  types |   # objects |   total size
+======= | =========== | ============
+   dict |        7430 |      8.72 MB
+    str |       58996 |      8.30 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |      1.93 GB
+                                 dict |        7803 |      8.86 MB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         592 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         179 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         180 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         180 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |      1.93 GB
+    <class 'pandas.core.series.Series |         618 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         630 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         185 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |      1.93 GB
+    <class 'pandas.core.series.Series |         604 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |      1.93 GB
+    <class 'pandas.core.series.Series |         592 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         641 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+    <class 'pandas.core.series.Series |         615 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         603 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         180 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         181 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+    <class 'pandas.core.series.Series |         629 |      1.93 GB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         641 |      1.93 GB
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |      1.93 GB
+    <class 'pandas.core.series.Series |         615 |      1.93 GB
+----------------------------
+
+

olapy/core/mdx/tools/bech_mem_with_psycopg.txt

+Memory summary:1 - before executing query //// _load_tables_db
+
+VM: 699.77Mb
+
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+
+VM: 1763.52Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 1763.52Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 1763.77Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 1763.77Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+
+VM: 1764.27Mb
+
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+
+VM: 1764.27Mb
+
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+
+VM: 1764.52Mb
+
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+
+VM: 1764.52Mb
+
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+
+VM: 1764.52Mb
+
+**********************************
+
+  types |   # objects |   total size
+======= | =========== | ============
+   dict |        7235 |      8.59 MB
+    str |       58455 |      8.18 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// _load_tables_db
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |    539.66 MB
+                                 dict |        7649 |      8.78 MB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         592 |    538.26 MB
+  <class 'pandas.core.frame.DataFrame |         179 |    538.22 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |    539.70 MB
+  <class 'pandas.core.frame.DataFrame |         180 |    539.66 MB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         618 |    539.70 MB
+  <class 'pandas.core.frame.DataFrame |         180 |    539.66 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |    540.42 MB
+    <class 'pandas.core.series.Series |         618 |    539.70 MB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         630 |    540.47 MB
+  <class 'pandas.core.frame.DataFrame |         185 |    540.42 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         185 |    540.83 MB
+    <class 'pandas.core.series.Series |         604 |    539.02 MB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         180 |    538.50 MB
+    <class 'pandas.core.series.Series |         592 |    538.26 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+  <class 'pandas.core.frame.DataFrame |         181 |    539.94 MB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+  <class 'pandas.core.frame.DataFrame |         181 |    539.94 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |    540.71 MB
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         641 |    540.83 MB
+  <class 'pandas.core.frame.DataFrame |         186 |    540.71 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |    541.11 MB
+    <class 'pandas.core.series.Series |         615 |    539.39 MB
+----------------------------
+Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         603 |    538.62 MB
+  <class 'pandas.core.frame.DataFrame |         180 |    538.50 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 222222222222 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+  <class 'pandas.core.frame.DataFrame |         181 |    539.94 MB
+----------------------------
+Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+  <class 'pandas.core.frame.DataFrame |         181 |    539.94 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 44444444 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |    540.71 MB
+    <class 'pandas.core.series.Series |         629 |    540.06 MB
+----------------------------
+Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file 
+                                types |   # objects |   total size
+===================================== | =========== | ============
+    <class 'pandas.core.series.Series |         641 |    540.83 MB
+  <class 'pandas.core.frame.DataFrame |         186 |    540.71 MB
+----------------------------
+Memory summary:2 - after query, before fetchall  /////// 6666666666 _construct_star_schema_config_file
+                                types |   # objects |   total size
+===================================== | =========== | ============
+  <class 'pandas.core.frame.DataFrame |         186 |    541.11 MB
+    <class 'pandas.core.series.Series |         615 |    539.39 MB
+
+

olapy/core/mdx/tools/config_file_parser.py

@@ -198,12 +198,18 @@ class ConfigParser:
         :param cube_path: path to cube (csv folders)
         :param file_name: config file name (DEFAULT = cubes-config.xml)
         """
-        if cube_path is None:
+        # home_directory = home_directory
+        if 'OLAPY_PATH' in os.environ:
+            home_directory = os.environ['OLAPY_PATH']
+        else:
             from os.path import expanduser
             home_directory = expanduser("~")
+
+        if cube_path is None:
             self.cube_path = os.path.join(home_directory, 'olapy-data', 'cubes')
         else:
             self.cube_path = cube_path
+
         self.file_name = file_name
         self.web_config_file_name = web_config_file_name
 
@@ -284,15 +290,21 @@ class ConfigParser:
                         ]) for xml_facts in tree.xpath('/cubes/cube/facts')
                 ]
 
+                # keys = {
+                #            key.text: key.attrib['ref']
+                #            for key in xml_facts.findall('keys/column_name')
+                #        },
+
                 dimensions = [
                     Dimension(
                         name=xml_dimension.find('name').text,
+                        # column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
                         displayName=xml_dimension.find('displayName').text,
-                        columns=[
-                            column_name.text
+                        columns={
+                            column_name.text : None if not column_name.attrib else column_name.attrib['column_new_name']
                             for column_name in xml_dimension.findall(
                                 'columns/name')
-                        ])
+                        })
                     for xml_dimension in tree.xpath(
                         '/cubes/cube/dimensions/dimension')
                 ]

olapy/core/mdx/tools/connection.py

-import psycopg2 as pg
+# import psycopg2 as pg
+from sqlalchemy import create_engine
 
 # postgres connection
 from olapy_config_file_parser import DbConfigParser
@@ -16,28 +17,32 @@ class MyDB(object):
     #         raise Exception('Missing database config file')
 
 
-    def __init__(self,db=None):
+    def __init__(self,db_config_file_path=None,db=None):
 
         # TODO temporary
-        db_config = DbConfigParser()
+        db_config = DbConfigParser(config_path=db_config_file_path)
         db_credentials = db_config.get_db_credentials()[0]
         username = db_credentials['user_name']
         password = db_credentials['password']
         host = db_credentials['host']
+        port = db_credentials['port']
 
         if db is None:
             # first i want to show all databases to user (in excel)
-            self.connection = pg.connect("user={0} password={1} host='{2}'".
-                                         format(username, password, host))
+            # self.engine = pg.connect("user={0} password={1} host='{2}'".
+            #                              format(username, password, host))
+            self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
+                username, password, 'postgres', host, port))
+
         else:
             # and then we connect to the user db
-            try:
-                self.connection = pg.connect(
-                    "user={0} password={1} dbname='{2}' host='{3}'".format(
-                        username, password, db, host))
-            except:
-                print("can't connect")
+            self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
+                username, password, db, host, port))
+            # self.connection = pg.connect(
+            #     "user={0} password={1} dbname='{2}' host='{3}'".format(
+            #         username, password, db, host))
+
 
     def __del__(self):
         if hasattr(self, 'connection'):
-            self.connection.close()
+            self.engine.dispose()

olapy/core/mdx/tools/mem_bench.py

+import os
+from os.path import expanduser
+
+from pympler import summary, muppy
+import psutil
+
+def get_virtual_memory_usage_kb():
+    """
+    The process's current virtual memory size in Kb, as a float.
+
+    """
+    return float(psutil.Process().memory_info_ex().vms) / 1024.0
+
+def memory_usage(where):
+    """
+    Print out a basic summary of memory usage.
+
+    """
+    with open(os.path.join(expanduser('~'), 'bech_mem.txt'), mode='a+') as file:
+        mem_summary = summary.summarize(muppy.get_objects())
+        file.write("Memory summary:" + where + '\n\n')
+        print("Memory summary:" + where )
+        summary.print_(mem_summary, limit=2)
+        print('----------------------------')
+        file.write("VM: %.2fMb" % (get_virtual_memory_usage_kb() / 1024.0) + '\n\n')

olapy/core/mdx/tools/olapy_config_file_parser.py

@@ -43,15 +43,13 @@ class DbConfigParser:
             parser = etree.XMLParser()
             tree = etree.parse(config_file, parser)
 
-            try:
-                return [
-                    {
-                        # 'sgbd': db.find('sgbd').text,
-                     'user_name': db.find('user_name').text,
-                     'password': db.find('password').text,
-                     'host': db.find('host').text,
-                     }
-                    for db in tree.xpath('/olapy/database')
-                ]
-            except:
-                raise ('missed name or source tags')
+            return [
+                {
+                    # 'sgbd': db.find('sgbd').text,
+                 'user_name': db.find('user_name').text,
+                 'password': db.find('password').text,
+                 'host': db.find('host').text,
+                 'port': db.find('port').text,
+                 }
+                for db in tree.xpath('/olapy/database')
+            ]

olapy/core/services/xmla.py

@@ -222,7 +222,7 @@ application = Application(
 wsgi_application = WsgiApplication(application)
 
 
-def start_server(write_on_file=False):
+def start_server(host='0.0.0.0',port=5000,write_on_file=False):
     """
     Start the xmla server.
 
@@ -254,7 +254,7 @@ def start_server(write_on_file=False):
     logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
     logging.info("listening to http://127.0.0.1:8000/xmla")
     logging.info("wsdl is at: http://localhost:8000/xmla?wsdl")
-    server = make_server('0.0.0.0', 8000, wsgi_application)
+    server = make_server(host, port, wsgi_application)
     server.serve_forever()
 
 

olapy/core/services/xmla_discover_tools.py

@@ -1830,8 +1830,10 @@ class XmlaDiscoverTools():
 
                 # TODO in another idea, change this
                 # TO CHANGE NAME DISPLAY THAT EXISTS IN CONFIG FILE
-                if MdxEngine.dimension_display_name != [] and tables in MdxEngine.dimension_display_name:
-                    continue
+
+                # if MdxEngine.dimension_display_name != [] and tables in MdxEngine.dimension_display_name:
+                #     continue
+
                 rows += """
                 <row>
                     <CATALOG_NAME>{0}</CATALOG_NAME>
@@ -1888,6 +1890,15 @@ class XmlaDiscoverTools():
                 for table_name, df in self.executer.tables_loaded.items():
                     if table_name == self.executer.facts:
                         continue
+
+                    # french caracteres
+                    # TODO encode dataframe
+                    if type(df.iloc[0][0]) == unicode:
+                        column_attribut = df.iloc[0][0].encode('utf-8')
+                    else:
+                        column_attribut = df.iloc[0][0]
+
+
                     rows += """
                     <row>
                         <CATALOG_NAME>{0}</CATALOG_NAME>
@@ -1910,8 +1921,11 @@ class XmlaDiscoverTools():
                         <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
                         <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
                     </row>
-                    """.format(self.selected_catalogue, table_name,
-                               df.columns[0], df.iloc[0][0])
+                    """.format(self.selected_catalogue,
+                               table_name,
+                               df.columns[0],
+                               column_attribut)
+
 
                 rows += """
                 <row>
@@ -1953,6 +1967,14 @@ class XmlaDiscoverTools():
                 for table_name, df in self.executer.tables_loaded.items():
                     if table_name == self.executer.facts:
                         continue
+
+                    # french caracteres
+                    # TODO encode dataframe
+                    if type(df.iloc[0][0]) == unicode:
+                        column_attribut = df.iloc[0][0].encode('utf-8')
+                    else:
+                        column_attribut = df.iloc[0][0]
+
                     rows += """
                     <row>
                         <CATALOG_NAME>{0}</CATALOG_NAME>
@@ -1975,8 +1997,10 @@ class XmlaDiscoverTools():
                         <HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
                         <INSTANCE_SELECTION>0</INSTANCE_SELECTION>
                     </row>
-                        """.format(self.selected_catalogue, table_name,
-                                   df.columns[0], df.iloc[0][0])
+                        """.format(self.selected_catalogue,
+                                   table_name,
+                                   df.columns[0],
+                                   column_attribut)
 
                 rows += """
                 <row>

olapy/core/services/xmla_execute_tools.py

@@ -3,6 +3,8 @@ from __future__ import absolute_import, division, print_function
 import itertools
 from collections import OrderedDict
 
+import numpy as np
+
 
 class XmlaExecuteTools():
     """XmlaExecuteTools for generating xmla execute responses."""
@@ -149,6 +151,12 @@ class XmlaExecuteTools():
 
             for tupl in tupls:
                 tuple_without_minus_1 = self.get_tuple_without_nan(tupl)
+
+                # french caracteres
+                # TODO encode dataframe
+                if type(tuple_without_minus_1[-1]) == unicode:
+                    tuple_without_minus_1 = [x.encode('utf-8') for x in tuple_without_minus_1]
+
                 axis0 += """
                 <Member Hierarchy="[{0}].[{0}]">
                     <UName>[{0}].[{0}].[{1}].{2}</UName>
@@ -324,7 +332,7 @@ class XmlaExecuteTools():
         cell_data = ""
         index = 0
         for value in columns_loop:
-            if value == -1:
+            if np.isnan(value) :
                 value = ''
             cell_data += """
             <Cell CellOrdinal="{0}">
@@ -550,6 +558,14 @@ class XmlaExecuteTools():
                 set(table_name
                     for table_name in mdx_execution_result['columns_desc']
                     ['all'])):
+
+            # TODO encode dataframe
+            # french caracteres
+            if type(self.executer.tables_loaded[dim_diff].iloc[0][0]) == unicode:
+                column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0].encode('utf-8')
+            else:
+                column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0]
+
             tuple += """
             <Member Hierarchy="[{0}].[{0}]">
                 <UName>[{0}].[{0}].[{1}].[{2}]</UName>
@@ -560,7 +576,7 @@ class XmlaExecuteTools():
             </Member>
             """.format(dim_diff,
                        self.executer.tables_loaded[dim_diff].columns[0],
-                       self.executer.tables_loaded[dim_diff].iloc[0][0])
+                       column_attribut)
 
         # if we have zero on one only measures used
         if len(self.executer.selected_measures) <= 1:

requirements.in

-#
-grako
+#grako
 pandas<1
 lxml==3.6.0  #lxml 3.7 causes problems in windows
 spyne<3
 treelib<2
+SQLAlchemy
 psycopg2
 
+
+# tools
+pympler
+psutil
+
 # Test
 werkzeug
 

requirements.txt

@@ -4,29 +4,14 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
-click==6.7                # via flask
-Flask-Login==0.3.2
-Flask-Script==2.0.5
-Flask-Session==0.3.0
-Flask-SQLAlchemy==2.1
-Flask-WTF==0.12
-flask==0.12.1
-grako==3.22.0
-itsdangerous==0.24        # via flask
-jinja2==2.9.6             # via flask
+grako==3.99.9
 lxml==3.6.0
-markupsafe==1.0           # via jinja2
 numpy==1.12.1             # via pandas
-pandas==0.19.2
-plotly==1.12.9
+pandas==0.20.1
 psycopg2==2.7.1
 python-dateutil==2.6.0    # via pandas
-pytz==2017.2              # via pandas, plotly, spyne
-requests==2.13.0          # via plotly
-six==1.10.0               # via plotly, python-dateutil
+pytz==2017.2              # via pandas, spyne
+six==1.10.0               # via python-dateutil
 spyne==2.12.14
-sqlalchemy==1.0.17
 treelib==1.3.5
-werkzeug==0.12.1          # via flask, flask-wtf
-wtforms==2.1
-XlsxWriter==0.9.3
+werkzeug==0.12.1

setup.py

@@ -29,7 +29,7 @@ setup(
     install_requires=install_requires,
     include_package_data=False,
     # cmdclass={
-    #     # 'develop': PostDevelopCommand,
+    #     'develop': PostDevelopCommand,
     #     'install': PostInstallCommand,
     # },
     classifiers=[
@@ -41,8 +41,9 @@ setup(
         # "Topic :: Business intelligence",
     ],)
 
-
-if RUNNING_TOX:
+if 'OLAPY_PATH' in os.environ:
+    home_directory = os.environ['OLAPY_PATH']
+elif RUNNING_TOX:
     home_directory = os.environ.get('HOME_DIR')
 else:
     home_directory = expanduser("~")
@@ -55,5 +56,3 @@ if not os.path.isdir(os.path.join(home_directory, 'olapy-data', 'cubes')):
 
 if not os.path.isfile(os.path.join(home_directory, 'olapy-data','olapy-config.xml')):
     copyfile('config/olapy-config.xml', os.path.join(home_directory, 'olapy-data','olapy-config.xml'))
-
-

tests/test_queries.py

-from __future__ import absolute_import, division, print_function
-
-import pandas as pd
-from pandas.util.testing import assert_frame_equal
-
-from olapy.core.mdx.executor.execute import MdxEngine
-from olapy.core.mdx.parser.parse import MdxParser
-
-CUBE = 'sales'
-
-query1 = "SELECT" \
-         "{[Measures].[Amount]} ON COLUMNS " \
-         "FROM [sales]"
-
-query2 = """SELECT
-         {[Geography].[Economy].[Partnership]} ON COLUMNS
-         FROM [sales]"""
-
-query3 = """SELECT
-         {[Measures].[Amount]} on 0,
-         non empty {[Geography].[Geo].[Country].members} ON COLUMNS
-         FROM [sales]"""
-
-query4 = """SELECT
-         {[Geography].[Economy].[Partnership]} ON COLUMNS,
-         non empty {[Geography].[Geo].[Country].members} on 1
-         from [sales]"""
-
-query5 = """select
-         {[Geography].[Economy].[Country]} on 0,
-         non empty {[Geography].[Geo].[Country].members} on 1
-         from [sales]"""
-
-query6 = """select
-         {[Geography].[Economy].[Partnership]} on 0,
-         {[Product].[Prod].[Company]} on 1
-         from [sales]"""
-
-query7 = """select
-         {[Geography].[Economy].[Partnership].[EU]} on 0,
-         {[Product].[Prod].[Company].[Crazy Development]} on 1
-         from [sales]"""
-
-query8 = """select
-         {[Geography].[Economy].[Partnership].[EU],
-         [Geography].[Economy].[Partnership].[None],
-         [Geography].[Economy].[Partnership].[NAFTA]} on 0,
-         {[Product].[Prod].[Company].[Crazy Development],
-         [Product].[Prod].[Company].[Company_test],
-         [Product].[Prod].[Company].[test_Development]} on 1
-         from [sales]"""
-
-query9 = """select
-         {[Geography].[Economy].[Partnership].[EU],
-         [Geography].[Economy].[Partnership].[None]} on 0
-         from [sales]"""
-
-query10 = """select
-          {[Geography].[Geo].[Country].[France],
-          [Geography].[Geo].[Country].[Spain]} on 0,
-          non empty {[Measures].[Amount]} on 1
-          from [sales]"""
-
-where1 = "Where [Time].[Calendar].[Day].[May 12,2010]"
-where2 = "Where[Product].[olapy].[Personal]"
-where3 = "Where[Time].[Calendar].[Year].[2010]"
-where4 = "Where [Measures].[Count]"
-where5 = "where [Count]"
-
-query11 = """
-          SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{DrilldownMember({{
-          [Time].[Time].[Year].Members}}, {
-          [Time].[Time].[Year].[2010]})}}, {
-          [Time].[Time].[Quarter].[2010].[Q2 2010]})}}, {
-          [Time].[Time].[Month].[2010].[Q2 2010].[May 2010]}))) DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME
-          ON COLUMNS
-          FROM [sales] WHERE ([Measures].[Amount])
-          CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
-          """
-
-query12 = """SELECT NON EMPTY Hierarchize(AddCalculatedMembers({
-            [Geography].[Geography].[Continent].Members}))
-            DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME ON COLUMNS
-            FROM [sales]
-            WHERE ([Measures].[Amount])
-            CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
-
-parser = MdxParser()
-executer = MdxEngine(CUBE)
-
-
-def test_parsing_query1():
-    assert parser.parsing_mdx_query(
-        'column', query=query1) == ['Measures', 'Amount']
-    assert parser.parsing_mdx_query('cube', query=query1) == "sales"
-    assert parser.parsing_mdx_query('row', query=query1) is None
-    query1_where = query1 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query1 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query1 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_parsing_query2():
-    assert parser.parsing_mdx_query(
-        'column', query=query2) == [u'Geography', u'Economy', u'Partnership']
-    assert parser.parsing_mdx_query('cube', query=query2) == "sales"
-    assert parser.parsing_mdx_query('row', query=query2) is None
-    query1_where = query2 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query2 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query2 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_parsing_query3():
-    assert parser.parsing_mdx_query(
-        'column', query=query3) == [u'Measures', u'Amount']
-    assert parser.parsing_mdx_query('cube', query=query3) == "sales"
-    assert parser.parsing_mdx_query(
-        'row', query=query3) == [u'Geography', u'Geo', u'Country', u'members']
-    query1_where = query3 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query3 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query3 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_parsing_query4():
-    query0 = query4
-    assert parser.parsing_mdx_query(
-        'column', query=query0) == [u'Geography', u'Economy', u'Partnership']
-    assert parser.parsing_mdx_query('cube', query=query0) == "sales"
-    assert parser.parsing_mdx_query(
-        'row', query=query0) == [u'Geography', u'Geo', u'Country', u'members']
-    query1_where = query0 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query0 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query0 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_parsing_query5():
-    query0 = query5
-    assert parser.parsing_mdx_query(
-        'column', query=query0) == [u'Geography', u'Economy', u'Country']
-    assert parser.parsing_mdx_query('cube', query=query0) == "sales"
-    assert parser.parsing_mdx_query(
-        'row', query=query0) == [u'Geography', u'Geo', u'Country', u'members']
-    query1_where = query0 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query0 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query0 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_parsing_query6():
-    query0 = query6
-    assert parser.parsing_mdx_query(
-        'column', query=query0) == [u'Geography', u'Economy', u'Partnership']
-    assert parser.parsing_mdx_query('cube', query=query0) == "sales"
-    assert parser.parsing_mdx_query(
-        'row', query=query0) == [u'Product', u'Prod', u'Company']
-    query1_where = query0 + '\n' + where1
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
-    query2_where = query0 + '\n' + where2
-    assert parser.parsing_mdx_query(
-        'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
-    query3_where = query0 + '\n' + where3
-    assert parser.parsing_mdx_query(
-        'condition',
-        query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
-
-
-def test_execution_query1():
-    executer.mdx_query = query1
-    assert executer.execute_mdx()['result']['Amount'][0] == 1023
-    executer.mdx_query = query11
-    assert executer.execute_mdx()['result']['Amount'][3] == 1
-    assert executer.execute_mdx()['result']['Amount'][4] == 2
-
-
-def test_execution_query3():
-    df = pd.DataFrame({
-        'Continent': ['America', 'Europe'],
-        'Amount': [768, 255]
-    }).groupby(['Continent']).sum()
-
-    executer.mdx_query = query12
-    assert assert_frame_equal(df, executer.execute_mdx()['result']) is None
-
-    executer.mdx_query = query11
-
-    assert list(executer.execute_mdx()['result'][
-        'Amount']) == [1023, 1023, 1023, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512]