Skip to content

O
olapy
Commit 1fb79bb3 authored by mouadh's avatar mouadh
Browse files
Options

Merge branch 'update_excel_config_file' 

# Conflicts:
#	requirements.txt
parents f4fd1e91 c85f307c
Hide whitespace changes
Inline Side-by-side
Showing with 1133 additions and 1042 deletions
config/olapy-config.xml
View file @ 1fb79bb3
<?xml version="1.0" encoding="UTF-8"?>
<!-- this config file will be deleted ASAP -->
<olapy>
<database>
......@@ -8,6 +8,7 @@
<user_name>postgres</user_name>
<password>root</password>
<host>localhost</host>
<port>5432</port>
</database>
......
olapy/core/mdx/executor/execute.py
View file @ 1fb79bb3
......@@ -33,12 +33,14 @@ class MdxEngine:
:param sep: separator in the csv files
"""
# DATA_FOLDER useful for olapy web (falsk instance_path)
# get olapy-data path with instance_path instead of 'expanduser'
DATA_FOLDER = None
CUBE_FOLDER = "cubes"
# (before instantiate MdxEngine I need to access cubes information)
csv_files_cubes = []
postgres_db_cubes = []
# to show just config file's dimensions
dimension_display_name = []
def __init__(self,
cube_name,
......@@ -64,8 +66,8 @@ class MdxEngine:
self.client = client_type
self.tables_loaded = self.load_tables()
# all measures
self.load_star_schema_dataframe = self.get_star_schema_dataframe()
self.measures = self.get_measures()
self.load_star_schema_dataframe = self.get_star_schema_dataframe()
self.tables_names = self._get_tables_name()
# default measure is the first one
self.selected_measures = [self.measures[0]]
......@@ -75,32 +77,42 @@ class MdxEngine:
""":return: list cubes name that exists in cubes folder (under ~/olapy-data/cubes) and postgres database (if connected)."""
# get csv files folders (cubes)
# toxworkdir does not expanduser properly under tox
if RUNNING_TOX:
# surrended with try, except and PASS so we continue getting cubes from different
# sources (db, csv...) without interruption
if cls.DATA_FOLDER is not None:
home_directory = os.path.dirname(cls.DATA_FOLDER)
elif RUNNING_TOX:
home_directory = os.environ.get('HOME_DIR')
else:
home_directory = expanduser("~")
location = os.path.join(home_directory, 'olapy-data', cls.CUBE_FOLDER)
# surrended with try, except and PASS so we continue getting cubes from different
# sources (db, csv...) without interruption
try:
MdxEngine.csv_files_cubes = [
file for file in os.listdir(location)
if os.path.isdir(os.path.join(location, file))
]
except Exception:
print('no csv folders')
pass
# get postgres databases
# surrended with try, except and PASS so we continue getting cubes from different
# sources (db, csv...) without interruption
try:
db = MyDB()
cursor = db.connection.cursor()
cursor.execute("""SELECT datname FROM pg_database
WHERE datistemplate = false;""")
db = MyDB(db_config_file_path=cls.DATA_FOLDER)
# TODO this work only with postgres
result = db.engine.execute('SELECT datname FROM pg_database WHERE datistemplate = false;')
available_tables = result.fetchall()
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
MdxEngine.postgres_db_cubes = [
database[0] for database in cursor.fetchall()
database[0] for database in available_tables
]
except Exception:
......@@ -112,12 +124,19 @@ class MdxEngine:
def _get_default_cube_directory(self):
# toxworkdir does not expanduser properly under tox
if RUNNING_TOX:
if 'OLAPY_PATH' in os.environ:
home_directory = os.environ.get('OLAPY_PATH')
elif MdxEngine.DATA_FOLDER is not None:
home_directory = MdxEngine.DATA_FOLDER
elif RUNNING_TOX:
home_directory = os.environ.get('HOME_DIR')
else:
home_directory = expanduser("~")
return os.path.join(home_directory, 'olapy-data', self.cube_folder)
if 'olapy-data' not in home_directory:
home_directory = os.path.join(home_directory, 'olapy-data')
return os.path.join(home_directory, self.cube_folder)
def _get_tables_name(self):
"""
......@@ -135,6 +154,7 @@ class MdxEngine:
"""
config_file_parser = ConfigParser(self.cube_path)
tables = {}
if config_file_parser.config_file_exist(
) and self.cube in config_file_parser.get_cubes_names(
) and self.client != 'web':
......@@ -155,6 +175,20 @@ class MdxEngine:
def get_measures(self):
""":return: all numerical columns in facts table."""
# col.lower()[-2:] != 'id' to ignore any id column
# if web get measures from config file
config_file_parser = ConfigParser(self.cube_path)
if self.client == 'web' and config_file_parser.config_file_exist('web'):
for cubes in config_file_parser.construct_cubes(self.client):
# TODO temp
# update facts name
self.facts = cubes.facts[0].table_name
if cubes.facts[0].measures:
return cubes.facts[0].measures
return [
col
for col in self.tables_loaded[self.facts].select_dtypes(
......@@ -169,7 +203,6 @@ class MdxEngine:
:return: star schema DataFrame
"""
fusion = None
config_file_parser = ConfigParser(self.cube_path)
if config_file_parser.config_file_exist(
self.client
......@@ -213,6 +246,8 @@ class MdxEngine:
:return: path to the cube
"""
if MdxEngine.DATA_FOLDER is not None:
return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER, self.cube)
return os.path.join(self.cube_path, self.cube)
# TODO temporary function
......
olapy/core/mdx/executor/execute_config_file.py
View file @ 1fb79bb3
from __future__ import absolute_import, division, print_function
from ..tools.mem_bench import memory_usage
from ..tools.connection import MyDB
import pandas.io.sql as psql
import os
def _load_table_config_file(executer_instance, cube_obj):
......@@ -15,22 +17,31 @@ def _load_table_config_file(executer_instance, cube_obj):
# just one facts table right now
executer_instance.facts = cube_obj.facts[0].table_name
db = MyDB(db=executer_instance.cube)
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
memory_usage("1 - before executing query //// _load_table_config_file")
for dimension in cube_obj.dimensions:
for table in cube_obj.dimensions:
value = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
db.connection)
df = psql.read_sql_query("SELECT * FROM {0}".format(dimension.name),
db.engine)
# only certain columns
if dimension.columns.keys():
df = df[dimension.columns.keys()]
tables[table.name] = value[[
col for col in value.columns if col.lower()[-3:] != '_id'
# change table display name
if dimension.displayName:
table_name = dimension.displayName
else:
table_name = dimension.name
# rename columns if value not None
df.rename(columns=(dict((k, v) for k, v in dimension.columns.items() if v)), inplace=True)
tables[table_name] = df[[
col for col in df.columns if col.lower()[-2:] != 'id'
]]
# update table display name
for dimension in cube_obj.dimensions:
if dimension.displayName and dimension.name and dimension.displayName != dimension.name:
tables[dimension.displayName] = tables[dimension.name][
dimension.columns]
executer_instance.dimension_display_name.append(dimension.name)
memory_usage("2 - after query, before fetchall /////// _load_table_config_file")
return tables
......@@ -44,22 +55,76 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
:return: star schema DataFrame
"""
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db=executer_instance.cube)
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
# load facts table
memory_usage("1 - before executing query //// _construct_star_schema_config_file")
fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.connection)
"SELECT * FROM {0}".format(executer_instance.facts), db.engine)
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
df = psql.read_sql_query(
"SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
db.connection)
db.engine)
for dimension in cubes_obj.dimensions:
if dimension_and_key.split('.')[0] == dimension.name:
df.rename(columns=dimension.columns, inplace=True)
# todo test with this
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
# TODO CHOSE BETWEEN THOSES DF
# if separated dimensions
# fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
# fusion = fusion.merge(
# df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
# # remove suffixe from dimension and keep the same column name for facts
# suffixes=('', '_y'))
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_config_file")
# TODO CHOSE BETWEEN THOSES DF
# if separated dimensions
# fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
# TODO CHOSE BETWEEN THOSES DF
# if facts contains all dimensions
# fusion = facts
# measures in config-file only
if cubes_obj.facts[0].measures:
executer_instance.measures = cubes_obj.facts[0].measures
return fusion
def _construct_star_schema_config_file_OLD(executer_instance, cubes_obj):
"""
Construct star schema DataFrame from configuration file.
:param cube_name: cube name (or database name)
:param cubes_obj: cubes object
:return: star schema DataFrame
"""
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db=executer_instance.cube)
# load facts table
memory_usage("1 - before executing query //// _construct_star_schema_config_file")
fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.engine)
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
df = psql.read_sql_query(
"SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
db.engine)
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1])
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_config_file")
# TODO CHOSE BETWEEN THOSES DF
# if separated dimensions
# fusion = fusion.merge(df, left_on=fact_key,right_on=dimension_and_key.split('.')[1])
# TODO CHOSE BETWEEN THOSES DF
# if facts contains all dimensions
......@@ -83,20 +148,24 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns = []
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db=executer_instance.cube)
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
# load facts table
if cubes_obj.facts[0].columns:
all_columns += cubes_obj.facts[0].columns
memory_usage("1 - before executing query //// 1111 _construct_web_star_schema_config_file ")
fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.connection)
"SELECT * FROM {0}".format(executer_instance.facts), db.engine)
memory_usage("2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file")
tables = {}
memory_usage("1 - before executing query //// 3333333333 _construct_web_star_schema_config_file ")
for table in cubes_obj.tables:
tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
db.connection)
db.engine)
try:
if table.columns:
......@@ -117,11 +186,14 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns += list(tab.columns)
tables.update({table.name: tab})
memory_usage("2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file")
# measures in config-file only
if cubes_obj.facts[0].measures:
executer_instance.measures = cubes_obj.facts[0].measures
all_columns += cubes_obj.facts[0].measures
memory_usage("1 - before executing query //// 55555555 _construct_web_star_schema_config_file ")
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
dimension_name = dimension_and_key.split('.')[0]
if dimension_name in tables.keys():
......@@ -129,13 +201,15 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
else:
df = psql.read_sql_query(
"SELECT * FROM {0}".format(dimension_and_key.split('.')[0]),
db.connection)
db.engine)
# TODO check merge (how)
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
# remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y'))
# remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y'))
memory_usage("2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file")
return fusion[[column for column in all_columns if 'id' != column[-2:]]]
olapy/core/mdx/executor/execute_db.py
View file @ 1fb79bb3
from __future__ import absolute_import, division, print_function
from sqlalchemy import inspect
import pandas as pd
from ..tools.mem_bench import memory_usage
from ..tools.connection import MyDB
import pandas.io.sql as psql
class StringFolder(object):
"""
Class that will fold strings. See 'fold_string'.
This object may be safely deleted or go out of scope when
strings have been folded.
"""
def __init__(self):
self.unicode_map = {}
def fold_string(self, s):
"""
Given a string (or unicode) parameter s, return a string object
that has the same value as s (and may be s). For all objects
with a given value, the same object will be returned. For unicode
objects that can be coerced to a string with the same value, a
string object will be returned.
If s is not a string or unicode object, it is returned unchanged.
:param s: a string or unicode object.
:return: a string or unicode object.
"""
# If s is not a string or unicode object, return it unchanged
if not isinstance(s, basestring):
return s
# If s is already a string, then str() has no effect.
# If s is Unicode, try and encode as a string and use intern.
# If s is Unicode and can't be encoded as a string, this try
# will raise a UnicodeEncodeError.
try:
return intern(str(s))
except UnicodeEncodeError:
# Fall through and handle s as Unicode
pass
# Look up the unicode value in the map and return
# the object from the map. If there is no matching entry,
# store this unicode object in the map and return it.
return self.unicode_map.setdefault(s, s)
def string_folding_wrapper(results):
"""
This generator yields rows from the results as tuples,
with all string values folded.
"""
# Get the list of keys so that we build tuples with all
# the values in key order.
keys = results.keys()
folder = StringFolder()
for row in results:
yield tuple(
folder.fold_string(row[key])
for key in keys
)
# TODO try pandas.read_sql_table and pandas.read_sql
def _load_tables_db(executer_instance):
"""
Load tables from database.
......@@ -11,18 +71,40 @@ def _load_tables_db(executer_instance):
:return: tables dict with table name as key and dataframe as value
"""
tables = {}
db = MyDB(db=executer_instance.cube)
cursor = db.connection.cursor()
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube)
inspector = inspect(db.engine)
memory_usage("1 - before executing query //// _load_tables_db")
for table_name in inspector.get_table_names():
for table_name in cursor.fetchall():
value = psql.read_sql_query(
'SELECT * FROM "{0}" '.format(table_name[0]), db.connection)
# value = psql.read_sql_query(
# 'SELECT * FROM "{0}"'.format(table_name), db.engine)
tables[table_name[0]] = value[[
# results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
results = db.engine.execution_options(stream_results=True).execute('SELECT * FROM "{0}"'.format(table_name))
# Fetch all the results of the query
value = pd.DataFrame(iter(results),columns=results.keys()) # Pass results as an iterator
# with string_folding_wrapper we loose response time
# value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
tables[table_name] = value[[
col for col in value.columns if col.lower()[-3:] != '_id'
]]
# tables = {}
# db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER, db=executer_instance.cube)
# # inspector = inspect(db.engine)
# cursor = db.engine.cursor()
# cursor.execute("""SELECT table_name FROM information_schema.tables
# WHERE table_schema = 'public'""")
#
# for table_name in cursor.fetchall():
# value = psql.read_sql_query(
# 'SELECT * FROM "{0}" '.format(table_name[0]), db.engine)
#
# tables[table_name[0]] = value[[
# col for col in value.columns if col.lower()[-3:] != '_id'
# ]]
memory_usage("2 - after query, before fetchall /////// _load_tables_db")
return tables
......@@ -34,21 +116,22 @@ def _construct_star_schema_db(executer_instance):
:return: star schema DataFrame
"""
db = MyDB(db=executer_instance.cube)
memory_usage("1 - before executing query //// _construct_star_schema_db")
# load facts table
fusion = psql.read_sql_query(
'SELECT * FROM "{0}" '.format(executer_instance.facts), db.connection)
with db.engine as connection:
fusion = psql.read_sql_query(
'SELECT * FROM "{0}" '.format(executer_instance.facts), connection)
cursor = db.connection.cursor()
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
for db_table_name in cursor.fetchall():
try:
fusion = fusion.merge(
psql.read_sql_query("SELECT * FROM {0}".format(
db_table_name[0]), db.connection))
except:
print('No common column')
pass
inspector = inspect(connection)
for db_table_name in inspector.get_table_names():
try:
fusion = fusion.merge(
psql.read_sql_query("SELECT * FROM {0}".format(
db_table_name[0]), connection))
except:
print('No common column')
pass
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_db")
return fusion
olapy/core/mdx/parser/gen_parser/mdxparser.py deleted 100644 → 0
View file @ f4fd1e91
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# CAVEAT UTILITOR
#
# This file was automatically generated by Grako.
#
# https://pypi.python.org/pypi/grako/
#
# Any changes you make to it will be overwritten the next time
# the file is generated.
from __future__ import (absolute_import, division, print_function,
unicode_literals)
from grako.buffering import Buffer
from grako.parsing import Parser, graken
from grako.util import RE_FLAGS, generic_main, re # noqa
__all__ = ['MdxParserGen', 'UnknownSemantics', 'main']
KEYWORDS = {}
class UnknownBuffer(Buffer):
"""
main class for parsing MDX query generated by grako
"""
def __init__(self,
text,
whitespace=None,
nameguard=None,
comments_re=None,
eol_comments_re=None,
ignorecase=None,
namechars='',
**kwargs):
super(UnknownBuffer, self).__init__(
text,
whitespace=whitespace,
nameguard=nameguard,
comments_re=comments_re,
eol_comments_re=eol_comments_re,
ignorecase=ignorecase,
namechars=namechars,
**kwargs)
class MdxParserGen(Parser):
def __init__(self,
whitespace=None,
nameguard=None,
comments_re=None,
eol_comments_re=None,
ignorecase=None,
left_recursion=False,
parseinfo=True,
keywords=None,
namechars='',
buffer_class=UnknownBuffer,
**kwargs):
if keywords is None:
keywords = KEYWORDS
super(MdxParserGen, self).__init__(
whitespace=whitespace,
nameguard=nameguard,
comments_re=comments_re,
eol_comments_re=eol_comments_re,
ignorecase=ignorecase,
left_recursion=left_recursion,
parseinfo=parseinfo,
keywords=keywords,
namechars=namechars,
buffer_class=buffer_class,
**kwargs)
@graken()
def _MDX_statement_(self):
self._select_statement_()
@graken('SelectStatement')
def _select_statement_(self):
self._token('SELECT')
self.name_last_node('name')
with self._optional():
self._axis_specification_()
self.name_last_node('axis_specification_columns')
with self._optional():
self._token(',')
self._axis_specification_()
self.name_last_node('axis_specification_rows')
self._token('FROM')
self._cube_specification_()
self.name_last_node('cube_specification')
with self._optional():
self._token('WHERE')
self._condition_specification_()
self.name_last_node('condition_specification')
self._check_eof()
self.ast._define([
'axis_specification_columns', 'axis_specification_rows',
'condition_specification', 'cube_specification', 'name'
], [])
@graken()
def _axis_specification_(self):
with self._optional():
self._left_accolade_()
with self._optional():
self._fetch_form_()
self._dim_props_place_()
self.name_last_node('@')
with self._optional():
self._right_accolade_()
self._token('ON')
self._axis_name_()
@graken()
def _dim_props_place_(self):
with self._optional():
self._left_parentheses_()
self._dim_props_type_()
self.name_last_node('@')
with self._optional():
self._right_parentheses_()
@graken()
def _dim_props_type_(self):
with self._optional():
self._left_accolade_()
with self._optional():
self._fetch_type_()
self._dim_props_op_l1_()
self.name_last_node('@')
with self._optional():
self._right_accolade_()
with self._optional():
def block1():
self._operator_()
self.name_last_node('@')
self._dim_props_type_()
self._closure(block1)
@graken()
def _dim_props_op_l1_(self):
with self._optional():
self._left_parentheses_()
self._dim_props_op_()
self.name_last_node('@')
with self._optional():
self._right_parentheses_()
@graken()
def _dim_props_op_(self):
with self._optional():
self._left_accolade_()
self._dim_props_ligne_()
self.name_last_node('@')
with self._optional():
self._right_accolade_()
with self._optional():
def block1():
self._comma_()
self.name_last_node('@')
self._dim_props_op_()
self._closure(block1)
@graken()
def _dim_props_ligne_(self):
with self._optional():
self._left_parentheses_()
self._dimension_place_()
self.name_last_node('@')
with self._optional():
self._right_parentheses_()
with self._optional():
def block1():
with self._group():
with self._choice():
with self._option():
self._comma_()
self.name_last_node('@')
with self._option():
self._dpoint_()
self.name_last_node('@')
self._error('no available options')
self._dim_props_ligne_()
self.name_last_node('@')
self._closure(block1)
@graken()
def _dimension_place_(self):
with self._choice():
with self._option():
with self._optional():
self._left_accolade_()
self._dim_props_()
self.name_last_node('@')
with self._optional():
self._point_()
self._laste_node_()
self.name_last_node('@')
with self._optional():
self._comma_()
self.name_last_node('@')
self._dim_props_()
self.name_last_node('@')
with self._optional():
self._point_()
self._laste_node_()
self.name_last_node('@')
with self._optional():
self._dpoint_()
self.name_last_node('@')
self._dim_props_()
self.name_last_node('@')
with self._optional():
self._point_()
self._laste_node_()
self.name_last_node('@')
with self._optional():
self._right_accolade_()
with self._option():
self._dimension_shortcut_()
self.name_last_node('@')
self._error('no available options')
@graken()
def _dim_props_(self):
def block0():
with self._optional():
self._point_()
with self._optional():
self._left_bracket_()
self._dimension_()
self.name_last_node('@')
with self._optional():
self._right_bracket_()
self._closure(block0)
@graken()
def _laste_node_(self):
with self._choice():
with self._option():
self._token('members')
with self._option():
self._token('children')
with self._option():
self._token('Members')
with self._option():
self._token('ALLMEMBERS')
self._error('expecting one of: ALLMEMBERS Members children members')
@graken()
def _dimension_(self):
self._pattern(r"[a-zA-Z0-9'_'' '',']*")
self.name_last_node('@')
@graken()
def _axis_name_(self):
with self._choice():
with self._option():
self._token('0')
with self._option():
self._token('1')
with self._option():
self._token('COLUMNS')
with self._option():
self._token('ROWS')
with self._option():
self._token('_ROWS')
self._error('expecting one of: 0 1 COLUMNS ROWS _ROWS')
@graken()
def _cube_specification_(self):
with self._optional():
self._left_bracket_()
self._dimension_()
with self._optional():
self._right_bracket_()
@graken()
def _condition_specification_(self):
with self._optional():
self._left_parentheses_()
def block0():
with self._optional():
self._point_()
with self._optional():
self._left_bracket_()
self._dimension_()
self.name_last_node('@')
with self._optional():
self._right_bracket_()
self._closure(block0)
with self._optional():
self._right_parentheses_()
@graken()
def _digit_(self):
with self._choice():
with self._option():
self._token('0')
with self._option():
self._token('1')
with self._option():
self._token('2')
with self._option():
self._token('3')
with self._option():
self._token('4')
with self._option():
self._token('5')
with self._option():
self._token('6')
with self._option():
self._token('7')
with self._option():
self._token('8')
with self._option():
self._token('9')
self._error('expecting one of: 0 1 2 3 4 5 6 7 8 9')
@graken()
def _fetch_type_(self):
with self._choice():
with self._option():
self._token('CROSSJOIN')
with self._option():
self._token('NONEMPTY')
with self._option():
self._token('union')
with self._option():
self._token('except')
with self._option():
self._token('extract')
self._error(
'expecting one of: CROSSJOIN NONEMPTY except extract union')
@graken()
def _dimension_shortcut_(self):
with self._choice():
with self._option():
self._token('all')
with self._option():
self._token('time')
self._error('expecting one of: all time')
@graken()
def _fetch_form_(self):
with self._choice():
with self._option():
self._token('NONEMPTY')
with self._option():
self._token('non_empty')
with self._option():
self._token('non empty')
self._error('expecting one of: NONEMPTY non empty non_empty')
@graken()
def _left_bracket_(self):
self._token('[')
@graken()
def _right_bracket_(self):
self._token(']')
@graken()
def _left_parentheses_(self):
self._token('(')
@graken()
def _right_parentheses_(self):
self._token(')')
@graken()
def _left_accolade_(self):
self._token('{')
@graken()
def _right_accolade_(self):
self._token('}')
@graken()
def _point_(self):
self._token('.')
@graken()
def _dpoint_(self):
self._token(':')
@graken()
def _comma_(self):
self._token(',')
@graken()
def _operator_(self):
with self._choice():
with self._option():
self._token('+')
with self._option():
self._token('-')
with self._option():
self._token('/')
with self._option():
self._token('*')
self._error('expecting one of: * + - /')
class UnknownSemantics(object):
def mdx_statement(self, ast):
return ast
def select_statement(self, ast):
return ast
def axis_specification(self, ast):
return ast
def dim_props_place(self, ast):
return ast
def dim_props_type(self, ast):
return ast
def dim_props_op_l1(self, ast):
return ast
def dim_props_op(self, ast):
return ast
def dim_props_ligne(self, ast):
return ast
def dimension_place(self, ast):
return ast
def dim_props(self, ast):
return ast
def laste_node(self, ast):
return ast
def dimension(self, ast):
return ast
def axis_name(self, ast):
return ast
def cube_specification(self, ast):
return ast
def condition_specification(self, ast):
return ast
def digit(self, ast):
return ast
def fetch_type(self, ast):
return ast
def dimension_shortcut(self, ast):
return ast
def fetch_form(self, ast):
return ast
def left_bracket(self, ast):
return ast
def right_bracket(self, ast):
return ast
def left_parentheses(self, ast):
return ast
def right_parentheses(self, ast):
return ast
def left_accolade(self, ast):
return ast
def right_accolade(self, ast):
return ast
def point(self, ast):
return ast
def dpoint(self, ast):
return ast
def comma(self, ast):
return ast
def operator(self, ast):
return ast
def main(filename, startrule, **kwargs):
with open(filename) as f:
text = f.read()
parser = MdxParserGen(parseinfo=False)
return parser.parse(text, startrule, filename=filename, **kwargs)
if __name__ == '__main__':
import json
ast = generic_main(main, MdxParserGen, name='Unknown')
print('AST:')
print(ast)
print()
print('JSON:')
print(json.dumps(ast, indent=2))
print()
olapy/core/mdx/parser/gen_parser/models.py deleted 100644 → 0
View file @ f4fd1e91
from __future__ import absolute_import, division, print_function
class SelectStatement:
def __init__(self, select_statement):
self.select_statement = select_statement
def __str__(self):
return '{}'.format(self.select_statement)
olapy/core/mdx/parser/grammar/ebnf_mdx.ebnf deleted 100644 → 0
View file @ f4fd1e91
(* The ebnf file is the translation and improvement of microsoft mdx's spec, from https://msdn.microsoft.com/fr-fr/library/windows/desktop/ms717923%28v=vs.85%29.aspx *)
(* The ebnf conversion was guided by http://stackoverflow.com/questions/14922242/how-to-convert-bnf-to-ebnf *)
(* The generation of mdx_parser.py is as easy as: /path/to/grako bnf_mdx.ebnf -o mdx_parser.py *)
mdx_statement =
select_statement;
select_statement::SelectStatement = name:'SELECT' [axis_specification_columns:axis_specification]
[',' axis_specification_rows:axis_specification]
'FROM' cube_specification:cube_specification
['WHERE' condition_specification:condition_specification]$
;
axis_specification = [left_accolade] [fetch_form] @:dim_props_place [right_accolade] 'ON' axis_name;
dim_props_place = [left_parentheses] @:dim_props_type [right_parentheses] ;
dim_props_type = [left_accolade] [fetch_type] @:dim_props_op_l1 [right_accolade] [{@:operator dim_props_type}*];
dim_props_op_l1 = [left_parentheses] @:dim_props_op [right_parentheses];
dim_props_op = [left_accolade] @:dim_props_ligne [right_accolade] [{@:comma dim_props_op}*];
dim_props_ligne = [left_parentheses] @:dimension_place [right_parentheses] [{ (@:comma | @:dpoint ) @:dim_props_ligne}*] ;
dimension_place = [left_accolade] @:dim_props [ point @:laste_node] [ @:comma @:dim_props [ point @:laste_node ]] [@:dpoint @:dim_props [ point @:laste_node ]] [right_accolade] | @:dimension_shortcut;
dim_props = {[point] [left_bracket] @:dimension [right_bracket]}* ;
laste_node = "members" | "children" | "Members" | 'ALLMEMBERS' ;
dimension = @:/[a-zA-Z0-9'_'' '',']*/ ;
axis_name = '0' | '1' |'COLUMNS' | 'ROWS' | '_ROWS';
cube_specification = [left_bracket] dimension [right_bracket];
condition_specification = [left_parentheses] {[point] [left_bracket] @:dimension [right_bracket]}* [right_parentheses] ;
digit =
"0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" ;
fetch_type = 'CROSSJOIN' | 'NONEMPTY' | 'union' | 'except' | 'extract' ;
dimension_shortcut = 'all' | 'time' ;
fetch_form = 'NONEMPTY' | 'non_empty' | 'non empty' ;
left_bracket = '[';
right_bracket = ']';
left_parentheses = '(';
right_parentheses = ')';
left_accolade = '{';
right_accolade = '}';
point = '.' ;
dpoint = ':' ;
comma = ',' ;
operator = '+' | '-' | '/' | '*' ;
olapy/core/mdx/parser/parse.py deleted 100644 → 0
View file @ f4fd1e91
from __future__ import absolute_import, division, print_function
from grako.model import ModelBuilderSemantics
from .gen_parser.mdxparser import MdxParserGen
from .gen_parser.models import SelectStatement
class MdxParser:
"""Parse the mdx query and split it into well-defined parts."""
START = 'MDX_statement'
@staticmethod
def parsing_mdx_query(axis, query):
"""Split the query into axis.
**Example**::
SELECT
{ [Geography].[Geo].[Country].[France],
[Geography].[Geo].[Country].[Spain] } ON COLUMNS,
{ [Product].[Prod].[Company].[Crazy Development] } ON ROWS
FROM [Sales]
WHERE [Time].[Calendar].[Year].[2010]
+------------+------------------------------------------------+
| | [Geography].[Geo].[Country].[France] |
| column | |
| | [Geography].[Geo].[Country].[Spain] |
+------------+------------------------------------------------+
| row | [Product].[Prod].[Company].[Crazy Development] |
+------------+------------------------------------------------+
| cube | [Sales] |
+------------+------------------------------------------------+
| condition | [Time].[Calendar].[Year].[2010] |
+------------+------------------------------------------------+
:param query: MDX Query
:param axis: column | row | cube | condition | all
:return: Tuples in the axis, from the MDX query
"""
model = MdxParserGen(semantics=ModelBuilderSemantics(
types=[SelectStatement]))
ast = model.parse(query, rule_name=MdxParser.START, ignorecase=True)
if axis == "column":
if ast.select_statement.axis_specification_columns is not None and \
u'' in ast.select_statement.axis_specification_columns:
ast.select_statement.axis_specification_columns.remove(u'')
return ast.select_statement.axis_specification_columns
elif axis == "row":
if ast.select_statement.axis_specification_rows is not None and \
u'' in ast.select_statement.axis_specification_rows:
ast.select_statement.axis_specification_rows.remove(u'')
return ast.select_statement.axis_specification_rows
elif axis == "cube":
if ast.select_statement.cube_specification is not None and \
u'' in ast.select_statement.cube_specification:
ast.select_statement.cube_specification.remove(u'')
return ast.select_statement.cube_specification[1] if \
isinstance(ast.select_statement.cube_specification, list) \
else ast.select_statement.cube_specification
elif axis == "condition":
if ast.select_statement.condition_specification is not None and \
type(ast.select_statement.condition_specification) not in (
unicode, str) and \
u'' in ast.select_statement.condition_specification:
ast.select_statement.condition_specification.remove(u'')
return ast.select_statement.condition_specification
elif axis == "all":
return 'Operation = {} \n' \
'Columns = {} \n' \
'Rows = {} \n' \
'From = {} \n' \
'Where = {} \n'.format(ast.select_statement.name,
ast.select_statement.from_,
ast.select_statement.axis_specification_columns,
ast.select_statement.axis_specification_rows,
ast.select_statement.cube_specification,
ast.select_statement.condition_specification,
)
olapy/core/mdx/tools/bech_mem_opt2.txt 0 → 100644
View file @ 1fb79bb3
Memory summary:1 - before executing query //// _load_tables_db
VM: 700.28Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 2781.79Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2782.04Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2782.29Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2782.29Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2784.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2784.04Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2783.79Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2783.79Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2784.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// _load_tables_db
types | # objects | total size
======= | =========== | ============
dict | 7429 | 8.72 MB
str | 58990 | 8.30 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
dict | 7802 | 8.88 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 179 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 618 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 604 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
<class 'pandas.core.series.Series | 592 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 629 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 615 | 2.01 GB
olapy/core/mdx/tools/bech_mem_opt3.txt 0 → 100644
View file @ 1fb79bb3
Memory summary:1 - before executing query //// _load_tables_db
VM: 700.27Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 2719.36Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2719.36Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2719.86Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2719.86Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.61Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.61Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.61Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2721.61Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2721.36Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2721.36Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2721.36Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2721.36Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.86Mb
****************************************
types | # objects | total size
======= | =========== | ============
dict | 7430 | 8.72 MB
str | 58996 | 8.30 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
dict | 7803 | 8.86 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 179 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
<class 'pandas.core.series.Series | 618 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
<class 'pandas.core.series.Series | 604 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
<class 'pandas.core.series.Series | 592 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 629 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 615 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 603 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 629 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 615 | 1.93 GB
----------------------------
olapy/core/mdx/tools/bech_mem_sqlalchemy_without_opt.txt 0 → 100644
View file @ 1fb79bb3
Memory summary:1 - before executing query //// _load_tables_db
VM: 700.27Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 3229.79Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 3229.79Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 3230.29Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 3230.29Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 3232.29Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 3232.29Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 3232.54Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 3232.54Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 3232.54Mb
*****************************************************************
Memory summary:1 - before executing query //// _load_tables_db
types | # objects | total size
======= | =========== | ============
dict | 7429 | 8.72 MB
str | 58990 | 8.30 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
dict | 7844 | 8.91 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 179 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 618 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 604 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
<class 'pandas.core.series.Series | 592 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 629 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 615 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 603 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 629 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 615 | 2.01 GB
----------------------------
olapy/core/mdx/tools/bech_mem_with_psycopg.txt 0 → 100644
View file @ 1fb79bb3
Memory summary:1 - before executing query //// _load_tables_db
VM: 699.77Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 1763.52Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1763.52Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1763.77Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1763.77Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.52Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.52Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.52Mb
**********************************
types | # objects | total size
======= | =========== | ============
dict | 7235 | 8.59 MB
str | 58455 | 8.18 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
dict | 7649 | 8.78 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 538.26 MB
<class 'pandas.core.frame.DataFrame | 179 | 538.22 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 539.70 MB
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 539.70 MB
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 540.42 MB
<class 'pandas.core.series.Series | 618 | 539.70 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 540.47 MB
<class 'pandas.core.frame.DataFrame | 185 | 540.42 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 540.83 MB
<class 'pandas.core.series.Series | 604 | 539.02 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 538.50 MB
<class 'pandas.core.series.Series | 592 | 538.26 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
<class 'pandas.core.series.Series | 629 | 540.06 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 540.83 MB
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 541.11 MB
<class 'pandas.core.series.Series | 615 | 539.39 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 603 | 538.62 MB
<class 'pandas.core.frame.DataFrame | 180 | 538.50 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
<class 'pandas.core.series.Series | 629 | 540.06 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 540.83 MB
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 541.11 MB
<class 'pandas.core.series.Series | 615 | 539.39 MB
olapy/core/mdx/tools/config_file_parser.py
View file @ 1fb79bb3
......@@ -198,12 +198,18 @@ class ConfigParser:
:param cube_path: path to cube (csv folders)
:param file_name: config file name (DEFAULT = cubes-config.xml)
"""
if cube_path is None:
# home_directory = home_directory
if 'OLAPY_PATH' in os.environ:
home_directory = os.environ['OLAPY_PATH']
else:
from os.path import expanduser
home_directory = expanduser("~")
if cube_path is None:
self.cube_path = os.path.join(home_directory, 'olapy-data', 'cubes')
else:
self.cube_path = cube_path
self.file_name = file_name
self.web_config_file_name = web_config_file_name
......@@ -284,15 +290,21 @@ class ConfigParser:
]) for xml_facts in tree.xpath('/cubes/cube/facts')
]
# keys = {
# key.text: key.attrib['ref']
# for key in xml_facts.findall('keys/column_name')
# },
dimensions = [
Dimension(
name=xml_dimension.find('name').text,
# column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
displayName=xml_dimension.find('displayName').text,
columns=[
column_name.text
columns={
column_name.text : None if not column_name.attrib else column_name.attrib['column_new_name']
for column_name in xml_dimension.findall(
'columns/name')
])
})
for xml_dimension in tree.xpath(
'/cubes/cube/dimensions/dimension')
]
......
olapy/core/mdx/tools/connection.py
View file @ 1fb79bb3
import psycopg2 as pg
# import psycopg2 as pg
from sqlalchemy import create_engine
# postgres connection
from olapy_config_file_parser import DbConfigParser
......@@ -16,28 +17,32 @@ class MyDB(object):
# raise Exception('Missing database config file')
def __init__(self,db=None):
def __init__(self,db_config_file_path=None,db=None):
# TODO temporary
db_config = DbConfigParser()
db_config = DbConfigParser(config_path=db_config_file_path)
db_credentials = db_config.get_db_credentials()[0]
username = db_credentials['user_name']
password = db_credentials['password']
host = db_credentials['host']
port = db_credentials['port']
if db is None:
# first i want to show all databases to user (in excel)
self.connection = pg.connect("user={0} password={1} host='{2}'".
format(username, password, host))
# self.engine = pg.connect("user={0} password={1} host='{2}'".
# format(username, password, host))
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, 'postgres', host, port))
else:
# and then we connect to the user db
try:
self.connection = pg.connect(
"user={0} password={1} dbname='{2}' host='{3}'".format(
username, password, db, host))
except:
print("can't connect")
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, db, host, port))
# self.connection = pg.connect(
# "user={0} password={1} dbname='{2}' host='{3}'".format(
# username, password, db, host))
def __del__(self):
if hasattr(self, 'connection'):
self.connection.close()
self.engine.dispose()
olapy/core/mdx/tools/mem_bench.py 0 → 100644
View file @ 1fb79bb3
import os
from os.path import expanduser
from pympler import summary, muppy
import psutil
def get_virtual_memory_usage_kb():
"""
The process's current virtual memory size in Kb, as a float.
"""
return float(psutil.Process().memory_info_ex().vms) / 1024.0
def memory_usage(where):
"""
Print out a basic summary of memory usage.
"""
with open(os.path.join(expanduser('~'), 'bech_mem.txt'), mode='a+') as file:
mem_summary = summary.summarize(muppy.get_objects())
file.write("Memory summary:" + where + '\n\n')
print("Memory summary:" + where )
summary.print_(mem_summary, limit=2)
print('----------------------------')
file.write("VM: %.2fMb" % (get_virtual_memory_usage_kb() / 1024.0) + '\n\n')
olapy/core/mdx/tools/olapy_config_file_parser.py
View file @ 1fb79bb3
......@@ -43,15 +43,13 @@ class DbConfigParser:
parser = etree.XMLParser()
tree = etree.parse(config_file, parser)
try:
return [
{
# 'sgbd': db.find('sgbd').text,
'user_name': db.find('user_name').text,
'password': db.find('password').text,
'host': db.find('host').text,
}
for db in tree.xpath('/olapy/database')
]
except:
raise ('missed name or source tags')
return [
{
# 'sgbd': db.find('sgbd').text,
'user_name': db.find('user_name').text,
'password': db.find('password').text,
'host': db.find('host').text,
'port': db.find('port').text,
}
for db in tree.xpath('/olapy/database')
]
olapy/core/services/xmla.py
View file @ 1fb79bb3
......@@ -222,7 +222,7 @@ application = Application(
wsgi_application = WsgiApplication(application)
def start_server(write_on_file=False):
def start_server(host='0.0.0.0',port=5000,write_on_file=False):
"""
Start the xmla server.
......@@ -254,7 +254,7 @@ def start_server(write_on_file=False):
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
logging.info("listening to http://127.0.0.1:8000/xmla")
logging.info("wsdl is at: http://localhost:8000/xmla?wsdl")
server = make_server('0.0.0.0', 8000, wsgi_application)
server = make_server(host, port, wsgi_application)
server.serve_forever()
......
olapy/core/services/xmla_discover_tools.py
View file @ 1fb79bb3
......@@ -1830,8 +1830,10 @@ class XmlaDiscoverTools():
# TODO in another idea, change this
# TO CHANGE NAME DISPLAY THAT EXISTS IN CONFIG FILE
if MdxEngine.dimension_display_name != [] and tables in MdxEngine.dimension_display_name:
continue
# if MdxEngine.dimension_display_name != [] and tables in MdxEngine.dimension_display_name:
# continue
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -1888,6 +1890,15 @@ class XmlaDiscoverTools():
for table_name, df in self.executer.tables_loaded.items():
if table_name == self.executer.facts:
continue
# french caracteres
# TODO encode dataframe
if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8')
else:
column_attribut = df.iloc[0][0]
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -1910,8 +1921,11 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue, table_name,
df.columns[0], df.iloc[0][0])
""".format(self.selected_catalogue,
table_name,
df.columns[0],
column_attribut)
rows += """
<row>
......@@ -1953,6 +1967,14 @@ class XmlaDiscoverTools():
for table_name, df in self.executer.tables_loaded.items():
if table_name == self.executer.facts:
continue
# french caracteres
# TODO encode dataframe
if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8')
else:
column_attribut = df.iloc[0][0]
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -1975,8 +1997,10 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue, table_name,
df.columns[0], df.iloc[0][0])
""".format(self.selected_catalogue,
table_name,
df.columns[0],
column_attribut)
rows += """
<row>
......
olapy/core/services/xmla_execute_tools.py
View file @ 1fb79bb3
......@@ -3,6 +3,8 @@ from __future__ import absolute_import, division, print_function
import itertools
from collections import OrderedDict
import numpy as np
class XmlaExecuteTools():
"""XmlaExecuteTools for generating xmla execute responses."""
......@@ -149,6 +151,12 @@ class XmlaExecuteTools():
for tupl in tupls:
tuple_without_minus_1 = self.get_tuple_without_nan(tupl)
# french caracteres
# TODO encode dataframe
if type(tuple_without_minus_1[-1]) == unicode:
tuple_without_minus_1 = [x.encode('utf-8') for x in tuple_without_minus_1]
axis0 += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].{2}</UName>
......@@ -324,7 +332,7 @@ class XmlaExecuteTools():
cell_data = ""
index = 0
for value in columns_loop:
if value == -1:
if np.isnan(value) :
value = ''
cell_data += """
<Cell CellOrdinal="{0}">
......@@ -550,6 +558,14 @@ class XmlaExecuteTools():
set(table_name
for table_name in mdx_execution_result['columns_desc']
['all'])):
# TODO encode dataframe
# french caracteres
if type(self.executer.tables_loaded[dim_diff].iloc[0][0]) == unicode:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0].encode('utf-8')
else:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0]
tuple += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].[{2}]</UName>
......@@ -560,7 +576,7 @@ class XmlaExecuteTools():
</Member>
""".format(dim_diff,
self.executer.tables_loaded[dim_diff].columns[0],
self.executer.tables_loaded[dim_diff].iloc[0][0])
column_attribut)
# if we have zero on one only measures used
if len(self.executer.selected_measures) <= 1:
......
requirements.in
View file @ 1fb79bb3
#
grako
#grako
pandas<1
lxml==3.6.0 #lxml 3.7 causes problems in windows
spyne<3
treelib<2
SQLAlchemy
psycopg2
# tools
pympler
psutil
# Test
werkzeug
......
requirements.txt
View file @ 1fb79bb3
......@@ -4,29 +4,14 @@
#
# pip-compile --output-file requirements.txt requirements.in
#
click==6.7 # via flask
Flask-Login==0.3.2
Flask-Script==2.0.5
Flask-Session==0.3.0
Flask-SQLAlchemy==2.1
Flask-WTF==0.12
flask==0.12.1
grako==3.22.0
itsdangerous==0.24 # via flask
jinja2==2.9.6 # via flask
grako==3.99.9
lxml==3.6.0
markupsafe==1.0 # via jinja2
numpy==1.12.1 # via pandas
pandas==0.19.2
plotly==1.12.9
pandas==0.20.1
psycopg2==2.7.1
python-dateutil==2.6.0 # via pandas
pytz==2017.2 # via pandas, plotly, spyne
requests==2.13.0 # via plotly
six==1.10.0 # via plotly, python-dateutil
pytz==2017.2 # via pandas, spyne
six==1.10.0 # via python-dateutil
spyne==2.12.14
sqlalchemy==1.0.17
treelib==1.3.5
werkzeug==0.12.1 # via flask, flask-wtf
wtforms==2.1
XlsxWriter==0.9.3
werkzeug==0.12.1
setup.py
View file @ 1fb79bb3
......@@ -29,7 +29,7 @@ setup(
install_requires=install_requires,
include_package_data=False,
# cmdclass={
# # 'develop': PostDevelopCommand,
# 'develop': PostDevelopCommand,
# 'install': PostInstallCommand,
# },
classifiers=[
......@@ -41,8 +41,9 @@ setup(
# "Topic :: Business intelligence",
],)
if RUNNING_TOX:
if 'OLAPY_PATH' in os.environ:
home_directory = os.environ['OLAPY_PATH']
elif RUNNING_TOX:
home_directory = os.environ.get('HOME_DIR')
else:
home_directory = expanduser("~")
......@@ -55,5 +56,3 @@ if not os.path.isdir(os.path.join(home_directory, 'olapy-data', 'cubes')):
if not os.path.isfile(os.path.join(home_directory, 'olapy-data','olapy-config.xml')):
copyfile('config/olapy-config.xml', os.path.join(home_directory, 'olapy-data','olapy-config.xml'))
tests/test_queries.py deleted 100644 → 0
View file @ f4fd1e91
from __future__ import absolute_import, division, print_function
import pandas as pd
from pandas.util.testing import assert_frame_equal
from olapy.core.mdx.executor.execute import MdxEngine
from olapy.core.mdx.parser.parse import MdxParser
CUBE = 'sales'
query1 = "SELECT" \
"{[Measures].[Amount]} ON COLUMNS " \
"FROM [sales]"
query2 = """SELECT
{[Geography].[Economy].[Partnership]} ON COLUMNS
FROM [sales]"""
query3 = """SELECT
{[Measures].[Amount]} on 0,
non empty {[Geography].[Geo].[Country].members} ON COLUMNS
FROM [sales]"""
query4 = """SELECT
{[Geography].[Economy].[Partnership]} ON COLUMNS,
non empty {[Geography].[Geo].[Country].members} on 1
from [sales]"""
query5 = """select
{[Geography].[Economy].[Country]} on 0,
non empty {[Geography].[Geo].[Country].members} on 1
from [sales]"""
query6 = """select
{[Geography].[Economy].[Partnership]} on 0,
{[Product].[Prod].[Company]} on 1
from [sales]"""
query7 = """select
{[Geography].[Economy].[Partnership].[EU]} on 0,
{[Product].[Prod].[Company].[Crazy Development]} on 1
from [sales]"""
query8 = """select
{[Geography].[Economy].[Partnership].[EU],
[Geography].[Economy].[Partnership].[None],
[Geography].[Economy].[Partnership].[NAFTA]} on 0,
{[Product].[Prod].[Company].[Crazy Development],
[Product].[Prod].[Company].[Company_test],
[Product].[Prod].[Company].[test_Development]} on 1
from [sales]"""
query9 = """select
{[Geography].[Economy].[Partnership].[EU],
[Geography].[Economy].[Partnership].[None]} on 0
from [sales]"""
query10 = """select
{[Geography].[Geo].[Country].[France],
[Geography].[Geo].[Country].[Spain]} on 0,
non empty {[Measures].[Amount]} on 1
from [sales]"""
where1 = "Where [Time].[Calendar].[Day].[May 12,2010]"
where2 = "Where[Product].[olapy].[Personal]"
where3 = "Where[Time].[Calendar].[Year].[2010]"
where4 = "Where [Measures].[Count]"
where5 = "where [Count]"
query11 = """
SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{DrilldownMember({{
[Time].[Time].[Year].Members}}, {
[Time].[Time].[Year].[2010]})}}, {
[Time].[Time].[Quarter].[2010].[Q2 2010]})}}, {
[Time].[Time].[Month].[2010].[Q2 2010].[May 2010]}))) DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME
ON COLUMNS
FROM [sales] WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
"""
query12 = """SELECT NON EMPTY Hierarchize(AddCalculatedMembers({
[Geography].[Geography].[Continent].Members}))
DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME ON COLUMNS
FROM [sales]
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
parser = MdxParser()
executer = MdxEngine(CUBE)
def test_parsing_query1():
assert parser.parsing_mdx_query(
'column', query=query1) == ['Measures', 'Amount']
assert parser.parsing_mdx_query('cube', query=query1) == "sales"
assert parser.parsing_mdx_query('row', query=query1) is None
query1_where = query1 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query1 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query1 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_parsing_query2():
assert parser.parsing_mdx_query(
'column', query=query2) == [u'Geography', u'Economy', u'Partnership']
assert parser.parsing_mdx_query('cube', query=query2) == "sales"
assert parser.parsing_mdx_query('row', query=query2) is None
query1_where = query2 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query2 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query2 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_parsing_query3():
assert parser.parsing_mdx_query(
'column', query=query3) == [u'Measures', u'Amount']
assert parser.parsing_mdx_query('cube', query=query3) == "sales"
assert parser.parsing_mdx_query(
'row', query=query3) == [u'Geography', u'Geo', u'Country', u'members']
query1_where = query3 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query3 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query3 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_parsing_query4():
query0 = query4
assert parser.parsing_mdx_query(
'column', query=query0) == [u'Geography', u'Economy', u'Partnership']
assert parser.parsing_mdx_query('cube', query=query0) == "sales"
assert parser.parsing_mdx_query(
'row', query=query0) == [u'Geography', u'Geo', u'Country', u'members']
query1_where = query0 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query0 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query0 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_parsing_query5():
query0 = query5
assert parser.parsing_mdx_query(
'column', query=query0) == [u'Geography', u'Economy', u'Country']
assert parser.parsing_mdx_query('cube', query=query0) == "sales"
assert parser.parsing_mdx_query(
'row', query=query0) == [u'Geography', u'Geo', u'Country', u'members']
query1_where = query0 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query0 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query0 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_parsing_query6():
query0 = query6
assert parser.parsing_mdx_query(
'column', query=query0) == [u'Geography', u'Economy', u'Partnership']
assert parser.parsing_mdx_query('cube', query=query0) == "sales"
assert parser.parsing_mdx_query(
'row', query=query0) == [u'Product', u'Prod', u'Company']
query1_where = query0 + '\n' + where1
assert parser.parsing_mdx_query(
'condition',
query=query1_where) == [u'Time', u'Calendar', u'Day', u'May 12,2010']
query2_where = query0 + '\n' + where2
assert parser.parsing_mdx_query(
'condition', query=query2_where) == [u'Product', u'olapy', u'Personal']
query3_where = query0 + '\n' + where3
assert parser.parsing_mdx_query(
'condition',
query=query3_where) == [u'Time', u'Calendar', u'Year', u'2010']
def test_execution_query1():
executer.mdx_query = query1
assert executer.execute_mdx()['result']['Amount'][0] == 1023
executer.mdx_query = query11
assert executer.execute_mdx()['result']['Amount'][3] == 1
assert executer.execute_mdx()['result']['Amount'][4] == 2
def test_execution_query3():
df = pd.DataFrame({
'Continent': ['America', 'Europe'],
'Amount': [768, 255]
}).groupby(['Continent']).sum()
executer.mdx_query = query12
assert assert_frame_equal(df, executer.execute_mdx()['result']) is None
executer.mdx_query = query11
assert list(executer.execute_mdx()['result'][
'Amount']) == [1023, 1023, 1023, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7