Skip to content

O
olapy
Commit 3c3ecadb authored by mouadh's avatar mouadh
Browse files
Options

Merge remote-tracking branch 'origin/master' 

# Conflicts:
#	requirements.txt
parents cedc74c8 16ebd261
Hide whitespace changes
Inline Side-by-side
Showing with 1359 additions and 2401 deletions
olapy/core/services/xmla.py
View file @ 3c3ecadb
......@@ -2,11 +2,13 @@
from __future__ import absolute_import, division, print_function
import HTMLParser
import xmlwitch
import os
from datetime import datetime
from os.path import expanduser
from lxml import etree
from spyne import AnyXml, Application, ServiceBase, rpc, Fault
from spyne.const.http import HTTP_200
from spyne.error import InvalidCredentialsError
......@@ -158,11 +160,13 @@ class XmlaProviderService(ServiceBase):
if request.Command.Statement == '':
# check if command contains a query
return etree.fromstring("""
<return>
<root xmlns="urn:schemas-microsoft-com:xml-analysis:empty"/>
</return>
""")
xml = xmlwitch.Builder()
with xml['return']:
xml.root(xmlns="urn:schemas-microsoft-com:xml-analysis:empty")
return str(xml)
else:
XmlaProviderService.discover_tools.change_catalogue(
request.Properties.PropertyList.Catalog)
......@@ -171,54 +175,45 @@ class XmlaProviderService(ServiceBase):
df = executer.execute_mdx()
xmla_tools = XmlaExecuteTools(executer)
return etree.fromstring("""
<return>
<root xmlns="urn:schemas-microsoft-com:xml-analysis:mddataset"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
{0}
<OlapInfo>
<CubeInfo>
<Cube>
<CubeName>Sales</CubeName>
<LastDataUpdate
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{7}</LastDataUpdate>
<LastSchemaUpdate
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{7}</LastSchemaUpdate>
</Cube>
</CubeInfo>
<AxesInfo>
{1}
{2}
</AxesInfo>
{3}
</OlapInfo>
<Axes>
{4}
{5}
</Axes>
<CellData>
{6}
</CellData>
</root>
</return>
""".format(execute_xsd,
xmla_tools.generate_axes_info(df),
xmla_tools.generate_axes_info_slicer(df),
xmla_tools.generate_cell_info(),
xmla_tools.generate_xs0(df),
xmla_tools.generate_slicer_axis(df),
xmla_tools.generate_cell_data(df),
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')).replace(
'&', '&amp;'))
# Problem:
# An XML parser returns the error “xmlParseEntityRef: noname”
#
# Cause:
# There is a stray ‘&’ (ampersand character) somewhere in the XML text eg. some text & some more text
# Solution
# .replace('&', '&amp;')
xml = xmlwitch.Builder()
with xml['return']:
with xml.root(
execute_xsd,
xmlns="urn:schemas-microsoft-com:xml-analysis:mddataset",
**{
'xmlns:xsd': 'http://www.w3.org/2001/XMLSchema',
'xmlns:xsi':
'http://www.w3.org/2001/XMLSchema-instance'
}):
with xml.OlapInfo(xmla_tools.generate_cell_info()):
with xml.CubeInfo:
with xml.Cube:
xml.CubeName('Sales')
xml.LastDataUpdate(
datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S'),
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine"
)
xml.LastSchemaUpdate(
datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S'),
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine"
)
xml.AxesInfo(
xmla_tools.generate_axes_info(df),
xmla_tools.generate_axes_info_slicer(df))
xml.Axes(
xmla_tools.generate_xs0(df),
xmla_tools.generate_slicer_axis(df))
xml.CellData(xmla_tools.generate_cell_data(df))
html_parser = HTMLParser.HTMLParser()
xml = html_parser.unescape(str(xml)).replace('&', '&amp;')
return xml
application = Application(
......
olapy/core/services/xmla_discover_tools.py
View file @ 3c3ecadb
This source diff could not be displayed because it is too large. You can view the blob instead.
olapy/core/services/xmla_execute_tools.py
View file @ 3c3ecadb
......@@ -4,6 +4,7 @@ import itertools
from collections import OrderedDict
import numpy as np
import xmlwitch
class XmlaExecuteTools():
......@@ -88,7 +89,8 @@ class XmlaExecuteTools():
:param splited_df:
:return:
"""
axis0 = ""
xml = xmlwitch.Builder()
# only measure selected
if mdx_execution_result['columns_desc'][mdx_query_axis].keys() == [
self.executer.facts
......@@ -128,78 +130,82 @@ class XmlaExecuteTools():
]
first_att = 3
for tupls in itertools.chain(*tuples):
axis0 += "<Tuple>\n"
if tupls[0][1] in self.executer.measures and len(
self.executer.selected_measures) > 1:
axis0 += """
<Member Hierarchy="[Measures]">
<UName>[Measures].[{0}]</UName>
<Caption>{0}</Caption>
<LName>[Measures]</LName>
<LNum>0</LNum>
<DisplayInfo>0</DisplayInfo>
<HIERARCHY_UNIQUE_NAME>[Measures]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(tupls[0][1])
if tupls[0][-1] in self.executer.measures:
axis0 += "</Tuple>\n"
continue
for tupl in tupls:
tuple_without_minus_1 = self.get_tuple_without_nan(tupl)
# french caracteres
# TODO encode dataframe
if type(tuple_without_minus_1[-1]) == unicode:
tuple_without_minus_1 = [
x.encode('utf-8', 'replace')
for x in tuple_without_minus_1
]
axis0 += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].{2}</UName>
<Caption>{3}</Caption>
<LName>[{0}].[{0}].[{1}]</LName>
<LNum>{4}</LNum>
<DisplayInfo>131076</DisplayInfo>""".format(
tuple_without_minus_1[0], splited_df[tuple_without_minus_1[
0]].columns[len(tuple_without_minus_1) - first_att],
'.'.join([
'[' + str(i) + ']'
for i in tuple_without_minus_1[first_att - 1:]
]), tuple_without_minus_1[-1],
len(tuple_without_minus_1) - first_att)
# PARENT_UNIQUE_NAME must be before HIERARCHY_UNIQUE_NAME
if len(tuple_without_minus_1[first_att - 1:]) > 1:
axis0 += """
<PARENT_UNIQUE_NAME>[{0}].[{0}].[{1}].{2}</PARENT_UNIQUE_NAME>""".format(
tuple_without_minus_1[0],
splited_df[tuple_without_minus_1[0]].columns[0],
'.'.join([
'[' + str(i) + ']'
for i in tuple_without_minus_1[first_att - 1:-1]
]))
axis0 += """
<HIERARCHY_UNIQUE_NAME>[{0}].[{0}]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(tuple_without_minus_1[0])
axis0 += "</Tuple>\n"
if axis0:
axis0 = """
<Axis name="{0}">
<Tuples>
{1}
</Tuples>
</Axis>
""".format(axis, axis0)
return axis0
if tuples:
with xml.Axis(name=axis):
with xml.Tuples:
for tupls in itertools.chain(*tuples):
with xml.Tuple:
if tupls[0][1] in self.executer.measures and len(
self.executer.selected_measures) > 1:
with xml.Member(Hierarchy="[Measures]"):
xml.UName(
'[Measures].[{0}]'.format(tupls[0][1]))
xml.Caption('{0}'.format(tupls[0][1]))
xml.LName('[Measures]')
xml.LNum('0')
xml.DisplayInfo('0')
xml.HIERARCHY_UNIQUE_NAME('[Measures]')
if tupls[0][-1] in self.executer.measures:
continue
for tupl in tupls:
tuple_without_minus_1 = self.get_tuple_without_nan(
tupl)
# french caracteres
# TODO encode dataframe
if type(tuple_without_minus_1[-1]) == unicode:
tuple_without_minus_1 = [
x.encode('utf-8', 'replace')
for x in tuple_without_minus_1
]
# todo ugly !!
with xml.Member(Hierarchy="[{0}].[{0}]".format(
tuple_without_minus_1[0])):
xml.UName('[{0}].[{0}].[{1}].{2}'.format(
tuple_without_minus_1[0], splited_df[
tuple_without_minus_1[0]].columns[
len(tuple_without_minus_1) -
first_att], '.'.join([
'[' + str(i) + ']'
for i in
tuple_without_minus_1[
first_att - 1:]
])))
xml.Caption('{0}'.format(
tuple_without_minus_1[-1]))
xml.LName('[{0}].[{0}].[{1}]'.format(
tuple_without_minus_1[0], splited_df[
tuple_without_minus_1[0]].columns[
len(tuple_without_minus_1) -
first_att]))
xml.LNum('{0}'.format(
len(tuple_without_minus_1) -
first_att))
xml.DisplayInfo('131076')
# PARENT_UNIQUE_NAME must be before HIERARCHY_UNIQUE_NAME (todo change it in xsd)
if len(tuple_without_minus_1[first_att -
1:]) > 1:
xml.PARENT_UNIQUE_NAME(
'[{0}].[{0}].[{1}].{2}'.format(
tuple_without_minus_1[0],
splited_df[
tuple_without_minus_1[0]]
.columns[0], '.'.join([
'[' + str(i) + ']'
for i in
tuple_without_minus_1[
first_att - 1:-1]
])))
xml.HIERARCHY_UNIQUE_NAME(
'[{0}].[{0}]'.format(
tuple_without_minus_1[0]))
return str(xml)
def generate_xs0(self, mdx_execution_result):
"""
......@@ -307,7 +313,6 @@ class XmlaExecuteTools():
:param mdx_execution_result: mdx_execute() result
:return: CellData as string
"""
columns_loop = []
if ((len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
^
......@@ -331,18 +336,16 @@ class XmlaExecuteTools():
index=False)
])
cell_data = ""
xml = xmlwitch.Builder()
index = 0
for value in columns_loop:
if np.isnan(value):
value = ''
cell_data += """
<Cell CellOrdinal="{0}">
<Value xsi:type="xsi:long">{1}</Value>
</Cell>
""".format(index, value)
with xml.Cell(CellOrdinal=str(index)):
xml.Value(str(value), **{'xsi:type': 'xsi:long'})
index += 1
return cell_data
return str(xml)
def generate_axes_info_slicer(self, mdx_execution_result):
"""
......@@ -377,43 +380,51 @@ class XmlaExecuteTools():
ignore_fact=True)
all_dimensions_names.append('Measures')
hierarchy_info_slicer = ""
xml = xmlwitch.Builder()
slicer_list = list(
set(all_dimensions_names) - set(
table_name
for table_name in mdx_execution_result['columns_desc']['all']))
# we have to write measures after dimensions !
# we have to write measures after dimensions ! (todo change xsd)
if 'Measures' in slicer_list:
slicer_list.insert(
len(slicer_list),
slicer_list.pop(slicer_list.index('Measures')))
for dim_diff in slicer_list:
to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures':
# if measures > 1 we don't have to write measure
if self.executer.facts in mdx_execution_result['columns_desc'][
'all'] and len(mdx_execution_result['columns_desc'][
'all'][self.executer.facts]) > 1:
continue
else:
to_write = "[Measures]"
hierarchy_info_slicer += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
""".format(to_write)
if hierarchy_info_slicer:
hierarchy_info_slicer = "<AxisInfo name='SlicerAxis'>\n" + hierarchy_info_slicer + "\n</AxisInfo>\n"
return hierarchy_info_slicer
if slicer_list:
with xml.AxisInfo(name='SlicerAxis'):
for dim_diff in slicer_list:
to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures':
# if measures > 1 we don't have to write measure
if self.executer.facts in mdx_execution_result[
'columns_desc']['all'] and len(
mdx_execution_result['columns_desc'][
'all'][self.executer.facts]) > 1:
continue
else:
to_write = "[Measures]"
with xml.HierarchyInfo(name=to_write):
xml.UName(
name="{0}.[MEMBER_UNIQUE_NAME]".format(to_write),
**{'type': 'xs:string'})
xml.Caption(
name="{0}.[MEMBER_CAPTION]".format(to_write),
**{'type': 'xs:string'})
xml.LName(
name="{0}.[LEVEL_UNIQUE_NAME]".format(to_write),
**{'type': 'xs:string'})
xml.LNum(
name="{0}.[LEVEL_NUMBER]".format(to_write),
**{'type': 'xs:int'})
xml.DisplayInfo(
name="{0}.[DISPLAY_INFO]".format(to_write),
**{'type': 'xs:unsignedInt'})
return str(xml)
def generate_one_axis_info(self,
mdx_execution_result,
......@@ -452,45 +463,72 @@ class XmlaExecuteTools():
:param Axis: Axis0 or Axis1 (Axis0 by default)
:return:
"""
hierarchy_info = ""
# measure must be written at the top
if self.executer.facts in mdx_execution_result['columns_desc'][
mdx_query_axis].keys() and len(mdx_execution_result[
'columns_desc'][mdx_query_axis][self.executer.facts]) > 1:
hierarchy_info += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
<PARENT_UNIQUE_NAME name="{0}.[PARENT_UNIQUE_NAME]" type="xs:string"/>
<HIERARCHY_UNIQUE_NAME name="{0}.[HIERARCHY_UNIQUE_NAME]" type="xs:string"/>
</HierarchyInfo>
""".format('[Measures]')
for table_name in mdx_execution_result['columns_desc'][mdx_query_axis]:
if table_name != self.executer.facts:
hierarchy_info += """
<HierarchyInfo name="[{0}].[{0}]">
<UName name="[{0}].[{0}].[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="[{0}].[{0}].[MEMBER_CAPTION]" type="xs:string"/>
<LName name="[{0}].[{0}].[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="[{0}].[{0}].[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="[{0}].[{0}].[DISPLAY_INFO]" type="xs:unsignedInt"/>
<PARENT_UNIQUE_NAME name="[{0}].[{0}].[PARENT_UNIQUE_NAME]" type="xs:string"/>
<HIERARCHY_UNIQUE_NAME name="[{0}].[{0}].[HIERARCHY_UNIQUE_NAME]" type="xs:string"/>
</HierarchyInfo>
""".format(table_name)
if hierarchy_info:
hierarchy_info = """
<AxisInfo name='{0}'>
{1}
</AxisInfo>
""".format(Axis, hierarchy_info)
return hierarchy_info
axis_tables = mdx_execution_result['columns_desc'][mdx_query_axis]
xml = xmlwitch.Builder()
# measure must be written at the top
if axis_tables:
with xml.AxisInfo(name=Axis):
# many measures , then write this on the top
if self.executer.facts in axis_tables.keys() and len(
axis_tables[self.executer.facts]) > 1:
with xml.HierarchyInfo(name='[Measures]'):
xml.UName(
name="[Measures].[MEMBER_UNIQUE_NAME]",
**{'type': 'xs:string'})
xml.Caption(
name="[Measures].[MEMBER_CAPTION]",
**{'type': 'xs:string'})
xml.LName(
name="[Measures].[LEVEL_UNIQUE_NAME]",
**{'type': 'xs:string'})
xml.LNum(
name="[Measures].[LEVEL_NUMBER]",
**{'type': 'xs:int'})
xml.DisplayInfo(
name="[Measures].[DISPLAY_INFO]",
**{'type': 'xs:unsignedInt'})
xml.PARENT_UNIQUE_NAME(
name="[Measures].[PARENT_UNIQUE_NAME]",
**{'type': 'xs:string'})
xml.HIERARCHY_UNIQUE_NAME(
name="[Measures].[HIERARCHY_UNIQUE_NAME]",
**{'type': 'xs:string'})
for table_name in axis_tables:
if table_name != self.executer.facts:
with xml.HierarchyInfo(
name='[{0}].[{0}]'.format(table_name)):
xml.UName(
name="[{0}].[{0}].[MEMBER_UNIQUE_NAME]".format(
table_name),
**{'type': 'xs:string'})
xml.Caption(
name="[{0}].[{0}].[MEMBER_CAPTION]".format(
table_name),
**{'type': 'xs:string'})
xml.LName(
name="[{0}].[{0}].[LEVEL_UNIQUE_NAME]".format(
table_name),
**{'type': 'xs:string'})
xml.LNum(
name="[{0}].[{0}].[LEVEL_NUMBER]".format(
table_name),
**{'type': 'xs:int'})
xml.DisplayInfo(
name="[{0}].[{0}].[DISPLAY_INFO]".format(
table_name),
**{'type': 'xs:unsignedInt'})
xml.PARENT_UNIQUE_NAME(
name="[{0}].[{0}].[PARENT_UNIQUE_NAME]".format(
table_name),
**{'type': 'xs:string'})
xml.HIERARCHY_UNIQUE_NAME(
name="[{0}].[{0}].[HIERARCHY_UNIQUE_NAME]".
format(table_name),
**{'type': 'xs:string'})
return str(xml)
def generate_axes_info(self, mdx_execution_result):
"""
......@@ -514,15 +552,16 @@ class XmlaExecuteTools():
@staticmethod
def generate_cell_info():
return """
<CellInfo>
<Value name="VALUE"/>
<FormatString name="FORMAT_STRING" type="xs:string"/>
<Language name="LANGUAGE" type="xs:unsignedInt"/>
<BackColor name="BACK_COLOR" type="xs:unsignedInt"/>
<ForeColor name="FORE_COLOR" type="xs:unsignedInt"/>
<FontFlags name="FONT_FLAGS" type="xs:int"/>
</CellInfo>"""
xml = xmlwitch.Builder()
with xml.CellInfo:
xml.Value(name="VALUE")
xml.FormatString(name="FORMAT_STRING", **{'type': 'xs:string'})
xml.Language(name="LANGUAGE", **{'type': 'xs:unsignedInt'})
xml.BackColor(name="BACK_COLOR", **{'type': 'xs:unsignedInt'})
xml.ForeColor(name="FORE_COLOR", **{'type': 'xs:unsignedInt'})
xml.FontFlags(name="FONT_FLAGS", **{'type': 'xs:int'})
return str(xml)
def generate_slicer_axis(self, mdx_execution_result):
"""
......@@ -553,57 +592,51 @@ class XmlaExecuteTools():
:param mdx_execution_result: mdx_execute() result
:return: SlicerAxis as string
"""
tuple = ""
# not used dimensions
for dim_diff in list(
set(self.executer.get_all_tables_names(ignore_fact=True)) -
set(table_name
for table_name in mdx_execution_result['columns_desc'][
'all'])):
# TODO encode dataframe
# french caracteres
if type(self.executer.tables_loaded[dim_diff].iloc[0][
0]) == unicode:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0].encode('utf-8', 'replace')
else:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0]
tuple += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].[{2}]</UName>
<Caption>{2}</Caption>
<LName>[{0}].[{0}].[{1}]</LName>
<LNum>0</LNum>
<DisplayInfo>2</DisplayInfo>
</Member>
""".format(dim_diff,
self.executer.tables_loaded[dim_diff].columns[0],
column_attribut)
# if we have zero on one only measures used
if len(self.executer.selected_measures) <= 1:
tuple += """
<Member Hierarchy="[Measures]">
<UName>[Measures].[{0}]</UName>
<Caption>{0}</Caption>
<LName>[Measures]</LName>
<LNum>0</LNum>
<DisplayInfo>0</DisplayInfo>
</Member>
""".format(self.executer.measures[0])
if tuple:
tuple = """
<Axis name="SlicerAxis">
<Tuples>
<Tuple>
{0}
</Tuple>
</Tuples>
</Axis>
""".format(tuple)
return tuple
unused_dimensions = list(
set(self.executer.get_all_tables_names(ignore_fact=True)) - set(
table_name
for table_name in mdx_execution_result['columns_desc']['all']))
xml = xmlwitch.Builder()
if unused_dimensions:
with xml.Axis(name="SlicerAxis"):
with xml.Tuples:
with xml.Tuple:
for dim_diff in unused_dimensions:
# TODO encode dataframe
# french caracteres
if type(self.executer.tables_loaded[dim_diff].iloc[
0][0]) == unicode:
column_attribut = self.executer.tables_loaded[
dim_diff].iloc[0][0].encode('utf-8',
'replace')
else:
column_attribut = self.executer.tables_loaded[
dim_diff].iloc[0][0]
with xml.Member(
Hierarchy="[{0}].[{0}]".format(dim_diff)):
xml.UName('[{0}].[{0}].[{1}].[{2}]'.format(
dim_diff, self.executer.tables_loaded[
dim_diff].columns[0], column_attribut))
xml.Caption(str(column_attribut))
xml.LName('[{0}].[{0}].[{1}]'.format(
dim_diff, self.executer.tables_loaded[
dim_diff].columns[0]))
xml.LNum('0')
xml.DisplayInfo('2')
# if we have zero on one only measures used
if len(self.executer.selected_measures) <= 1:
with xml.Member(
Hierarchy="[Measures]".format(dim_diff)):
xml.UName('[Measures].[{0}]'.format(
self.executer.measures[0]))
xml.Caption(
'{0}'.format(self.executer.measures[0]))
xml.LName('[Measures]')
xml.LNum('0')
xml.DisplayInfo('0')
return str(xml)
requirements.in
View file @ 3c3ecadb
......@@ -5,6 +5,7 @@ spyne<3
treelib<2
SQLAlchemy
psycopg2
xmlwitch
# Test
werkzeug
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7