Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
O
olapy
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
nexedi
olapy
Commits
71fe3e64
Commit
71fe3e64
authored
Jun 01, 2017
by
mouadh
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
change to sqlalchemy
parent
99d84a3b
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
45 additions
and
39 deletions
+45
-39
config/olapy-config.xml
config/olapy-config.xml
+2
-1
olapy/core/mdx/executor/execute.py
olapy/core/mdx/executor/execute.py
+7
-4
olapy/core/mdx/executor/execute_config_file.py
olapy/core/mdx/executor/execute_config_file.py
+2
-2
olapy/core/mdx/executor/execute_db.py
olapy/core/mdx/executor/execute_db.py
+11
-12
olapy/core/mdx/tools/connection.py
olapy/core/mdx/tools/connection.py
+13
-8
olapy/core/mdx/tools/olapy_config_file_parser.py
olapy/core/mdx/tools/olapy_config_file_parser.py
+10
-12
No files found.
config/olapy-config.xml
View file @
71fe3e64
<?xml version="1.0" encoding="UTF-8"?>
<?xml version="1.0" encoding="UTF-8"?>
<!-- this config file will be deleted ASAP -->
<olapy>
<olapy>
<database>
<database>
...
@@ -8,6 +8,7 @@
...
@@ -8,6 +8,7 @@
<user_name>
postgres
</user_name>
<user_name>
postgres
</user_name>
<password>
root
</password>
<password>
root
</password>
<host>
localhost
</host>
<host>
localhost
</host>
<port>
5432
</port>
</database>
</database>
...
...
olapy/core/mdx/executor/execute.py
View file @
71fe3e64
...
@@ -95,12 +95,15 @@ class MdxEngine:
...
@@ -95,12 +95,15 @@ class MdxEngine:
# get postgres databases
# get postgres databases
try
:
try
:
db
=
MyDB
()
db
=
MyDB
()
cursor
=
db
.
connection
.
cursor
()
connection
=
db
.
engine
cursor
.
execute
(
"""SELECT datname FROM pg_database
# TODO this work only with postgres
WHERE datistemplate = false;"""
)
result
=
connection
.
execute
(
'SELECT datname FROM pg_database WHERE datistemplate = false;'
)
available_tables
=
result
.
fetchall
()
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
MdxEngine
.
postgres_db_cubes
=
[
MdxEngine
.
postgres_db_cubes
=
[
database
[
0
]
for
database
in
cursor
.
fetchall
()
database
[
0
]
for
database
in
available_tables
]
]
except
Exception
:
except
Exception
:
...
...
olapy/core/mdx/executor/execute_config_file.py
View file @
71fe3e64
...
@@ -90,13 +90,13 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
...
@@ -90,13 +90,13 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns
+=
cubes_obj
.
facts
[
0
].
columns
all_columns
+=
cubes_obj
.
facts
[
0
].
columns
fusion
=
psql
.
read_sql_query
(
fusion
=
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
executer_instance
.
facts
),
db
.
connection
)
"SELECT * FROM {0}"
.
format
(
executer_instance
.
facts
),
db
.
engine
)
tables
=
{}
tables
=
{}
for
table
in
cubes_obj
.
tables
:
for
table
in
cubes_obj
.
tables
:
tab
=
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
table
.
name
),
tab
=
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
table
.
name
),
db
.
connection
)
db
.
engine
)
try
:
try
:
if
table
.
columns
:
if
table
.
columns
:
...
...
olapy/core/mdx/executor/execute_db.py
View file @
71fe3e64
from
__future__
import
absolute_import
,
division
,
print_function
from
__future__
import
absolute_import
,
division
,
print_function
from
sqlalchemy
import
inspect
from
..tools.connection
import
MyDB
from
..tools.connection
import
MyDB
import
pandas.io.sql
as
psql
import
pandas.io.sql
as
psql
...
@@ -12,15 +14,13 @@ def _load_tables_db(executer_instance):
...
@@ -12,15 +14,13 @@ def _load_tables_db(executer_instance):
"""
"""
tables
=
{}
tables
=
{}
db
=
MyDB
(
db
=
executer_instance
.
cube
)
db
=
MyDB
(
db
=
executer_instance
.
cube
)
cursor
=
db
.
connection
.
cursor
()
inspector
=
inspect
(
db
.
engine
)
cursor
.
execute
(
"""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'"""
)
for
table_name
in
cursor
.
fetchall
():
for
table_name
in
inspector
.
get_table_names
():
value
=
psql
.
read_sql_query
(
value
=
psql
.
read_sql_query
(
'SELECT * FROM "{0}"
'
.
format
(
table_name
[
0
]),
db
.
connection
)
'SELECT * FROM "{0}"
'
.
format
(
table_name
),
db
.
engine
)
tables
[
table_name
[
0
]
]
=
value
[[
tables
[
table_name
]
=
value
[[
col
for
col
in
value
.
columns
if
col
.
lower
()[
-
3
:]
!=
'_id'
col
for
col
in
value
.
columns
if
col
.
lower
()[
-
3
:]
!=
'_id'
]]
]]
return
tables
return
tables
...
@@ -37,16 +37,15 @@ def _construct_star_schema_db(executer_instance):
...
@@ -37,16 +37,15 @@ def _construct_star_schema_db(executer_instance):
# load facts table
# load facts table
fusion
=
psql
.
read_sql_query
(
fusion
=
psql
.
read_sql_query
(
'SELECT * FROM "{0}" '
.
format
(
executer_instance
.
facts
),
db
.
connection
)
'SELECT * FROM "{0}" '
.
format
(
executer_instance
.
facts
),
db
.
engine
)
inspector
=
inspect
(
db
.
engine
)
cursor
=
db
.
connection
.
cursor
()
for
db_table_name
in
inspector
.
get_table_names
():
cursor
.
execute
(
"""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'"""
)
for
db_table_name
in
cursor
.
fetchall
():
try
:
try
:
fusion
=
fusion
.
merge
(
fusion
=
fusion
.
merge
(
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
db_table_name
[
0
]),
db
.
connection
))
db_table_name
[
0
]),
db
.
engine
))
except
:
except
:
print
(
'No common column'
)
print
(
'No common column'
)
pass
pass
...
...
olapy/core/mdx/tools/connection.py
View file @
71fe3e64
import
psycopg2
as
pg
import
psycopg2
as
pg
from
sqlalchemy
import
create_engine
# postgres connection
# postgres connection
from
olapy_config_file_parser
import
DbConfigParser
from
olapy_config_file_parser
import
DbConfigParser
...
@@ -24,20 +25,24 @@ class MyDB(object):
...
@@ -24,20 +25,24 @@ class MyDB(object):
username
=
db_credentials
[
'user_name'
]
username
=
db_credentials
[
'user_name'
]
password
=
db_credentials
[
'password'
]
password
=
db_credentials
[
'password'
]
host
=
db_credentials
[
'host'
]
host
=
db_credentials
[
'host'
]
port
=
db_credentials
[
'port'
]
if
db
is
None
:
if
db
is
None
:
# first i want to show all databases to user (in excel)
# first i want to show all databases to user (in excel)
self
.
connection
=
pg
.
connect
(
"user={0} password={1} host='{2}'"
.
self
.
engine
=
pg
.
connect
(
"user={0} password={1} host='{2}'"
.
format
(
username
,
password
,
host
))
format
(
username
,
password
,
host
))
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
username
,
password
,
'postgres'
,
host
,
port
))
else
:
else
:
# and then we connect to the user db
# and then we connect to the user db
try
:
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
self
.
connection
=
pg
.
connect
(
username
,
password
,
db
,
host
,
port
))
"user={0} password={1} dbname='{2}' host='{3}'"
.
forma
t
(
# self.connection = pg.connec
t(
username
,
password
,
db
,
host
))
# "user={0} password={1} dbname='{2}' host='{3}'".format(
except
:
# username, password, db, host))
print
(
"can't connect"
)
def
__del__
(
self
):
def
__del__
(
self
):
if
hasattr
(
self
,
'connection'
):
if
hasattr
(
self
,
'connection'
):
self
.
connection
.
cl
ose
()
self
.
engine
.
disp
ose
()
olapy/core/mdx/tools/olapy_config_file_parser.py
View file @
71fe3e64
...
@@ -43,15 +43,13 @@ class DbConfigParser:
...
@@ -43,15 +43,13 @@ class DbConfigParser:
parser
=
etree
.
XMLParser
()
parser
=
etree
.
XMLParser
()
tree
=
etree
.
parse
(
config_file
,
parser
)
tree
=
etree
.
parse
(
config_file
,
parser
)
try
:
return
[
return
[
{
{
# 'sgbd': db.find('sgbd').text,
# 'sgbd': db.find('sgbd').text,
'user_name'
:
db
.
find
(
'user_name'
).
text
,
'user_name'
:
db
.
find
(
'user_name'
).
text
,
'password'
:
db
.
find
(
'password'
).
text
,
'password'
:
db
.
find
(
'password'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
'port'
:
db
.
find
(
'port'
).
text
,
}
}
for
db
in
tree
.
xpath
(
'/olapy/database'
)
for
db
in
tree
.
xpath
(
'/olapy/database'
)
]
]
except
:
raise
(
'missed name or source tags'
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment