Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
O
olapy
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
nexedi
olapy
Commits
71fe3e64
Commit
71fe3e64
authored
Jun 01, 2017
by
mouadh
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
change to sqlalchemy
parent
99d84a3b
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
45 additions
and
39 deletions
+45
-39
config/olapy-config.xml
config/olapy-config.xml
+2
-1
olapy/core/mdx/executor/execute.py
olapy/core/mdx/executor/execute.py
+7
-4
olapy/core/mdx/executor/execute_config_file.py
olapy/core/mdx/executor/execute_config_file.py
+2
-2
olapy/core/mdx/executor/execute_db.py
olapy/core/mdx/executor/execute_db.py
+11
-12
olapy/core/mdx/tools/connection.py
olapy/core/mdx/tools/connection.py
+13
-8
olapy/core/mdx/tools/olapy_config_file_parser.py
olapy/core/mdx/tools/olapy_config_file_parser.py
+10
-12
No files found.
config/olapy-config.xml
View file @
71fe3e64
<?xml version="1.0" encoding="UTF-8"?>
<!-- this config file will be deleted ASAP -->
<olapy>
<database>
...
...
@@ -8,6 +8,7 @@
<user_name>
postgres
</user_name>
<password>
root
</password>
<host>
localhost
</host>
<port>
5432
</port>
</database>
...
...
olapy/core/mdx/executor/execute.py
View file @
71fe3e64
...
...
@@ -95,12 +95,15 @@ class MdxEngine:
# get postgres databases
try
:
db
=
MyDB
()
cursor
=
db
.
connection
.
cursor
()
cursor
.
execute
(
"""SELECT datname FROM pg_database
WHERE datistemplate = false;"""
)
connection
=
db
.
engine
# TODO this work only with postgres
result
=
connection
.
execute
(
'SELECT datname FROM pg_database WHERE datistemplate = false;'
)
available_tables
=
result
.
fetchall
()
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
MdxEngine
.
postgres_db_cubes
=
[
database
[
0
]
for
database
in
cursor
.
fetchall
()
database
[
0
]
for
database
in
available_tables
]
except
Exception
:
...
...
olapy/core/mdx/executor/execute_config_file.py
View file @
71fe3e64
...
...
@@ -90,13 +90,13 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns
+=
cubes_obj
.
facts
[
0
].
columns
fusion
=
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
executer_instance
.
facts
),
db
.
connection
)
"SELECT * FROM {0}"
.
format
(
executer_instance
.
facts
),
db
.
engine
)
tables
=
{}
for
table
in
cubes_obj
.
tables
:
tab
=
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
table
.
name
),
db
.
connection
)
db
.
engine
)
try
:
if
table
.
columns
:
...
...
olapy/core/mdx/executor/execute_db.py
View file @
71fe3e64
from
__future__
import
absolute_import
,
division
,
print_function
from
sqlalchemy
import
inspect
from
..tools.connection
import
MyDB
import
pandas.io.sql
as
psql
...
...
@@ -12,15 +14,13 @@ def _load_tables_db(executer_instance):
"""
tables
=
{}
db
=
MyDB
(
db
=
executer_instance
.
cube
)
cursor
=
db
.
connection
.
cursor
()
cursor
.
execute
(
"""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'"""
)
inspector
=
inspect
(
db
.
engine
)
for
table_name
in
cursor
.
fetchall
():
for
table_name
in
inspector
.
get_table_names
():
value
=
psql
.
read_sql_query
(
'SELECT * FROM "{0}"
'
.
format
(
table_name
[
0
]),
db
.
connection
)
'SELECT * FROM "{0}"
'
.
format
(
table_name
),
db
.
engine
)
tables
[
table_name
[
0
]
]
=
value
[[
tables
[
table_name
]
=
value
[[
col
for
col
in
value
.
columns
if
col
.
lower
()[
-
3
:]
!=
'_id'
]]
return
tables
...
...
@@ -37,16 +37,15 @@ def _construct_star_schema_db(executer_instance):
# load facts table
fusion
=
psql
.
read_sql_query
(
'SELECT * FROM "{0}" '
.
format
(
executer_instance
.
facts
),
db
.
connection
)
'SELECT * FROM "{0}" '
.
format
(
executer_instance
.
facts
),
db
.
engine
)
inspector
=
inspect
(
db
.
engine
)
cursor
=
db
.
connection
.
cursor
()
cursor
.
execute
(
"""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'"""
)
for
db_table_name
in
cursor
.
fetchall
():
for
db_table_name
in
inspector
.
get_table_names
():
try
:
fusion
=
fusion
.
merge
(
psql
.
read_sql_query
(
"SELECT * FROM {0}"
.
format
(
db_table_name
[
0
]),
db
.
connection
))
db_table_name
[
0
]),
db
.
engine
))
except
:
print
(
'No common column'
)
pass
...
...
olapy/core/mdx/tools/connection.py
View file @
71fe3e64
import
psycopg2
as
pg
from
sqlalchemy
import
create_engine
# postgres connection
from
olapy_config_file_parser
import
DbConfigParser
...
...
@@ -24,20 +25,24 @@ class MyDB(object):
username
=
db_credentials
[
'user_name'
]
password
=
db_credentials
[
'password'
]
host
=
db_credentials
[
'host'
]
port
=
db_credentials
[
'port'
]
if
db
is
None
:
# first i want to show all databases to user (in excel)
self
.
connection
=
pg
.
connect
(
"user={0} password={1} host='{2}'"
.
self
.
engine
=
pg
.
connect
(
"user={0} password={1} host='{2}'"
.
format
(
username
,
password
,
host
))
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
username
,
password
,
'postgres'
,
host
,
port
))
else
:
# and then we connect to the user db
try
:
self
.
connection
=
pg
.
connect
(
"user={0} password={1} dbname='{2}' host='{3}'"
.
forma
t
(
username
,
password
,
db
,
host
))
except
:
print
(
"can't connect"
)
self
.
engine
=
create_engine
(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'
.
format
(
username
,
password
,
db
,
host
,
port
))
# self.connection = pg.connec
t(
# "user={0} password={1} dbname='{2}' host='{3}'".format(
# username, password, db, host))
def
__del__
(
self
):
if
hasattr
(
self
,
'connection'
):
self
.
connection
.
cl
ose
()
self
.
engine
.
disp
ose
()
olapy/core/mdx/tools/olapy_config_file_parser.py
View file @
71fe3e64
...
...
@@ -43,15 +43,13 @@ class DbConfigParser:
parser
=
etree
.
XMLParser
()
tree
=
etree
.
parse
(
config_file
,
parser
)
try
:
return
[
{
# 'sgbd': db.find('sgbd').text,
'user_name'
:
db
.
find
(
'user_name'
).
text
,
'password'
:
db
.
find
(
'password'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
}
for
db
in
tree
.
xpath
(
'/olapy/database'
)
]
except
:
raise
(
'missed name or source tags'
)
return
[
{
# 'sgbd': db.find('sgbd').text,
'user_name'
:
db
.
find
(
'user_name'
).
text
,
'password'
:
db
.
find
(
'password'
).
text
,
'host'
:
db
.
find
(
'host'
).
text
,
'port'
:
db
.
find
(
'port'
).
text
,
}
for
db
in
tree
.
xpath
(
'/olapy/database'
)
]
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment