Commit f49a9959 authored by mouadh's avatar mouadh

remove df optimization

parent 217c6233
...@@ -9,60 +9,62 @@ import pandas.io.sql as psql ...@@ -9,60 +9,62 @@ import pandas.io.sql as psql
# execute csv files if they respect olapy's start schema model, # execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model) # and execute data base tables if they respect olapy's start schema model)
class StringFolder(object):
"""
Class that will fold strings. See 'fold_string'. # class StringFolder(object):
This object may be safely deleted or go out of scope when # """
strings have been folded. # Class that will fold strings. See 'fold_string'.
""" # This object may be safely deleted or go out of scope when
def __init__(self): # strings have been folded.
self.unicode_map = {} # """
# def __init__(self):
def fold_string(self, s): # self.unicode_map = {}
""" #
Given a string (or unicode) parameter s, return a string object # def fold_string(self, s):
that has the same value as s (and may be s). For all objects # """
with a given value, the same object will be returned. For unicode # Given a string (or unicode) parameter s, return a string object
objects that can be coerced to a string with the same value, a # that has the same value as s (and may be s). For all objects
string object will be returned. # with a given value, the same object will be returned. For unicode
If s is not a string or unicode object, it is returned unchanged. # objects that can be coerced to a string with the same value, a
:param s: a string or unicode object. # string object will be returned.
:return: a string or unicode object. # If s is not a string or unicode object, it is returned unchanged.
""" # :param s: a string or unicode object.
# If s is not a string or unicode object, return it unchanged # :return: a string or unicode object.
if not isinstance(s, basestring): # """
return s # # If s is not a string or unicode object, return it unchanged
# if not isinstance(s, basestring):
# If s is already a string, then str() has no effect. # return s
# If s is Unicode, try and encode as a string and use intern. #
# If s is Unicode and can't be encoded as a string, this try # # If s is already a string, then str() has no effect.
# will raise a UnicodeEncodeError. # # If s is Unicode, try and encode as a string and use intern.
try: # # If s is Unicode and can't be encoded as a string, this try
return intern(str(s)) # # will raise a UnicodeEncodeError.
except UnicodeEncodeError: # try:
# Fall through and handle s as Unicode # return intern(str(s))
pass # except UnicodeEncodeError:
# # Fall through and handle s as Unicode
# Look up the unicode value in the map and return # pass
# the object from the map. If there is no matching entry, #
# store this unicode object in the map and return it. # # Look up the unicode value in the map and return
return self.unicode_map.setdefault(s, s) # # the object from the map. If there is no matching entry,
# # store this unicode object in the map and return it.
# return self.unicode_map.setdefault(s, s)
def string_folding_wrapper(results): #
""" #
This generator yields rows from the results as tuples, # def string_folding_wrapper(results):
with all string values folded. # """
""" # This generator yields rows from the results as tuples,
# Get the list of keys so that we build tuples with all # with all string values folded.
# the values in key order. # """
keys = results.keys() # # Get the list of keys so that we build tuples with all
folder = StringFolder() # # the values in key order.
for row in results: # keys = results.keys()
yield tuple( # folder = StringFolder()
folder.fold_string(row[key]) # for row in results:
for key in keys # yield tuple(
) # folder.fold_string(row[key])
# for key in keys
# )
# TODO try pandas.read_sql_table and pandas.read_sql # TODO try pandas.read_sql_table and pandas.read_sql
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment