Commit f49a9959 authored by mouadh's avatar mouadh

remove df optimization

parent 217c6233
......@@ -9,60 +9,62 @@ import pandas.io.sql as psql
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
class StringFolder(object):
"""
Class that will fold strings. See 'fold_string'.
This object may be safely deleted or go out of scope when
strings have been folded.
"""
def __init__(self):
self.unicode_map = {}
def fold_string(self, s):
"""
Given a string (or unicode) parameter s, return a string object
that has the same value as s (and may be s). For all objects
with a given value, the same object will be returned. For unicode
objects that can be coerced to a string with the same value, a
string object will be returned.
If s is not a string or unicode object, it is returned unchanged.
:param s: a string or unicode object.
:return: a string or unicode object.
"""
# If s is not a string or unicode object, return it unchanged
if not isinstance(s, basestring):
return s
# If s is already a string, then str() has no effect.
# If s is Unicode, try and encode as a string and use intern.
# If s is Unicode and can't be encoded as a string, this try
# will raise a UnicodeEncodeError.
try:
return intern(str(s))
except UnicodeEncodeError:
# Fall through and handle s as Unicode
pass
# Look up the unicode value in the map and return
# the object from the map. If there is no matching entry,
# store this unicode object in the map and return it.
return self.unicode_map.setdefault(s, s)
def string_folding_wrapper(results):
"""
This generator yields rows from the results as tuples,
with all string values folded.
"""
# Get the list of keys so that we build tuples with all
# the values in key order.
keys = results.keys()
folder = StringFolder()
for row in results:
yield tuple(
folder.fold_string(row[key])
for key in keys
)
# class StringFolder(object):
# """
# Class that will fold strings. See 'fold_string'.
# This object may be safely deleted or go out of scope when
# strings have been folded.
# """
# def __init__(self):
# self.unicode_map = {}
#
# def fold_string(self, s):
# """
# Given a string (or unicode) parameter s, return a string object
# that has the same value as s (and may be s). For all objects
# with a given value, the same object will be returned. For unicode
# objects that can be coerced to a string with the same value, a
# string object will be returned.
# If s is not a string or unicode object, it is returned unchanged.
# :param s: a string or unicode object.
# :return: a string or unicode object.
# """
# # If s is not a string or unicode object, return it unchanged
# if not isinstance(s, basestring):
# return s
#
# # If s is already a string, then str() has no effect.
# # If s is Unicode, try and encode as a string and use intern.
# # If s is Unicode and can't be encoded as a string, this try
# # will raise a UnicodeEncodeError.
# try:
# return intern(str(s))
# except UnicodeEncodeError:
# # Fall through and handle s as Unicode
# pass
#
# # Look up the unicode value in the map and return
# # the object from the map. If there is no matching entry,
# # store this unicode object in the map and return it.
# return self.unicode_map.setdefault(s, s)
#
#
# def string_folding_wrapper(results):
# """
# This generator yields rows from the results as tuples,
# with all string values folded.
# """
# # Get the list of keys so that we build tuples with all
# # the values in key order.
# keys = results.keys()
# folder = StringFolder()
# for row in results:
# yield tuple(
# folder.fold_string(row[key])
# for key in keys
# )
# TODO try pandas.read_sql_table and pandas.read_sql
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment