Skip to content

Z
Zope
Commit ac18b517 authored by Fred Drake's avatar Fred Drake
Browse files
Options

Jim explained that these are old and no longer relevant.

parent 1f97e260
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 1465 deletions
utilities/ExtensionBuilder.py deleted 100755 → 0
View file @ 1f97e260
"""Build Python extension modules from Setup files. Particularly
designed for building extensions for Zope in a way that works
the same way for *nix and win32. Note that for building Zope
you need to get the Zope source distribution (binary releases
do not contain everything needed to build the Zope extensions).
To use, drop this file into the 'utilities' directory of your
Zope source installation. Run
'python ./utilities/ExtensionBuilder.py -h' for usage details.
Note that this is not supported code. I am making this available
mainly because the question "how do I build extensions on win32?"
keeps coming up on the zope mailing list. Hopefully those of you
wanting to build on win32 will find this useful. As Zope moves to
support Python 2, I expect distutils to be the standard for Zope
extensions, so hopefully this is just a short-term hack.
brian@digicool.com
"""
import sys, os, regex, string, getopt
from exceptions import StandardError
class ExtensionBuilder:
"""An tool for building extension modules. All filesystem path usage
is relative to the current working directory when the script is
invoked. Usage: instantiate an ExtensionBuilder and call make(path),
passing a directory to build, or call makeAll() to build all Python
extensions found in the current directory and all subdirectories."""
def getPythonSourcePath(self):
"""Return the path to the root of the Python source installation,
raises BuildError if no valid source installation can be found."""
if hasattr(self, 'py_src_path'):
return self.py_src_path
py_src_path = sys.exec_prefix
if (os.path.exists(os.path.join(py_src_path, 'Include')) or
os.path.exists(os.path.join(py_src_path, 'include'))):
self.py_src_path = py_src_path
return py_src_path
raise BuildError(
'A Python source installation must be available to build ' \
'extensions. No valid source installation was found.'
)
def getPythonIncludePath(self):
py_src_path = self.getPythonSourcePath()
py_inc_path = os.path.join(py_src_path, 'include')
if os.path.exists(py_inc_path):
self.py_inc_path = py_inc_path
return py_inc_path
py_inc_path = os.path.join(py_src_path, 'Include')
if os.path.exists(py_inc_path):
self.py_inc_path = py_inc_path
return py_inc_path
raise BuildError(
'Unable to determine include directory path.'
)
def getMakefileTemplatePath(self):
"""Return the path to the prototype extension makefile from the
Python distribution (Makefile.pre.in). Raises BuildError if
the prototype makefile cannot be found."""
if hasattr(self, 'py_makefile_path'):
return self.py_makefile_path
py_src_path = self.getPythonSourcePath()
py_ver_path = 'python%s' % sys.version[:3]
py_makefile_path = os.path.join(py_src_path, 'lib', py_ver_path,
'config', 'Makefile.pre.in'
)
if os.path.exists(py_makefile_path):
self.py_makefile_path = py_makefile_path
return py_makefile_path
raise BuildError(
'The prototype extension makefile (Makefile.pre.in) could not ' \
'be found.'
)
def isValidSetupFile(self, filename):
"""Return true if the file named by filename smells valid."""
file = open(filename, 'r')
line = file.readline()
file.close()
if line[:9] == '# install':
return 0
return 1
def doCommand(self, command, fail=1, echo=1):
"""Relay command to the system. If fail is true, raise BuildError
if the system command returns a non-zero value. If echo is true,
print commands as they are executed."""
if echo: print command
result = os.system(command)
if result and fail:
raise BuildError(
'Command "%s" failed with error code: %d' % (command, result)
)
def printMessage(self, message):
"""Print a nice little status message."""
print
print '-' * 78
print message
print '-' * 78
print
def makePosix(self, path):
"""Make the extensions in the directory specified by path."""
path_items = string.split(path, os.sep)
for name in path_items:
os.chdir(name)
self.doCommand('cp %s .' % self.getMakefileTemplatePath())
self.doCommand('make -f Makefile.pre.in boot PYTHON=%s' % (
sys.executable
))
self.doCommand('make')
self.doCommand('make clean')
for name in path_items:
os.chdir('..')
return
def makeMakefileWin32(self, module, source, srcdir, optlist, args):
"""Create .def and .mak files for an extension."""
params = {'module': module,
'pyhome': self.getPythonSourcePath(),
'source': source,
'srcdir': srcdir,
'cfg': 'Release',
}
includepath = self.getPythonIncludePath()
params['includes']=string.join(
['/I "%s" /I "%s\\PC"' % (includepath, params['pyhome'])] +
map(lambda i: '/I "%s"' % i, opt(optlist,'I')) +
map(lambda i: '/D "%s"' % i, opt(optlist,'D')),
' ')
# Note: handling of 'L' or 'l' is currently not working :(
tmplist = []
for item in opt(optlist, 'L'):
item = os.path.join(os.getcwd(), item)
item = os.path.normpath(item)
tmplist.append('/libpath:"%s"' % item)
params['libdirs'] = string.join(tmplist, '')
tmplist = []
for item in opt(optlist, 'l'):
item = os.path.join(os.getcwd(), item)
item = os.path.normpath(item)
tmplist.append('/defaultlib:"%s.lib"' % item)
params['other_libs'] = string.join(tmplist, '')
params['other_clean_release']=string.join(
map(lambda o: '\n\t-@erase "%s\\Release\\%s.obj"' % (
filepath(o), filebase(o)
), args),'')
params['other_clean_debug']=string.join(
map(lambda o: '\n\t-@erase "%s\\Debug\\%s.obj"' % (
filepath(o), filebase(o)
), args),'')
params['other_rule']=string.join(
map(lambda o:
'"$(INTDIR)\%s.obj" : %s $(DEP_CPP_MOD) "$(INTDIR)"\n'
'\t$(CPP) $(CPP_PROJ) %s '
% (filebase(o), o, o),
args),'\n')
params['other_link']=string.join(
map(lambda o: '\n\t"$(INTDIR)\%s.obj" \\' % filebase(o),
args),'')
# Figure out lib file based on py version.
libname = 'python%s.lib' % string.replace(sys.version[:3], '.', '')
params['pythonlib'] = libname
file = open('%s.def' % module, 'w')
file.write(win32_def % params)
file.close()
file = open('%s.mak' % module, 'w')
file.write(win32_mak % params)
file.close()
def makeWin32(self, path):
"""Make the extensions in the directory specified by path.
Assumes an installation of MS Visual C++ 4.0 or later."""
ext_line = regex.compile('\([a-z_][a-z0-9_~]*\)[ \t]*'
'\([.a-z_][./a-z0-9_~]*[.]c\)[ \t\n]',
regex.casefold)
path_items = string.split(path, os.sep)
for name in path_items:
os.chdir(name)
current_dir = os.getcwd()
if 'Setup.in' in os.listdir(current_dir):
self.doCommand('copy Setup.in Setup')
# Parse the Setup file. We have to do a lot of work because
# NMAKE is very stupid about relative paths :( The only way
# to get things to work relatively reliably is to turn all
# path references into absolute paths and actually cd to the
# extension directory before running NMAKE.
setupfile = open('Setup', 'r')
for line in setupfile.readlines():
if line[0] == '#':
continue
if ext_line.match(line) >= 0:
args = string.split(line)
module = args[0]
source = args[1]
if source[:2] == './':
continue
optlist, args = getopt.getopt(args[2:], 'I:D:L:l:')
for arg in args:
if (arg[:1] == '-') or (string.find(arg, '.') < 0):
raise BuildError(
'Invalid arguments in Setup: %s' % (
string.join(args)
))
tmplist = []
for key, val in optlist:
value = string.replace(val, '/', '\\')
if key == '-I':
value = os.path.join(current_dir, value)
value = os.path.normpath(value)
tmplist.append((key, value))
optlist = tmplist
tmplist = []
for arg in args:
arg = string.replace(arg, '/', '\\')
if '\\' in arg:
arg = os.path.join(current_dir, arg)
arg = os.path.normpath(arg)
tmplist.append(arg)
args = tmplist
source = string.replace(source, '/', '\\')
srcdir = filepath(source)
srcdir = os.path.normpath(os.path.join(os.getcwd(), srcdir))
source = os.path.split(source)[-1]
os.chdir(srcdir)
# Build .def and .mak files
self.makeMakefileWin32(module, source, srcdir, optlist, args)
# Build extension dlls
config = 'CFG="%s - Win32 Release"' % module
self.doCommand('nmake /nologo /f %s.mak %s' % (
module, config
))
self.doCommand('del *.def')
self.doCommand('del *.mak')
os.chdir(current_dir)
self.doCommand('copy %s\\Release\\%s.dll %s.pyd' % (
srcdir, module, module
))
for name in path_items:
os.chdir('..')
return
if sys.platform == 'win32':
makeMethod = makeWin32
else:
makeMethod = makePosix
def make(self, path, make_sub=0):
"""Make the extensions in the directory specified by path. If
the make_sub argument is passed as a true value, build any
extensions found in all subdirectories."""
name_list = os.listdir(path)
if 'Setup' in name_list:
if self.isValidSetupFile(os.path.join(path, 'Setup')):
self.printMessage('Building extensions in: %s' % path)
self.makeMethod(path)
if not make_sub:
return
for name in name_list:
pathname = os.path.join(path, name)
if os.path.isdir(pathname):
self.make(pathname, make_sub)
class BuildError(StandardError):
"""A fatal build exception."""
pass
def filebase(file):
return os.path.splitext(os.path.split(file)[1])[0]
def filepath(file):
parts = os.path.split(file)
if parts[0] == '':
return '.'
return parts[0]
def opt(optlist, name):
l=filter(lambda t, name='-'+name: t[0]==name, optlist)
return map(lambda t: t[1], l)
usage="""Usage: %s [options] [path]
This script builds Python extensions on most platforms, including
win32 provided you have MSVC++ 4.0 or higher installed. To compile
on windows, you need to run the vcvars32.bat file included with
MSVC++ to set your shell environment for using command line VC tools
before running this script.
This script is most commonly used to build a Zope source release. To
do that, cd to the root of your Zope installation and run:
python ./utilities/ExtensionBuilder.py -z
Options:
-p Specify the location of your Python source tree. If not
specified, the script will try to figure it out itself
and complain if it can't.
-z Build the Zope source release. Run this from the root of
a Zope source installation to build all of the standard
Zope extensions (ignoring others that may be around).
-a Build all extensions found in the current working directory
and all subdirectories.
-m Build only a specific extension module (or set of modules)
found in the directory passed via the -m value. This value
must be a path *relative* to the current working directory
when the script is invoked.
"""
def build_all(path):
builder = ExtensionBuilder()
builder.make(path, make_sub=1)
print 'Done!'
def build_dir(path):
builder = ExtensionBuilder()
builder.make(path)
print 'Done!'
def build_zope(path):
# path is assumed to be Zope root directory
builder = ExtensionBuilder()
def system(command, builder=builder):
command = string.join(string.split(command, '/'), os.sep)
builder.doCommand(command)
if path != '.': system('cd %s' % path)
cp = (sys.platform == 'win32') and 'copy' or 'cp'
if os.path.exists('./lib/python/Setup20') and sys.version[:1]=='2':
system('%s ./lib/python/Setup20 ./lib/python/Setup' % cp)
if os.path.exists('./lib/python/Setup15') and sys.version[:1]=='1':
system('%s ./lib/python/Setup15 ./lib/python/Setup' % cp)
# Note that this assumes Zope 2.3.1 b2 or later!
for item in ('lib/python',
'lib/python/DocumentTemplate',
'lib/python/ZODB',
'lib/python/BTrees',
'lib/python/AccessControl',
'lib/python/SearchIndex',
'lib/python/Shared/DC/xml/pyexpat',
'lib/python/Products/PluginIndexes/TextIndex/Splitter/ZopeSplitter',
'lib/python/Products/PluginIndexes/TextIndex/Splitter/ISO_8859_1_Splitter',
'lib/python/Products/PluginIndexes/TextIndex/Splitter/UnicodeSplitter',
'lib/python/Products/ZCTextIndex'
):
dirpath = string.join(string.split(item, '/'), os.sep)
builder.make(dirpath)
if sys.version[:1] != '2':
os.chdir('lib')
files=filter(
lambda f: string.lower(f[:8])=='cpickle.',
os.listdir('python')
)
if files:
os.chdir('python')
os.chdir('ZODB')
for f in files:
src=os.path.join('..',f)
try: os.link(src,f)
except: open(f,'wb').write(open(src,'rb').read())
os.chdir('..')
os.chdir('..')
os.chdir('..')
print 'Done!'
def main():
optlist, args=getopt.getopt(sys.argv[1:], 'p:m:azh')
for key, val in optlist:
if key == '-p':
ExtensionBuilder.py_src_path = val
elif key == '-h':
print usage
sys.exit(0)
elif key == '-a':
build_all('.')
elif key == '-m':
build_dir(val)
elif key == '-z':
build_zope('.')
else:
print 'Unrecognized option: %s' % key
print usage
sys.exit(0)
win32_def="""EXPORTS
init%(module)s
"""
win32_mak="""# Microsoft Developer Studio Generated NMAKE File, Format Version 4.00
# ** DO NOT EDIT **
# TARGTYPE "Win32 (x86) Dynamic-Link Library" 0x0102
!IF "$(CFG)" == ""
CFG=%(module)s - Win32 Debug
!MESSAGE No configuration specified. Defaulting to %(module)s - Win32 Debug.
!ENDIF
!IF "$(CFG)" != "%(module)s - Win32 Release" && "$(CFG)" !=\\
"%(module)s - Win32 Debug"
!MESSAGE Invalid configuration "$(CFG)" specified.
!MESSAGE You can specify a configuration when running NMAKE on this makefile
!MESSAGE by defining the macro CFG on the command line. For example:
!MESSAGE
!MESSAGE NMAKE /f "%(module)s.mak" CFG="%(module)s - Win32 Debug"
!MESSAGE
!MESSAGE Possible choices for configuration are:
!MESSAGE
!MESSAGE "%(module)s - Win32 Release" (based on\\
"Win32 (x86) Dynamic-Link Library")
!MESSAGE "%(module)s - Win32 Debug" (based on "Win32 (x86) Dynamic-Link Library")
!MESSAGE
!ERROR An invalid configuration is specified.
!ENDIF
!IF "$(OS)" == "Windows_NT"
NULL=
!ELSE
NULL=nul
!ENDIF
################################################################################
# Begin Project
# PROP Target_Last_Scanned "%(module)s - Win32 Debug"
CPP=cl.exe
RSC=rc.exe
MTL=mktyplib.exe
!IF "$(CFG)" == "%(module)s - Win32 Release"
# PROP BASE Use_MFC 0
# PROP BASE Use_Debug_Libraries 0
# PROP BASE Output_Dir "Release"
# PROP BASE Intermediate_Dir "Release"
# PROP BASE Target_Dir ""
# PROP Use_MFC 0
# PROP Use_Debug_Libraries 0
# PROP Output_Dir "Release"
# PROP Intermediate_Dir "Release"
# PROP Target_Dir ""
OUTDIR=%(srcdir)s\\Release
INTDIR=%(srcdir)s\\Release
ALL : "$(OUTDIR)\\%(module)s.dll"
CLEAN :
-@erase "$(OUTDIR)\\%(module)s.dll"
-@erase "$(OUTDIR)\\%(module)s.obj"%(other_clean_release)s
-@erase "$(OUTDIR)\\%(module)s.lib"
-@erase "$(OUTDIR)\\%(module)s.exp"
-@erase "$(OUTDIR)\\%(module)s.pch"
"$(OUTDIR)" :
if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"
# ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
# ADD CPP /nologo /MD /W3 /GX /O2 %(includes)s /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
CPP_PROJ=/nologo /MD /W3 /GX /O2 %(includes)s /D "WIN32" /D\\
"NDEBUG" /D "_WINDOWS" /Fp"$(INTDIR)\\%(module)s.pch" /YX /Fo"$(INTDIR)/" /c
CPP_OBJS=$(OUTDIR)/
CPP_SBRS=
# ADD BASE MTL /nologo /D "NDEBUG" /win32
# ADD MTL /nologo /D "NDEBUG" /win32
MTL_PROJ=/nologo /D "NDEBUG" /win32
# ADD BASE RSC /l 0x409 /d "NDEBUG"
# ADD RSC /l 0x409 /d "NDEBUG"
BSC32=bscmake.exe
# ADD BASE BSC32 /nologo
# ADD BSC32 /nologo
BSC32_FLAGS=/nologo /o"$(OUTDIR)\\%(module)s.bsc"
BSC32_SBRS=
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /machine:I386
# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /machine:I386
LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\\
advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\\
odbccp32.lib /nologo /subsystem:windows /dll /incremental:no\\
/pdb:"$(OUTDIR)\\%(module)s.pdb" /machine:I386 /def:".\\%(module)s.def"\\
/out:"$(OUTDIR)\\%(module)s.dll" /implib:"$(OUTDIR)\\%(module)s.lib" %(libdirs)s
DEF_FILE= \\
".\\%(module)s.def"
LINK32_OBJS= \\
"$(INTDIR)\\%(module)s.obj" \\%(other_link)s
"%(pyhome)s\\libs\\%(pythonlib)s"
"$(OUTDIR)\\%(module)s.dll" : "$(OUTDIR)" $(DEF_FILE) $(LINK32_OBJS)
$(LINK32) @<<
$(LINK32_FLAGS) $(LINK32_OBJS)
<<
!ELSEIF "$(CFG)" == "%(module)s - Win32 Debug"
# PROP BASE Use_MFC 0
# PROP BASE Use_Debug_Libraries 1
# PROP BASE Output_Dir "Debug"
# PROP BASE Intermediate_Dir "Debug"
# PROP BASE Target_Dir ""
# PROP Use_MFC 0
# PROP Use_Debug_Libraries 1
# PROP Output_Dir "Debug"
# PROP Intermediate_Dir "Debug"
# PROP Target_Dir ""
OUTDIR=%(srcdir)s\\Debug
INTDIR=%(srcdir)s\\Debug
ALL : "$(OUTDIR)\\%(module)s.dll"
CLEAN :
-@erase "$(OUTDIR)\\%(module)s.dll"
-@erase "$(OUTDIR)\\%(module)s.obj"%(other_clean_debug)s
-@erase "$(OUTDIR)\\%(module)s.ilk"
-@erase "$(OUTDIR)\\%(module)s.lib"
-@erase "$(OUTDIR)\\%(module)s.exp"
-@erase "$(OUTDIR)\\%(module)s.pdb"
-@erase "$(OUTDIR)\\%(module)s.pch"
-@erase "$(OUTDIR)\\pcbuild.pdb"
-@erase "$(OUTDIR)\\pcbuild.idb"
"$(OUTDIR)" :
if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"
# ADD BASE CPP /nologo /MDd /W3 /Gm /GX /Zi /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /YX /c
# ADD CPP /nologo /MDd /W3 /Gm /GX /Zi /Od %(includes)s /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /YX /c
CPP_PROJ=/nologo /MDd /W3 /Gm /GX /Zi /Od %(includes)s /D "WIN32"\\
/D "_DEBUG" /D "_WINDOWS" /Fp"$(INTDIR)/%(module)s.pch" /YX /Fo"$(INTDIR)/"\\
/Fd"$(INTDIR)/" /c
CPP_OBJS=$(OUTDIR)/
CPP_SBRS=
# ADD BASE MTL /nologo /D "_DEBUG" /win32
# ADD MTL /nologo /D "_DEBUG" /win32
MTL_PROJ=/nologo /D "_DEBUG" /win32
# ADD BASE RSC /l 0x409 /d "_DEBUG"
# ADD RSC /l 0x409 /d "_DEBUG"
BSC32=bscmake.exe
# ADD BASE BSC32 /nologo
# ADD BSC32 /nologo
BSC32_FLAGS=/nologo /o"$(OUTDIR)/%(module)s.bsc"
BSC32_SBRS=
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /debug /machine:I386
# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /debug /machine:I386
LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\\
advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\\
odbccp32.lib /nologo /subsystem:windows /dll /incremental:yes\\
/pdb:"$(OUTDIR)\\%(module)s.pdb" /debug /machine:I386 /def:".\\%(module)s.def"\\
/out:"$(OUTDIR)\\%(module)s.dll" /implib:"$(OUTDIR)\\%(module)s.lib" %(libdirs)s
DEF_FILE= \\
".\\%(module)s.def"
LINK32_OBJS= \\
"$(INTDIR)\\%(module)s.obj" \\%(other_link)s
"%(pyhome)s\\libs\\%(pythonlib)s"
"$(OUTDIR)\\%(module)s.dll" : "$(OUTDIR)" $(DEF_FILE) $(LINK32_OBJS)
$(LINK32) @<<
$(LINK32_FLAGS) $(LINK32_OBJS)
<<
!ENDIF
.c{$(CPP_OBJS)}.obj:
$(CPP) $(CPP_PROJ) $<
.cpp{$(CPP_OBJS)}.obj:
$(CPP) $(CPP_PROJ) $<
.cxx{$(CPP_OBJS)}.obj:
$(CPP) $(CPP_PROJ) $<
.c{$(CPP_SBRS)}.sbr:
$(CPP) $(CPP_PROJ) $<
.cpp{$(CPP_SBRS)}.sbr:
$(CPP) $(CPP_PROJ) $<
.cxx{$(CPP_SBRS)}.sbr:
$(CPP) $(CPP_PROJ) $<
################################################################################
# Begin Target
# Name "%(module)s - Win32 Release"
# Name "%(module)s - Win32 Debug"
!IF "$(CFG)" == "%(module)s - Win32 Release"
!ELSEIF "$(CFG)" == "%(module)s - Win32 Debug"
!ENDIF
################################################################################
# Begin Source File
SOURCE="%(srcdir)s\\%(source)s"
"$(INTDIR)\\%(module)s.obj" : $(SOURCE) "$(INTDIR)"
%(other_rule)s
# End Source File
################################################################################
# Begin Source File
SOURCE=.\\%(module)s.def
!IF "$(CFG)" == "%(module)s - Win32 Release"
!ELSEIF "$(CFG)" == "%(module)s - Win32 Debug"
!ENDIF
# End Source File
################################################################################
# Begin Source File
SOURCE=%(pyhome)s\\libs\\%(pythonlib)s
!IF "$(CFG)" == "%(module)s - Win32 Release"
!ELSEIF "$(CFG)" == "%(module)s - Win32 Debug"
!ENDIF
# End Source File
################################################################################
# Begin Source File
SOURCE=.\\readme.txt
!IF "$(CFG)" == "%(module)s - Win32 Release"
!ELSEIF "$(CFG)" == "%(module)s - Win32 Debug"
!ENDIF
# End Source File
# End Target
# End Project
################################################################################
"""
if __name__ == '__main__':
main()
utilities/FS.py deleted 100644 → 0
View file @ 1f97e260
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import struct, tempfile, string, time, pickle, os, sys
from struct import pack, unpack
from cStringIO import StringIO
class FS:
"""FileStorage 'report' writer for converting BoboPOS to FileStorage
"""
def __init__(self, fname, file):
file.seek(0,2)
self._input_size=file.tell()
file.seek(0)
self.__name__=fname
self._file=open(fname,'w+b')
self._file.write('FS21')
self._index={}
self._indexpos=self._index.get
self._tindex=[]
self._tappend=self._tindex.append
self._tfile=tempfile.TemporaryFile()
self._pos=4
self._progress=None
def rpt(self, pos, oid, start, tname, user, t, p, first, newtrans):
if pos is None:
self.tpc_finish()
sys.stderr.write(' %% 100 \n')
return
else:
progress=' %% %.1f \r' % (pos*100.0/self._input_size)
if progress != self._progress:
sys.stderr.write(progress)
self._progress=progress
if newtrans:
try: string.atof(tname)
except:
# Ugh, we have a weird tname. We'll just ignore the transaction
# boundary and merge transactions
if first:
# But we can't ignore the first one, so we'll hack in a
# bogus start date
self.tpc_begin('100', user, t)
else:
if not first: self.tpc_finish()
self.tpc_begin(tname, user, t)
self.store(oid, p)
def store(self, oid, data):
old=self._indexpos(oid, 0)
pnv=None
tfile=self._tfile
write=tfile.write
pos=self._pos
here=tfile.tell()+pos+self._thl
self._tappend(oid, here)
serial=self._serial
data=fixpickle(data, oid)
write(pack(">8s8s8s8sH8s",
p64(oid+1),serial,p64(old),p64(pos),
0,p64(len(data))
)
)
write(data)
def tpc_begin(self, tname, user, desc):
del self._tindex[:] # Just to be sure!
self._tfile.seek(0)
t=string.atof(tname)
y,m,d,h,mi=time.gmtime(t)[:5]
s=t%60
self._serial=struct.pack(
">II",
(((((y-1900)*12)+m-1)*31+d-1)*24+h)*60+mi,
long(s * (1L << 32) / 60)
)
# Ugh, we have to record the transaction header length
# so that we can get version pointers right.
self._thl=23+len(user)+len(desc)
# And we have to save the data used to compute the
# header length. It's unlikely that this stuff would
# change, but if it did, it would be a disaster.
self._ud=user, desc
def tpc_finish(self):
file=self._file
write=file.write
tfile=self._tfile
dlen=tfile.tell()
tfile.seek(0)
id=self._serial
user, desc = self._ud
self._ud=None
tlen=self._thl
pos=self._pos
tl=tlen+dlen
stl=p64(tl)
write(pack(
">8s" "8s" "c" "H" "H" "H"
,id, stl, ' ', len(user), len(desc), 0,
))
if user: write(user)
if desc: write(desc)
cp(tfile, file, dlen)
write(stl)
self._pos=pos+tl+8
tindex=self._tindex
index=self._index
for oid, pos in tindex: index[oid]=pos
del tindex[:]
class ZEXP:
"""Zope Export format 'report' writer
"""
def __init__(self, fname, file):
file.seek(0,2)
self._input_size=file.tell()
file.seek(0)
self.__name__=fname
self._file=open(fname,'w+b')
self._file.write('ZEXP')
self._pos=4
self._progress=None
def rpt(self, pos, oid, start, tname, user, t, p, first, newtrans):
write=self._file.write
if pos is None:
write('\377'*16) # end marker
sys.stderr.write(' %% 100 \n')
return
else:
progress=' %% %.1f \r' % (pos*100.0/self._input_size)
if progress != self._progress:
sys.stderr.write(progress)
self._progress=progress
data=fixpickle(p, oid)
l=len(data)
write(p64(oid+1)+p64(l))
write(data)
self._pos=self._pos+l+16
t32 = 1L << 32
def p64(v, pack=struct.pack):
if v < t32: h=0
else:
h=v/t32
v=v%t32
return pack(">II", h, v)
def cp(f1, f2, l):
read=f1.read
write=f2.write
n=8192
while l > 0:
if n > l: n=l
d=read(n)
write(d)
l = l - len(d)
class Ghost: pass
class Global:
__safe_for_unpickling__=1
def __init__(self, m, n):
self._module, self._name = m, n
def __call__(self, *args):
return Inst(self, args)
def __basicnew__(self):
return Inst(self, None)
def _global(m, n):
if m[:8]=='BoboPOS.':
if m=='BoboPOS.PickleDictionary' and n=='Root':
m='ZODB.conversionhack'
n='hack'
elif m=='BoboPOS.PersistentMapping': m='Persistence'
elif m=='BoboPOS.cPickleJar' and n=='ec':
m=n='ExtensionClass'
else:
raise 'Unexpected BoboPOS class', (m, n)
elif m=='PersistentMapping': m='Persistence'
return Global(m,n)
class Inst:
_state=None
def __init__(self, c, args):
self._cls=c
self._args=args
def __setstate__(self, state): self._state=state
from pickle import INST, GLOBAL, MARK, BUILD, OBJ, REDUCE
InstanceType=type(Ghost())
class Unpickler(pickle.Unpickler):
dispatch={}
dispatch.update(pickle.Unpickler.dispatch)
def load_inst(self):
k = self.marker()
args = tuple(self.stack[k+1:])
del self.stack[k:]
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = _global(module, name)
value=Inst(klass, args)
self.append(value)
dispatch[INST] = load_inst
def load_global(self):
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = _global(module, name)
self.append(klass)
dispatch[GLOBAL] = load_global
def persistent_load(self, oid,
TupleType=type(()), Ghost=Ghost, p64=p64):
"Remap object ids from ZODB 2 stype to ZODB 3 style"
if type(oid) is TupleType:
oid, klass = oid
oid = p64(oid+1), klass
else:
oid = p64(oid+1)
Ghost=Ghost()
Ghost.oid=oid
return Ghost
class Pickler(pickle.Pickler):
dispatch={}
dispatch.update(pickle.Pickler.dispatch)
def persistent_id(self, object, Ghost=Ghost):
if hasattr(object, '__class__') and object.__class__ is Ghost:
return object.oid
def save_inst(self, object):
d = id(object)
cls = object.__class__
memo = self.memo
write = self.write
save = self.save
if cls is Global:
memo_len = len(memo)
write(GLOBAL + object._module + '\n' + object._name + '\n' +
self.put(memo_len))
memo[d] = (memo_len, object)
return
self.save_inst(object._cls)
save(object._args)
memo_len = len(memo)
write(REDUCE + self.put(memo_len))
memo[d] = (memo_len, object)
stuff=object._state
save(stuff)
write(BUILD)
dispatch[InstanceType] = save_inst
def save_global(self, object, name = None):
write = self.write
memo = self.memo
if (name is None):
name = object._name
module=object._module
memo_len = len(memo)
write(GLOBAL + module + '\n' + name + '\n' +
self.put(memo_len))
memo[id(object)] = (memo_len, object)
dispatch[type(Ghost)] = save_global
def fixpickle(p, oid):
pfile=StringIO(p)
unpickler=Unpickler(pfile)
newp=StringIO()
pickler=Pickler(newp,1)
pickler.dump(unpickler.load())
state=unpickler.load()
if oid==-1: state={'_container': state}
pickler.dump(state)
p=newp.getvalue()
return p
utilities/bbb.py deleted 100644 → 0
View file @ 1f97e260
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Read and (re-)format BoboPOS 2 database files
"""
import struct, string, sys, time, os
try: from cStringIO import StringIO
except: from StringIO import StringIO
ppml=None
file__version__=3.0
packed_version='SDBMV'+struct.pack(">f",file__version__)
def error(message, fatal=0, exc=0):
if exc:
sys.stderr.write('%s: %s' % (sys.exc_info()[0], sys.exc_info()[1]))
sys.stderr.write("\n%s\n" % message)
if fatal: sys.exit(fatal)
def _read_and_report(file, rpt=None, fromEnd=0, both=0, n=99999999, show=0,
forgive=0, export=0):
"""\
Read a file's index up to the given time.
"""
first=1
seek=file.seek
read=file.read
unpack=struct.unpack
split=string.split
join=string.join
find=string.find
seek(0,2)
file_size=file.tell()
if not export:
seek(0)
h=read(len(packed_version))
if h != packed_version:
if h[:4]=='FS21':
error("The input file is a ZODB File Storage\n"
"This script only works with ZODB 2 (BoboPOS) "
"data or export files."
,1)
else:
error("The input file is not a ZODB 2 database file.\n"
"This script only works with ZODB 2 (BoboPOS) "
"data or export files."
,1)
gmtime=time.gmtime
if fromEnd: pos=file_size
else: pos=newpos=(not export) and len(packed_version)
tlast=0
err=0
tnamelast=None
while 1:
if fromEnd:
seek(pos-4)
l=unpack(">i", read(4))[0]
if l==0:
b=pos
p=pos-4
while l==0:
p=p-4
seek(p)
l=unpack(">i", read(4))[0]
pos=p+4
error("nulls skipped from %s to %s" % (pos,b))
p=pos-l
if p < 0:
error('Corrupted data before %s' % pos)
if show > 0:
p=pos-show
if p < 0: p=0
seek(p)
p=read(pos-p)
else:
p=''
error(p,1)
pos=p
else:
pos=newpos
seek(pos)
h=read(24) # 24=header_size
if not h: break
if len(h) != 24: break
oid,prev,start,tlen,plen=unpack(">iidii",h)
if prev < 0 or (prev and (prev >= pos)):
error('Bad previous record pointer (%s) at %s' % (prev, pos))
if show > 0: error(read(show))
if not forgive:
err=1
break
if start < tlast:
error('record time stamps are not chronological at %s' % pos)
if show > 0: error(read(show))
if not forgive:
err=1
break
if plen > tlen or plen < 0 or oid < -999:
error('Corrupted data record at %s' % pos)
if show > 0: error(read(show))
err=1
break
newpos=pos+tlen
if newpos > file_size:
error('Truncated data record at %s' % pos)
if show > 0: error(read(show))
if not forgive:
err=1
break
seek(newpos-4)
if read(4) != h[16:20]:
__traceback_info__=pos, oid,prev,start,tlen,plen
error('Corrupted data record at %s' % pos)
if show > 0:
seek(pos+24)
error(read(show))
err=1
break
tlast=start-100
if rpt is None: continue
n=n-1
if n < 1: break
seek(pos+24)
if plen > 0:
p=read(plen)
if p[-1:] != '.':
error('Corrupted pickle at %s %s %s' % (pos,plen,len(p)))
if show > 0:
seek(pos+24)
error(read(show))
err=1
break
else: p=''
t=split(read(tlen-plen-28),'\t')
tname, user = (t+[''])[:2]
t=join(t[2:],'\t')
start,f=divmod(start,1)
y,m,d,h,mn,s=gmtime(start)[:6]
s=s+f
start="%.4d-%.2d-%.2d %.2d:%.2d:%.3f" % (y,m,d,h,mn,s)
rpt(pos,oid,start,tname,user,t,p,first,tname!=tnamelast)
first=0
tnamelast=tname
if err and both and not fromEnd:
_read_and_report(file, rpt, 1, 0, n, show)
rpt(None, None, None, None, None, None, None, None, None)
def none(*ignored): pass
def positions(pos, *ignored):
if pos is not None: sys.stdout.write("%s\n" % pos)
def oids(pos, oid, *ignored):
if pos is not None: sys.stdout.write("%s\n" % oid)
def tab_delimited(*args):
sys.stdout.write("%s\n" % string.join(args[:6],'\t'))
def undo_log(pos, oid, start, tname, user, t, p, first, newtrans):
if not newtrans: return
sys.stdout.write("%s:\t%s\t%s\t%s\n" % (pos, start, user, t))
reports={
'none': (none,
('Read a database file checking for errors',
'but producing no output')
),
'oids': (oids,
('Read the database and output object ids',)),
'positions': (positions,
('Read the database and output record positions',)),
'tab_delimited': (tab_delimited,
('Output record meta-data in tab-delimited format',)),
'undo': (undo_log,
(
'Output a transaction summary that shows the position of',
'each transaction. This is useful for undoing ',
'transactions from the OS command line when',
'some programming error has caused objects to get to',
'a state where Zope can\'t start up.',
'',
'Eventually, there will be an undo utility for undoing',
'individual transactions. For now, you can simply',
'truncate the file at the position of a problem',
'transaction to return the database to the state it',
'was in before the transaction',
)),
}
def main(argv):
import getopt
items=reports.items()
items.sort()
usage="""Usage: python %s [options] filename
where filename is the name of the database file.
options:
-r report
Specify an output report.
The valid reports are:\n\n\t\t%s
-e
Read the file from back to front
-l n
Show only n records
-b
If an error is encountered while reading from front,
ret reading from the back.
-s n
If a corrupted data record is found, show the first n
bytes of the corrupted record.
-f filename
Convert to ZODB 3 File-Storage format
-p path
Add a directory to the Python path.
-x
The input file is a ZODB 2 export file.
""" % (sys.argv[0],
string.join(map(
lambda i:
("%s --\n\t\t\t%s" % (i[0], string.join(i[1][1],'\n\t\t\t'))),
items),
',\n\n\t\t'))
sys.path.append(os.path.split(sys.argv[0])[0])
try:
opts, args = getopt.getopt(argv,'r:ebl:s:f:p:x')
filename,=args
except: error(usage,1,1)
try: file=open(filename,'rb')
except: error('Coud not open %s' % filename,1,1)
rpt=none
fromEnd=0
both=0
n=99999999
show=0
export=0
convert=0
for o, v in opts:
o=o[1:]
if o=='r':
try: rpt=reports[v][0]
except: error('Invalid report: %s' % v, 1)
elif o=='l':
try: n=string.atoi(v)
except: error('The number of records, %s, shuld ne an integer'
% v, 1)
elif o=='s':
try: show=string.atoi(v)
except: error('The number of bytes, %s, shuld ne an integer'
% v, 1)
elif o=='e': fromEnd=1
elif o=='x': export=1
elif o=='f': convert=1
elif o=='b': both=1
elif o=='p':
if v=='-':
v=os.path.join(
os.path.split(sys.argv[0])[0],
'..','lib','python')
sys.path.insert(0,v)
print sys.path
else: error('Unrecognized option: -%s' % o, 1)
if convert:
import FS
if export:
rpt=FS.ZEXP(v, file).rpt
else:
rpt=FS.FS(v, file).rpt
_read_and_report(file, rpt, fromEnd, both, n, show,
forgive=1, export=export)
if __name__=='__main__': main(sys.argv[1:])
utilities/fixbbbts.py deleted 100644 → 0
View file @ 1f97e260
import sys
__doc__="""Fix BoboPOS time stamps
If a system has a problem with it's clock setting, it may cause
database records to be written with time stamps in the future. This
causes problems when the clock is fixed or when the data are moved to
a system that doesn't have a broken clock.
The database has a requirement that records should be chronologically
ordered and that times not be in the future.
This copies a database, restamping the times as it goes.
%s [options] file1 file2
Copy file1 to file2 restamping the records.
options:
-o offset
Records that are later offset seconds in the past
are moved back to offset seconds in the past plus
some small offset chosen so that times are not the same and
are chronological.
""" % sys.argv[0]
InvalidFormat='Format Error'
Corrupted='Data Corruption'
def main():
import getopt, string, struct, time
file__version__=3.0
packed_version='SDBMV'+struct.pack(">f",file__version__)
try:
opts, args = getopt.getopt(sys.argv[1:], 'o:')
file1, file2 = args
offset=86400
for o, v in opts:
if o=='-o':
offset=string.atoi(v)
except:
print __doc__
print "%s: %s" % sys.exc_info()[:2]
start=time.time()-offset
next=start+0.001
input=open(file1,'rb')
read=input.read
output=open(file2,'wb')
write=output.write
pack=struct.pack
unpack=struct.unpack
h=read(len(packed_version))
if h != packed_version:
raise InvalidFormat, 'This is not a BoboPOS file'
write(h)
pos=len(h)
while 1:
h=read(24)
if not h: break
if len(h) < 24: raise Corrupted, pos
oid, prev, t, tlen, plen = unpack(">iidii", h)
if start is None or t > start:
t=next
next=next+0.001
start=None
if plen > tlen or tlen < 28: raise Corrupted, pos
write(pack(">iidii", oid, prev, t, tlen, plen))
l=tlen-28
s=8196
while l > 0:
if s > l: s=l
d=read(s)
if not d: raise Corrupted, pos
write(d)
l=l-len(d)
d=read(4)
if d != h[16:20]: raise Corrupted, pos
write(d)
pos=pos+tlen
if __name__=='__main__': main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7