matrix 3.32 KB
Newer Older
1 2 3 4 5 6
#!/usr/bin/python

import sys
import os
import math
import optparse
7
import traceback
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
from time import time

from neo.tests.functional import NEOCluster
from ZODB.FileStorage import FileStorage

def run(masters, storages, replicas, partitions, datafs, verbose):
    print "Import of %s with m=%s, s=%s, r=%s, p=%s" % (
            datafs, masters, storages, replicas, partitions)
    # cluster
    neo = NEOCluster(
        db_list=['test_import_%d' % i for i in xrange(storages)],
        clear_databases=True,
        partitions=partitions,
        replicas=replicas,
        master_node_count=masters,
        verbose=verbose,
    )
    # import
    neo_storage = neo.getZODBStorage()
    dfs_storage = FileStorage(file_name=datafs)
    neo.start()
    start = time()
30 31 32 33 34
    try:
        try:
            neo_storage.copyTransactionsFrom(dfs_storage)
            return time() - start
        except:
35 36
            traceback.print_exc()
            return None
37 38
    finally:
        neo.stop()
39 40 41 42 43 44 45

def runMatrix(datafs, storages, replicas, verbose):
    stats = {}
    size = float(os.path.getsize(datafs))
    for s in storages:
        for r in [r for r in replicas if r < s]:
            stats.setdefault(s, {})
46 47 48 49
            result = run(1, s, r, 100, datafs, verbose)
            if result is not None:
                result = size / result / 1024
            stats[s][r] = result
50 51 52 53 54 55 56 57 58 59 60 61 62 63
    return stats

def buildArray(storages, replicas, results):
    # draw an array with results
    fmt = '|' + '|'.join(['  %8s  '] * (len(replicas) + 1)) + '|\n'
    sep = '+' + '+'.join(['-' * 12] * (len(replicas) + 1)) + '+\n'
    report = sep
    report += fmt % tuple(['S\R'] + range(0, len(replicas)))
    report += sep
    for s in storages:
        values = []
        assert s in results
        for r in replicas:
            if r in results[s]:
64 65 66 67
                if results[s][r] is None:
                    values.append('FAIL')
                else:
                    values.append('%8.1f' % results[s][r])
68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
            else:
                values.append('N/A')
        report += fmt % (tuple([s] + values))
        report += sep
    return report

if __name__ == "__main__":

    # options
    parser = optparse.OptionParser()
    parser.add_option('-d', '--datafs')
    parser.add_option('', '--min-storages')
    parser.add_option('', '--max-storages')
    parser.add_option('', '--min-replicas')
    parser.add_option('', '--max-replicas')
    parser.add_option('-v', '--verbose', action='store_true')
    (options, args) = parser.parse_args()

    # check arguments
    if not options.datafs or not os.path.exists(options.datafs):
        sys.exit('Missing or wrong data.fs argument')

    # parse args
    min_s = int(options.min_storages or 1)
    max_s = int(options.max_storages or 2)
    min_r = int(options.min_replicas or 0)
    max_r = int(options.max_replicas or 1)
    datafs = options.datafs
    verbose = options.verbose or False

    # build storage (logarithm) & replicas (linear) lists
    min_s2 = int(math.log(min_s, 2))
    max_s2 = int(math.log(max_s, 2))
    storages = [2 ** x for x in range(min_s2, max_s2 + 1)]
    if storages[0] < min_s:
        storages[0] = min_s
    if storages[-1] < max_s:
        storages.append(max_s)
    replicas = range(min_r, max_r + 1)
    results = runMatrix(datafs, storages, replicas, verbose)
    print buildArray(storages, replicas, results)