Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
143
Merge Requests
143
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
erp5
Commits
a595132a
Commit
a595132a
authored
Aug 13, 2013
by
Benjamin Blanc
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
scalability: runScalabilityTestSuite: Restructure code.
parent
fd014995
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
59 additions
and
89 deletions
+59
-89
erp5/util/scalability/runScalabilityTestSuite.py
erp5/util/scalability/runScalabilityTestSuite.py
+59
-89
No files found.
erp5/util/scalability/runScalabilityTestSuite.py
View file @
a595132a
...
@@ -2,6 +2,7 @@
...
@@ -2,6 +2,7 @@
import
argparse
import
argparse
import
os
import
os
import
shutil
import
time
import
time
import
sys
import
sys
import
multiprocessing
import
multiprocessing
...
@@ -25,7 +26,9 @@ MAX_TESTING_TIME = 60
...
@@ -25,7 +26,9 @@ MAX_TESTING_TIME = 60
MAX_GETTING_CONNECTION_TIME
=
60
*
5
MAX_GETTING_CONNECTION_TIME
=
60
*
5
def
getConnection
(
erp5_url
,
log
):
def
getConnection
(
erp5_url
,
log
):
"""
Return a connection with the erp5 instance.
"""
start_time
=
time
.
time
()
start_time
=
time
.
time
()
count
=
0
count
=
0
while
MAX_GETTING_CONNECTION_TIME
>
time
.
time
()
-
start_time
:
while
MAX_GETTING_CONNECTION_TIME
>
time
.
time
()
-
start_time
:
...
@@ -45,6 +48,9 @@ def getConnection(erp5_url, log):
...
@@ -45,6 +48,9 @@ def getConnection(erp5_url, log):
raise
ValueError
(
"Cannot get new connection after %d try (for %s s)"
%
(
count
,
str
(
time
.
time
()
-
start_time
)))
raise
ValueError
(
"Cannot get new connection after %d try (for %s s)"
%
(
count
,
str
(
time
.
time
()
-
start_time
)))
def
waitFor0PendingActivities
(
erp5_url
,
log
):
def
waitFor0PendingActivities
(
erp5_url
,
log
):
"""
Waiting while there are no pending activities on the erp5 instance.
"""
start_time
=
time
.
time
()
start_time
=
time
.
time
()
parsed
=
urlparse
.
urlparse
(
erp5_url
)
parsed
=
urlparse
.
urlparse
(
erp5_url
)
user
=
parsed
.
username
;
user
=
parsed
.
username
;
...
@@ -82,9 +88,10 @@ def waitFor0PendingActivities(erp5_url, log):
...
@@ -82,9 +88,10 @@ def waitFor0PendingActivities(erp5_url, log):
if
not
ok
:
if
not
ok
:
raise
ValueError
(
"Cannot waitFor0PendingActivities after %d try (for %s s)"
%
(
count
,
str
(
time
.
time
()
-
start_time
)))
raise
ValueError
(
"Cannot waitFor0PendingActivities after %d try (for %s s)"
%
(
count
,
str
(
time
.
time
()
-
start_time
)))
def
getCreatedDocumentNumberFromERP5
(
erp5_url
,
log
):
def
getCreatedDocumentNumberFromERP5
(
erp5_url
,
log
):
"""
"""
Get the number of created documents
Get the number of created documents
from erp5 instance.
"""
"""
log
(
"count docs number from ERP5 instance"
)
log
(
"count docs number from ERP5 instance"
)
count_retry
=
0
count_retry
=
0
...
@@ -233,45 +240,19 @@ class ScalabilityLauncher(object):
...
@@ -233,45 +240,19 @@ class ScalabilityLauncher(object):
ScalabilityLauncher
.
_checkParsedArguments
(
namespace
)
ScalabilityLauncher
.
_checkParsedArguments
(
namespace
)
return
namespace
return
namespace
def
checkERP5Instance
(
self
):
def
moveLogs
(
self
,
folder_name
):
"""
# Get file paths
Check if erp5_instance is accessible
file_to_move_list
=
glob
.
glob
(
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
"""
pass
def
_returnFileContentList
(
self
,
path
,
scheme
):
"""
"""
complete_scheme
=
os
.
path
.
join
(
path
,
scheme
)
file_path_list
=
glob
.
glob
(
complete_scheme
)
content_list
=
[]
for
file_path
in
file_path_list
:
opened_file
=
open
(
file_path
,
'r'
)
content_list
.
append
(
''
.
join
(
opened_file
.
readlines
()))
opened_file
.
close
()
return
content_list
def
returnLogList
(
self
):
return
self
.
_returnFileContentList
(
self
.
__argumentNamespace
.
log_path
,
"%s*.log"
%
LOG_FILE_PREFIX
)
def
returnCsvList
(
self
):
return
self
.
_returnFileContentList
(
self
.
__argumentNamespace
.
log_path
,
"%s*.csv"
%
LOG_FILE_PREFIX
)
def
getCreatedDocumentNumber
(
self
):
number
=
0
complete_scheme
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
"%s*.csv"
%
LOG_FILE_PREFIX
)
file_path_list
=
glob
.
glob
(
complete_scheme
)
for
file_path
in
file_path_list
:
number
=
number
+
sum
(
1
for
line
in
open
(
file_path
))
return
number
def
cleanUpCsv
(
self
):
files_to_delete
=
glob
.
glob
(
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
"%s*.csv"
%
LOG_FILE_PREFIX
))
"%s*.csv"
%
LOG_FILE_PREFIX
))
for
file_path
in
files_to_delete
:
file_to_move_list
+=
glob
.
glob
(
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
os
.
remove
(
file_path
)
"%s*.log"
%
LOG_FILE_PREFIX
))
# Create folder
new_directory_path
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
folder_name
)
if
not
os
.
path
.
exists
(
new_directory_path
):
os
.
makedirs
(
new_directory_path
)
# Move files
for
file_to_move
in
file_to_move_list
:
shutil
.
move
(
file_to_move
,
new_directory_path
)
def
getNextTest
(
self
):
def
getNextTest
(
self
):
"""
"""
...
@@ -287,16 +268,6 @@ class ScalabilityLauncher(object):
...
@@ -287,16 +268,6 @@ class ScalabilityLauncher(object):
next_test
=
ScalabilityTest
(
decoded_data
,
self
.
test_result
)
next_test
=
ScalabilityTest
(
decoded_data
,
self
.
test_result
)
return
next_test
return
next_test
def
getCreatedDocumentNumber
(
self
):
# First file line is corresponding to header
number
=
-
1
complete_scheme
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
log_path
,
"%s*.csv"
%
LOG_FILE_PREFIX
)
file_path_list
=
glob
.
glob
(
complete_scheme
)
for
file_path
in
file_path_list
:
number
=
number
+
sum
(
1
for
line
in
open
(
file_path
))
return
number
def
run
(
self
):
def
run
(
self
):
self
.
log
(
"Scalability Launcher started, with:"
)
self
.
log
(
"Scalability Launcher started, with:"
)
self
.
log
(
"Test suite master url: %s"
%
self
.
__argumentNamespace
.
test_suite_master_url
)
self
.
log
(
"Test suite master url: %s"
%
self
.
__argumentNamespace
.
test_suite_master_url
)
...
@@ -306,34 +277,33 @@ class ScalabilityLauncher(object):
...
@@ -306,34 +277,33 @@ class ScalabilityLauncher(object):
self
.
log
(
"Node title: %s"
%
self
.
__argumentNamespace
.
node_title
)
self
.
log
(
"Node title: %s"
%
self
.
__argumentNamespace
.
node_title
)
self
.
log
(
"ERP5 url: %s"
%
self
.
__argumentNamespace
.
erp5_url
)
self
.
log
(
"ERP5 url: %s"
%
self
.
__argumentNamespace
.
erp5_url
)
max_time
=
36000
start_time
=
time
.
time
()
error_message_set
,
exit_status
=
set
(),
0
error_message_set
,
exit_status
=
set
(),
0
# Get suite informations
# Get suite informations
suite
=
makeSuite
(
self
.
__argumentNamespace
.
test_suite
,
self
.
log
)
suite
=
makeSuite
(
self
.
__argumentNamespace
.
test_suite
,
self
.
log
)
test_suites
=
suite
.
getTestList
()
test_suite_list
=
suite
.
getTestList
()
# Main loop
while
True
:
while
time
.
time
()
-
start_time
<
max_time
:
time
.
sleep
(
5
)
current_test
=
self
.
getNextTest
()
current_test
=
self
.
getNextTest
()
if
current_test
==
None
:
if
current_test
==
None
:
self
.
log
(
"No Test Case Ready"
)
self
.
log
(
"No Test Case Ready"
)
else
:
else
:
error_count
=
1
error_count
=
1
#
Waiting for 0-
pending activities
#
Do not run a test while there are
pending activities
waitFor0PendingActivities
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
waitFor0PendingActivities
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
# Get the number of documents present before running the test.
previous_document_number
=
getCreatedDocumentNumberFromERP5
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
previous_document_number
=
getCreatedDocumentNumberFromERP5
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
self
.
log
(
"previous_document_number: %d"
%
previous_document_number
)
self
.
log
(
"previous_document_number: %d"
%
previous_document_number
)
# Here call a runScalabilityTest ( placed on product/ERP5Type/tests ) ?
self
.
log
(
"Test Case %s is running..."
%
(
current_test
.
title
))
self
.
log
(
"Test Case %s is running..."
%
(
current_test
.
title
))
try
:
try
:
# Prepare command parameters
current_test_number
=
int
(
current_test
.
title
)
current_test_number
=
int
(
current_test
.
title
)
test_duration
=
suite
.
getTestDuration
(
current_test_number
)
test_duration
=
suite
.
getTestDuration
(
current_test_number
)
benchmarks_path
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
erp5_location
,
suite
.
getTestPath
())
benchmarks_path
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
erp5_location
,
suite
.
getTestPath
())
#TODO: generate a basic user file with all scalability users.
user_file_full_path
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
erp5_location
,
suite
.
getUsersFilePath
())
user_file_full_path
=
os
.
path
.
join
(
self
.
__argumentNamespace
.
erp5_location
,
suite
.
getUsersFilePath
())
user_file_path
=
os
.
path
.
split
(
user_file_full_path
)[
0
]
user_file_path
=
os
.
path
.
split
(
user_file_full_path
)[
0
]
user_file
=
os
.
path
.
split
(
user_file_full_path
)[
1
]
user_file
=
os
.
path
.
split
(
user_file_full_path
)[
1
]
...
@@ -343,35 +313,33 @@ class ScalabilityLauncher(object):
...
@@ -343,35 +313,33 @@ class ScalabilityLauncher(object):
self
.
log
(
"user_number: %s"
%
str
(
user_number
))
self
.
log
(
"user_number: %s"
%
str
(
user_number
))
self
.
log
(
"test_duration: %ss"
%
str
(
test_duration
))
self
.
log
(
"test_duration: %ss"
%
str
(
test_duration
))
# Generate commands
# Generate commands
to run
command_list
=
[]
command_list
=
[]
i
=
0
for
test_suite
in
test_suite_list
:
for
suite
in
test_suites
:
command_list
.
append
([
tester_path
,
command_list
[
i
]
=
[
tester_path
,
self
.
__argumentNamespace
.
erp5_url
,
self
.
__argumentNamespace
.
erp5_url
,
str
(
user_number
),
str
(
user_number
/
len
(
test_suite_list
)
),
suite
,
test_
suite
,
'--benchmark-path-list'
,
benchmarks_path
,
'--benchmark-path-list'
,
benchmarks_path
,
'--users-file-path'
,
user_file_path
,
'--users-file-path'
,
user_file_path
,
'--users-file'
,
user_file
,
'--users-file'
,
user_file
,
'--filename-prefix'
,
"%s_%s_"
%
(
LOG_FILE_PREFIX
,
current_test
.
title
),
'--filename-prefix'
,
"%s_%s_"
%
(
LOG_FILE_PREFIX
,
current_test
.
title
),
'--report-directory'
,
self
.
__argumentNamespace
.
log_path
,
'--report-directory'
,
self
.
__argumentNamespace
.
log_path
,
'--repeat'
,
"%s"
%
str
(
MAX_DOCUMENTS
),
'--repeat'
,
"%s"
%
str
(
MAX_DOCUMENTS
),
]
])
i
+=
1
# Launch
# Launch
tester_process_list
=
[]
tester_process_list
=
[]
for
i
in
range
(
i
,
len
(
command_list
))
:
for
command
in
command_list
:
self
.
log
(
"command: %s"
%
str
(
command
_list
[
i
]
))
self
.
log
(
"command: %s"
%
str
(
command
))
tester_process_list
[
i
]
=
subprocess
.
Popen
(
command_list
[
i
]
)
tester_process_list
.
append
(
subprocess
.
Popen
(
command
)
)
# Sleep
# Sleep
time
.
sleep
(
test_duration
)
time
.
sleep
(
test_duration
)
# Stop
# Stop
for
i
in
range
(
i
,
len
(
tester_process_list
))
:
for
tester_process
in
tester_process_list
:
tester_process
_list
[
i
]
.
send_signal
(
signal
.
SIGINT
)
tester_process
.
send_signal
(
signal
.
SIGINT
)
# Ok
# Ok
error_count
=
0
error_count
=
0
...
@@ -381,20 +349,21 @@ class ScalabilityLauncher(object):
...
@@ -381,20 +349,21 @@ class ScalabilityLauncher(object):
raise
ValueError
(
"Tester call failed"
)
raise
ValueError
(
"Tester call failed"
)
self
.
log
(
"Test Case %s is finish"
%
(
current_test
.
title
))
self
.
log
(
"Test Case %s is finish"
%
(
current_test
.
title
))
self
.
log
(
"Going to count the number of created documents"
)
self
.
log
(
"Going to count the number of created documents"
)
time
.
sleep
(
120
)
# Wait for 0 pending activities before counting
waitFor0PendingActivities
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
# Count created documents
current_document_number
=
getCreatedDocumentNumberFromERP5
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
current_document_number
=
getCreatedDocumentNumberFromERP5
(
self
.
__argumentNamespace
.
erp5_url
,
self
.
log
)
created_document_number
=
current_document_number
-
previous_document_number
created_document_number
=
current_document_number
-
previous_document_number
self
.
log
(
"previous_document_number: %d"
%
previous_document_number
)
self
.
log
(
"previous_document_number: %d"
%
previous_document_number
)
self
.
log
(
"current_document_number: %d"
%
current_document_number
)
self
.
log
(
"current_document_number: %d"
%
current_document_number
)
self
.
log
(
"created_document_number: %d"
%
created_document_number
)
self
.
log
(
"created_document_number: %d"
%
created_document_number
)
created_document_per_hour_number
=
(
(
float
(
created_document_number
)
*
60
*
60
)
/
float
(
test_duration
)
)
created_document_per_hour_number
=
(
(
float
(
created_document_number
)
*
60
*
60
)
/
float
(
test_duration
)
)
#log_contents = self.returnLogList()
# Move csv/logs
#csv_contents = self.returnCsvList()
self
.
moveLogs
(
current_test
.
title
)
self
.
cleanUpCsv
()
# Make a connection with ERP5 master
retry_time
=
2.0
retry_time
=
2.0
proxy
=
taskdistribution
.
ServerProxy
(
proxy
=
taskdistribution
.
ServerProxy
(
self
.
__argumentNamespace
.
test_suite_master_url
,
self
.
__argumentNamespace
.
test_suite_master_url
,
...
@@ -405,26 +374,27 @@ class ScalabilityLauncher(object):
...
@@ -405,26 +374,27 @@ class ScalabilityLauncher(object):
current_test
.
relative_path
,
current_test
.
relative_path
,
current_test
.
title
current_test
.
title
)
)
# Generate output
results
=
"created docs=%d
\
n
"
\
results
=
"created docs=%d
\
n
"
\
"duration=%d
\
n
"
\
"duration=%d
\
n
"
\
"number of tests=%d
\
n
"
\
"number of tests=%d
\
n
"
\
%
(
%
(
created_document_number
,
created_document_number
,
test_duration
,
test_duration
,
len
(
test_suite
s
)
len
(
test_suite
_list
)
)
)
self
.
log
(
"results: %s"
%
results
)
self
.
log
(
"results:"
)
self
.
log
(
"%s"
%
results
)
self
.
log
(
"%s doc in %s secs = %s docs per hour"
%
(
created_document_number
,
test_duration
,
created_document_per_hour_number
))
self
.
log
(
"%s doc in %s secs = %s docs per hour"
%
(
created_document_number
,
test_duration
,
created_document_per_hour_number
))
# Stop test case
test_result_line_test
.
stop
(
stdout
=
results
,
test_result_line_test
.
stop
(
stdout
=
results
,
test_count
=
created_document_number
,
test_count
=
len
(
test_suite_list
)
,
error_count
=
error_count
,
error_count
=
error_count
,
duration
=
test_duration
)
duration
=
test_duration
)
self
.
log
(
"Test Case Stopped"
)
self
.
log
(
"Test Case Stopped"
)
# Sleep between two loops
time
.
sleep
(
5
)
return
error_message_set
,
exit_status
return
error_message_set
,
exit_status
def
main
():
def
main
():
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment