Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
22b05a1f
Commit
22b05a1f
authored
Mar 30, 2018
by
Francisco Javier López
Committed by
Douwe Maan
Mar 30, 2018
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Extend API for exporting a project with direct upload URL
parent
7c36e856
Changes
19
Show whitespace changes
Inline
Side-by-side
Showing
19 changed files
with
671 additions
and
25 deletions
+671
-25
app/models/project.rb
app/models/project.rb
+14
-2
app/services/projects/import_export/export_service.rb
app/services/projects/import_export/export_service.rb
+25
-8
app/workers/project_export_worker.rb
app/workers/project_export_worker.rb
+11
-3
changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml
...gs/unreleased/fj-42685-extend-project-export-endpoint.yml
+5
-0
doc/api/project_import_export.md
doc/api/project_import_export.md
+17
-2
lib/api/project_export.rb
lib/api/project_export.rb
+18
-1
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
...ort/after_export_strategies/base_after_export_strategy.rb
+83
-0
lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
...after_export_strategies/download_notification_strategy.rb
+17
-0
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
...ort_export/after_export_strategies/web_upload_strategy.rb
+61
-0
lib/gitlab/import_export/after_export_strategy_builder.rb
lib/gitlab/import_export/after_export_strategy_builder.rb
+24
-0
lib/gitlab/import_export/shared.rb
lib/gitlab/import_export/shared.rb
+13
-1
spec/fixtures/api/schemas/public_api/v4/project/export_status.json
...ures/api/schemas/public_api/v4/project/export_status.json
+9
-2
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
...fter_export_strategies/base_after_export_strategy_spec.rb
+104
-0
spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
...xport/after_export_strategies/web_upload_strategy_spec.rb
+36
-0
spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb
...itlab/import_export/after_export_strategy_builder_spec.rb
+29
-0
spec/models/project_spec.rb
spec/models/project_spec.rb
+18
-1
spec/requests/api/project_export_spec.rb
spec/requests/api/project_export_spec.rb
+74
-5
spec/services/projects/import_export/export_service_spec.rb
spec/services/projects/import_export/export_service_spec.rb
+85
-0
spec/workers/project_export_worker_spec.rb
spec/workers/project_export_worker_spec.rb
+28
-0
No files found.
app/models/project.rb
View file @
22b05a1f
...
@@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base
...
@@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base
@errors
=
original_errors
@errors
=
original_errors
end
end
def
add_export_job
(
current_user
:,
params:
{})
def
add_export_job
(
current_user
:,
after_export_strategy:
nil
,
params:
{})
job_id
=
ProjectExportWorker
.
perform_async
(
current_user
.
id
,
self
.
id
,
params
)
job_id
=
ProjectExportWorker
.
perform_async
(
current_user
.
id
,
self
.
id
,
after_export_strategy
,
params
)
if
job_id
if
job_id
Rails
.
logger
.
info
"Export job started for project ID
#{
self
.
id
}
with job ID
#{
job_id
}
"
Rails
.
logger
.
info
"Export job started for project ID
#{
self
.
id
}
with job ID
#{
job_id
}
"
...
@@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base
...
@@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base
def
export_status
def
export_status
if
export_in_progress?
if
export_in_progress?
:started
:started
elsif
after_export_in_progress?
:after_export_action
elsif
export_project_path
elsif
export_project_path
:finished
:finished
else
else
...
@@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base
...
@@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base
import_export_shared
.
active_export_count
>
0
import_export_shared
.
active_export_count
>
0
end
end
def
after_export_in_progress?
import_export_shared
.
after_export_in_progress?
end
def
remove_exports
def
remove_exports
return
nil
unless
export_path
.
present?
return
nil
unless
export_path
.
present?
FileUtils
.
rm_rf
(
export_path
)
FileUtils
.
rm_rf
(
export_path
)
end
end
def
remove_exported_project_file
return
unless
export_project_path
.
present?
FileUtils
.
rm_f
(
export_project_path
)
end
def
full_path_slug
def
full_path_slug
Gitlab
::
Utils
.
slugify
(
full_path
.
to_s
)
Gitlab
::
Utils
.
slugify
(
full_path
.
to_s
)
end
end
...
...
app/services/projects/import_export/export_service.rb
View file @
22b05a1f
module
Projects
module
Projects
module
ImportExport
module
ImportExport
class
ExportService
<
BaseService
class
ExportService
<
BaseService
def
execute
(
_
options
=
{})
def
execute
(
after_export_strategy
=
nil
,
options
=
{})
@shared
=
project
.
import_export_shared
@shared
=
project
.
import_export_shared
save_all
save_all!
execute_after_export_action
(
after_export_strategy
)
end
end
private
private
def
save_all
def
execute_after_export_action
(
after_export_strategy
)
if
[
version_saver
,
avatar_saver
,
project_tree_saver
,
uploads_saver
,
repo_saver
,
wiki_repo_saver
].
all?
(
&
:save
)
return
unless
after_export_strategy
unless
after_export_strategy
.
execute
(
current_user
,
project
)
cleanup_and_notify_error
end
end
def
save_all!
if
save_services
Gitlab
::
ImportExport
::
Saver
.
save
(
project:
project
,
shared:
@shared
)
Gitlab
::
ImportExport
::
Saver
.
save
(
project:
project
,
shared:
@shared
)
notify_success
notify_success
else
else
cleanup_and_notify
cleanup_and_notify
_error!
end
end
end
end
def
save_services
[
version_saver
,
avatar_saver
,
project_tree_saver
,
uploads_saver
,
repo_saver
,
wiki_repo_saver
].
all?
(
&
:save
)
end
def
version_saver
def
version_saver
Gitlab
::
ImportExport
::
VersionSaver
.
new
(
shared:
@shared
)
Gitlab
::
ImportExport
::
VersionSaver
.
new
(
shared:
@shared
)
end
end
...
@@ -41,19 +55,22 @@ module Projects
...
@@ -41,19 +55,22 @@ module Projects
Gitlab
::
ImportExport
::
WikiRepoSaver
.
new
(
project:
project
,
shared:
@shared
)
Gitlab
::
ImportExport
::
WikiRepoSaver
.
new
(
project:
project
,
shared:
@shared
)
end
end
def
cleanup_and_notify
def
cleanup_and_notify
_error
Rails
.
logger
.
error
(
"Import/Export - Project
#{
project
.
name
}
with ID:
#{
project
.
id
}
export error -
#{
@shared
.
errors
.
join
(
', '
)
}
"
)
Rails
.
logger
.
error
(
"Import/Export - Project
#{
project
.
name
}
with ID:
#{
project
.
id
}
export error -
#{
@shared
.
errors
.
join
(
', '
)
}
"
)
FileUtils
.
rm_rf
(
@shared
.
export_path
)
FileUtils
.
rm_rf
(
@shared
.
export_path
)
notify_error
notify_error
end
def
cleanup_and_notify_error!
cleanup_and_notify_error
raise
Gitlab
::
ImportExport
::
Error
.
new
(
@shared
.
errors
.
join
(
', '
))
raise
Gitlab
::
ImportExport
::
Error
.
new
(
@shared
.
errors
.
join
(
', '
))
end
end
def
notify_success
def
notify_success
Rails
.
logger
.
info
(
"Import/Export - Project
#{
project
.
name
}
with ID:
#{
project
.
id
}
successfully exported"
)
Rails
.
logger
.
info
(
"Import/Export - Project
#{
project
.
name
}
with ID:
#{
project
.
id
}
successfully exported"
)
notification_service
.
project_exported
(
@project
,
@current_user
)
end
end
def
notify_error
def
notify_error
...
...
app/workers/project_export_worker.rb
View file @
22b05a1f
...
@@ -4,11 +4,19 @@ class ProjectExportWorker
...
@@ -4,11 +4,19 @@ class ProjectExportWorker
sidekiq_options
retry:
3
sidekiq_options
retry:
3
def
perform
(
current_user_id
,
project_id
,
params
=
{})
def
perform
(
current_user_id
,
project_id
,
after_export_strategy
=
{},
params
=
{})
params
=
params
.
with_indifferent_access
current_user
=
User
.
find
(
current_user_id
)
current_user
=
User
.
find
(
current_user_id
)
project
=
Project
.
find
(
project_id
)
project
=
Project
.
find
(
project_id
)
after_export
=
build!
(
after_export_strategy
)
::
Projects
::
ImportExport
::
ExportService
.
new
(
project
,
current_user
,
params
).
execute
::
Projects
::
ImportExport
::
ExportService
.
new
(
project
,
current_user
,
params
).
execute
(
after_export
)
end
private
def
build!
(
after_export_strategy
)
strategy_klass
=
after_export_strategy
&
.
delete
(
'klass'
)
Gitlab
::
ImportExport
::
AfterExportStrategyBuilder
.
build!
(
strategy_klass
,
after_export_strategy
)
end
end
end
end
changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml
0 → 100644
View file @
22b05a1f
---
title
:
Extend API for exporting a project with direct upload URL
merge_request
:
17686
author
:
type
:
added
doc/api/project_import_export.md
View file @
22b05a1f
...
@@ -8,6 +8,14 @@
...
@@ -8,6 +8,14 @@
Start a new export.
Start a new export.
The endpoint also accepts an
`upload`
param. This param is a hash that contains
all the necessary information to upload the exported project to a web server or
to any S3-compatible platform. At the moment we only support binary
data file uploads to the final server.
If the
`upload`
params is present,
`upload[url]`
param is required.
(
**Note:**
This feature was introduced in GitLab 10.7)
```
http
```
http
POST /projects/:id/export
POST /projects/:id/export
```
```
...
@@ -16,9 +24,12 @@ POST /projects/:id/export
...
@@ -16,9 +24,12 @@ POST /projects/:id/export
| --------- | -------------- | -------- | ---------------------------------------- |
| --------- | -------------- | -------- | ---------------------------------------- |
|
`id`
| integer/string | yes | The ID or
[
URL-encoded path of the project
](
README.md#namespaced-path-encoding
)
owned by the authenticated user |
|
`id`
| integer/string | yes | The ID or
[
URL-encoded path of the project
](
README.md#namespaced-path-encoding
)
owned by the authenticated user |
|
`description`
| string | no | Overrides the project description |
|
`description`
| string | no | Overrides the project description |
|
`upload`
| hash | no | Hash that contains the information to upload the exported project to a web server |
|
`upload[url]`
| string | yes | The URL to upload the project |
|
`upload[http_method]`
| string | no | The HTTP method to upload the exported project. Only
`PUT`
and
`POST`
methods allowed. Default is
`PUT`
|
```
console
```
console
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK"
--form "description=Foo Bar" https://gitlab.example.com/api/v4/projects/1/export
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK"
https://gitlab.example.com/api/v4/projects/1/export --data "description=FooBar&upload[http_method]=PUT&upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd"
```
```
```
json
```
json
...
@@ -43,7 +54,11 @@ GET /projects/:id/export
...
@@ -43,7 +54,11 @@ GET /projects/:id/export
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
```
```
Status can be one of
`none`
,
`started`
, or
`finished`
.
Status can be one of
`none`
,
`started`
,
`after_export_action`
or
`finished`
. The
`after_export_action`
state represents that the export process has been completed successfully and
the platform is performing some actions on the resulted file. For example, sending
an email notifying the user to download the file, uploading the exported file
to a web server, etc.
`_links`
are only present when export has finished.
`_links`
are only present when export has finished.
...
...
lib/api/project_export.rb
View file @
22b05a1f
...
@@ -33,11 +33,28 @@ module API
...
@@ -33,11 +33,28 @@ module API
end
end
params
do
params
do
optional
:description
,
type:
String
,
desc:
'Override the project description'
optional
:description
,
type:
String
,
desc:
'Override the project description'
optional
:upload
,
type:
Hash
do
optional
:url
,
type:
String
,
desc:
'The URL to upload the project'
optional
:http_method
,
type:
String
,
default:
'PUT'
,
desc:
'HTTP method to upload the exported project'
end
end
end
post
':id/export'
do
post
':id/export'
do
project_export_params
=
declared_params
(
include_missing:
false
)
project_export_params
=
declared_params
(
include_missing:
false
)
after_export_params
=
project_export_params
.
delete
(
:upload
)
||
{}
export_strategy
=
if
after_export_params
[
:url
].
present?
params
=
after_export_params
.
slice
(
:url
,
:http_method
).
symbolize_keys
user_project
.
add_export_job
(
current_user:
current_user
,
params:
project_export_params
)
Gitlab
::
ImportExport
::
AfterExportStrategies
::
WebUploadStrategy
.
new
(
params
)
end
if
export_strategy
&
.
invalid?
render_validation_error!
(
export_strategy
)
else
user_project
.
add_export_job
(
current_user:
current_user
,
after_export_strategy:
export_strategy
,
params:
project_export_params
)
end
accepted!
accepted!
end
end
...
...
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
0 → 100644
View file @
22b05a1f
module
Gitlab
module
ImportExport
module
AfterExportStrategies
class
BaseAfterExportStrategy
include
ActiveModel
::
Validations
extend
Forwardable
StrategyError
=
Class
.
new
(
StandardError
)
AFTER_EXPORT_LOCK_FILE_NAME
=
'.after_export_action'
.
freeze
private
attr_reader
:project
,
:current_user
public
def
initialize
(
attributes
=
{})
@options
=
OpenStruct
.
new
(
attributes
)
self
.
class
.
instance_eval
do
def_delegators
:@options
,
*
attributes
.
keys
end
end
def
execute
(
current_user
,
project
)
return
unless
project
&
.
export_project_path
@project
=
project
@current_user
=
current_user
if
invalid?
log_validation_errors
return
end
create_or_update_after_export_lock
strategy_execute
true
rescue
=>
e
project
.
import_export_shared
.
error
(
e
)
false
ensure
delete_after_export_lock
end
def
to_json
(
options
=
{})
@options
.
to_h
.
merge!
(
klass:
self
.
class
.
name
).
to_json
end
def
self
.
lock_file_path
(
project
)
return
unless
project
&
.
export_path
File
.
join
(
project
.
export_path
,
AFTER_EXPORT_LOCK_FILE_NAME
)
end
protected
def
strategy_execute
raise
NotImplementedError
end
private
def
create_or_update_after_export_lock
FileUtils
.
touch
(
self
.
class
.
lock_file_path
(
project
))
end
def
delete_after_export_lock
lock_file
=
self
.
class
.
lock_file_path
(
project
)
FileUtils
.
rm
(
lock_file
)
if
lock_file
.
present?
&&
File
.
exist?
(
lock_file
)
end
def
log_validation_errors
errors
.
full_messages
.
each
{
|
msg
|
project
.
import_export_shared
.
add_error_message
(
msg
)
}
end
end
end
end
end
lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
0 → 100644
View file @
22b05a1f
module
Gitlab
module
ImportExport
module
AfterExportStrategies
class
DownloadNotificationStrategy
<
BaseAfterExportStrategy
private
def
strategy_execute
notification_service
.
project_exported
(
project
,
current_user
)
end
def
notification_service
@notification_service
||=
NotificationService
.
new
end
end
end
end
end
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
0 → 100644
View file @
22b05a1f
module
Gitlab
module
ImportExport
module
AfterExportStrategies
class
WebUploadStrategy
<
BaseAfterExportStrategy
PUT_METHOD
=
'PUT'
.
freeze
POST_METHOD
=
'POST'
.
freeze
INVALID_HTTP_METHOD
=
'invalid. Only PUT and POST methods allowed.'
.
freeze
validates
:url
,
url:
true
validate
do
unless
[
PUT_METHOD
,
POST_METHOD
].
include?
(
http_method
.
upcase
)
errors
.
add
(
:http_method
,
INVALID_HTTP_METHOD
)
end
end
def
initialize
(
url
:,
http_method:
PUT_METHOD
)
super
end
protected
def
strategy_execute
handle_response_error
(
send_file
)
project
.
remove_exported_project_file
end
def
handle_response_error
(
response
)
unless
response
.
success?
error_code
=
response
.
dig
(
'Error'
,
'Code'
)
||
response
.
code
error_message
=
response
.
dig
(
'Error'
,
'Message'
)
||
response
.
message
raise
StrategyError
.
new
(
"Error uploading the project. Code
#{
error_code
}
:
#{
error_message
}
"
)
end
end
private
def
send_file
export_file
=
File
.
open
(
project
.
export_project_path
)
Gitlab
::
HTTP
.
public_send
(
http_method
.
downcase
,
url
,
send_file_options
(
export_file
))
# rubocop:disable GitlabSecurity/PublicSend
ensure
export_file
.
close
if
export_file
end
def
send_file_options
(
export_file
)
{
body_stream:
export_file
,
headers:
headers
}
end
def
headers
{
'Content-Length'
=>
File
.
size
(
project
.
export_project_path
).
to_s
}
end
end
end
end
end
lib/gitlab/import_export/after_export_strategy_builder.rb
0 → 100644
View file @
22b05a1f
module
Gitlab
module
ImportExport
class
AfterExportStrategyBuilder
StrategyNotFoundError
=
Class
.
new
(
StandardError
)
def
self
.
build!
(
strategy_klass
,
attributes
=
{})
return
default_strategy
.
new
unless
strategy_klass
attributes
||=
{}
klass
=
strategy_klass
.
constantize
rescue
nil
unless
klass
&&
klass
<
AfterExportStrategies
::
BaseAfterExportStrategy
raise
StrategyNotFoundError
.
new
(
"Strategy
#{
strategy_klass
}
not found"
)
end
klass
.
new
(
**
attributes
.
symbolize_keys
)
end
def
self
.
default_strategy
AfterExportStrategies
::
DownloadNotificationStrategy
end
end
end
end
lib/gitlab/import_export/shared.rb
View file @
22b05a1f
...
@@ -22,7 +22,7 @@ module Gitlab
...
@@ -22,7 +22,7 @@ module Gitlab
def
error
(
error
)
def
error
(
error
)
error_out
(
error
.
message
,
caller
[
0
].
dup
)
error_out
(
error
.
message
,
caller
[
0
].
dup
)
@errors
<<
error
.
message
add_error_message
(
error
.
message
)
# Debug:
# Debug:
if
error
.
backtrace
if
error
.
backtrace
...
@@ -32,6 +32,14 @@ module Gitlab
...
@@ -32,6 +32,14 @@ module Gitlab
end
end
end
end
def
add_error_message
(
error_message
)
@errors
<<
error_message
end
def
after_export_in_progress?
File
.
exist?
(
after_export_lock_file
)
end
private
private
def
relative_path
def
relative_path
...
@@ -45,6 +53,10 @@ module Gitlab
...
@@ -45,6 +53,10 @@ module Gitlab
def
error_out
(
message
,
caller
)
def
error_out
(
message
,
caller
)
Rails
.
logger
.
error
(
"Import/Export error raised on
#{
caller
}
:
#{
message
}
"
)
Rails
.
logger
.
error
(
"Import/Export error raised on
#{
caller
}
:
#{
message
}
"
)
end
end
def
after_export_lock_file
AfterExportStrategies
::
BaseAfterExportStrategy
.
lock_file_path
(
project
)
end
end
end
end
end
end
end
spec/fixtures/api/schemas/public_api/v4/project/export_status.json
View file @
22b05a1f
{
{
"type"
:
"object"
,
"type"
:
"object"
,
"allOf"
:
[
"allOf"
:
[
{
"$ref"
:
"identity.json"
},
{
"$ref"
:
"identity.json"
},
{
{
"required"
:
[
"required"
:
[
"export_status"
"export_status"
...
@@ -9,7 +11,12 @@
...
@@ -9,7 +11,12 @@
"properties"
:
{
"properties"
:
{
"export_status"
:
{
"export_status"
:
{
"type"
:
"string"
,
"type"
:
"string"
,
"enum"
:
[
"none"
,
"started"
,
"finished"
]
"enum"
:
[
"none"
,
"started"
,
"finished"
,
"after_export_action"
]
}
}
}
}
}
}
...
...
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
0 → 100644
View file @
22b05a1f
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
AfterExportStrategies
::
BaseAfterExportStrategy
do
let!
(
:service
)
{
described_class
.
new
}
let!
(
:project
)
{
create
(
:project
,
:with_export
)
}
let
(
:shared
)
{
project
.
import_export_shared
}
let!
(
:user
)
{
create
(
:user
)
}
describe
'#execute'
do
before
do
allow
(
service
).
to
receive
(
:strategy_execute
)
end
it
'returns if project exported file is not found'
do
allow
(
project
).
to
receive
(
:export_project_path
).
and_return
(
nil
)
expect
(
service
).
not_to
receive
(
:strategy_execute
)
service
.
execute
(
user
,
project
)
end
it
'creates a lock file in the export dir'
do
allow
(
service
).
to
receive
(
:delete_after_export_lock
)
service
.
execute
(
user
,
project
)
expect
(
lock_path_exist?
).
to
be_truthy
end
context
'when the method succeeds'
do
it
'removes the lock file'
do
service
.
execute
(
user
,
project
)
expect
(
lock_path_exist?
).
to
be_falsey
end
end
context
'when the method fails'
do
before
do
allow
(
service
).
to
receive
(
:strategy_execute
).
and_call_original
end
context
'when validation fails'
do
before
do
allow
(
service
).
to
receive
(
:invalid?
).
and_return
(
true
)
end
it
'does not create the lock file'
do
expect
(
service
).
not_to
receive
(
:create_or_update_after_export_lock
)
service
.
execute
(
user
,
project
)
end
it
'does not execute main logic'
do
expect
(
service
).
not_to
receive
(
:strategy_execute
)
service
.
execute
(
user
,
project
)
end
it
'logs validation errors in shared context'
do
expect
(
service
).
to
receive
(
:log_validation_errors
)
service
.
execute
(
user
,
project
)
end
end
context
'when an exception is raised'
do
it
'removes the lock'
do
expect
{
service
.
execute
(
user
,
project
)
}.
to
raise_error
(
NotImplementedError
)
expect
(
lock_path_exist?
).
to
be_falsey
end
end
end
end
describe
'#log_validation_errors'
do
it
'add the message to the shared context'
do
errors
=
%w(test_message test_message2)
allow
(
service
).
to
receive
(
:invalid?
).
and_return
(
true
)
allow
(
service
.
errors
).
to
receive
(
:full_messages
).
and_return
(
errors
)
expect
(
shared
).
to
receive
(
:add_error_message
).
twice
.
and_call_original
service
.
execute
(
user
,
project
)
expect
(
shared
.
errors
).
to
eq
errors
end
end
describe
'#to_json'
do
it
'adds the current strategy class to the serialized attributes'
do
params
=
{
param1:
1
}
result
=
params
.
merge
(
klass:
described_class
.
to_s
).
to_json
expect
(
described_class
.
new
(
params
).
to_json
).
to
eq
result
end
end
def
lock_path_exist?
File
.
exist?
(
described_class
.
lock_file_path
(
project
))
end
end
spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
0 → 100644
View file @
22b05a1f
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
AfterExportStrategies
::
WebUploadStrategy
do
let
(
:example_url
)
{
'http://www.example.com'
}
let
(
:strategy
)
{
subject
.
new
(
url:
example_url
,
http_method:
'post'
)
}
let!
(
:project
)
{
create
(
:project
,
:with_export
)
}
let!
(
:user
)
{
build
(
:user
)
}
subject
{
described_class
}
describe
'validations'
do
it
'only POST and PUT method allowed'
do
%w(POST post PUT put)
.
each
do
|
method
|
expect
(
subject
.
new
(
url:
example_url
,
http_method:
method
)).
to
be_valid
end
expect
(
subject
.
new
(
url:
example_url
,
http_method:
'whatever'
)).
not_to
be_valid
end
it
'onyl allow urls as upload urls'
do
expect
(
subject
.
new
(
url:
example_url
)).
to
be_valid
expect
(
subject
.
new
(
url:
'whatever'
)).
not_to
be_valid
end
end
describe
'#execute'
do
it
'removes the exported project file after the upload'
do
allow
(
strategy
).
to
receive
(
:send_file
)
allow
(
strategy
).
to
receive
(
:handle_response_error
)
expect
(
project
).
to
receive
(
:remove_exported_project_file
)
strategy
.
execute
(
user
,
project
)
end
end
end
spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb
0 → 100644
View file @
22b05a1f
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
AfterExportStrategyBuilder
do
let!
(
:strategies_namespace
)
{
'Gitlab::ImportExport::AfterExportStrategies'
}
describe
'.build!'
do
context
'when klass param is'
do
it
'null it returns the default strategy'
do
expect
(
described_class
.
build!
(
nil
).
class
).
to
eq
described_class
.
default_strategy
end
it
'not a valid class it raises StrategyNotFoundError exception'
do
expect
{
described_class
.
build!
(
'Whatever'
)
}.
to
raise_error
(
described_class
::
StrategyNotFoundError
)
end
it
'not a descendant of AfterExportStrategy'
do
expect
{
described_class
.
build!
(
'User'
)
}.
to
raise_error
(
described_class
::
StrategyNotFoundError
)
end
end
it
'initializes strategy with attributes param'
do
params
=
{
param1:
1
,
param2:
2
,
param3:
3
}
strategy
=
described_class
.
build!
(
"
#{
strategies_namespace
}
::DownloadNotificationStrategy"
,
params
)
params
.
each
{
|
k
,
v
|
expect
(
strategy
.
public_send
(
k
)).
to
eq
v
}
end
end
end
spec/models/project_spec.rb
View file @
22b05a1f
...
@@ -2560,7 +2560,7 @@ describe Project do
...
@@ -2560,7 +2560,7 @@ describe Project do
end
end
end
end
describe
'#remove_export
s
'
do
describe
'#remove_export'
do
let
(
:legacy_project
)
{
create
(
:project
,
:legacy_storage
,
:with_export
)
}
let
(
:legacy_project
)
{
create
(
:project
,
:legacy_storage
,
:with_export
)
}
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
...
@@ -2608,6 +2608,23 @@ describe Project do
...
@@ -2608,6 +2608,23 @@ describe Project do
end
end
end
end
describe
'#remove_exported_project_file'
do
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
it
'removes the exported project file'
do
exported_file
=
project
.
export_project_path
expect
(
File
.
exist?
(
exported_file
)).
to
be_truthy
allow
(
FileUtils
).
to
receive
(
:rm_f
).
and_call_original
expect
(
FileUtils
).
to
receive
(
:rm_f
).
with
(
exported_file
).
and_call_original
project
.
remove_exported_project_file
expect
(
File
.
exist?
(
exported_file
)).
to
be_falsy
end
end
describe
'#forks_count'
do
describe
'#forks_count'
do
it
'returns the number of forks'
do
it
'returns the number of forks'
do
project
=
build
(
:project
)
project
=
build
(
:project
)
...
...
spec/requests/api/project_export_spec.rb
View file @
22b05a1f
...
@@ -5,6 +5,7 @@ describe API::ProjectExport do
...
@@ -5,6 +5,7 @@ describe API::ProjectExport do
set
(
:project_none
)
{
create
(
:project
)
}
set
(
:project_none
)
{
create
(
:project
)
}
set
(
:project_started
)
{
create
(
:project
)
}
set
(
:project_started
)
{
create
(
:project
)
}
set
(
:project_finished
)
{
create
(
:project
)
}
set
(
:project_finished
)
{
create
(
:project
)
}
set
(
:project_after_export
)
{
create
(
:project
)
}
set
(
:user
)
{
create
(
:user
)
}
set
(
:user
)
{
create
(
:user
)
}
set
(
:admin
)
{
create
(
:admin
)
}
set
(
:admin
)
{
create
(
:admin
)
}
...
@@ -12,11 +13,13 @@ describe API::ProjectExport do
...
@@ -12,11 +13,13 @@ describe API::ProjectExport do
let
(
:path_none
)
{
"/projects/
#{
project_none
.
id
}
/export"
}
let
(
:path_none
)
{
"/projects/
#{
project_none
.
id
}
/export"
}
let
(
:path_started
)
{
"/projects/
#{
project_started
.
id
}
/export"
}
let
(
:path_started
)
{
"/projects/
#{
project_started
.
id
}
/export"
}
let
(
:path_finished
)
{
"/projects/
#{
project_finished
.
id
}
/export"
}
let
(
:path_finished
)
{
"/projects/
#{
project_finished
.
id
}
/export"
}
let
(
:path_after_export
)
{
"/projects/
#{
project_after_export
.
id
}
/export"
}
let
(
:download_path
)
{
"/projects/
#{
project
.
id
}
/export/download"
}
let
(
:download_path
)
{
"/projects/
#{
project
.
id
}
/export/download"
}
let
(
:download_path_none
)
{
"/projects/
#{
project_none
.
id
}
/export/download"
}
let
(
:download_path_none
)
{
"/projects/
#{
project_none
.
id
}
/export/download"
}
let
(
:download_path_started
)
{
"/projects/
#{
project_started
.
id
}
/export/download"
}
let
(
:download_path_started
)
{
"/projects/
#{
project_started
.
id
}
/export/download"
}
let
(
:download_path_finished
)
{
"/projects/
#{
project_finished
.
id
}
/export/download"
}
let
(
:download_path_finished
)
{
"/projects/
#{
project_finished
.
id
}
/export/download"
}
let
(
:download_path_export_action
)
{
"/projects/
#{
project_after_export
.
id
}
/export/download"
}
let
(
:export_path
)
{
"
#{
Dir
.
tmpdir
}
/project_export_spec"
}
let
(
:export_path
)
{
"
#{
Dir
.
tmpdir
}
/project_export_spec"
}
...
@@ -29,6 +32,11 @@ describe API::ProjectExport do
...
@@ -29,6 +32,11 @@ describe API::ProjectExport do
# simulate exported
# simulate exported
FileUtils
.
mkdir_p
project_finished
.
export_path
FileUtils
.
mkdir_p
project_finished
.
export_path
FileUtils
.
touch
File
.
join
(
project_finished
.
export_path
,
'_export.tar.gz'
)
FileUtils
.
touch
File
.
join
(
project_finished
.
export_path
,
'_export.tar.gz'
)
# simulate in after export action
FileUtils
.
mkdir_p
project_after_export
.
export_path
FileUtils
.
touch
File
.
join
(
project_after_export
.
export_path
,
'_export.tar.gz'
)
FileUtils
.
touch
Gitlab
::
ImportExport
::
AfterExportStrategies
::
BaseAfterExportStrategy
.
lock_file_path
(
project_after_export
)
end
end
after
do
after
do
...
@@ -73,6 +81,14 @@ describe API::ProjectExport do
...
@@ -73,6 +81,14 @@ describe API::ProjectExport do
expect
(
json_response
[
'export_status'
]).
to
eq
(
'started'
)
expect
(
json_response
[
'export_status'
]).
to
eq
(
'started'
)
end
end
it
'is after_export'
do
get
api
(
path_after_export
,
user
)
expect
(
response
).
to
have_gitlab_http_status
(
200
)
expect
(
response
).
to
match_response_schema
(
'public_api/v4/project/export_status'
)
expect
(
json_response
[
'export_status'
]).
to
eq
(
'after_export_action'
)
end
it
'is finished'
do
it
'is finished'
do
get
api
(
path_finished
,
user
)
get
api
(
path_finished
,
user
)
...
@@ -99,6 +115,7 @@ describe API::ProjectExport do
...
@@ -99,6 +115,7 @@ describe API::ProjectExport do
project_none
.
add_master
(
user
)
project_none
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_after_export
.
add_master
(
user
)
end
end
it_behaves_like
'get project export status ok'
it_behaves_like
'get project export status ok'
...
@@ -163,6 +180,36 @@ describe API::ProjectExport do
...
@@ -163,6 +180,36 @@ describe API::ProjectExport do
end
end
end
end
shared_examples_for
'get project export upload after action'
do
context
'and is uploading'
do
it
'downloads'
do
get
api
(
download_path_export_action
,
user
)
expect
(
response
).
to
have_gitlab_http_status
(
200
)
end
end
context
'when upload complete'
do
before
do
FileUtils
.
rm_rf
(
project_after_export
.
export_path
)
end
it_behaves_like
'404 response'
do
let
(
:request
)
{
get
api
(
download_path_export_action
,
user
)
}
end
end
end
shared_examples_for
'get project download by strategy'
do
context
'when upload strategy set'
do
it_behaves_like
'get project export upload after action'
end
context
'when download strategy set'
do
it_behaves_like
'get project export download'
end
end
it_behaves_like
'when project export is disabled'
do
it_behaves_like
'when project export is disabled'
do
let
(
:request
)
{
get
api
(
download_path
,
admin
)
}
let
(
:request
)
{
get
api
(
download_path
,
admin
)
}
end
end
...
@@ -171,7 +218,7 @@ describe API::ProjectExport do
...
@@ -171,7 +218,7 @@ describe API::ProjectExport do
context
'when user is an admin'
do
context
'when user is an admin'
do
let
(
:user
)
{
admin
}
let
(
:user
)
{
admin
}
it_behaves_like
'get project
export download
'
it_behaves_like
'get project
download by strategy
'
end
end
context
'when user is a master'
do
context
'when user is a master'
do
...
@@ -180,9 +227,10 @@ describe API::ProjectExport do
...
@@ -180,9 +227,10 @@ describe API::ProjectExport do
project_none
.
add_master
(
user
)
project_none
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_after_export
.
add_master
(
user
)
end
end
it_behaves_like
'get project
export download
'
it_behaves_like
'get project
download by strategy
'
end
end
context
'when user is a developer'
do
context
'when user is a developer'
do
...
@@ -229,12 +277,32 @@ describe API::ProjectExport do
...
@@ -229,12 +277,32 @@ describe API::ProjectExport do
end
end
shared_examples_for
'post project export start'
do
shared_examples_for
'post project export start'
do
context
'with upload strategy'
do
context
'when params invalid'
do
it_behaves_like
'400 response'
do
let
(
:request
)
{
post
(
api
(
path
,
user
),
'upload[url]'
=>
'whatever'
)
}
end
end
it
'starts'
do
allow_any_instance_of
(
Gitlab
::
ImportExport
::
AfterExportStrategies
::
WebUploadStrategy
).
to
receive
(
:send_file
)
post
(
api
(
path
,
user
),
'upload[url]'
=>
'http://gitlab.com'
)
expect
(
response
).
to
have_gitlab_http_status
(
202
)
end
end
context
'with download strategy'
do
it
'starts'
do
it
'starts'
do
expect_any_instance_of
(
Gitlab
::
ImportExport
::
AfterExportStrategies
::
WebUploadStrategy
).
not_to
receive
(
:send_file
)
post
api
(
path
,
user
)
post
api
(
path
,
user
)
expect
(
response
).
to
have_gitlab_http_status
(
202
)
expect
(
response
).
to
have_gitlab_http_status
(
202
)
end
end
end
end
end
it_behaves_like
'when project export is disabled'
do
it_behaves_like
'when project export is disabled'
do
let
(
:request
)
{
post
api
(
path
,
admin
)
}
let
(
:request
)
{
post
api
(
path
,
admin
)
}
...
@@ -253,6 +321,7 @@ describe API::ProjectExport do
...
@@ -253,6 +321,7 @@ describe API::ProjectExport do
project_none
.
add_master
(
user
)
project_none
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_started
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_finished
.
add_master
(
user
)
project_after_export
.
add_master
(
user
)
end
end
it_behaves_like
'post project export start'
it_behaves_like
'post project export start'
...
...
spec/services/projects/import_export/export_service_spec.rb
0 → 100644
View file @
22b05a1f
require
'spec_helper'
describe
Projects
::
ImportExport
::
ExportService
do
describe
'#execute'
do
let!
(
:user
)
{
create
(
:user
)
}
let
(
:project
)
{
create
(
:project
)
}
let
(
:shared
)
{
project
.
import_export_shared
}
let
(
:service
)
{
described_class
.
new
(
project
,
user
)
}
let!
(
:after_export_strategy
)
{
Gitlab
::
ImportExport
::
AfterExportStrategies
::
DownloadNotificationStrategy
.
new
}
context
'when all saver services succeed'
do
before
do
allow
(
service
).
to
receive
(
:save_services
).
and_return
(
true
)
end
it
'saves the project in the file system'
do
expect
(
Gitlab
::
ImportExport
::
Saver
).
to
receive
(
:save
).
with
(
project:
project
,
shared:
shared
)
service
.
execute
end
it
'calls the after export strategy'
do
expect
(
after_export_strategy
).
to
receive
(
:execute
)
service
.
execute
(
after_export_strategy
)
end
context
'when after export strategy fails'
do
before
do
allow
(
after_export_strategy
).
to
receive
(
:execute
).
and_return
(
false
)
end
after
do
service
.
execute
(
after_export_strategy
)
end
it
'removes the remaining exported data'
do
allow
(
shared
).
to
receive
(
:export_path
).
and_return
(
'whatever'
)
allow
(
FileUtils
).
to
receive
(
:rm_rf
)
expect
(
FileUtils
).
to
receive
(
:rm_rf
).
with
(
shared
.
export_path
)
end
it
'notifies the user'
do
expect_any_instance_of
(
NotificationService
).
to
receive
(
:project_not_exported
)
end
it
'notifies logger'
do
allow
(
Rails
.
logger
).
to
receive
(
:error
)
expect
(
Rails
.
logger
).
to
receive
(
:error
)
end
end
end
context
'when saver services fail'
do
before
do
allow
(
service
).
to
receive
(
:save_services
).
and_return
(
false
)
end
after
do
expect
{
service
.
execute
}.
to
raise_error
(
Gitlab
::
ImportExport
::
Error
)
end
it
'removes the remaining exported data'
do
allow
(
shared
).
to
receive
(
:export_path
).
and_return
(
'whatever'
)
allow
(
FileUtils
).
to
receive
(
:rm_rf
)
expect
(
FileUtils
).
to
receive
(
:rm_rf
).
with
(
shared
.
export_path
)
end
it
'notifies the user'
do
expect_any_instance_of
(
NotificationService
).
to
receive
(
:project_not_exported
)
end
it
'notifies logger'
do
expect
(
Rails
.
logger
).
to
receive
(
:error
)
end
it
'the after export strategy is not called'
do
expect
(
service
).
not_to
receive
(
:execute_after_export_action
)
end
end
end
end
spec/workers/project_export_worker_spec.rb
0 → 100644
View file @
22b05a1f
require
'spec_helper'
describe
ProjectExportWorker
do
let!
(
:user
)
{
create
(
:user
)
}
let!
(
:project
)
{
create
(
:project
)
}
subject
{
described_class
.
new
}
describe
'#perform'
do
context
'when it succeeds'
do
it
'calls the ExportService'
do
expect_any_instance_of
(
::
Projects
::
ImportExport
::
ExportService
).
to
receive
(
:execute
)
subject
.
perform
(
user
.
id
,
project
.
id
,
{
'klass'
=>
'Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy'
})
end
end
context
'when it fails'
do
it
'raises an exception when params are invalid'
do
expect_any_instance_of
(
::
Projects
::
ImportExport
::
ExportService
).
not_to
receive
(
:execute
)
expect
{
subject
.
perform
(
1234
,
project
.
id
,
{})
}.
to
raise_exception
(
ActiveRecord
::
RecordNotFound
)
expect
{
subject
.
perform
(
user
.
id
,
1234
,
{})
}.
to
raise_exception
(
ActiveRecord
::
RecordNotFound
)
expect
{
subject
.
perform
(
user
.
id
,
project
.
id
,
{
'klass'
=>
'Whatever'
})
}.
to
raise_exception
(
Gitlab
::
ImportExport
::
AfterExportStrategyBuilder
::
StrategyNotFoundError
)
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment