Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
e564fe97
Commit
e564fe97
authored
Jun 01, 2017
by
Bob Van Landuyt
Committed by
Robert Speicher
Jun 01, 2017
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Rename `Gitlab::Git::EncodingHelper` to `Gitlab::EncodingHelper`
parent
8d131eb8
Changes
12
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
73 additions
and
75 deletions
+73
-75
app/models/merge_request_diff.rb
app/models/merge_request_diff.rb
+1
-1
app/validators/dynamic_path_validator.rb
app/validators/dynamic_path_validator.rb
+1
-1
lib/gitlab/encoding_helper.rb
lib/gitlab/encoding_helper.rb
+62
-0
lib/gitlab/git/blame.rb
lib/gitlab/git/blame.rb
+1
-1
lib/gitlab/git/blob.rb
lib/gitlab/git/blob.rb
+1
-1
lib/gitlab/git/commit.rb
lib/gitlab/git/commit.rb
+1
-1
lib/gitlab/git/diff.rb
lib/gitlab/git/diff.rb
+1
-1
lib/gitlab/git/encoding_helper.rb
lib/gitlab/git/encoding_helper.rb
+0
-64
lib/gitlab/git/ref.rb
lib/gitlab/git/ref.rb
+1
-1
lib/gitlab/git/tree.rb
lib/gitlab/git/tree.rb
+1
-1
spec/lib/gitlab/encoding_helper_spec.rb
spec/lib/gitlab/encoding_helper_spec.rb
+2
-2
spec/lib/gitlab/git/repository_spec.rb
spec/lib/gitlab/git/repository_spec.rb
+1
-1
No files found.
app/models/merge_request_diff.rb
View file @
e564fe97
class
MergeRequestDiff
<
ActiveRecord
::
Base
include
Sortable
include
Importable
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
# Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE
=
100
...
...
app/validators/dynamic_path_validator.rb
View file @
e564fe97
...
...
@@ -6,7 +6,7 @@
# Values are checked for formatting and exclusion from a list of illegal path
# names.
class
DynamicPathValidator
<
ActiveModel
::
EachValidator
extend
Gitlab
::
Git
::
EncodingHelper
extend
Gitlab
::
EncodingHelper
class
<<
self
def
valid_user_path?
(
path
)
...
...
lib/gitlab/encoding_helper.rb
0 → 100644
View file @
e564fe97
module
Gitlab
module
EncodingHelper
extend
self
# This threshold is carefully tweaked to prevent usage of encodings detected
# by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
# we're better off sticking with utf8 encoding.
# Reason: git diff can return strings with invalid utf8 byte sequences if it
# truncates a diff in the middle of a multibyte character. In this case
# CharlockHolmes will try to guess the encoding and will likely suggest an
# obscure encoding with low confidence.
# There is a lot more info with this merge request:
# https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
ENCODING_CONFIDENCE_THRESHOLD
=
40
def
encode!
(
message
)
return
nil
unless
message
.
respond_to?
:force_encoding
# if message is utf-8 encoding, just return it
message
.
force_encoding
(
"UTF-8"
)
return
message
if
message
.
valid_encoding?
# return message if message type is binary
detect
=
CharlockHolmes
::
EncodingDetector
.
detect
(
message
)
return
message
.
force_encoding
(
"BINARY"
)
if
detect
&&
detect
[
:type
]
==
:binary
# force detected encoding if we have sufficient confidence.
if
detect
&&
detect
[
:encoding
]
&&
detect
[
:confidence
]
>
ENCODING_CONFIDENCE_THRESHOLD
message
.
force_encoding
(
detect
[
:encoding
])
end
# encode and clean the bad chars
message
.
replace
clean
(
message
)
rescue
encoding
=
detect
?
detect
[
:encoding
]
:
"unknown"
"--broken encoding:
#{
encoding
}
"
end
def
encode_utf8
(
message
)
detect
=
CharlockHolmes
::
EncodingDetector
.
detect
(
message
)
if
detect
begin
CharlockHolmes
::
Converter
.
convert
(
message
,
detect
[
:encoding
],
'UTF-8'
)
rescue
ArgumentError
=>
e
Rails
.
logger
.
warn
(
"Ignoring error converting
#{
detect
[
:encoding
]
}
into UTF8:
#{
e
.
message
}
"
)
''
end
else
clean
(
message
)
end
end
private
def
clean
(
message
)
message
.
encode
(
"UTF-16BE"
,
undef: :replace
,
invalid: :replace
,
replace:
""
)
.
encode
(
"UTF-8"
)
.
gsub
(
"
\0
"
.
encode
(
"UTF-8"
),
""
)
end
end
end
lib/gitlab/git/blame.rb
View file @
e564fe97
module
Gitlab
module
Git
class
Blame
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
attr_reader
:lines
,
:blames
...
...
lib/gitlab/git/blob.rb
View file @
e564fe97
...
...
@@ -2,7 +2,7 @@ module Gitlab
module
Git
class
Blob
include
Linguist
::
BlobHelper
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
# This number is the maximum amount of data that we want to display to
# the user. We load as much as we can for encoding detection
...
...
lib/gitlab/git/commit.rb
View file @
e564fe97
...
...
@@ -2,7 +2,7 @@
module
Gitlab
module
Git
class
Commit
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
attr_accessor
:raw_commit
,
:head
,
:refs
...
...
lib/gitlab/git/diff.rb
View file @
e564fe97
...
...
@@ -3,7 +3,7 @@ module Gitlab
module
Git
class
Diff
TimeoutError
=
Class
.
new
(
StandardError
)
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
# Diff properties
attr_accessor
:old_path
,
:new_path
,
:a_mode
,
:b_mode
,
:diff
...
...
lib/gitlab/git/encoding_helper.rb
deleted
100644 → 0
View file @
8d131eb8
module
Gitlab
module
Git
module
EncodingHelper
extend
self
# This threshold is carefully tweaked to prevent usage of encodings detected
# by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
# we're better off sticking with utf8 encoding.
# Reason: git diff can return strings with invalid utf8 byte sequences if it
# truncates a diff in the middle of a multibyte character. In this case
# CharlockHolmes will try to guess the encoding and will likely suggest an
# obscure encoding with low confidence.
# There is a lot more info with this merge request:
# https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
ENCODING_CONFIDENCE_THRESHOLD
=
40
def
encode!
(
message
)
return
nil
unless
message
.
respond_to?
:force_encoding
# if message is utf-8 encoding, just return it
message
.
force_encoding
(
"UTF-8"
)
return
message
if
message
.
valid_encoding?
# return message if message type is binary
detect
=
CharlockHolmes
::
EncodingDetector
.
detect
(
message
)
return
message
.
force_encoding
(
"BINARY"
)
if
detect
&&
detect
[
:type
]
==
:binary
# force detected encoding if we have sufficient confidence.
if
detect
&&
detect
[
:encoding
]
&&
detect
[
:confidence
]
>
ENCODING_CONFIDENCE_THRESHOLD
message
.
force_encoding
(
detect
[
:encoding
])
end
# encode and clean the bad chars
message
.
replace
clean
(
message
)
rescue
encoding
=
detect
?
detect
[
:encoding
]
:
"unknown"
"--broken encoding:
#{
encoding
}
"
end
def
encode_utf8
(
message
)
detect
=
CharlockHolmes
::
EncodingDetector
.
detect
(
message
)
if
detect
begin
CharlockHolmes
::
Converter
.
convert
(
message
,
detect
[
:encoding
],
'UTF-8'
)
rescue
ArgumentError
=>
e
Rails
.
logger
.
warn
(
"Ignoring error converting
#{
detect
[
:encoding
]
}
into UTF8:
#{
e
.
message
}
"
)
''
end
else
clean
(
message
)
end
end
private
def
clean
(
message
)
message
.
encode
(
"UTF-16BE"
,
undef: :replace
,
invalid: :replace
,
replace:
""
)
.
encode
(
"UTF-8"
)
.
gsub
(
"
\0
"
.
encode
(
"UTF-8"
),
""
)
end
end
end
end
lib/gitlab/git/ref.rb
View file @
e564fe97
module
Gitlab
module
Git
class
Ref
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
# Branch or tag name
# without "refs/tags|heads" prefix
...
...
lib/gitlab/git/tree.rb
View file @
e564fe97
module
Gitlab
module
Git
class
Tree
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
attr_accessor
:id
,
:root_id
,
:name
,
:path
,
:type
,
:mode
,
:commit_id
,
:submodule_url
...
...
spec/lib/gitlab/
git/
encoding_helper_spec.rb
→
spec/lib/gitlab/encoding_helper_spec.rb
View file @
e564fe97
require
"spec_helper"
describe
Gitlab
::
Git
::
EncodingHelper
do
let
(
:ext_class
)
{
Class
.
new
{
extend
Gitlab
::
Git
::
EncodingHelper
}
}
describe
Gitlab
::
EncodingHelper
do
let
(
:ext_class
)
{
Class
.
new
{
extend
Gitlab
::
EncodingHelper
}
}
let
(
:binary_string
)
{
File
.
read
(
Rails
.
root
+
"spec/fixtures/dk.png"
)
}
describe
'#encode!'
do
...
...
spec/lib/gitlab/git/repository_spec.rb
View file @
e564fe97
require
"spec_helper"
describe
Gitlab
::
Git
::
Repository
,
seed_helper:
true
do
include
Gitlab
::
Git
::
EncodingHelper
include
Gitlab
::
EncodingHelper
let
(
:repository
)
{
Gitlab
::
Git
::
Repository
.
new
(
'default'
,
TEST_REPO_PATH
)
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment