Commit 1e1e69b9 authored by Matthew Holt's avatar Matthew Holt

Discard byte order mark (fix #962)

parent cf1b355d
...@@ -26,9 +26,20 @@ type ( ...@@ -26,9 +26,20 @@ type (
) )
// load prepares the lexer to scan an input for tokens. // load prepares the lexer to scan an input for tokens.
// It discards any leading byte order mark.
func (l *lexer) load(input io.Reader) error { func (l *lexer) load(input io.Reader) error {
l.reader = bufio.NewReader(input) l.reader = bufio.NewReader(input)
l.line = 1 l.line = 1
// discard byte order mark, if present
firstCh, _, err := l.reader.ReadRune()
if err == nil && firstCh != 0xFEFF {
err := l.reader.UnreadRune()
if err != nil {
return err
}
}
return nil return nil
} }
......
...@@ -128,6 +128,12 @@ func TestLexer(t *testing.T) { ...@@ -128,6 +128,12 @@ func TestLexer(t *testing.T) {
{Line: 2, Text: "characters"}, {Line: 2, Text: "characters"},
}, },
}, },
{
input: "\xEF\xBB\xBF:8080", // test with leading byte order mark
expected: []Token{
{Line: 1, Text: ":8080"},
},
},
} }
for i, testCase := range testCases { for i, testCase := range testCases {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment