Commit 1e1e69b9 authored by Matthew Holt's avatar Matthew Holt

Discard byte order mark (fix #962)

parent cf1b355d
......@@ -26,9 +26,20 @@ type (
)
// load prepares the lexer to scan an input for tokens.
// It discards any leading byte order mark.
func (l *lexer) load(input io.Reader) error {
l.reader = bufio.NewReader(input)
l.line = 1
// discard byte order mark, if present
firstCh, _, err := l.reader.ReadRune()
if err == nil && firstCh != 0xFEFF {
err := l.reader.UnreadRune()
if err != nil {
return err
}
}
return nil
}
......
......@@ -128,6 +128,12 @@ func TestLexer(t *testing.T) {
{Line: 2, Text: "characters"},
},
},
{
input: "\xEF\xBB\xBF:8080", // test with leading byte order mark
expected: []Token{
{Line: 1, Text: ":8080"},
},
},
}
for i, testCase := range testCases {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment