maddy

 1/*
 2Maddy Mail Server - Composable all-in-one email server.
 3Copyright © 2019-2020 Max Mazurov <fox.cpp@disroot.org>, Maddy Mail Server contributors
 4
 5This program is free software: you can redistribute it and/or modify
 6it under the terms of the GNU General Public License as published by
 7the Free Software Foundation, either version 3 of the License, or
 8(at your option) any later version.
 9
10This program is distributed in the hope that it will be useful,
11but WITHOUT ANY WARRANTY; without even the implied warranty of
12MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13GNU General Public License for more details.
14
15You should have received a copy of the GNU General Public License
16along with this program.  If not, see <https://www.gnu.org/licenses/>.
17*/
18
19package lexer
20
21import (
22	"io"
23)
24
25// allTokens lexes the entire input, but does not parse it.
26// It returns all the tokens from the input, unstructured
27// and in order.
28func allTokens(input io.Reader) ([]Token, error) {
29	l := new(lexer)
30	err := l.load(input)
31	if err != nil {
32		return nil, err
33	}
34	var tokens []Token
35	for l.next() {
36		tokens = append(tokens, l.token)
37	}
38	if err := l.err(); err != nil {
39		return nil, err
40	}
41	return tokens, nil
42}