1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests v8::internal::Scanner. Note that presently most unit tests for the
// Scanner are in cctest/test-parsing.cc, rather than here.
#include "src/handles/handles-inl.h"
#include "src/objects/objects-inl.h"
#include "src/parsing/parse-info.h"
#include "src/parsing/scanner-character-streams.h"
#include "src/parsing/scanner.h"
#include "test/cctest/cctest.h"
namespace v8 {
namespace internal {
namespace {
const char src_simple[] = "function foo() { var x = 2 * a() + b; }";
struct ScannerTestHelper {
ScannerTestHelper() = default;
ScannerTestHelper(ScannerTestHelper&& other) V8_NOEXCEPT
: stream(std::move(other.stream)),
scanner(std::move(other.scanner)) {}
std::unique_ptr<Utf16CharacterStream> stream;
std::unique_ptr<Scanner> scanner;
Scanner* operator->() const { return scanner.get(); }
Scanner* get() const { return scanner.get(); }
};
ScannerTestHelper make_scanner(const char* src) {
ScannerTestHelper helper;
helper.stream = ScannerStream::ForTesting(src);
helper.scanner = std::unique_ptr<Scanner>(
new Scanner(helper.stream.get(),
UnoptimizedCompileFlags::ForTest(CcTest::i_isolate())));
helper.scanner->Initialize();
return helper;
}
} // anonymous namespace
// CHECK_TOK checks token equality, but by checking for equality of the token
// names. That should have the same result, but has much nicer error messaages.
#define CHECK_TOK(a, b) CHECK_EQ(Token::Name(a), Token::Name(b))
TEST(Bookmarks) {
// Scan through the given source and record the tokens for use as reference
// below.
std::vector<Token::Value> tokens;
{
auto scanner = make_scanner(src_simple);
do {
tokens.push_back(scanner->Next());
} while (scanner->current_token() != Token::EOS);
}
// For each position:
// - Scan through file,
// - set a bookmark once the position is reached,
// - scan a bit more,
// - reset to the bookmark, and
// - scan until the end.
// At each step, compare to the reference token sequence generated above.
for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) {
auto scanner = make_scanner(src_simple);
Scanner::BookmarkScope bookmark(scanner.get());
for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) {
if (i == bookmark_pos) {
bookmark.Set(scanner->peek_location().beg_pos);
}
CHECK_TOK(tokens[i], scanner->Next());
}
bookmark.Apply();
for (size_t i = bookmark_pos; i < tokens.size(); i++) {
CHECK_TOK(tokens[i], scanner->Next());
}
}
}
TEST(AllThePushbacks) {
const struct {
const char* src;
const Token::Value tokens[5]; // Large enough for any of the test cases.
} test_cases[] = {
{"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
{"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}},
{"<!-x",
{Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
{"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}},
};
for (const auto& test_case : test_cases) {
auto scanner = make_scanner(test_case.src);
for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) {
CHECK_TOK(test_case.tokens[i], scanner->Next());
}
CHECK_TOK(Token::EOS, scanner->Next());
}
}
} // namespace internal
} // namespace v8
|
