Skip to content

Lexer: return a struct-of-arrays TokenList (BC break, ~26% faster lexing) #314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JanTvrdik wants to merge 1 commit into
base: 2.3.x
Choose a base branch
from
Open

Lexer: return a struct-of-arrays TokenList (BC break, ~26% faster lexing) #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 14 additions & 17 deletions src/Lexer/Lexer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,6 @@ class Lexer
self::TOKEN_WILDCARD => '*',
];

public const VALUE_OFFSET = 0;
public const TYPE_OFFSET = 1;
public const LINE_OFFSET = 2;

private ParserConfig $config; // @phpstan-ignore property.onlyWritten

private ?string $regexp = null;
Expand All @@ -106,10 +102,7 @@ public function __construct(ParserConfig $config)
$this->config = $config;
}

/**
* @return list<array{string, int, int}>
*/
public function tokenize(string $s): array
public function tokenize(string $s): TokenList
{
if ($this->regexp === null) {
$this->regexp = $this->generateRegexp();
Expand All @@ -123,26 +116,30 @@ public function tokenize(string $s): array

$values = $matches[0];
if ($values === []) {
return [['', self::TOKEN_END, 1]];
return new TokenList([''], [self::TOKEN_END], [1]);
}

$marks = $matches['MARK'];

$tokens = [];
// $matches[0] is already the values array of the TokenList, so the
// values never get copied out of it token by token.
$types = [];
$lines = [];
$line = 1;
foreach ($values as $i => $value) {
$type = (int) $marks[$i];
$tokens[] = [$value, $type, $line];
foreach ($matches['MARK'] as $mark) {
$type = (int) $mark;
$types[] = $type;
$lines[] = $line;
if ($type !== self::TOKEN_PHPDOC_EOL) {
continue;
}

$line++;
}

$tokens[] = ['', self::TOKEN_END, $line];
$values[] = '';
$types[] = self::TOKEN_END;
$lines[] = $line;

return $tokens;
return new TokenList($values, $types, $lines);
}

private function generateRegexp(): string
Expand Down
50 changes: 50 additions & 0 deletions src/Lexer/TokenList.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
<?php declare(strict_types = 1);

namespace PHPStan\PhpDocParser\Lexer;

use Countable;
use function count;

/**
* Tokens in a struct-of-arrays layout: three parallel packed arrays instead of
* one array (or object) per token.
*/
final class TokenList implements Countable
{

/** @var list<string> */
public array $values;

/** @var list<int> */
public array $types;

/** @var list<int> */
public array $lines;

/** @var int<0, max> */
public int $count;

/**
* @param list<string> $values
* @param list<int> $types
* @param list<int> $lines
*/
public function __construct(array $values, array $types, array $lines)
{
$this->values = $values;
$this->types = $types;
$this->lines = $lines;
$this->count = count($values);
}

/**
* Only so that code written against the array this replaces keeps working.
* Read the $count property directly in hot paths -- reading a property is
* cheaper than the call this makes.
*/
public function count(): int
{
return $this->count;
}

}
91 changes: 48 additions & 43 deletions src/Parser/TokenIterator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,28 @@
use LogicException;
use PHPStan\PhpDocParser\Ast\Comment;
use PHPStan\PhpDocParser\Lexer\Lexer;
use PHPStan\PhpDocParser\Lexer\TokenList;
use function array_pop;
use function assert;
use function count;
use function in_array;
use function strlen;
use function substr;

class TokenIterator
{

/** @var list<array{string, int, int}> */
private array $tokens;
private TokenList $tokens;

/** @var list<string> */
private array $values;

/** @var list<int> */
private array $types;

/** @var list<int> */
private array $lines;

private int $count;

private int $index;

Expand All @@ -31,67 +41,65 @@ class TokenIterator

private ?string $newline = null;

/**
* @param list<array{string, int, int}> $tokens
*/
public function __construct(array $tokens, int $index = 0)
public function __construct(TokenList $tokens, int $index = 0)
{
$this->tokens = $tokens;
$this->values = $tokens->values;
$this->types = $tokens->types;
$this->lines = $tokens->lines;
$this->count = $tokens->count;
$this->index = $index;

$this->skipIrrelevantTokens();
}

/**
* @return list<array{string, int, int}>
*/
public function getTokens(): array
public function getTokens(): TokenList
{
return $this->tokens;
}
Comment on lines +56 to 59

public function getContentBetween(int $startPos, int $endPos): string
{
if ($startPos < 0 || $endPos > count($this->tokens)) {
if ($startPos < 0 || $endPos > $this->count) {
throw new LogicException();
}

$content = '';
for ($i = $startPos; $i < $endPos; $i++) {
$content .= $this->tokens[$i][Lexer::VALUE_OFFSET];
$content .= $this->values[$i];
}

return $content;
}

public function getTokenCount(): int
{
return count($this->tokens);
return $this->count;
}

public function currentTokenValue(): string
{
return $this->tokens[$this->index][Lexer::VALUE_OFFSET];
return $this->values[$this->index];
}

public function currentTokenType(): int
{
return $this->tokens[$this->index][Lexer::TYPE_OFFSET];
return $this->types[$this->index];
}

public function currentTokenOffset(): int
{
$offset = 0;
for ($i = 0; $i < $this->index; $i++) {
$offset += strlen($this->tokens[$i][Lexer::VALUE_OFFSET]);
$offset += strlen($this->values[$i]);
}

return $offset;
}

public function currentTokenLine(): int
{
return $this->tokens[$this->index][Lexer::LINE_OFFSET];
return $this->lines[$this->index];
}

public function currentTokenIndex(): int
Expand All @@ -103,8 +111,8 @@ public function endIndexOfLastRelevantToken(): int
{
$endIndex = $this->currentTokenIndex();
$endIndex--;
while (in_array($this->tokens[$endIndex][Lexer::TYPE_OFFSET], $this->skippedTokenTypes, true)) {
if (!isset($this->tokens[$endIndex - 1])) {
while (in_array($this->types[$endIndex], $this->skippedTokenTypes, true)) {
if ($endIndex - 1 < 0) {
break;
}
$endIndex--;
Expand All @@ -115,25 +123,25 @@ public function endIndexOfLastRelevantToken(): int

public function isCurrentTokenValue(string $tokenValue): bool
{
return $this->tokens[$this->index][Lexer::VALUE_OFFSET] === $tokenValue;
return $this->values[$this->index] === $tokenValue;
}

public function isCurrentTokenType(int ...$tokenType): bool
{
return in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $tokenType, true);
return in_array($this->types[$this->index], $tokenType, true);
}

public function isPrecededByHorizontalWhitespace(): bool
{
return ($this->tokens[$this->index - 1][Lexer::TYPE_OFFSET] ?? -1) === Lexer::TOKEN_HORIZONTAL_WS;
return ($this->types[$this->index - 1] ?? -1) === Lexer::TOKEN_HORIZONTAL_WS;
}

/**
* @throws ParserException
*/
public function consumeTokenType(int $tokenType): void
{
if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] !== $tokenType) {
if ($this->types[$this->index] !== $tokenType) {
$this->throwError($tokenType);
}

Expand All @@ -151,7 +159,7 @@ public function consumeTokenType(int $tokenType): void
*/
public function consumeTokenValue(int $tokenType, string $tokenValue): void
{
if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] !== $tokenType || $this->tokens[$this->index][Lexer::VALUE_OFFSET] !== $tokenValue) {
if ($this->types[$this->index] !== $tokenType || $this->values[$this->index] !== $tokenValue) {
$this->throwError($tokenType, $tokenValue);
}

Expand All @@ -161,7 +169,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
/** @phpstan-impure */
public function tryConsumeTokenValue(string $tokenValue): bool
{
if ($this->tokens[$this->index][Lexer::VALUE_OFFSET] !== $tokenValue) {
if ($this->values[$this->index] !== $tokenValue) {
return false;
}

Expand All @@ -183,7 +191,7 @@ public function flushComments(): array
/** @phpstan-impure */
public function tryConsumeTokenType(int $tokenType): bool
{
if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] !== $tokenType) {
if ($this->types[$this->index] !== $tokenType) {
return false;
}

Expand Down Expand Up @@ -246,8 +254,8 @@ private function detectNewline(): void

public function getSkippedHorizontalWhiteSpaceIfAny(): string
{
if ($this->index > 0 && $this->tokens[$this->index - 1][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) {
return $this->tokens[$this->index - 1][Lexer::VALUE_OFFSET];
if ($this->index > 0 && $this->types[$this->index - 1] === Lexer::TOKEN_HORIZONTAL_WS) {
return $this->values[$this->index - 1];
}

return '';
Expand All @@ -257,8 +265,8 @@ public function getSkippedHorizontalWhiteSpaceIfAny(): string
public function joinUntil(int ...$tokenType): string
{
$s = '';
while (!in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $tokenType, true)) {
$s .= $this->tokens[$this->index++][Lexer::VALUE_OFFSET];
while (!in_array($this->types[$this->index], $tokenType, true)) {
$s .= $this->values[$this->index++];
}
return $s;
}
Expand All @@ -271,12 +279,12 @@ public function next(): void

private function skipIrrelevantTokens(): void
{
if (!isset($this->tokens[$this->index])) {
if ($this->index >= $this->count) {
return;
}

while (in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $this->skippedTokenTypes, true)) {
if (!isset($this->tokens[$this->index + 1])) {
while (in_array($this->types[$this->index], $this->skippedTokenTypes, true)) {
if ($this->index + 1 >= $this->count) {
break;
}
$this->index++;
Expand All @@ -296,8 +304,7 @@ public function removeEndOfLineFromSkippedTokens(): void
/** @phpstan-impure */
public function forwardToTheEnd(): void
{
$lastToken = count($this->tokens) - 1;
$this->index = $lastToken;
$this->index = $this->count - 1;
}

public function pushSavePoint(): void
Expand Down Expand Up @@ -339,11 +346,10 @@ private function throwError(int $expectedTokenType, ?string $expectedTokenValue
*/
public function hasTokenImmediatelyBefore(int $pos, int $expectedTokenType): bool
{
$tokens = $this->tokens;
$types = $this->types;
$pos--;
for (; $pos >= 0; $pos--) {
$token = $tokens[$pos];
$type = $token[Lexer::TYPE_OFFSET];
$type = $types[$pos];
if ($type === $expectedTokenType) {
return true;
}
Expand All @@ -364,11 +370,10 @@ public function hasTokenImmediatelyBefore(int $pos, int $expectedTokenType): boo
*/
public function hasTokenImmediatelyAfter(int $pos, int $expectedTokenType): bool
{
$tokens = $this->tokens;
$types = $this->types;
$pos++;
for ($c = count($tokens); $pos < $c; $pos++) {
$token = $tokens[$pos];
$type = $token[Lexer::TYPE_OFFSET];
for ($c = $this->count; $pos < $c; $pos++) {
$type = $types[$pos];
if ($type === $expectedTokenType) {
return true;
}
Expand Down
Loading
Loading