import Lexer from 'perplex/lib/lexer.js'

public class | source

Lexer

Lexes a source-string into tokens.

Example:

const lex = perplex('...')
  .token('ID', /my-id-regex/)
  .token('(', /\(/)
  .token(')', /\)/)
  .token('WS', /\s+/, true) // true means 'skip'

while ((let t = lex.next()).type != 'EOF') {
  console.log(t)
}
// alternatively:
console.log(lex.toArray())

Constructor Summary

Public Constructor
public	constructor(source: string) Creates a new Lexer instance

Member Summary

Public Members
public get	position: number: * Gets the current lexer position
public set	position(i: number): * Sets the current lexer position
public get	source: string: * Gets the source the lexer is operating on
public set	source(s: string): * Sets the source the lexer is operating on

Method Summary

Public Methods
public	attachTo(other: Lexer<T>) Attaches this lexer to another lexer's state
public	disable(type: T): Lexer<T> Disables a token type
public	enable(type: T, enabled: boolean): Lexer<T> Enables a token type
public	expect(type: T): Token<T> Like next, but throws an exception if the next token is not of the required type.
public	next(): Token<T> Consumes and returns the next Token in the source string.
public	peek(position: number): Token<T> Returns the next Token in the source string, but does not consume it.
public	strpos(i: number): Position Converts a string-index (relative to the source string) to a line and a column.
public	toArray(): Token<T>[] Converts the token stream to an array of Tokens
public	token(type: T, pattern: string \| RegExp, skip: boolean): Lexer<T> Creates a new token type

Public Constructors

public constructor(source: string) source

Creates a new Lexer instance

Params:

Name	Type	Attribute	Description
source	string	optional default: ''	The source string to operate on.

Public Members

public get position: number: * source

Gets the current lexer position

Return:

number

Returns the position

public set position(i: number): * source

Sets the current lexer position

public get source: string: * source

Gets the source the lexer is operating on

Return:

string

Returns the source

public set source(s: string): * source

Sets the source the lexer is operating on

Public Methods

public attachTo(other: Lexer<T>) source

Attaches this lexer to another lexer's state

Params:

Name	Type	Attribute	Description
other	Lexer<T>		The other lexer to attach to

public disable(type: T): Lexer<T> source

Disables a token type

Params:

Name	Type	Attribute	Description
type	T		The token type to disable

Return:

Lexer<T>

public enable(type: T, enabled: boolean): Lexer<T> source

Enables a token type

Params:

Name	Type	Attribute	Description
type	T		The token type to enalbe
enabled	boolean	optional default: true nullable: true	Whether to enable/disable the specified token type

Return:

Lexer<T>

public expect(type: T): Token<T> source

Like next, but throws an exception if the next token is not of the required type.

Params:

Name	Type	Attribute	Description
type	T		The token type expected from next

Return:

Token<T>

Returns the Token on success

public next(): Token<T> source

Consumes and returns the next Token in the source string. If there are no more tokens, it returns a Token of type $EOF

Return:

Token<T>

public peek(position: number): Token<T> source

Returns the next Token in the source string, but does not consume it. If there are no more tokens, it returns a Token of type $EOF

Params:

Name	Type	Attribute	Description
position	number	optional default: `this.position`	The position at which to start reading

Return:

Token<T>

public strpos(i: number): Position source

Converts a string-index (relative to the source string) to a line and a column.

Params:

Name	Type	Attribute	Description
i	number		The index to compute

Return:

Position

public toArray(): Token<T>[] source

Converts the token stream to an array of Tokens

Return:

Token<T>[]

The array of tokens (not including (EOF))

public token(type: T, pattern: string | RegExp, skip: boolean): Lexer<T> source

Creates a new token type

Params:

Name	Type	Attribute	Description
type	T		The token type
pattern	string \| RegExp		The pattern to match
skip	boolean	nullable: true	Whether this type of token should be skipped

Return:

Lexer<T>