class GraphQL::Language::Lexer
Constants
- BLOCK_QUOTE
- BLOCK_STRING_REGEXP
- COMMENT_REGEXP
- ESCAPED
- ESCAPED_QUOTE
- ESCAPES
- ESCAPES_REPLACE
- FIRST_BYTES
-
Use this array to check, for a given byte that will start a token, what kind of token might it start?
- FLOAT_DECIMAL_REGEXP
- FLOAT_EXP_REGEXP
- FOUR_DIGIT_UNICODE
- IDENTIFIER_REGEXP
- IGNORE_REGEXP
- INT_REGEXP
- KEYWORDS
- KEYWORD_BY_TWO_BYTES
- KEYWORD_REGEXP
- NUMERIC_REGEXP
-
TODO:
FLOAT_EXP_REGEXPshould not be allowed to followINT_REGEXP, integers are not allowed to have exponent parts. - N_DIGIT_UNICODE
- PUNCTUATION_NAME_FOR_BYTE
-
A sparse array mapping the bytes for each punctuation to a symbol name for that punctuation
- QUOTE
- QUOTED_STRING_REGEXP
- STRING_CHAR
- STRING_ESCAPE
- UNICODE_DIGIT
- UNICODE_ESCAPE
- UTF_8
- VALID_STRING
Attributes
Public Class Methods
# File lib/graphql/language/lexer.rb, line 7 def initialize(graphql_str, filename: nil, max_tokens: nil) if !(graphql_str.encoding == Encoding::UTF_8 || graphql_str.ascii_only?) graphql_str = graphql_str.dup.force_encoding(Encoding::UTF_8) end @string = graphql_str @filename = filename if !@string.valid_encoding? raise_parse_error("Parse error on bad Unicode escape sequence", nil, nil) end @scanner = StringScanner.new(graphql_str) @pos = nil @max_tokens = max_tokens || Float::INFINITY @tokens_count = 0 @finished = false end
# File lib/graphql/language/lexer.rb, line 350 def self.replace_escaped_characters_in_place(raw_string) raw_string.gsub!(ESCAPED) do |matched_str| if (point_str_1 = $1 || $2) codepoint_1 = point_str_1.to_i(16) if (codepoint_2 = $3) codepoint_2 = codepoint_2.to_i(16) if (codepoint_1 >= 0xD800 && codepoint_1 <= 0xDBFF) && # leading surrogate (codepoint_2 >= 0xDC00 && codepoint_2 <= 0xDFFF) # trailing surrogate # A surrogate pair combined = ((codepoint_1 - 0xD800) * 0x400) + (codepoint_2 - 0xDC00) + 0x10000 [combined].pack('U'.freeze) else # Two separate code points [codepoint_1].pack('U'.freeze) + [codepoint_2].pack('U'.freeze) end else [codepoint_1].pack('U'.freeze) end else ESCAPES_REPLACE[matched_str] end end nil end
Replace any escaped unicode or whitespace with the actual characters To avoid allocating more strings, this modifies the string passed into it
Source
# File lib/graphql/language/lexer.rb, line 377 def self.tokenize(string) lexer = GraphQL::Language::Lexer.new(string) tokens = [] while (token_name = lexer.advance) new_token = [ token_name, lexer.line_number, lexer.column_number, lexer.debug_token_value(token_name), ] tokens << new_token end tokens end
This is not used during parsing because the parser doesnβt actually need tokens.
Public Instance Methods
Source
# File lib/graphql/language/lexer.rb, line 269 def _hash key (key * 18592990) >> 27 & 0x1f end
This produces a unique integer for bytes 2 and 3 of each keyword string See tenderlovemaking.com/2023/09/02/fast-tokenizers-with-stringscanner.html
Source
# File lib/graphql/language/lexer.rb, line 34 def advance loop do @scanner.skip(IGNORE_REGEXP) if @scanner.skip(COMMENT_REGEXP) @tokens_count += 1 if @tokens_count > @max_tokens @pos = @scanner.pos raise_parse_error("This query is too large to execute.") end next end break end if @scanner.eos? @finished = true return false end @tokens_count += 1 if @tokens_count > @max_tokens raise_parse_error("This query is too large to execute.") end @pos = @scanner.pos next_byte = @string.getbyte(@pos) next_byte_is_for = FIRST_BYTES[next_byte] case next_byte_is_for when ByteFor::PUNCTUATION @scanner.pos += 1 PUNCTUATION_NAME_FOR_BYTE[next_byte] when ByteFor::NAME if len = @scanner.skip(KEYWORD_REGEXP) case len when 2 :ON when 12 :SUBSCRIPTION else pos = @pos # Use bytes 2 and 3 as a unique identifier for this keyword bytes = (@string.getbyte(pos + 2) << 8) | @string.getbyte(pos + 1) KEYWORD_BY_TWO_BYTES[_hash(bytes)] end else @scanner.skip(IDENTIFIER_REGEXP) :IDENTIFIER end when ByteFor::IDENTIFIER @scanner.skip(IDENTIFIER_REGEXP) :IDENTIFIER when ByteFor::NUMBER if len = @scanner.skip(NUMERIC_REGEXP) if GraphQL.reject_numbers_followed_by_names new_pos = @scanner.pos peek_byte = @string.getbyte(new_pos) next_first_byte = FIRST_BYTES[peek_byte] if next_first_byte == ByteFor::NAME || next_first_byte == ByteFor::IDENTIFIER number_part = token_value name_part = @scanner.scan(IDENTIFIER_REGEXP) raise_parse_error("Name after number is not allowed (in `#{number_part}#{name_part}`)") end end # Check for a matched decimal: @scanner[1] ? :FLOAT : :INT else # Attempt to find the part after the `-` value = @scanner.scan(/-\s?[a-z0-9]*/i) invalid_byte_for_number_error_message = "Expected type 'number', but it was malformed#{value.nil? ? "" : ": #{value.inspect}"}." raise_parse_error(invalid_byte_for_number_error_message) end when ByteFor::ELLIPSIS if @string.getbyte(@pos + 1) != 46 || @string.getbyte(@pos + 2) != 46 raise_parse_error("Expected `...`, actual: #{@string[@pos..@pos + 2].inspect}") end @scanner.pos += 3 :ELLIPSIS when ByteFor::STRING if @scanner.skip(BLOCK_STRING_REGEXP) || @scanner.skip(QUOTED_STRING_REGEXP) :STRING else raise_parse_error("Expected string or block string, but it was malformed") end else @scanner.pos += 1 :UNKNOWN_CHAR end end
Source
# File lib/graphql/language/lexer.rb, line 189 def column_number line_prefix = @scanner.string.byteslice(0, @pos) line_prefix = line_prefix.b unless line_prefix.valid_encoding? newline_index = line_prefix.rindex("\n") newline_index ? line_prefix.length - newline_index : line_prefix.length + 1 end
# File lib/graphql/language/lexer.rb, line 129 def debug_token_value(token_name) if token_name && Lexer::Punctuation.const_defined?(token_name) Lexer::Punctuation.const_get(token_name) elsif token_name == :ELLIPSIS "..." elsif token_name == :STRING string_value elsif @scanner.matched_size.nil? @string.byteslice(@scanner.pos - 1, 1) else token_value end end
Source
# File lib/graphql/language/lexer.rb, line 27 def freeze @scanner = nil super end
Calls superclass method
Source
# File lib/graphql/language/lexer.rb, line 185 def line_number @scanner.string.byteslice(0, @pos).b.count("\n".b) + 1 end
# File lib/graphql/language/lexer.rb, line 196 def raise_parse_error(message, line = line_number, col = column_number) raise GraphQL::ParseError.new(message, line, col, @string, filename: @filename) end
Source
# File lib/graphql/language/lexer.rb, line 158 def string_value str = token_value is_block = str.start_with?('"""') if is_block str.gsub!(/\A"""|"""\z/, '') return Language::BlockString.trim_whitespace(str) else str.gsub!(/\A"|"\z/, '') if !str.valid_encoding? || !str.match?(VALID_STRING) raise_parse_error("Bad unicode escape in #{str.inspect}") else begin Lexer.replace_escaped_characters_in_place(str) rescue RangeError raise_parse_error("Bad unicode escape in #{str.inspect}") end if !str.valid_encoding? raise_parse_error("Bad unicode escape in #{str.inspect}") else str end end end end
Source
# File lib/graphql/language/lexer.rb, line 123 def token_value @string.byteslice(@scanner.pos - @scanner.matched_size, @scanner.matched_size) rescue StandardError => err raise GraphQL::Error, "(token_value failed: #{err.class}: #{err.message})" end