Class CodeRay::Scanners::JSON
In: lib/coderay/scanners/json.rb
Parent: Scanner

Methods

Included Modules

Streamable

Constants

CONSTANTS = %w( true false null )
IDENT_KIND = WordList.new(:key).add(CONSTANTS, :reserved)
ESCAPE = / [bfnrt\\"\/] /x
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x

Public Instance methods

[Source]

     # File lib/coderay/scanners/json.rb, line 16
 16:     def scan_tokens tokens, options
 17:       
 18:       state = :initial
 19:       stack = []
 20:       string_delimiter = nil
 21:       key_expected = false
 22:       
 23:       until eos?
 24:         
 25:         kind = nil
 26:         match = nil
 27:         
 28:         case state
 29:         
 30:         when :initial
 31:           if match = scan(/ \s+ | \\\n /x)
 32:             tokens << [match, :space]
 33:             next
 34:           elsif match = scan(/ [:,\[{\]}] /x)
 35:             kind = :operator
 36:             case match
 37:             when '{' then stack << :object; key_expected = true
 38:             when '[' then stack << :array
 39:             when ':' then key_expected = false
 40:             when ',' then key_expected = true if stack.last == :object
 41:             when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
 42:             end
 43:           elsif match = scan(/ true | false | null /x)
 44:             kind = IDENT_KIND[match]
 45:           elsif match = scan(/-?(?:0|[1-9]\d*)/)
 46:             kind = :integer
 47:             if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
 48:               match << matched
 49:               kind = :float
 50:             end
 51:           elsif match = scan(/"/)
 52:             state = key_expected ? :key : :string
 53:             tokens << [:open, state]
 54:             kind = :delimiter
 55:           else
 56:             getch
 57:             kind = :error
 58:           end
 59:           
 60:         when :string, :key
 61:           if scan(/[^\\"]+/)
 62:             kind = :content
 63:           elsif scan(/"/)
 64:             tokens << ['"', :delimiter]
 65:             tokens << [:close, state]
 66:             state = :initial
 67:             next
 68:           elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
 69:             kind = :char
 70:           elsif scan(/\\./m)
 71:             kind = :content
 72:           elsif scan(/ \\ | $ /x)
 73:             tokens << [:close, :delimiter]
 74:             kind = :error
 75:             state = :initial
 76:           else
 77:             raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
 78:           end
 79:           
 80:         else
 81:           raise_inspect 'Unknown state', tokens
 82:           
 83:         end
 84:         
 85:         match ||= matched
 86:         if $DEBUG and not kind
 87:           raise_inspect 'Error token %p in line %d' %
 88:             [[match, kind], line], tokens
 89:         end
 90:         raise_inspect 'Empty token', tokens unless match
 91:         
 92:         tokens << [match, kind]
 93:         
 94:       end
 95:       
 96:       if [:string, :key].include? state
 97:         tokens << [:close, state]
 98:       end
 99:       
100:       tokens
101:     end

[Validate]