16: def scan_tokens tokens, options
17:
18: state = :initial
19: stack = []
20: string_delimiter = nil
21: key_expected = false
22:
23: until eos?
24:
25: kind = nil
26: match = nil
27:
28: case state
29:
30: when :initial
31: if match = scan(/ \s+ | \\\n /x)
32: tokens << [match, :space]
33: next
34: elsif match = scan(/ [:,\[{\]}] /x)
35: kind = :operator
36: case match
37: when '{' then stack << :object; key_expected = true
38: when '[' then stack << :array
39: when ':' then key_expected = false
40: when ',' then key_expected = true if stack.last == :object
41: when '}', ']' then stack.pop
42: end
43: elsif match = scan(/ true | false | null /x)
44: kind = IDENT_KIND[match]
45: elsif match = scan(/-?(?:0|[1-9]\d*)/)
46: kind = :integer
47: if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
48: match << matched
49: kind = :float
50: end
51: elsif match = scan(/"/)
52: state = key_expected ? :key : :string
53: tokens << [:open, state]
54: kind = :delimiter
55: else
56: getch
57: kind = :error
58: end
59:
60: when :string, :key
61: if scan(/[^\\"]+/)
62: kind = :content
63: elsif scan(/"/)
64: tokens << ['"', :delimiter]
65: tokens << [:close, state]
66: state = :initial
67: next
68: elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
69: kind = :char
70: elsif scan(/\\./m)
71: kind = :content
72: elsif scan(/ \\ | $ /x)
73: tokens << [:close, :delimiter]
74: kind = :error
75: state = :initial
76: else
77: raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
78: end
79:
80: else
81: raise_inspect 'Unknown state', tokens
82:
83: end
84:
85: match ||= matched
86: if $DEBUG and not kind
87: raise_inspect 'Error token %p in line %d' %
88: [[match, kind], line], tokens
89: end
90: raise_inspect 'Empty token', tokens unless match
91:
92: tokens << [match, kind]
93:
94: end
95:
96: if [:string, :key].include? state
97: tokens << [:close, state]
98: end
99:
100: tokens
101: end