0001"""
0002Implementation of JSONDecoder
0003"""
0004import re
0005import sys
0006
0007from simplejson.scanner import Scanner, pattern
0008try:
0009 from simplejson import _speedups
0010except:
0011 _speedups = None
0012
0013FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
0014
0015def _floatconstants():
0016 import struct
0017 import sys
0018 _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
0019 if sys.byteorder != 'big':
0020 _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
0021 nan, inf = struct.unpack('dd', _BYTES)
0022 return nan, inf, -inf
0023
0024NaN, PosInf, NegInf = _floatconstants()
0025
0026
0027def linecol(doc, pos):
0028 lineno = doc.count('\n', 0, pos) + 1
0029 if lineno == 1:
0030 colno = pos
0031 else:
0032 colno = pos - doc.rindex('\n', 0, pos)
0033 return lineno, colno
0034
0035
0036def errmsg(msg, doc, pos, end=None):
0037 lineno, colno = linecol(doc, pos)
0038 if end is None:
0039 return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
0040 endlineno, endcolno = linecol(doc, end)
0041 return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
0042 msg, lineno, colno, endlineno, endcolno, pos, end)
0043
0044
0045_CONSTANTS = {
0046 '-Infinity': NegInf,
0047 'Infinity': PosInf,
0048 'NaN': NaN,
0049 'true': True,
0050 'false': False,
0051 'null': None,
0052}
0053
0054def JSONConstant(match, context, c=_CONSTANTS):
0055 s = match.group(0)
0056 fn = getattr(context, 'parse_constant', None)
0057 if fn is None:
0058 rval = c[s]
0059 else:
0060 rval = fn(s)
0061 return rval, None
0062pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
0063
0064
0065def JSONNumber(match, context):
0066 match = JSONNumber.regex.match(match.string, *match.span())
0067 integer, frac, exp = match.groups()
0068 if frac or exp:
0069 fn = getattr(context, 'parse_float', None) or float
0070 res = fn(integer + (frac or '') + (exp or ''))
0071 else:
0072 fn = getattr(context, 'parse_int', None) or int
0073 res = fn(integer)
0074 return res, None
0075pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
0076
0077
0078STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
0079BACKSLASH = {
0080 '"': u'"', '\\': u'\\', '/': u'/',
0081 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
0082}
0083
0084DEFAULT_ENCODING = "utf-8"
0085
0086def scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
0087 if encoding is None:
0088 encoding = DEFAULT_ENCODING
0089 chunks = []
0090 _append = chunks.append
0091 begin = end - 1
0092 while 1:
0093 chunk = _m(s, end)
0094 if chunk is None:
0095 raise ValueError(
0096 errmsg("Unterminated string starting at", s, begin))
0097 end = chunk.end()
0098 content, terminator = chunk.groups()
0099 if content:
0100 if not isinstance(content, unicode):
0101 content = unicode(content, encoding)
0102 _append(content)
0103 if terminator == '"':
0104 break
0105 elif terminator != '\\':
0106 if strict:
0107 raise ValueError(errmsg("Invalid control character %r at", s, end))
0108 else:
0109 _append(terminator)
0110 continue
0111 try:
0112 esc = s[end]
0113 except IndexError:
0114 raise ValueError(
0115 errmsg("Unterminated string starting at", s, begin))
0116 if esc != 'u':
0117 try:
0118 m = _b[esc]
0119 except KeyError:
0120 raise ValueError(
0121 errmsg("Invalid \\escape: %r" % (esc,), s, end))
0122 end += 1
0123 else:
0124 esc = s[end + 1:end + 5]
0125 next_end = end + 5
0126 msg = "Invalid \\uXXXX escape"
0127 try:
0128 if len(esc) != 4:
0129 raise ValueError
0130 uni = int(esc, 16)
0131 if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
0132 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
0133 if not s[end + 5:end + 7] == '\\u':
0134 raise ValueError
0135 esc2 = s[end + 7:end + 11]
0136 if len(esc2) != 4:
0137 raise ValueError
0138 uni2 = int(esc2, 16)
0139 uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
0140 next_end += 6
0141 m = unichr(uni)
0142 except ValueError:
0143 raise ValueError(errmsg(msg, s, end))
0144 end = next_end
0145 _append(m)
0146 return u''.join(chunks), end
0147
0148
0149
0150if _speedups is not None:
0151 scanstring = _speedups.scanstring
0152
0153def JSONString(match, context):
0154 encoding = getattr(context, 'encoding', None)
0155 strict = getattr(context, 'strict', True)
0156 return scanstring(match.string, match.end(), encoding, strict)
0157pattern(r'"')(JSONString)
0158
0159
0160WHITESPACE = re.compile(r'\s*', FLAGS)
0161
0162def JSONObject(match, context, _w=WHITESPACE.match):
0163 pairs = {}
0164 s = match.string
0165 end = _w(s, match.end()).end()
0166 nextchar = s[end:end + 1]
0167
0168 if nextchar == '}':
0169 return pairs, end + 1
0170 if nextchar != '"':
0171 raise ValueError(errmsg("Expecting property name", s, end))
0172 end += 1
0173 encoding = getattr(context, 'encoding', None)
0174 strict = getattr(context, 'strict', True)
0175 iterscan = JSONScanner.iterscan
0176 while True:
0177 key, end = scanstring(s, end, encoding, strict)
0178 end = _w(s, end).end()
0179 if s[end:end + 1] != ':':
0180 raise ValueError(errmsg("Expecting : delimiter", s, end))
0181 end = _w(s, end + 1).end()
0182 try:
0183 value, end = iterscan(s, idx=end, context=context).next()
0184 except StopIteration:
0185 raise ValueError(errmsg("Expecting object", s, end))
0186 pairs[key] = value
0187 end = _w(s, end).end()
0188 nextchar = s[end:end + 1]
0189 end += 1
0190 if nextchar == '}':
0191 break
0192 if nextchar != ',':
0193 raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
0194 end = _w(s, end).end()
0195 nextchar = s[end:end + 1]
0196 end += 1
0197 if nextchar != '"':
0198 raise ValueError(errmsg("Expecting property name", s, end - 1))
0199 object_hook = getattr(context, 'object_hook', None)
0200 if object_hook is not None:
0201 pairs = object_hook(pairs)
0202 return pairs, end
0203pattern(r'{')(JSONObject)
0204
0205
0206def JSONArray(match, context, _w=WHITESPACE.match):
0207 values = []
0208 s = match.string
0209 end = _w(s, match.end()).end()
0210
0211 nextchar = s[end:end + 1]
0212 if nextchar == ']':
0213 return values, end + 1
0214 iterscan = JSONScanner.iterscan
0215 while True:
0216 try:
0217 value, end = iterscan(s, idx=end, context=context).next()
0218 except StopIteration:
0219 raise ValueError(errmsg("Expecting object", s, end))
0220 values.append(value)
0221 end = _w(s, end).end()
0222 nextchar = s[end:end + 1]
0223 end += 1
0224 if nextchar == ']':
0225 break
0226 if nextchar != ',':
0227 raise ValueError(errmsg("Expecting , delimiter", s, end))
0228 end = _w(s, end).end()
0229 return values, end
0230pattern(r'\[')(JSONArray)
0231
0232
0233ANYTHING = [
0234 JSONObject,
0235 JSONArray,
0236 JSONString,
0237 JSONConstant,
0238 JSONNumber,
0239]
0240
0241JSONScanner = Scanner(ANYTHING)
0242
0243
0244class JSONDecoder(object):
0245 """
0246 Simple JSON <http://json.org> decoder
0247
0248 Performs the following translations in decoding by default:
0249
0250 +---------------+-------------------+
0251 | JSON | Python |
0252 +===============+===================+
0253 | object | dict |
0254 +---------------+-------------------+
0255 | array | list |
0256 +---------------+-------------------+
0257 | string | unicode |
0258 +---------------+-------------------+
0259 | number (int) | int, long |
0260 +---------------+-------------------+
0261 | number (real) | float |
0262 +---------------+-------------------+
0263 | true | True |
0264 +---------------+-------------------+
0265 | false | False |
0266 +---------------+-------------------+
0267 | null | None |
0268 +---------------+-------------------+
0269
0270 It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
0271 their corresponding ``float`` values, which is outside the JSON spec.
0272 """
0273
0274 _scanner = Scanner(ANYTHING)
0275 __all__ = ['__init__', 'decode', 'raw_decode']
0276
0277 def __init__(self, encoding=None, object_hook=None, parse_float=None,
0278 parse_int=None, parse_constant=None, strict=True):
0279 """
0280 ``encoding`` determines the encoding used to interpret any ``str``
0281 objects decoded by this instance (utf-8 by default). It has no
0282 effect when decoding ``unicode`` objects.
0283
0284 Note that currently only encodings that are a superset of ASCII work,
0285 strings of other encodings should be passed in as ``unicode``.
0286
0287 ``object_hook``, if specified, will be called with the result
0288 of every JSON object decoded and its return value will be used in
0289 place of the given ``dict``. This can be used to provide custom
0290 deserializations (e.g. to support JSON-RPC class hinting).
0291
0292 ``parse_float``, if specified, will be called with the string
0293 of every JSON float to be decoded. By default this is equivalent to
0294 float(num_str). This can be used to use another datatype or parser
0295 for JSON floats (e.g. decimal.Decimal).
0296
0297 ``parse_int``, if specified, will be called with the string
0298 of every JSON int to be decoded. By default this is equivalent to
0299 int(num_str). This can be used to use another datatype or parser
0300 for JSON integers (e.g. float).
0301
0302 ``parse_constant``, if specified, will be called with one of the
0303 following strings: -Infinity, Infinity, NaN, null, true, false.
0304 This can be used to raise an exception if invalid JSON numbers
0305 are encountered.
0306 """
0307 self.encoding = encoding
0308 self.object_hook = object_hook
0309 self.parse_float = parse_float
0310 self.parse_int = parse_int
0311 self.parse_constant = parse_constant
0312 self.strict = strict
0313
0314 def decode(self, s, _w=WHITESPACE.match):
0315 """
0316 Return the Python representation of ``s`` (a ``str`` or ``unicode``
0317 instance containing a JSON document)
0318 """
0319 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
0320 end = _w(s, end).end()
0321 if end != len(s):
0322 raise ValueError(errmsg("Extra data", s, end, len(s)))
0323 return obj
0324
0325 def raw_decode(self, s, **kw):
0326 """
0327 Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
0328 with a JSON document) and return a 2-tuple of the Python
0329 representation and the index in ``s`` where the document ended.
0330
0331 This can be used to decode a JSON document from a string that may
0332 have extraneous data at the end.
0333 """
0334 kw.setdefault('context', self)
0335 try:
0336 obj, end = self._scanner.iterscan(s, **kw).next()
0337 except StopIteration:
0338 raise ValueError("No JSON object could be decoded")
0339 return obj, end
0340
0341__all__ = ['JSONDecoder']