--- JSON-XS/XS.xs 2008/03/26 22:54:38 1.83 +++ JSON-XS/XS.xs 2013/10/28 23:19:54 1.119 @@ -14,12 +14,13 @@ #endif // some old perls do not have this, try to make it work, no -// guarentees, though. if it breaks, you get to keep the pieces. +// guarantees, though. if it breaks, you get to keep the pieces. #ifndef UTF8_MAXBYTES # define UTF8_MAXBYTES 13 #endif -#define IVUV_MAXCHARS (sizeof (UV) * CHAR_BIT * 28 / 93 + 2) +// three extra for rounding, sign, and end of string +#define IVUV_MAXCHARS (sizeof (UV) * CHAR_BIT * 28 / 93 + 3) #define F_ASCII 0x00000001UL #define F_LATIN1 0x00000002UL @@ -33,24 +34,19 @@ #define F_ALLOW_BLESSED 0x00000400UL #define F_CONV_BLESSED 0x00000800UL #define F_RELAXED 0x00001000UL - -#define F_MAXDEPTH 0xf8000000UL -#define S_MAXDEPTH 27 -#define F_MAXSIZE 0x01f00000UL -#define S_MAXSIZE 20 +#define F_ALLOW_UNKNOWN 0x00002000UL +#define F_ALLOW_TAGS 0x00004000UL #define F_HOOK 0x00080000UL // some hooks exist, so slow-path processing -#define DEC_DEPTH(flags) (1UL << ((flags & F_MAXDEPTH) >> S_MAXDEPTH)) -#define DEC_SIZE(flags) (1UL << ((flags & F_MAXSIZE ) >> S_MAXSIZE )) - #define F_PRETTY F_INDENT | F_SPACE_BEFORE | F_SPACE_AFTER -#define F_DEFAULT (9UL << S_MAXDEPTH) #define INIT_SIZE 32 // initial scalar size to be allocated #define INDENT_STEP 3 // spaces per indentation level #define SHORT_STRING_LEN 16384 // special-case strings of up to this size +#define DECODE_WANTS_OCTETS(json) ((json)->flags & F_UTF8) + #define SB do { #define SE } while (0) @@ -69,6 +65,8 @@ ((unsigned type)((unsigned type)(val) - (unsigned type)(beg)) \ <= (unsigned type)((unsigned type)(end) - (unsigned type)(beg))) +#define ERR_NESTING_EXCEEDED "json text or perl structure exceeds maximum nesting level (max_depth set too low?)" + #ifdef USE_ITHREADS # define JSON_SLOW 1 # define JSON_STASH (json_stash ? json_stash : gv_stashpv ("JSON::XS", 1)) @@ -77,20 +75,28 @@ # define JSON_STASH json_stash #endif -static HV *json_stash, *json_boolean_stash; // JSON::XS:: -static SV *json_true, *json_false; +// the amount of HEs to allocate on the stack, when sorting keys +#define STACK_HES 64 + +static HV *json_stash, *types_boolean_stash; // JSON::XS:: +static SV *types_true, *types_false, *sv_json; enum { INCR_M_WS = 0, // initial whitespace skipping, must be 0 INCR_M_STR, // inside string INCR_M_BS, // inside backslash + INCR_M_C0, // inside comment in initial whitespace sequence + INCR_M_C1, // inside comment in other places INCR_M_JSON // outside anything, count nesting }; -#define INCR_DONE(json) (!(json)->incr_nest && (json)->incr_mode == INCR_M_JSON) +#define INCR_DONE(json) ((json)->incr_nest <= 0 && (json)->incr_mode == INCR_M_JSON) typedef struct { U32 flags; + U32 max_depth; + STRLEN max_size; + SV *cb_object; HV *cb_sk_object; @@ -98,9 +104,16 @@ SV *incr_text; // the source text so far STRLEN incr_pos; // the current offset into the text int incr_nest; // {[]}-nesting level - int incr_mode; + unsigned char incr_mode; } JSON; +INLINE void +json_init (JSON *json) +{ + Zero (json, 1, JSON); + json->max_depth = 512; +} + ///////////////////////////////////////////////////////////////////////////// // utility functions @@ -119,6 +132,7 @@ shrink (SV *sv) { sv_utf8_downgrade (sv, 1); + if (SvLEN (sv) > SvCUR (sv) + 1) { #ifdef SvPV_shrink_to_cur @@ -172,6 +186,110 @@ return s; } +// convert offset pointer to character index, sv must be string +static STRLEN +ptr_to_index (SV *sv, char *offset) +{ + return SvUTF8 (sv) + ? utf8_distance (offset, SvPVX (sv)) + : offset - SvPVX (sv); +} + +///////////////////////////////////////////////////////////////////////////// +// fp hell + +// scan a group of digits, and a trailing exponent +static void +json_atof_scan1 (const char *s, NV *accum, int *expo, int postdp, int maxdepth) +{ + UV uaccum = 0; + int eaccum = 0; + + // if we recurse too deep, skip all remaining digits + // to avoid a stack overflow attack + if (expect_false (--maxdepth <= 0)) + while (((U8)*s - '0') < 10) + ++s; + + for (;;) + { + U8 dig = (U8)*s - '0'; + + if (expect_false (dig >= 10)) + { + if (dig == (U8)((U8)'.' - (U8)'0')) + { + ++s; + json_atof_scan1 (s, accum, expo, 1, maxdepth); + } + else if ((dig | ' ') == 'e' - '0') + { + int exp2 = 0; + int neg = 0; + + ++s; + + if (*s == '-') + { + ++s; + neg = 1; + } + else if (*s == '+') + ++s; + + while ((dig = (U8)*s - '0') < 10) + exp2 = exp2 * 10 + *s++ - '0'; + + *expo += neg ? -exp2 : exp2; + } + + break; + } + + ++s; + + uaccum = uaccum * 10 + dig; + ++eaccum; + + // if we have too many digits, then recurse for more + // we actually do this for rather few digits + if (uaccum >= (UV_MAX - 9) / 10) + { + if (postdp) *expo -= eaccum; + json_atof_scan1 (s, accum, expo, postdp, maxdepth); + if (postdp) *expo += eaccum; + + break; + } + } + + // this relies greatly on the quality of the pow () + // implementation of the platform, but a good + // implementation is hard to beat. + // (IEEE 754 conformant ones are required to be exact) + if (postdp) *expo -= eaccum; + *accum += uaccum * Perl_pow (10., *expo); + *expo += eaccum; +} + +static NV +json_atof (const char *s) +{ + NV accum = 0.; + int expo = 0; + int neg = 0; + + if (*s == '-') + { + ++s; + neg = 1; + } + + // a recursion depth of ten gives us >>500 bits + json_atof_scan1 (s, &accum, &expo, 0, 10); + + return neg ? -accum : accum; +} ///////////////////////////////////////////////////////////////////////////// // encoder @@ -183,7 +301,6 @@ SV *sv; // result scalar JSON json; U32 indent; // indentation level - U32 maxdepth; // max. indentation/recursion level UV limit; // escape character values >= this value when encoding } enc_t; @@ -192,8 +309,8 @@ { if (expect_false (enc->cur + len >= enc->end)) { - STRLEN cur = enc->cur - SvPVX (enc->sv); - SvGROW (enc->sv, cur + len + 1); + STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); enc->cur = SvPVX (enc->sv) + cur; enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; } @@ -278,14 +395,13 @@ } else { - static char hexdigit [16] = "0123456789abcdef"; need (enc, len += 5); *enc->cur++ = '\\'; *enc->cur++ = 'u'; - *enc->cur++ = hexdigit [ uch >> 12 ]; - *enc->cur++ = hexdigit [(uch >> 8) & 15]; - *enc->cur++ = hexdigit [(uch >> 4) & 15]; - *enc->cur++ = hexdigit [(uch >> 0) & 15]; + *enc->cur++ = PL_hexdigit [ uch >> 12 ]; + *enc->cur++ = PL_hexdigit [(uch >> 8) & 15]; + *enc->cur++ = PL_hexdigit [(uch >> 4) & 15]; + *enc->cur++ = PL_hexdigit [(uch >> 0) & 15]; } str += clen; @@ -366,11 +482,11 @@ { int i, len = av_len (av); - if (enc->indent >= enc->maxdepth) - croak ("data structure too deep (hit recursion limit)"); + if (enc->indent >= enc->json.max_depth) + croak (ERR_NESTING_EXCEEDED); encode_ch (enc, '['); - + if (len >= 0) { encode_nl (enc); ++enc->indent; @@ -392,7 +508,7 @@ encode_nl (enc); --enc->indent; encode_indent (enc); } - + encode_ch (enc, ']'); } @@ -406,7 +522,7 @@ SV *sv = HeSVKEY (he); STRLEN len; char *str; - + SvGETMAGIC (sv); str = SvPV (sv, len); @@ -452,16 +568,16 @@ { HE *he; - if (enc->indent >= enc->maxdepth) - croak ("data structure too deep (hit recursion limit)"); + if (enc->indent >= enc->json.max_depth) + croak (ERR_NESTING_EXCEEDED); encode_ch (enc, '{'); // for canonical output we have to sort by keys first // actually, this is mostly due to the stupid so-called - // security workaround added somewhere in 5.8.x. + // security workaround added somewhere in 5.8.x // that randomises hash orderings - if (enc->json.flags & F_CANONICAL) + if (enc->json.flags & F_CANONICAL && !SvRMAGICAL (hv)) { int count = hv_iterinit (hv); @@ -482,11 +598,15 @@ if (count) { int i, fast = 1; -#if defined(__BORLANDC__) || defined(_MSC_VER) - HE **hes = _alloca (count * sizeof (HE)); -#else - HE *hes [count]; // if your compiler dies here, you need to enable C99 mode -#endif + HE *hes_stack [STACK_HES]; + HE **hes = hes_stack; + + // allocate larger arrays on the heap + if (count > STACK_HES) + { + SV *sv = sv_2mortal (NEWSV (0, count * sizeof (*hes))); + hes = (HE **)SvPVX (sv); + } i = 0; while ((he = hv_iternext (hv))) @@ -571,65 +691,107 @@ if (expect_false (SvOBJECT (sv))) { - HV *stash = !JSON_SLOW || json_boolean_stash - ? json_boolean_stash - : gv_stashpv ("JSON::XS::Boolean", 1); + HV *boolean_stash = !JSON_SLOW || types_boolean_stash + ? types_boolean_stash + : gv_stashpv ("Types::Serialiser::Boolean", 1); + HV *stash = SvSTASH (sv); - if (SvSTASH (sv) == stash) + if (stash == boolean_stash) { if (SvIV (sv)) encode_str (enc, "true", 4, 0); else encode_str (enc, "false", 5, 0); } - else + else if (enc->json.flags & F_CONV_BLESSED) { -#if 0 - if (0 && sv_derived_from (rv, "JSON::Literal")) - { - // not yet - } -#endif - if (enc->json.flags & F_CONV_BLESSED) + GV *to_json = gv_fetchmethod_autoload (stash, "TO_JSON", 0); + + if (to_json) { + dSP; + + ENTER; SAVETMPS; PUSHMARK (SP); // we re-bless the reference to get overload and other niceties right - GV *to_json = gv_fetchmethod_autoload (SvSTASH (sv), "TO_JSON", 0); + XPUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), stash)); - if (to_json) - { - dSP; + // calling with G_SCALAR ensures that we always get a 1 return value + PUTBACK; + call_sv ((SV *)GvCV (to_json), G_SCALAR); + SPAGAIN; + + // catch this surprisingly common error + if (SvROK (TOPs) && SvRV (TOPs) == sv) + croak ("%s::TO_JSON method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); - ENTER; SAVETMPS; PUSHMARK (SP); - XPUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), SvSTASH (sv))); + sv = POPs; + PUTBACK; + + encode_sv (enc, sv); + + FREETMPS; LEAVE; + } + else if (enc->json.flags & F_ALLOW_BLESSED) + encode_str (enc, "null", 4, 0); + else + croak ("encountered object '%s' when conv_object is enabled, but TO_JSON is missing and allow_blessed disabled", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); + } + else if (enc->json.flags & F_ALLOW_TAGS) + { + GV *method = gv_fetchmethod_autoload (stash, "FREEZE", 0); - // calling with G_SCALAR ensures that we always get a 1 return value - PUTBACK; - call_sv ((SV *)GvCV (to_json), G_SCALAR); - SPAGAIN; + if (method) + { + int count; + dSP; - // catch this surprisingly common error - if (SvROK (TOPs) && SvRV (TOPs) == sv) - croak ("%s::TO_JSON method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); + ENTER; SAVETMPS; PUSHMARK (SP); + EXTEND (SP, 2); + // we re-bless the reference to get overload and other niceties right + PUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), stash)); + PUSHs (sv_json); - sv = POPs; - PUTBACK; + PUTBACK; + count = call_sv ((SV *)GvCV (method), G_ARRAY); + SPAGAIN; + + // catch this surprisingly common error + if (SvROK (TOPs) && SvRV (TOPs) == sv) + croak ("%s::TO_JSON method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); + + encode_ch (enc, '('); + encode_ch (enc, '"'); + encode_str (enc, HvNAME (stash), HvNAMELEN (stash), HvNAMEUTF8 (stash)); + encode_ch (enc, '"'); + encode_ch (enc, ')'); + encode_ch (enc, '['); - encode_sv (enc, sv); + while (count) + { + encode_sv (enc, SP[1 - count--]); - FREETMPS; LEAVE; + if (count) + encode_ch (enc, ','); } - else if (enc->json.flags & F_ALLOW_BLESSED) - encode_str (enc, "null", 4, 0); - else - croak ("encountered object '%s', but neither allow_blessed enabled nor TO_JSON method available on it", - SvPV_nolen (sv_2mortal (newRV_inc (sv)))); + + encode_ch (enc, ']'); + + PUTBACK; + + FREETMPS; LEAVE; } else if (enc->json.flags & F_ALLOW_BLESSED) encode_str (enc, "null", 4, 0); else - croak ("encountered object '%s', but neither allow_blessed nor convert_blessed settings are enabled", + croak ("encountered object '%s' when allow_tags is enabled, but FREEZE is missing and allow_blessed disabled", SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } + else if (enc->json.flags & F_ALLOW_BLESSED) + encode_str (enc, "null", 4, 0); + else + croak ("encountered object '%s', but neither allow_blessed, convert_blessed nor allow_tags settings are enabled", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } else if (svt == SVt_PVHV) encode_hv (enc, (HV *)sv); @@ -644,10 +806,14 @@ encode_str (enc, "true", 4, 0); else if (len == 1 && *pv == '0') encode_str (enc, "false", 5, 0); + else if (enc->json.flags & F_ALLOW_UNKNOWN) + encode_str (enc, "null", 4, 0); else croak ("cannot encode reference to scalar '%s' unless the scalar is 0 or 1", SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } + else if (enc->json.flags & F_ALLOW_UNKNOWN) + encode_str (enc, "null", 4, 0); else croak ("encountered %s, but JSON can only represent references to arrays or hashes", SvPV_nolen (sv_2mortal (newRV_inc (sv)))); @@ -711,7 +877,7 @@ { // large integer, use the (rather slow) snprintf way. need (enc, IVUV_MAXCHARS); - enc->cur += + enc->cur += SvIsUV(sv) ? snprintf (enc->cur, IVUV_MAXCHARS, "%"UVuf, (UV)SvUVX (sv)) : snprintf (enc->cur, IVUV_MAXCHARS, "%"IVdf, (IV)SvIVX (sv)); @@ -719,11 +885,11 @@ } else if (SvROK (sv)) encode_rv (enc, SvRV (sv)); - else if (!SvOK (sv)) + else if (!SvOK (sv) || enc->json.flags & F_ALLOW_UNKNOWN) encode_str (enc, "null", 4, 0); else - croak ("encountered perl type (%s,0x%x) that JSON cannot handle, you might want to report this", - SvPV_nolen (sv), SvFLAGS (sv)); + croak ("encountered perl type (%s,0x%x) that JSON cannot handle, check your input data", + SvPV_nolen (sv), (unsigned int)SvFLAGS (sv)); } static SV * @@ -739,13 +905,13 @@ enc.cur = SvPVX (enc.sv); enc.end = SvEND (enc.sv); enc.indent = 0; - enc.maxdepth = DEC_DEPTH (enc.json.flags); enc.limit = enc.json.flags & F_ASCII ? 0x000080UL : enc.json.flags & F_LATIN1 ? 0x000100UL : 0x110000UL; SvPOK_only (enc.sv); encode_sv (&enc, scalar); + encode_nl (&enc); SvCUR_set (enc.sv, enc.cur - SvPVX (enc.sv)); *SvEND (enc.sv) = 0; // many xs functions expect a trailing 0 for text strings @@ -816,7 +982,7 @@ ++dec->cur; \ SE -#define DEC_INC_DEPTH if (++dec->depth > dec->maxdepth) ERR ("json datastructure exceeds maximum nesting level (set a higher max_depth)") +#define DEC_INC_DEPTH if (++dec->depth > dec->json.max_depth) ERR (ERR_NESTING_EXCEEDED) #define DEC_DEC_DEPTH --dec->depth static SV *decode_sv (dec_t *dec); @@ -935,11 +1101,10 @@ else if (ch >= 0x80) { STRLEN clen; - UV uch; --dec_cur; - uch = decode_utf8 (dec_cur, dec->end - dec_cur, &clen); + decode_utf8 (dec_cur, dec->end - dec_cur, &clen); if (clen == (STRLEN)-1) ERR ("malformed UTF-8 character in JSON string"); @@ -966,7 +1131,11 @@ if (sv) { - SvGROW (sv, SvCUR (sv) + len + 1); + STRLEN cur = SvCUR (sv); + + if (SvLEN (sv) <= cur + len) + SvGROW (sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + memcpy (SvPVX (sv) + SvCUR (sv), buf, len); SvCUR_set (sv, SvCUR (sv) + len); } @@ -1067,20 +1236,20 @@ if (*start == '-') switch (len) { - case 2: return newSViv (-( start [1] - '0' * 1)); - case 3: return newSViv (-( start [1] * 10 + start [2] - '0' * 11)); - case 4: return newSViv (-( start [1] * 100 + start [2] * 10 + start [3] - '0' * 111)); - case 5: return newSViv (-( start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 1111)); - case 6: return newSViv (-(start [1] * 10000 + start [2] * 1000 + start [3] * 100 + start [4] * 10 + start [5] - '0' * 11111)); + case 2: return newSViv (-(IV)( start [1] - '0' * 1)); + case 3: return newSViv (-(IV)( start [1] * 10 + start [2] - '0' * 11)); + case 4: return newSViv (-(IV)( start [1] * 100 + start [2] * 10 + start [3] - '0' * 111)); + case 5: return newSViv (-(IV)( start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 1111)); + case 6: return newSViv (-(IV)(start [1] * 10000 + start [2] * 1000 + start [3] * 100 + start [4] * 10 + start [5] - '0' * 11111)); } else switch (len) { - case 1: return newSViv ( start [0] - '0' * 1); - case 2: return newSViv ( start [0] * 10 + start [1] - '0' * 11); - case 3: return newSViv ( start [0] * 100 + start [1] * 10 + start [2] - '0' * 111); - case 4: return newSViv ( start [0] * 1000 + start [1] * 100 + start [2] * 10 + start [3] - '0' * 1111); - case 5: return newSViv ( start [0] * 10000 + start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 11111); + case 1: return newSViv ( start [0] - '0' * 1); + case 2: return newSViv ( start [0] * 10 + start [1] - '0' * 11); + case 3: return newSViv ( start [0] * 100 + start [1] * 10 + start [2] - '0' * 111); + case 4: return newSViv ( start [0] * 1000 + start [1] * 100 + start [2] * 10 + start [3] - '0' * 1111); + case 5: return newSViv ( start [0] * 10000 + start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 11111); } { @@ -1099,20 +1268,16 @@ len -= *start == '-' ? 1 : 0; // does not fit into IV or UV, try NV - if ((sizeof (NV) == sizeof (double) && DBL_DIG >= len) - #if defined (LDBL_DIG) - || (sizeof (NV) == sizeof (long double) && LDBL_DIG >= len) - #endif - ) + if (len <= NV_DIG) // fits into NV without loss of precision - return newSVnv (Atof (start)); + return newSVnv (json_atof (start)); // everything else fails, convert it to a string return newSVpvn (start, dec->cur - start); } // loss of precision here - return newSVnv (Atof (start)); + return newSVnv (json_atof (start)); fail: return 0; @@ -1146,7 +1311,7 @@ ++dec->cur; break; } - + if (*dec->cur != ',') ERR (", or ] expected while parsing array"); @@ -1291,6 +1456,7 @@ ENTER; SAVETMPS; PUSHMARK (SP); XPUSHs (HeVAL (he)); + sv_2mortal (sv); PUTBACK; count = call_sv (HeVAL (cb), G_ARRAY); SPAGAIN; @@ -1301,6 +1467,7 @@ return sv; } + SvREFCNT_inc (sv); FREETMPS; LEAVE; } } @@ -1336,15 +1503,92 @@ } static SV * +decode_tag (dec_t *dec) +{ + SV *tag = 0; + SV *val = 0; + + if (!(dec->json.flags & F_ALLOW_TAGS)) + ERR ("malformed JSON string, neither array, object, number, string or atom"); + + ++dec->cur; + + tag = decode_sv (dec); + if (!tag) + goto fail; + + if (!SvPOK (tag)) + ERR ("malformed JSON string, (tag) must be a string"); + + if (*dec->cur != ')') + ERR (") expected after tag"); + + ++dec->cur; + + val = decode_sv (dec); + if (!val) + goto fail; + + if (!SvROK (val) || SvTYPE (SvRV (val)) != SVt_PVAV) + ERR ("malformed JSON string, tag value must be an array"); + + { + AV *av = (AV *)SvRV (val); + int i, len = av_len (av) + 1; + HV *stash = gv_stashsv (tag, 0); + SV *sv; + + if (!stash) + ERR ("cannot decode perl-object (package does not exist)"); + + GV *method = gv_fetchmethod_autoload (stash, "THAW", 0); + + if (!method) + ERR ("cannot decode perl-object (package does not have a THAW method)"); + + dSP; + + ENTER; SAVETMPS; PUSHMARK (SP); + EXTEND (SP, len + 2); + // we re-bless the reference to get overload and other niceties right + PUSHs (tag); + PUSHs (sv_json); + + for (i = 0; i < len; ++i) + PUSHs (*av_fetch (av, i, 1)); + + PUTBACK; + call_sv ((SV *)GvCV (method), G_SCALAR); + SPAGAIN; + + SvREFCNT_dec (tag); + SvREFCNT_dec (val); + sv = SvREFCNT_inc (POPs); + + PUTBACK; + + FREETMPS; LEAVE; + + return sv; + } + +fail: + SvREFCNT_dec (tag); + SvREFCNT_dec (val); + return 0; +} + +static SV * decode_sv (dec_t *dec) { // the beauty of JSON: you need exactly one character lookahead // to parse everything. switch (*dec->cur) { - case '"': ++dec->cur; return decode_str (dec); - case '[': ++dec->cur; return decode_av (dec); + case '"': ++dec->cur; return decode_str (dec); + case '[': ++dec->cur; return decode_av (dec); case '{': ++dec->cur; return decode_hv (dec); + case '(': return decode_tag (dec); case '-': case '0': case '1': case '2': case '3': case '4': @@ -1356,9 +1600,9 @@ { dec->cur += 4; #if JSON_SLOW - json_true = get_bool ("JSON::XS::true"); + types_true = get_bool ("Types::Serialiser::true"); #endif - return newSVsv (json_true); + return newSVsv (types_true); } else ERR ("'true' expected"); @@ -1370,9 +1614,9 @@ { dec->cur += 5; #if JSON_SLOW - json_false = get_bool ("JSON::XS::false"); + types_false = get_bool ("Types::Serialiser::false"); #endif - return newSVsv (json_false); + return newSVsv (types_false); } else ERR ("'false' expected"); @@ -1391,7 +1635,7 @@ break; default: - ERR ("malformed JSON string, neither array, object, number, string or atom"); + ERR ("malformed JSON string, neither tag, array, object, number, string or atom"); break; } @@ -1400,32 +1644,56 @@ } static SV * -decode_json (SV *string, JSON *json, STRLEN *offset_return) +decode_json (SV *string, JSON *json, char **offset_return) { dec_t dec; - STRLEN offset; SV *sv; - SvGETMAGIC (string); + /* work around bugs in 5.10 where manipulating magic values + * makes perl ignore the magic in subsequent accesses. + * also make a copy of non-PV values, to get them into a clean + * state (SvPV should do that, but it's buggy, see below). + */ + /*SvGETMAGIC (string);*/ + if (SvMAGICAL (string) || !SvPOK (string)) + string = sv_2mortal (newSVsv (string)); + SvUPGRADE (string, SVt_PV); - if (json->flags & F_MAXSIZE && SvCUR (string) > DEC_SIZE (json->flags)) - croak ("attempted decode of JSON text of %lu bytes size, but max_size is set to %lu", - (unsigned long)SvCUR (string), (unsigned long)DEC_SIZE (json->flags)); + /* work around a bug in perl 5.10, which causes SvCUR to fail an + * assertion with -DDEBUGGING, although SvCUR is documented to + * return the xpv_cur field which certainly exists after upgrading. + * according to nicholas clark, calling SvPOK fixes this. + * But it doesn't fix it, so try another workaround, call SvPV_nolen + * and hope for the best. + * Damnit, SvPV_nolen still trips over yet another assertion. This + * assertion business is seriously broken, try yet another workaround + * for the broken -DDEBUGGING. + */ + { +#ifdef DEBUGGING + STRLEN offset = SvOK (string) ? sv_len (string) : 0; +#else + STRLEN offset = SvCUR (string); +#endif + + if (offset > json->max_size && json->max_size) + croak ("attempted decode of JSON text of %lu bytes size, but max_size is set to %lu", + (unsigned long)SvCUR (string), (unsigned long)json->max_size); + } - if (json->flags & F_UTF8) + if (DECODE_WANTS_OCTETS (json)) sv_utf8_downgrade (string, 0); else sv_utf8_upgrade (string); SvGROW (string, SvCUR (string) + 1); // should basically be a NOP - dec.json = *json; - dec.cur = SvPVX (string); - dec.end = SvEND (string); - dec.err = 0; - dec.depth = 0; - dec.maxdepth = DEC_DEPTH (dec.json.flags); + dec.json = *json; + dec.cur = SvPVX (string); + dec.end = SvEND (string); + dec.err = 0; + dec.depth = 0; if (dec.json.cb_object || dec.json.cb_sk_object) dec.json.flags |= F_HOOK; @@ -1435,6 +1703,9 @@ decode_ws (&dec); sv = decode_sv (&dec); + if (offset_return) + *offset_return = dec.cur; + if (!(offset_return || !sv)) { // check for trailing garbage @@ -1448,16 +1719,6 @@ } } - if (offset_return || !sv) - { - offset = dec.json.flags & F_UTF8 - ? dec.cur - SvPVX (string) - : utf8_distance (dec.cur, SvPVX (string)); - - if (offset_return) - *offset_return = offset; - } - if (!sv) { SV *uni = sv_newmortal (); @@ -1471,9 +1732,9 @@ pv_uni_display (uni, dec.cur, dec.end - dec.cur, 20, UNI_DISPLAY_QQ); LEAVE; - croak ("%s, at character offset %d [\"%s\"]", + croak ("%s, at character offset %d (before \"%s\")", dec.err, - (int)offset, + (int)ptr_to_index (string, dec.cur), dec.cur != dec.end ? SvPV_nolen (uni) : "(end of string)"); } @@ -1493,19 +1754,30 @@ { const char *p = SvPVX (self->incr_text) + self->incr_pos; + // the state machine here is a bit convoluted and could be simplified a lot + // but this would make it slower, so... + for (;;) { //printf ("loop pod %d *p<%c><%s>, mode %d nest %d\n", p - SvPVX (self->incr_text), *p, p, self->incr_mode, self->incr_nest);//D switch (self->incr_mode) { - // only used for intiial whitespace skipping + // only used for initial whitespace skipping case INCR_M_WS: for (;;) { if (*p > 0x20) { - self->incr_mode = INCR_M_JSON; - goto incr_m_json; + if (*p == '#') + { + self->incr_mode = INCR_M_C0; + goto incr_m_c; + } + else + { + self->incr_mode = INCR_M_JSON; + goto incr_m_json; + } } else if (!*p) goto interrupt; @@ -1522,6 +1794,25 @@ self->incr_mode = INCR_M_STR; goto incr_m_str; + // inside #-style comments + case INCR_M_C0: + case INCR_M_C1: + incr_m_c: + for (;;) + { + if (*p == '\n') + { + self->incr_mode = self->incr_mode == INCR_M_C0 ? INCR_M_WS : INCR_M_JSON; + break; + } + else if (!*p) + goto interrupt; + + ++p; + } + + break; + // inside a string case INCR_M_STR: incr_m_str: @@ -1581,13 +1872,24 @@ case '[': case '{': - ++self->incr_nest; + case '(': + if (++self->incr_nest > self->max_depth) + croak (ERR_NESTING_EXCEEDED); break; case ']': case '}': - if (!--self->incr_nest) + if (--self->incr_nest <= 0) goto interrupt; + break; + + case ')': + --self->incr_nest; + break; + + case '#': + self->incr_mode = INCR_M_C1; + goto incr_m_c; } } } @@ -1598,6 +1900,7 @@ interrupt: self->incr_pos = p - SvPVX (self->incr_text); + //printf ("interrupt<%.*s>\n", self->incr_pos, SvPVX(self->incr_text));//D //printf ("return pos %d mode %d nest %d\n", self->incr_pos, self->incr_mode, self->incr_nest);//D } @@ -1617,27 +1920,31 @@ : i >= 'A' && i <= 'F' ? i - 'A' + 10 : -1; - json_stash = gv_stashpv ("JSON::XS" , 1); - json_boolean_stash = gv_stashpv ("JSON::XS::Boolean", 1); + json_stash = gv_stashpv ("JSON::XS" , 1); + types_boolean_stash = gv_stashpv ("Types::Serialiser::Boolean", 1); + + types_true = get_bool ("Types::Serialiser::true"); + types_false = get_bool ("Types::Serialiser::false"); + + sv_json = newSVpv ("JSON", 0); + SvREADONLY_on (sv_json); - json_true = get_bool ("JSON::XS::true"); - json_false = get_bool ("JSON::XS::false"); + CvNODEBUG_on (get_cv ("JSON::XS::incr_text", 0)); /* the debugger completely breaks lvalue subs */ } PROTOTYPES: DISABLE void CLONE (...) CODE: - json_stash = 0; - json_boolean_stash = 0; + json_stash = 0; + types_boolean_stash = 0; void new (char *klass) PPCODE: { - SV *pv = NEWSV (0, sizeof (JSON)); + SV *pv = NEWSV (0, sizeof (JSON)); SvPOK_only (pv); - Zero (SvPVX (pv), 1, JSON); - ((JSON *)SvPVX (pv))->flags = F_DEFAULT; + json_init ((JSON *)SvPVX (pv)); XPUSHs (sv_2mortal (sv_bless ( newRV_noinc (pv), strEQ (klass, "JSON::XS") ? JSON_STASH : gv_stashpv (klass, 1) @@ -1659,6 +1966,8 @@ allow_blessed = F_ALLOW_BLESSED convert_blessed = F_CONV_BLESSED relaxed = F_RELAXED + allow_unknown = F_ALLOW_UNKNOWN + allow_tags = F_ALLOW_TAGS PPCODE: { if (enable) @@ -1683,49 +1992,30 @@ get_allow_blessed = F_ALLOW_BLESSED get_convert_blessed = F_CONV_BLESSED get_relaxed = F_RELAXED + get_allow_unknown = F_ALLOW_UNKNOWN + get_allow_tags = F_ALLOW_TAGS PPCODE: XPUSHs (boolSV (self->flags & ix)); -void max_depth (JSON *self, UV max_depth = 0x80000000UL) +void max_depth (JSON *self, U32 max_depth = 0x80000000UL) PPCODE: -{ - UV log2 = 0; - - if (max_depth > 0x80000000UL) max_depth = 0x80000000UL; - - while ((1UL << log2) < max_depth) - ++log2; - - self->flags = self->flags & ~F_MAXDEPTH | (log2 << S_MAXDEPTH); - + self->max_depth = max_depth; XPUSHs (ST (0)); -} U32 get_max_depth (JSON *self) CODE: - RETVAL = DEC_DEPTH (self->flags); + RETVAL = self->max_depth; OUTPUT: RETVAL -void max_size (JSON *self, UV max_size = 0) +void max_size (JSON *self, U32 max_size = 0) PPCODE: -{ - UV log2 = 0; - - if (max_size > 0x80000000UL) max_size = 0x80000000UL; - if (max_size == 1) max_size = 2; - - while ((1UL << log2) < max_size) - ++log2; - - self->flags = self->flags & ~F_MAXSIZE | (log2 << S_MAXSIZE); - + self->max_size = max_size; XPUSHs (ST (0)); -} int get_max_size (JSON *self) CODE: - RETVAL = DEC_SIZE (self->flags); + RETVAL = self->max_size; OUTPUT: RETVAL @@ -1741,7 +2031,7 @@ void filter_json_single_key_object (JSON *self, SV *key, SV *cb = &PL_sv_undef) PPCODE: { - if (!self->cb_sk_object) + if (!self->cb_sk_object) self->cb_sk_object = newHV (); if (SvOK (cb)) @@ -1762,19 +2052,23 @@ void encode (JSON *self, SV *scalar) PPCODE: - XPUSHs (encode_json (scalar, self)); + PUTBACK; scalar = encode_json (scalar, self); SPAGAIN; + XPUSHs (scalar); void decode (JSON *self, SV *jsonstr) PPCODE: - XPUSHs (decode_json (jsonstr, self, 0)); + PUTBACK; jsonstr = decode_json (jsonstr, self, 0); SPAGAIN; + XPUSHs (jsonstr); void decode_prefix (JSON *self, SV *jsonstr) PPCODE: { - STRLEN offset; + SV *sv; + char *offset; + PUTBACK; sv = decode_json (jsonstr, self, &offset); SPAGAIN; EXTEND (SP, 2); - PUSHs (decode_json (jsonstr, self, &offset)); - PUSHs (sv_2mortal (newSVuv (offset))); + PUSHs (sv); + PUSHs (sv_2mortal (newSVuv (ptr_to_index (jsonstr, offset)))); } void incr_parse (JSON *self, SV *jsonstr = 0) @@ -1783,23 +2077,44 @@ if (!self->incr_text) self->incr_text = newSVpvn ("", 0); + /* if utf8-ness doesn't match the decoder, need to upgrade/downgrade */ + if (!DECODE_WANTS_OCTETS (self) == !SvUTF8 (self->incr_text)) + if (DECODE_WANTS_OCTETS (self)) + { + if (self->incr_pos) + self->incr_pos = utf8_length ((U8 *)SvPVX (self->incr_text), + (U8 *)SvPVX (self->incr_text) + self->incr_pos); + + sv_utf8_downgrade (self->incr_text, 0); + } + else + { + sv_utf8_upgrade (self->incr_text); + + if (self->incr_pos) + self->incr_pos = utf8_hop ((U8 *)SvPVX (self->incr_text), self->incr_pos) + - (U8 *)SvPVX (self->incr_text); + } + // append data, if any if (jsonstr) { - if (SvUTF8 (jsonstr) && !SvUTF8 (self->incr_text)) - { - /* utf-8-ness differs, need to upgrade */ - sv_utf8_upgrade (self->incr_text); - - if (self->incr_pos) - self->incr_pos = utf8_hop ((U8 *)SvPVX (self->incr_text), self->incr_pos) - - (U8 *)SvPVX (self->incr_text); - } + /* make sure both strings have same encoding */ + if (SvUTF8 (jsonstr) != SvUTF8 (self->incr_text)) + if (SvUTF8 (jsonstr)) + sv_utf8_downgrade (jsonstr, 0); + else + sv_utf8_upgrade (jsonstr); + /* and then just blindly append */ { STRLEN len; const char *str = SvPV (jsonstr, len); - SvGROW (self->incr_text, SvCUR (self->incr_text) + len + 1); + STRLEN cur = SvCUR (self->incr_text); + + if (SvLEN (self->incr_text) <= cur + len) + SvGROW (self->incr_text, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + Move (str, SvEND (self->incr_text), len, char); SvCUR_set (self->incr_text, SvCUR (self->incr_text) + len); *SvEND (self->incr_text) = 0; // this should basically be a nop, too, but make sure it's there @@ -1809,21 +2124,38 @@ if (GIMME_V != G_VOID) do { - STRLEN offset; + SV *sv; + char *offset; if (!INCR_DONE (self)) { incr_parse (self); + + if (self->incr_pos > self->max_size && self->max_size) + croak ("attempted decode of JSON text of %lu bytes size, but max_size is set to %lu", + (unsigned long)self->incr_pos, (unsigned long)self->max_size); + if (!INCR_DONE (self)) - break; + { + // as an optimisation, do not accumulate white space in the incr buffer + if (self->incr_mode == INCR_M_WS && self->incr_pos) + { + self->incr_pos = 0; + SvCUR_set (self->incr_text, 0); + } + + break; + } } - XPUSHs (decode_json (self->incr_text, self, &offset)); + PUTBACK; sv = decode_json (self->incr_text, self, &offset); SPAGAIN; + XPUSHs (sv); - sv_chop (self->incr_text, SvPV_nolen (self->incr_text) + offset); - self->incr_pos -= offset; + self->incr_pos -= offset - SvPVX (self->incr_text); self->incr_nest = 0; self->incr_mode = 0; + + sv_chop (self->incr_text, offset); } while (GIMME_V == G_ARRAY); } @@ -1852,6 +2184,16 @@ } } +void incr_reset (JSON *self) + CODE: +{ + SvREFCNT_dec (self->incr_text); + self->incr_text = 0; + self->incr_pos = 0; + self->incr_nest = 0; + self->incr_mode = 0; +} + void DESTROY (JSON *self) CODE: SvREFCNT_dec (self->cb_sk_object); @@ -1861,23 +2203,22 @@ PROTOTYPES: ENABLE void encode_json (SV *scalar) - ALIAS: - to_json_ = 0 - encode_json = F_UTF8 PPCODE: { - JSON json = { F_DEFAULT | ix }; - XPUSHs (encode_json (scalar, &json)); + JSON json; + json_init (&json); + json.flags |= F_UTF8; + PUTBACK; scalar = encode_json (scalar, &json); SPAGAIN; + XPUSHs (scalar); } void decode_json (SV *jsonstr) - ALIAS: - from_json_ = 0 - decode_json = F_UTF8 PPCODE: { - JSON json = { F_DEFAULT | ix }; - XPUSHs (decode_json (jsonstr, &json, 0)); + JSON json; + json_init (&json); + json.flags |= F_UTF8; + PUTBACK; jsonstr = decode_json (jsonstr, &json, 0); SPAGAIN; + XPUSHs (jsonstr); } -