--- JSON-XS/XS.xs 2008/09/29 03:09:27 1.93 +++ JSON-XS/XS.xs 2016/02/21 16:18:19 1.126 @@ -14,12 +14,24 @@ #endif // some old perls do not have this, try to make it work, no -// guarentees, though. if it breaks, you get to keep the pieces. +// guarantees, though. if it breaks, you get to keep the pieces. #ifndef UTF8_MAXBYTES # define UTF8_MAXBYTES 13 #endif -#define IVUV_MAXCHARS (sizeof (UV) * CHAR_BIT * 28 / 93 + 2) +// compatibility with perl <5.18 +#ifndef HvNAMELEN_get +# define HvNAMELEN_get(hv) strlen (HvNAME (hv)) +#endif +#ifndef HvNAMELEN +# define HvNAMELEN(hv) HvNAMELEN_get (hv) +#endif +#ifndef HvNAMEUTF8 +# define HvNAMEUTF8(hv) 0 +#endif + +// three extra for rounding, sign, and end of string +#define IVUV_MAXCHARS (sizeof (UV) * CHAR_BIT * 28 / 93 + 3) #define F_ASCII 0x00000001UL #define F_LATIN1 0x00000002UL @@ -34,6 +46,7 @@ #define F_CONV_BLESSED 0x00000800UL #define F_RELAXED 0x00001000UL #define F_ALLOW_UNKNOWN 0x00002000UL +#define F_ALLOW_TAGS 0x00004000UL #define F_HOOK 0x00080000UL // some hooks exist, so slow-path processing #define F_PRETTY F_INDENT | F_SPACE_BEFORE | F_SPACE_AFTER @@ -43,6 +56,8 @@ #define SHORT_STRING_LEN 16384 // special-case strings of up to this size +#define DECODE_WANTS_OCTETS(json) ((json)->flags & F_UTF8) + #define SB do { #define SE } while (0) @@ -66,18 +81,25 @@ #ifdef USE_ITHREADS # define JSON_SLOW 1 # define JSON_STASH (json_stash ? json_stash : gv_stashpv ("JSON::XS", 1)) +# define BOOL_STASH (bool_stash ? bool_stash : gv_stashpv ("Types::Serialiser::Boolean", 1)) #else # define JSON_SLOW 0 # define JSON_STASH json_stash +# define BOOL_STASH bool_stash #endif -static HV *json_stash, *json_boolean_stash; // JSON::XS:: -static SV *json_true, *json_false; +// the amount of HEs to allocate on the stack, when sorting keys +#define STACK_HES 64 + +static HV *json_stash, *bool_stash; // JSON::XS::, Types::Serialiser::Boolean:: +static SV *bool_true, *bool_false, *sv_json; enum { INCR_M_WS = 0, // initial whitespace skipping, must be 0 INCR_M_STR, // inside string INCR_M_BS, // inside backslash + INCR_M_C0, // inside comment in initial whitespace sequence + INCR_M_C1, // inside comment in other places INCR_M_JSON // outside anything, count nesting }; @@ -177,6 +199,151 @@ return s; } +// convert offset pointer to character index, sv must be string +static STRLEN +ptr_to_index (SV *sv, char *offset) +{ + return SvUTF8 (sv) + ? utf8_distance (offset, SvPVX (sv)) + : offset - SvPVX (sv); +} + +///////////////////////////////////////////////////////////////////////////// +// fp hell + +// scan a group of digits, and a trailing exponent +static void +json_atof_scan1 (const char *s, NV *accum, int *expo, int postdp, int maxdepth) +{ + UV uaccum = 0; + int eaccum = 0; + + // if we recurse too deep, skip all remaining digits + // to avoid a stack overflow attack + if (expect_false (--maxdepth <= 0)) + while (((U8)*s - '0') < 10) + ++s; + + for (;;) + { + U8 dig = (U8)*s - '0'; + + if (expect_false (dig >= 10)) + { + if (dig == (U8)((U8)'.' - (U8)'0')) + { + ++s; + json_atof_scan1 (s, accum, expo, 1, maxdepth); + } + else if ((dig | ' ') == 'e' - '0') + { + int exp2 = 0; + int neg = 0; + + ++s; + + if (*s == '-') + { + ++s; + neg = 1; + } + else if (*s == '+') + ++s; + + while ((dig = (U8)*s - '0') < 10) + exp2 = exp2 * 10 + *s++ - '0'; + + *expo += neg ? -exp2 : exp2; + } + + break; + } + + ++s; + + uaccum = uaccum * 10 + dig; + ++eaccum; + + // if we have too many digits, then recurse for more + // we actually do this for rather few digits + if (uaccum >= (UV_MAX - 9) / 10) + { + if (postdp) *expo -= eaccum; + json_atof_scan1 (s, accum, expo, postdp, maxdepth); + if (postdp) *expo += eaccum; + + break; + } + } + + // this relies greatly on the quality of the pow () + // implementation of the platform, but a good + // implementation is hard to beat. + // (IEEE 754 conformant ones are required to be exact) + if (postdp) *expo -= eaccum; + *accum += uaccum * Perl_pow (10., *expo); + *expo += eaccum; +} + +static NV +json_atof (const char *s) +{ + NV accum = 0.; + int expo = 0; + int neg = 0; + + if (*s == '-') + { + ++s; + neg = 1; + } + + // a recursion depth of ten gives us >>500 bits + json_atof_scan1 (s, &accum, &expo, 0, 10); + + return neg ? -accum : accum; +} + +// target of scalar reference is bool? -1 == nope, 0 == false, 1 == true +static int +ref_bool_type (SV *sv) +{ + svtype svt = SvTYPE (sv); + + if (svt < SVt_PVAV) + { + STRLEN len = 0; + char *pv = svt ? SvPV (sv, len) : 0; + + if (len == 1) + if (*pv == '1') + return 1; + else if (*pv == '0') + return 0; + + } + + return -1; +} + +// returns whether scalar is not a reference in the sense of allow_nonref +static int +json_nonref (SV *scalar) +{ + if (!SvROK (scalar)) + return 1; + + scalar = SvRV (scalar); + + if (SvSTASH (scalar) == bool_stash) + return 1; + + if (!SvOBJECT (scalar) && ref_bool_type (scalar) >= 0) + return 1; + + return 0; +} + ///////////////////////////////////////////////////////////////////////////// // encoder @@ -197,7 +364,7 @@ if (expect_false (enc->cur + len >= enc->end)) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); - SvGROW (enc->sv, cur + len + 1); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); enc->cur = SvPVX (enc->sv) + cur; enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; } @@ -373,7 +540,7 @@ croak (ERR_NESTING_EXCEEDED); encode_ch (enc, '['); - + if (len >= 0) { encode_nl (enc); ++enc->indent; @@ -395,7 +562,7 @@ encode_nl (enc); --enc->indent; encode_indent (enc); } - + encode_ch (enc, ']'); } @@ -409,7 +576,7 @@ SV *sv = HeSVKEY (he); STRLEN len; char *str; - + SvGETMAGIC (sv); str = SvPV (sv, len); @@ -464,7 +631,7 @@ // actually, this is mostly due to the stupid so-called // security workaround added somewhere in 5.8.x // that randomises hash orderings - if (enc->json.flags & F_CANONICAL) + if (enc->json.flags & F_CANONICAL && !SvRMAGICAL (hv)) { int count = hv_iterinit (hv); @@ -485,11 +652,15 @@ if (count) { int i, fast = 1; -#if defined(__BORLANDC__) || defined(_MSC_VER) - HE **hes = _alloca (count * sizeof (HE)); -#else - HE *hes [count]; // if your compiler dies here, you need to enable C99 mode -#endif + HE *hes_stack [STACK_HES]; + HE **hes = hes_stack; + + // allocate larger arrays on the heap + if (count > STACK_HES) + { + SV *sv = sv_2mortal (NEWSV (0, count * sizeof (*hes))); + hes = (HE **)SvPVX (sv); + } i = 0; while ((he = hv_iternext (hv))) @@ -568,71 +739,92 @@ encode_rv (enc_t *enc, SV *sv) { svtype svt; + GV *method; SvGETMAGIC (sv); svt = SvTYPE (sv); if (expect_false (SvOBJECT (sv))) { - HV *stash = !JSON_SLOW || json_boolean_stash - ? json_boolean_stash - : gv_stashpv ("JSON::XS::Boolean", 1); + HV *stash = SvSTASH (sv); - if (SvSTASH (sv) == stash) + if (stash == bool_stash) { if (SvIV (sv)) encode_str (enc, "true", 4, 0); else encode_str (enc, "false", 5, 0); } - else + else if ((enc->json.flags & F_ALLOW_TAGS) && (method = gv_fetchmethod_autoload (stash, "FREEZE", 0))) { -#if 0 - if (0 && sv_derived_from (rv, "JSON::Literal")) - { - // not yet - } -#endif - if (enc->json.flags & F_CONV_BLESSED) + int count; + dSP; + + ENTER; SAVETMPS; + SAVESTACK_POS (); + PUSHMARK (SP); + EXTEND (SP, 2); + // we re-bless the reference to get overload and other niceties right + PUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), stash)); + PUSHs (sv_json); + + PUTBACK; + count = call_sv ((SV *)GvCV (method), G_ARRAY); + SPAGAIN; + + // catch this surprisingly common error + if (SvROK (TOPs) && SvRV (TOPs) == sv) + croak ("%s::FREEZE method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); + + encode_ch (enc, '('); + encode_ch (enc, '"'); + encode_str (enc, HvNAME (stash), HvNAMELEN (stash), HvNAMEUTF8 (stash)); + encode_ch (enc, '"'); + encode_ch (enc, ')'); + encode_ch (enc, '['); + + while (count) { - // we re-bless the reference to get overload and other niceties right - GV *to_json = gv_fetchmethod_autoload (SvSTASH (sv), "TO_JSON", 0); + encode_sv (enc, SP[1 - count--]); - if (to_json) - { - dSP; + if (count) + encode_ch (enc, ','); + } - ENTER; SAVETMPS; PUSHMARK (SP); - XPUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), SvSTASH (sv))); + encode_ch (enc, ']'); - // calling with G_SCALAR ensures that we always get a 1 return value - PUTBACK; - call_sv ((SV *)GvCV (to_json), G_SCALAR); - SPAGAIN; + FREETMPS; LEAVE; + } + else if ((enc->json.flags & F_CONV_BLESSED) && (method = gv_fetchmethod_autoload (stash, "TO_JSON", 0))) + { + dSP; - // catch this surprisingly common error - if (SvROK (TOPs) && SvRV (TOPs) == sv) - croak ("%s::TO_JSON method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); + ENTER; SAVETMPS; + PUSHMARK (SP); + // we re-bless the reference to get overload and other niceties right + XPUSHs (sv_bless (sv_2mortal (newRV_inc (sv)), stash)); + + // calling with G_SCALAR ensures that we always get a 1 return value + PUTBACK; + call_sv ((SV *)GvCV (method), G_SCALAR); + SPAGAIN; + + // catch this surprisingly common error + if (SvROK (TOPs) && SvRV (TOPs) == sv) + croak ("%s::TO_JSON method returned same object as was passed instead of a new one", HvNAME (SvSTASH (sv))); - sv = POPs; - PUTBACK; + sv = POPs; + PUTBACK; - encode_sv (enc, sv); + encode_sv (enc, sv); - FREETMPS; LEAVE; - } - else if (enc->json.flags & F_ALLOW_BLESSED) - encode_str (enc, "null", 4, 0); - else - croak ("encountered object '%s', but neither allow_blessed enabled nor TO_JSON method available on it", - SvPV_nolen (sv_2mortal (newRV_inc (sv)))); - } - else if (enc->json.flags & F_ALLOW_BLESSED) - encode_str (enc, "null", 4, 0); - else - croak ("encountered object '%s', but neither allow_blessed nor convert_blessed settings are enabled", - SvPV_nolen (sv_2mortal (newRV_inc (sv)))); + FREETMPS; LEAVE; } + else if (enc->json.flags & F_ALLOW_BLESSED) + encode_str (enc, "null", 4, 0); + else + croak ("encountered object '%s', but neither allow_blessed, convert_blessed nor allow_tags settings are enabled (or TO_JSON/FREEZE method missing)", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } else if (svt == SVt_PVHV) encode_hv (enc, (HV *)sv); @@ -640,12 +832,11 @@ encode_av (enc, (AV *)sv); else if (svt < SVt_PVAV) { - STRLEN len = 0; - char *pv = svt ? SvPV (sv, len) : 0; + int bool_type = ref_bool_type (sv); - if (len == 1 && *pv == '1') + if (bool_type == 1) encode_str (enc, "true", 4, 0); - else if (len == 1 && *pv == '0') + else if (bool_type == 0) encode_str (enc, "false", 5, 0); else if (enc->json.flags & F_ALLOW_UNKNOWN) encode_str (enc, "null", 4, 0); @@ -718,7 +909,7 @@ { // large integer, use the (rather slow) snprintf way. need (enc, IVUV_MAXCHARS); - enc->cur += + enc->cur += SvIsUV(sv) ? snprintf (enc->cur, IVUV_MAXCHARS, "%"UVuf, (UV)SvUVX (sv)) : snprintf (enc->cur, IVUV_MAXCHARS, "%"IVdf, (IV)SvIVX (sv)); @@ -729,8 +920,8 @@ else if (!SvOK (sv) || enc->json.flags & F_ALLOW_UNKNOWN) encode_str (enc, "null", 4, 0); else - croak ("encountered perl type (%s,0x%x) that JSON cannot handle, you might want to report this", - SvPV_nolen (sv), SvFLAGS (sv)); + croak ("encountered perl type (%s,0x%x) that JSON cannot handle, check your input data", + SvPV_nolen (sv), (unsigned int)SvFLAGS (sv)); } static SV * @@ -738,7 +929,7 @@ { enc_t enc; - if (!(json->flags & F_ALLOW_NONREF) && !SvROK (scalar)) + if (!(json->flags & F_ALLOW_NONREF) && json_nonref (scalar)) croak ("hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this)"); enc.json = *json; @@ -752,6 +943,7 @@ SvPOK_only (enc.sv); encode_sv (&enc, scalar); + encode_nl (&enc); SvCUR_set (enc.sv, enc.cur - SvPVX (enc.sv)); *SvEND (enc.sv) = 0; // many xs functions expect a trailing 0 for text strings @@ -941,11 +1133,10 @@ else if (ch >= 0x80) { STRLEN clen; - UV uch; --dec_cur; - uch = decode_utf8 (dec_cur, dec->end - dec_cur, &clen); + decode_utf8 (dec_cur, dec->end - dec_cur, &clen); if (clen == (STRLEN)-1) ERR ("malformed UTF-8 character in JSON string"); @@ -955,6 +1146,8 @@ utf8 = 1; } + else if (ch == '\t' && dec->json.flags & F_RELAXED) + *cur++ = ch; else { --dec_cur; @@ -972,7 +1165,11 @@ if (sv) { - SvGROW (sv, SvCUR (sv) + len + 1); + STRLEN cur = SvCUR (sv); + + if (SvLEN (sv) <= cur + len) + SvGROW (sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + memcpy (SvPVX (sv) + SvCUR (sv), buf, len); SvCUR_set (sv, SvCUR (sv) + len); } @@ -1073,20 +1270,20 @@ if (*start == '-') switch (len) { - case 2: return newSViv (-( start [1] - '0' * 1)); - case 3: return newSViv (-( start [1] * 10 + start [2] - '0' * 11)); - case 4: return newSViv (-( start [1] * 100 + start [2] * 10 + start [3] - '0' * 111)); - case 5: return newSViv (-( start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 1111)); - case 6: return newSViv (-(start [1] * 10000 + start [2] * 1000 + start [3] * 100 + start [4] * 10 + start [5] - '0' * 11111)); + case 2: return newSViv (-(IV)( start [1] - '0' * 1)); + case 3: return newSViv (-(IV)( start [1] * 10 + start [2] - '0' * 11)); + case 4: return newSViv (-(IV)( start [1] * 100 + start [2] * 10 + start [3] - '0' * 111)); + case 5: return newSViv (-(IV)( start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 1111)); + case 6: return newSViv (-(IV)(start [1] * 10000 + start [2] * 1000 + start [3] * 100 + start [4] * 10 + start [5] - '0' * 11111)); } else switch (len) { - case 1: return newSViv ( start [0] - '0' * 1); - case 2: return newSViv ( start [0] * 10 + start [1] - '0' * 11); - case 3: return newSViv ( start [0] * 100 + start [1] * 10 + start [2] - '0' * 111); - case 4: return newSViv ( start [0] * 1000 + start [1] * 100 + start [2] * 10 + start [3] - '0' * 1111); - case 5: return newSViv ( start [0] * 10000 + start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 11111); + case 1: return newSViv ( start [0] - '0' * 1); + case 2: return newSViv ( start [0] * 10 + start [1] - '0' * 11); + case 3: return newSViv ( start [0] * 100 + start [1] * 10 + start [2] - '0' * 111); + case 4: return newSViv ( start [0] * 1000 + start [1] * 100 + start [2] * 10 + start [3] - '0' * 1111); + case 5: return newSViv ( start [0] * 10000 + start [1] * 1000 + start [2] * 100 + start [3] * 10 + start [4] - '0' * 11111); } { @@ -1105,20 +1302,16 @@ len -= *start == '-' ? 1 : 0; // does not fit into IV or UV, try NV - if ((sizeof (NV) == sizeof (double) && DBL_DIG >= len) - #if defined (LDBL_DIG) - || (sizeof (NV) == sizeof (long double) && LDBL_DIG >= len) - #endif - ) + if (len <= NV_DIG) // fits into NV without loss of precision - return newSVnv (Atof (start)); + return newSVnv (json_atof (start)); // everything else fails, convert it to a string return newSVpvn (start, dec->cur - start); } // loss of precision here - return newSVnv (Atof (start)); + return newSVnv (json_atof (start)); fail: return 0; @@ -1152,7 +1345,7 @@ ++dec->cur; break; } - + if (*dec->cur != ',') ERR (", or ] expected while parsing array"); @@ -1286,7 +1479,7 @@ he = hv_iternext (hv); hv_iterinit (hv); - // the next line creates a mortal sv each time its called. + // the next line creates a mortal sv each time it's called. // might want to optimise this for common cases. cb = hv_fetch_ent (dec->json.cb_sk_object, hv_iterkeysv (he), 0, 0); @@ -1295,8 +1488,11 @@ dSP; int count; - ENTER; SAVETMPS; PUSHMARK (SP); + ENTER; SAVETMPS; + SAVESTACK_POS (); + PUSHMARK (SP); XPUSHs (HeVAL (he)); + sv_2mortal (sv); PUTBACK; count = call_sv (HeVAL (cb), G_ARRAY); SPAGAIN; @@ -1307,6 +1503,7 @@ return sv; } + SvREFCNT_inc (sv); FREETMPS; LEAVE; } } @@ -1316,7 +1513,9 @@ dSP; int count; - ENTER; SAVETMPS; PUSHMARK (SP); + ENTER; SAVETMPS; + SAVESTACK_POS (); + PUSHMARK (SP); XPUSHs (sv_2mortal (sv)); PUTBACK; count = call_sv (dec->json.cb_object, G_ARRAY); SPAGAIN; @@ -1342,15 +1541,99 @@ } static SV * +decode_tag (dec_t *dec) +{ + SV *tag = 0; + SV *val = 0; + + if (!(dec->json.flags & F_ALLOW_TAGS)) + ERR ("malformed JSON string, neither array, object, number, string or atom"); + + ++dec->cur; + + decode_ws (dec); + + tag = decode_sv (dec); + if (!tag) + goto fail; + + if (!SvPOK (tag)) + ERR ("malformed JSON string, (tag) must be a string"); + + decode_ws (dec); + + if (*dec->cur != ')') + ERR (") expected after tag"); + + ++dec->cur; + + decode_ws (dec); + + val = decode_sv (dec); + if (!val) + goto fail; + + if (!SvROK (val) || SvTYPE (SvRV (val)) != SVt_PVAV) + ERR ("malformed JSON string, tag value must be an array"); + + { + AV *av = (AV *)SvRV (val); + int i, len = av_len (av) + 1; + HV *stash = gv_stashsv (tag, 0); + SV *sv; + + if (!stash) + ERR ("cannot decode perl-object (package does not exist)"); + + GV *method = gv_fetchmethod_autoload (stash, "THAW", 0); + + if (!method) + ERR ("cannot decode perl-object (package does not have a THAW method)"); + + dSP; + + ENTER; SAVETMPS; + PUSHMARK (SP); + EXTEND (SP, len + 2); + // we re-bless the reference to get overload and other niceties right + PUSHs (tag); + PUSHs (sv_json); + + for (i = 0; i < len; ++i) + PUSHs (*av_fetch (av, i, 1)); + + PUTBACK; + call_sv ((SV *)GvCV (method), G_SCALAR); + SPAGAIN; + + SvREFCNT_dec (tag); + SvREFCNT_dec (val); + sv = SvREFCNT_inc (POPs); + + PUTBACK; + + FREETMPS; LEAVE; + + return sv; + } + +fail: + SvREFCNT_dec (tag); + SvREFCNT_dec (val); + return 0; +} + +static SV * decode_sv (dec_t *dec) { // the beauty of JSON: you need exactly one character lookahead // to parse everything. switch (*dec->cur) { - case '"': ++dec->cur; return decode_str (dec); - case '[': ++dec->cur; return decode_av (dec); + case '"': ++dec->cur; return decode_str (dec); + case '[': ++dec->cur; return decode_av (dec); case '{': ++dec->cur; return decode_hv (dec); + case '(': return decode_tag (dec); case '-': case '0': case '1': case '2': case '3': case '4': @@ -1362,9 +1645,9 @@ { dec->cur += 4; #if JSON_SLOW - json_true = get_bool ("JSON::XS::true"); + bool_true = get_bool ("Types::Serialiser::true"); #endif - return newSVsv (json_true); + return newSVsv (bool_true); } else ERR ("'true' expected"); @@ -1376,9 +1659,9 @@ { dec->cur += 5; #if JSON_SLOW - json_false = get_bool ("JSON::XS::false"); + bool_false = get_bool ("Types::Serialiser::false"); #endif - return newSVsv (json_false); + return newSVsv (bool_false); } else ERR ("'false' expected"); @@ -1397,7 +1680,7 @@ break; default: - ERR ("malformed JSON string, neither array, object, number, string or atom"); + ERR ("malformed JSON string, neither tag, array, object, number, string or atom"); break; } @@ -1406,13 +1689,20 @@ } static SV * -decode_json (SV *string, JSON *json, STRLEN *offset_return) +decode_json (SV *string, JSON *json, char **offset_return) { dec_t dec; - STRLEN offset; SV *sv; - SvGETMAGIC (string); + /* work around bugs in 5.10 where manipulating magic values + * makes perl ignore the magic in subsequent accesses. + * also make a copy of non-PV values, to get them into a clean + * state (SvPV should do that, but it's buggy, see below). + */ + /*SvGETMAGIC (string);*/ + if (SvMAGICAL (string) || !SvPOK (string)) + string = sv_2mortal (newSVsv (string)); + SvUPGRADE (string, SVt_PV); /* work around a bug in perl 5.10, which causes SvCUR to fail an @@ -1425,17 +1715,19 @@ * assertion business is seriously broken, try yet another workaround * for the broken -DDEBUGGING. */ + { #ifdef DEBUGGING - offset = SvOK (string) ? sv_len (string) : 0; + STRLEN offset = SvOK (string) ? sv_len (string) : 0; #else - offset = SvCUR (string); + STRLEN offset = SvCUR (string); #endif - if (offset > json->max_size && json->max_size) - croak ("attempted decode of JSON text of %lu bytes size, but max_size is set to %lu", - (unsigned long)SvCUR (string), (unsigned long)json->max_size); + if (offset > json->max_size && json->max_size) + croak ("attempted decode of JSON text of %lu bytes size, but max_size is set to %lu", + (unsigned long)SvCUR (string), (unsigned long)json->max_size); + } - if (json->flags & F_UTF8) + if (DECODE_WANTS_OCTETS (json)) sv_utf8_downgrade (string, 0); else sv_utf8_upgrade (string); @@ -1456,6 +1748,9 @@ decode_ws (&dec); sv = decode_sv (&dec); + if (offset_return) + *offset_return = dec.cur; + if (!(offset_return || !sv)) { // check for trailing garbage @@ -1469,16 +1764,6 @@ } } - if (offset_return || !sv) - { - offset = dec.json.flags & F_UTF8 - ? dec.cur - SvPVX (string) - : utf8_distance (dec.cur, SvPVX (string)); - - if (offset_return) - *offset_return = offset; - } - if (!sv) { SV *uni = sv_newmortal (); @@ -1492,15 +1777,15 @@ pv_uni_display (uni, dec.cur, dec.end - dec.cur, 20, UNI_DISPLAY_QQ); LEAVE; - croak ("%s, at character offset %d [\"%s\"]", + croak ("%s, at character offset %d (before \"%s\")", dec.err, - (int)offset, + (int)ptr_to_index (string, dec.cur), dec.cur != dec.end ? SvPV_nolen (uni) : "(end of string)"); } sv = sv_2mortal (sv); - if (!(dec.json.flags & F_ALLOW_NONREF) && !SvROK (sv)) + if (!(dec.json.flags & F_ALLOW_NONREF) && json_nonref (sv)) croak ("JSON text must be an object or array (but found number, string, true, false or null, use allow_nonref to allow this)"); return sv; @@ -1514,19 +1799,30 @@ { const char *p = SvPVX (self->incr_text) + self->incr_pos; + // the state machine here is a bit convoluted and could be simplified a lot + // but this would make it slower, so... + for (;;) { //printf ("loop pod %d *p<%c><%s>, mode %d nest %d\n", p - SvPVX (self->incr_text), *p, p, self->incr_mode, self->incr_nest);//D switch (self->incr_mode) { - // only used for intiial whitespace skipping + // only used for initial whitespace skipping case INCR_M_WS: for (;;) { if (*p > 0x20) { - self->incr_mode = INCR_M_JSON; - goto incr_m_json; + if (*p == '#') + { + self->incr_mode = INCR_M_C0; + goto incr_m_c; + } + else + { + self->incr_mode = INCR_M_JSON; + goto incr_m_json; + } } else if (!*p) goto interrupt; @@ -1543,6 +1839,25 @@ self->incr_mode = INCR_M_STR; goto incr_m_str; + // inside #-style comments + case INCR_M_C0: + case INCR_M_C1: + incr_m_c: + for (;;) + { + if (*p == '\n') + { + self->incr_mode = self->incr_mode == INCR_M_C0 ? INCR_M_WS : INCR_M_JSON; + break; + } + else if (!*p) + goto interrupt; + + ++p; + } + + break; + // inside a string case INCR_M_STR: incr_m_str: @@ -1602,6 +1917,7 @@ case '[': case '{': + case '(': if (++self->incr_nest > self->max_depth) croak (ERR_NESTING_EXCEEDED); break; @@ -1610,6 +1926,15 @@ case '}': if (--self->incr_nest <= 0) goto interrupt; + break; + + case ')': + --self->incr_nest; + break; + + case '#': + self->incr_mode = INCR_M_C1; + goto incr_m_c; } } } @@ -1620,6 +1945,7 @@ interrupt: self->incr_pos = p - SvPVX (self->incr_text); + //printf ("interrupt<%.*s>\n", self->incr_pos, SvPVX(self->incr_text));//D //printf ("return pos %d mode %d nest %d\n", self->incr_pos, self->incr_mode, self->incr_nest);//D } @@ -1639,24 +1965,28 @@ : i >= 'A' && i <= 'F' ? i - 'A' + 10 : -1; - json_stash = gv_stashpv ("JSON::XS" , 1); - json_boolean_stash = gv_stashpv ("JSON::XS::Boolean", 1); + json_stash = gv_stashpv ("JSON::XS" , 1); + bool_stash = gv_stashpv ("Types::Serialiser::Boolean", 1); + bool_true = get_bool ("Types::Serialiser::true"); + bool_false = get_bool ("Types::Serialiser::false"); - json_true = get_bool ("JSON::XS::true"); - json_false = get_bool ("JSON::XS::false"); + sv_json = newSVpv ("JSON", 0); + SvREADONLY_on (sv_json); + + CvNODEBUG_on (get_cv ("JSON::XS::incr_text", 0)); /* the debugger completely breaks lvalue subs */ } PROTOTYPES: DISABLE void CLONE (...) CODE: - json_stash = 0; - json_boolean_stash = 0; + json_stash = 0; + bool_stash = 0; void new (char *klass) PPCODE: { - SV *pv = NEWSV (0, sizeof (JSON)); + SV *pv = NEWSV (0, sizeof (JSON)); SvPOK_only (pv); json_init ((JSON *)SvPVX (pv)); XPUSHs (sv_2mortal (sv_bless ( @@ -1681,6 +2011,7 @@ convert_blessed = F_CONV_BLESSED relaxed = F_RELAXED allow_unknown = F_ALLOW_UNKNOWN + allow_tags = F_ALLOW_TAGS PPCODE: { if (enable) @@ -1706,6 +2037,7 @@ get_convert_blessed = F_CONV_BLESSED get_relaxed = F_RELAXED get_allow_unknown = F_ALLOW_UNKNOWN + get_allow_tags = F_ALLOW_TAGS PPCODE: XPUSHs (boolSV (self->flags & ix)); @@ -1743,7 +2075,7 @@ void filter_json_single_key_object (JSON *self, SV *key, SV *cb = &PL_sv_undef) PPCODE: { - if (!self->cb_sk_object) + if (!self->cb_sk_object) self->cb_sk_object = newHV (); if (SvOK (cb)) @@ -1764,19 +2096,23 @@ void encode (JSON *self, SV *scalar) PPCODE: - XPUSHs (encode_json (scalar, self)); + PUTBACK; scalar = encode_json (scalar, self); SPAGAIN; + XPUSHs (scalar); void decode (JSON *self, SV *jsonstr) PPCODE: - XPUSHs (decode_json (jsonstr, self, 0)); + PUTBACK; jsonstr = decode_json (jsonstr, self, 0); SPAGAIN; + XPUSHs (jsonstr); void decode_prefix (JSON *self, SV *jsonstr) PPCODE: { - STRLEN offset; + SV *sv; + char *offset; + PUTBACK; sv = decode_json (jsonstr, self, &offset); SPAGAIN; EXTEND (SP, 2); - PUSHs (decode_json (jsonstr, self, &offset)); - PUSHs (sv_2mortal (newSVuv (offset))); + PUSHs (sv); + PUSHs (sv_2mortal (newSVuv (ptr_to_index (jsonstr, offset)))); } void incr_parse (JSON *self, SV *jsonstr = 0) @@ -1785,23 +2121,44 @@ if (!self->incr_text) self->incr_text = newSVpvn ("", 0); + /* if utf8-ness doesn't match the decoder, need to upgrade/downgrade */ + if (!DECODE_WANTS_OCTETS (self) == !SvUTF8 (self->incr_text)) + if (DECODE_WANTS_OCTETS (self)) + { + if (self->incr_pos) + self->incr_pos = utf8_length ((U8 *)SvPVX (self->incr_text), + (U8 *)SvPVX (self->incr_text) + self->incr_pos); + + sv_utf8_downgrade (self->incr_text, 0); + } + else + { + sv_utf8_upgrade (self->incr_text); + + if (self->incr_pos) + self->incr_pos = utf8_hop ((U8 *)SvPVX (self->incr_text), self->incr_pos) + - (U8 *)SvPVX (self->incr_text); + } + // append data, if any if (jsonstr) { - if (SvUTF8 (jsonstr) && !SvUTF8 (self->incr_text)) - { - /* utf-8-ness differs, need to upgrade */ - sv_utf8_upgrade (self->incr_text); - - if (self->incr_pos) - self->incr_pos = utf8_hop ((U8 *)SvPVX (self->incr_text), self->incr_pos) - - (U8 *)SvPVX (self->incr_text); - } + /* make sure both strings have same encoding */ + if (SvUTF8 (jsonstr) != SvUTF8 (self->incr_text)) + if (SvUTF8 (jsonstr)) + sv_utf8_downgrade (jsonstr, 0); + else + sv_utf8_upgrade (jsonstr); + /* and then just blindly append */ { STRLEN len; const char *str = SvPV (jsonstr, len); - SvGROW (self->incr_text, SvCUR (self->incr_text) + len + 1); + STRLEN cur = SvCUR (self->incr_text); + + if (SvLEN (self->incr_text) <= cur + len) + SvGROW (self->incr_text, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + Move (str, SvEND (self->incr_text), len, char); SvCUR_set (self->incr_text, SvCUR (self->incr_text) + len); *SvEND (self->incr_text) = 0; // this should basically be a nop, too, but make sure it's there @@ -1811,7 +2168,8 @@ if (GIMME_V != G_VOID) do { - STRLEN offset; + SV *sv; + char *offset; if (!INCR_DONE (self)) { @@ -1822,15 +2180,26 @@ (unsigned long)self->incr_pos, (unsigned long)self->max_size); if (!INCR_DONE (self)) - break; + { + // as an optimisation, do not accumulate white space in the incr buffer + if (self->incr_mode == INCR_M_WS && self->incr_pos) + { + self->incr_pos = 0; + SvCUR_set (self->incr_text, 0); + } + + break; + } } - XPUSHs (decode_json (self->incr_text, self, &offset)); + PUTBACK; sv = decode_json (self->incr_text, self, &offset); SPAGAIN; + XPUSHs (sv); - sv_chop (self->incr_text, SvPV_nolen (self->incr_text) + offset); - self->incr_pos -= offset; + self->incr_pos -= offset - SvPVX (self->incr_text); self->incr_nest = 0; self->incr_mode = 0; + + sv_chop (self->incr_text, offset); } while (GIMME_V == G_ARRAY); } @@ -1878,27 +2247,22 @@ PROTOTYPES: ENABLE void encode_json (SV *scalar) - ALIAS: - to_json_ = 0 - encode_json = F_UTF8 PPCODE: { JSON json; json_init (&json); - json.flags |= ix; - XPUSHs (encode_json (scalar, &json)); + json.flags |= F_UTF8; + PUTBACK; scalar = encode_json (scalar, &json); SPAGAIN; + XPUSHs (scalar); } void decode_json (SV *jsonstr) - ALIAS: - from_json_ = 0 - decode_json = F_UTF8 PPCODE: { JSON json; json_init (&json); - json.flags |= ix; - XPUSHs (decode_json (jsonstr, &json, 0)); + json.flags |= F_UTF8; + PUTBACK; jsonstr = decode_json (jsonstr, &json, 0); SPAGAIN; + XPUSHs (jsonstr); } -