ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/CBOR-XS/XS.xs
(Generate patch)

Comparing CBOR-XS/XS.xs (file contents):
Revision 1.36 by root, Sat Nov 30 17:37:45 2013 UTC vs.
Revision 1.49 by root, Mon Feb 8 04:11:11 2016 UTC

… …
7#include <stdlib.h> 7#include <stdlib.h>
8#include <stdio.h> 8#include <stdio.h>
9#include <limits.h> 9#include <limits.h>
10#include <float.h> 10#include <float.h>
11 11
12#define ECB_NO_THREADS 1
12#include "ecb.h" 13#include "ecb.h"
13 14
14// compatibility with perl <5.18 15// compatibility with perl <5.18
15#ifndef HvNAMELEN_get 16#ifndef HvNAMELEN_get
16# define HvNAMELEN_get(hv) strlen (HvNAME (hv)) 17# define HvNAMELEN_get(hv) strlen (HvNAME (hv))
… …
97}; 98};
98 99
99#define F_SHRINK 0x00000001UL 100#define F_SHRINK 0x00000001UL
100#define F_ALLOW_UNKNOWN 0x00000002UL 101#define F_ALLOW_UNKNOWN 0x00000002UL
101#define F_ALLOW_SHARING 0x00000004UL 102#define F_ALLOW_SHARING 0x00000004UL
103#define F_ALLOW_CYCLES 0x00000008UL
102#define F_PACK_STRINGS 0x00000008UL 104#define F_PACK_STRINGS 0x00000010UL
105#define F_VALIDATE_UTF8 0x00000020UL
103 106
104#define INIT_SIZE 32 // initial scalar size to be allocated 107#define INIT_SIZE 32 // initial scalar size to be allocated
105 108
106#define SB do { 109#define SB do {
107#define SE } while (0) 110#define SE } while (0)
… …
126typedef struct { 129typedef struct {
127 U32 flags; 130 U32 flags;
128 U32 max_depth; 131 U32 max_depth;
129 STRLEN max_size; 132 STRLEN max_size;
130 SV *filter; 133 SV *filter;
134
135 // for the incremental parser
136 STRLEN incr_pos; // the current offset into the text
137 STRLEN incr_need; // minimum bytes needed to decode
138 AV *incr_count; // for every nesting level, the number of outstanding values, or -1 for indef.
131} CBOR; 139} CBOR;
132 140
133ecb_inline void 141ecb_inline void
134cbor_init (CBOR *cbor) 142cbor_init (CBOR *cbor)
135{ 143{
… …
139 147
140ecb_inline void 148ecb_inline void
141cbor_free (CBOR *cbor) 149cbor_free (CBOR *cbor)
142{ 150{
143 SvREFCNT_dec (cbor->filter); 151 SvREFCNT_dec (cbor->filter);
152 SvREFCNT_dec (cbor->incr_count);
144} 153}
145 154
146///////////////////////////////////////////////////////////////////////////// 155/////////////////////////////////////////////////////////////////////////////
147// utility functions 156// utility functions
148 157
… …
315 324
316 ++enc->depth; 325 ++enc->depth;
317 326
318 encode_uint (enc, MAJOR_ARRAY, len + 1); 327 encode_uint (enc, MAJOR_ARRAY, len + 1);
319 328
329 if (SvMAGICAL (av))
320 for (i = 0; i <= len; ++i) 330 for (i = 0; i <= len; ++i)
321 { 331 {
322 SV **svp = av_fetch (av, i, 0); 332 SV **svp = av_fetch (av, i, 0);
323 encode_sv (enc, svp ? *svp : &PL_sv_undef); 333 encode_sv (enc, svp ? *svp : &PL_sv_undef);
324 } 334 }
335 else
336 for (i = 0; i <= len; ++i)
337 {
338 SV *sv = AvARRAY (av)[i];
339 encode_sv (enc, sv ? sv : &PL_sv_undef);
340 }
325 341
326 --enc->depth; 342 --enc->depth;
327} 343}
328 344
329static void 345static void
… …
568} 584}
569 585
570static SV * 586static SV *
571encode_cbor (SV *scalar, CBOR *cbor) 587encode_cbor (SV *scalar, CBOR *cbor)
572{ 588{
573 enc_t enc = { }; 589 enc_t enc = { 0 };
574 590
575 enc.cbor = *cbor; 591 enc.cbor = *cbor;
576 enc.sv = sv_2mortal (NEWSV (0, INIT_SIZE)); 592 enc.sv = sv_2mortal (NEWSV (0, INIT_SIZE));
577 enc.cur = SvPVX (enc.sv); 593 enc.cur = SvPVX (enc.sv);
578 enc.end = SvEND (enc.sv); 594 enc.end = SvEND (enc.sv);
… …
728{ 744{
729 // for speed reasons, we specialcase single-string 745 // for speed reasons, we specialcase single-string
730 // byte or utf-8 strings as keys, but only when !stringref 746 // byte or utf-8 strings as keys, but only when !stringref
731 747
732 if (ecb_expect_true (!dec->stringref)) 748 if (ecb_expect_true (!dec->stringref))
733 if (ecb_expect_true ((*dec->cur - MAJOR_BYTES) <= LENGTH_EXT8)) 749 if (ecb_expect_true ((U8)(*dec->cur - MAJOR_BYTES) <= LENGTH_EXT8))
734 { 750 {
735 I32 len = decode_uint (dec); 751 I32 len = decode_uint (dec);
736 char *key = (char *)dec->cur; 752 char *key = (char *)dec->cur;
737 753
754 WANT (len);
738 dec->cur += len; 755 dec->cur += len;
739 756
740 if (ecb_expect_false (dec->stringref))
741 av_push (dec->stringref, newSVpvn (key, len));
742
743 hv_store (hv, key, len, decode_sv (dec), 0); 757 hv_store (hv, key, len, decode_sv (dec), 0);
744 758
745 return; 759 return;
746 } 760 }
747 else if (ecb_expect_true ((*dec->cur - MAJOR_TEXT) <= LENGTH_EXT8)) 761 else if (ecb_expect_true ((U8)(*dec->cur - MAJOR_TEXT) <= LENGTH_EXT8))
748 { 762 {
749 I32 len = decode_uint (dec); 763 I32 len = decode_uint (dec);
750 char *key = (char *)dec->cur; 764 char *key = (char *)dec->cur;
751 765
766 WANT (len);
752 dec->cur += len; 767 dec->cur += len;
753 768
754 if (ecb_expect_false (dec->stringref)) 769 if (ecb_expect_false (dec->cbor.flags & F_VALIDATE_UTF8))
755 av_push (dec->stringref, newSVpvn_utf8 (key, len, 1)); 770 if (!is_utf8_string (key, len))
771 ERR ("corrupted CBOR data (invalid UTF-8 in map key)");
756 772
757 hv_store (hv, key, -len, decode_sv (dec), 0); 773 hv_store (hv, key, -len, decode_sv (dec), 0);
758 774
759 return; 775 return;
760 } 776 }
… …
762 SV *k = decode_sv (dec); 778 SV *k = decode_sv (dec);
763 SV *v = decode_sv (dec); 779 SV *v = decode_sv (dec);
764 780
765 hv_store_ent (hv, k, v, 0); 781 hv_store_ent (hv, k, v, 0);
766 SvREFCNT_dec (k); 782 SvREFCNT_dec (k);
783
784fail:
785 ;
767} 786}
768 787
769static SV * 788static SV *
770decode_hv (dec_t *dec) 789decode_hv (dec_t *dec)
771{ 790{
… …
853 && SvCUR (sv) >= minimum_string_length (AvFILLp (dec->stringref) + 1)) 872 && SvCUR (sv) >= minimum_string_length (AvFILLp (dec->stringref) + 1))
854 av_push (dec->stringref, SvREFCNT_inc_NN (sv)); 873 av_push (dec->stringref, SvREFCNT_inc_NN (sv));
855 } 874 }
856 875
857 if (utf8) 876 if (utf8)
877 {
878 if (ecb_expect_false (dec->cbor.flags & F_VALIDATE_UTF8))
879 if (!is_utf8_string (SvPVX (sv), SvCUR (sv)))
880 ERR ("corrupted CBOR data (invalid UTF-8 in text string)");
881
858 SvUTF8_on (sv); 882 SvUTF8_on (sv);
883 }
859 884
860 return sv; 885 return sv;
861 886
862fail: 887fail:
863 SvREFCNT_dec (sv); 888 SvREFCNT_dec (sv);
… …
912 case CBOR_TAG_VALUE_SHAREABLE: 937 case CBOR_TAG_VALUE_SHAREABLE:
913 { 938 {
914 if (ecb_expect_false (!dec->shareable)) 939 if (ecb_expect_false (!dec->shareable))
915 dec->shareable = (AV *)sv_2mortal ((SV *)newAV ()); 940 dec->shareable = (AV *)sv_2mortal ((SV *)newAV ());
916 941
942 if (dec->cbor.flags & F_ALLOW_CYCLES)
943 {
917 sv = newSV (0); 944 sv = newSV (0);
918 av_push (dec->shareable, SvREFCNT_inc_NN (sv)); 945 av_push (dec->shareable, SvREFCNT_inc_NN (sv));
919 946
920 SV *osv = decode_sv (dec); 947 SV *osv = decode_sv (dec);
921 sv_setsv (sv, osv); 948 sv_setsv (sv, osv);
922 SvREFCNT_dec_NN (osv); 949 SvREFCNT_dec_NN (osv);
950 }
951 else
952 {
953 av_push (dec->shareable, &PL_sv_undef);
954 int idx = AvFILLp (dec->shareable);
955 sv = decode_sv (dec);
956 av_store (dec->shareable, idx, SvREFCNT_inc_NN (sv));
957 }
923 } 958 }
924 break; 959 break;
925 960
926 case CBOR_TAG_VALUE_SHAREDREF: 961 case CBOR_TAG_VALUE_SHAREDREF:
927 { 962 {
… …
932 967
933 if (!dec->shareable || (int)idx > AvFILLp (dec->shareable)) 968 if (!dec->shareable || (int)idx > AvFILLp (dec->shareable))
934 ERR ("corrupted CBOR data (sharedref index out of bounds)"); 969 ERR ("corrupted CBOR data (sharedref index out of bounds)");
935 970
936 sv = SvREFCNT_inc_NN (AvARRAY (dec->shareable)[idx]); 971 sv = SvREFCNT_inc_NN (AvARRAY (dec->shareable)[idx]);
972
973 if (sv == &PL_sv_undef)
974 ERR ("cyclic CBOR data structure found, but allow_cycles is not enabled");
937 } 975 }
938 break; 976 break;
939 977
940 case CBOR_TAG_PERL_OBJECT: 978 case CBOR_TAG_PERL_OBJECT:
941 { 979 {
… …
1110 1148
1111 return newSVnv (ecb_binary64_to_double (fp)); 1149 return newSVnv (ecb_binary64_to_double (fp));
1112 } 1150 }
1113 1151
1114 // 0..19 unassigned simple 1152 // 0..19 unassigned simple
1115 // 24 reserved + unassigned (reserved values are not encodable) 1153 // 24 reserved + unassigned simple (reserved values are not encodable)
1154 // 28-30 unassigned misc
1155 // 31 break code
1116 default: 1156 default:
1117 ERR ("corrupted CBOR data (reserved/unassigned major 7 value)"); 1157 ERR ("corrupted CBOR data (reserved/unassigned/unexpected major 7 value)");
1118 } 1158 }
1119 1159
1120 break; 1160 break;
1121 } 1161 }
1122 1162
… …
1125} 1165}
1126 1166
1127static SV * 1167static SV *
1128decode_cbor (SV *string, CBOR *cbor, char **offset_return) 1168decode_cbor (SV *string, CBOR *cbor, char **offset_return)
1129{ 1169{
1130 dec_t dec = { }; 1170 dec_t dec = { 0 };
1131 SV *sv; 1171 SV *sv;
1132 STRLEN len; 1172 STRLEN len;
1133 char *data = SvPVbyte (string, len); 1173 char *data = SvPVbyte (string, len);
1134 1174
1135 if (len > cbor->max_size && cbor->max_size) 1175 if (len > cbor->max_size && cbor->max_size)
… …
1149 if (dec.cur != dec.end && !dec.err) 1189 if (dec.cur != dec.end && !dec.err)
1150 dec.err = "garbage after CBOR object"; 1190 dec.err = "garbage after CBOR object";
1151 1191
1152 if (dec.err) 1192 if (dec.err)
1153 { 1193 {
1194 if (dec.shareable)
1195 {
1196 // need to break cyclic links, which whould all be in shareable
1197 int i;
1198 SV **svp;
1199
1200 for (i = av_len (dec.shareable) + 1; i--; )
1201 if ((svp = av_fetch (dec.shareable, i, 0)))
1202 sv_setsv (*svp, &PL_sv_undef);
1203 }
1204
1154 SvREFCNT_dec (sv); 1205 SvREFCNT_dec (sv);
1155 croak ("%s, at offset %d (octet 0x%02x)", dec.err, dec.cur - (U8 *)data, (int)(uint8_t)*dec.cur); 1206 croak ("%s, at offset %d (octet 0x%02x)", dec.err, dec.cur - (U8 *)data, (int)(uint8_t)*dec.cur);
1156 } 1207 }
1157 1208
1158 sv = sv_2mortal (sv); 1209 sv = sv_2mortal (sv);
1159 1210
1160 return sv; 1211 return sv;
1161} 1212}
1162 1213
1214/////////////////////////////////////////////////////////////////////////////
1215// incremental parser
1216
1217#define INCR_DONE(cbor) (AvFILLp (cbor->incr_count) < 0)
1218
1219// returns 0 for notyet, 1 for success or error
1220static int
1221incr_parse (CBOR *self, SV *cborstr)
1222{
1223 STRLEN cur;
1224 SvPV (cborstr, cur);
1225
1226 while (ecb_expect_true (self->incr_need <= cur))
1227 {
1228 // table of integer count bytes
1229 static I8 incr_len[MINOR_MASK + 1] = {
1230 0, 0, 0, 0, 0, 0, 0, 0,
1231 0, 0, 0, 0, 0, 0, 0, 0,
1232 0, 0, 0, 0, 0, 0, 0, 0,
1233 1, 2, 4, 8,-1,-1,-1,-2
1234 };
1235
1236 const U8 *p = SvPVX (cborstr) + self->incr_pos;
1237 U8 m = *p & MINOR_MASK;
1238 IV count = SvIVX (AvARRAY (self->incr_count)[AvFILLp (self->incr_count)]);
1239 I8 ilen = incr_len[m];
1240
1241 self->incr_need = self->incr_pos + 1;
1242
1243 if (ecb_expect_false (ilen < 0))
1244 {
1245 if (m != MINOR_INDEF)
1246 return 1; // error
1247
1248 if (*p == (MAJOR_MISC | MINOR_INDEF))
1249 {
1250 if (count >= 0)
1251 return 1; // error
1252
1253 count = 1;
1254 }
1255 else
1256 {
1257 av_push (self->incr_count, newSViv (-1)); //TODO: nest
1258 count = -1;
1259 }
1260 }
1261 else
1262 {
1263 self->incr_need += ilen;
1264 if (ecb_expect_false (self->incr_need > cur))
1265 return 0;
1266
1267 int major = *p >> MAJOR_SHIFT;
1268
1269 switch (major)
1270 {
1271 case MAJOR_TAG >> MAJOR_SHIFT:
1272 ++count; // tags merely prefix another value
1273 break;
1274
1275 case MAJOR_BYTES >> MAJOR_SHIFT:
1276 case MAJOR_TEXT >> MAJOR_SHIFT:
1277 case MAJOR_ARRAY >> MAJOR_SHIFT:
1278 case MAJOR_MAP >> MAJOR_SHIFT:
1279 {
1280 UV len;
1281
1282 if (ecb_expect_false (ilen))
1283 {
1284 len = 0;
1285
1286 do {
1287 len = (len << 8) | *++p;
1288 } while (--ilen);
1289 }
1290 else
1291 len = m;
1292
1293 switch (major)
1294 {
1295 case MAJOR_BYTES >> MAJOR_SHIFT:
1296 case MAJOR_TEXT >> MAJOR_SHIFT:
1297 self->incr_need += len;
1298 if (ecb_expect_false (self->incr_need > cur))
1299 return 0;
1300
1301 break;
1302
1303 case MAJOR_MAP >> MAJOR_SHIFT:
1304 len <<= 1;
1305 case MAJOR_ARRAY >> MAJOR_SHIFT:
1306 if (len)
1307 {
1308 av_push (self->incr_count, newSViv (len + 1)); //TODO: nest
1309 count = len + 1;
1310 }
1311 break;
1312 }
1313 }
1314 }
1315 }
1316
1317 self->incr_pos = self->incr_need;
1318
1319 if (count > 0)
1320 {
1321 while (!--count)
1322 {
1323 if (!AvFILLp (self->incr_count))
1324 return 1; // done
1325
1326 SvREFCNT_dec_NN (av_pop (self->incr_count));
1327 count = SvIVX (AvARRAY (self->incr_count)[AvFILLp (self->incr_count)]);
1328 }
1329
1330 SvIVX (AvARRAY (self->incr_count)[AvFILLp (self->incr_count)]) = count;
1331 }
1332 }
1333
1334 return 0;
1335}
1336
1337
1163///////////////////////////////////////////////////////////////////////////// 1338/////////////////////////////////////////////////////////////////////////////
1164// XS interface functions 1339// XS interface functions
1165 1340
1166MODULE = CBOR::XS PACKAGE = CBOR::XS 1341MODULE = CBOR::XS PACKAGE = CBOR::XS
1167 1342
… …
1207void shrink (CBOR *self, int enable = 1) 1382void shrink (CBOR *self, int enable = 1)
1208 ALIAS: 1383 ALIAS:
1209 shrink = F_SHRINK 1384 shrink = F_SHRINK
1210 allow_unknown = F_ALLOW_UNKNOWN 1385 allow_unknown = F_ALLOW_UNKNOWN
1211 allow_sharing = F_ALLOW_SHARING 1386 allow_sharing = F_ALLOW_SHARING
1387 allow_cycles = F_ALLOW_CYCLES
1212 pack_strings = F_PACK_STRINGS 1388 pack_strings = F_PACK_STRINGS
1389 validate_utf8 = F_VALIDATE_UTF8
1213 PPCODE: 1390 PPCODE:
1214{ 1391{
1215 if (enable) 1392 if (enable)
1216 self->flags |= ix; 1393 self->flags |= ix;
1217 else 1394 else
… …
1223void get_shrink (CBOR *self) 1400void get_shrink (CBOR *self)
1224 ALIAS: 1401 ALIAS:
1225 get_shrink = F_SHRINK 1402 get_shrink = F_SHRINK
1226 get_allow_unknown = F_ALLOW_UNKNOWN 1403 get_allow_unknown = F_ALLOW_UNKNOWN
1227 get_allow_sharing = F_ALLOW_SHARING 1404 get_allow_sharing = F_ALLOW_SHARING
1405 get_allow_cycles = F_ALLOW_CYCLES
1228 get_pack_strings = F_PACK_STRINGS 1406 get_pack_strings = F_PACK_STRINGS
1407 get_validate_utf8 = F_VALIDATE_UTF8
1229 PPCODE: 1408 PPCODE:
1230 XPUSHs (boolSV (self->flags & ix)); 1409 XPUSHs (boolSV (self->flags & ix));
1231 1410
1232void max_depth (CBOR *self, U32 max_depth = 0x80000000UL) 1411void max_depth (CBOR *self, U32 max_depth = 0x80000000UL)
1233 PPCODE: 1412 PPCODE:
… …
1282 EXTEND (SP, 2); 1461 EXTEND (SP, 2);
1283 PUSHs (sv); 1462 PUSHs (sv);
1284 PUSHs (sv_2mortal (newSVuv (offset - SvPVX (cborstr)))); 1463 PUSHs (sv_2mortal (newSVuv (offset - SvPVX (cborstr))));
1285} 1464}
1286 1465
1466void incr_parse (CBOR *self, SV *cborstr)
1467 ALIAS:
1468 incr_parse_multiple = 1
1469 PPCODE:
1470{
1471 if (SvUTF8 (cborstr))
1472 sv_utf8_downgrade (cborstr, 0);
1473
1474 if (!self->incr_count)
1475 {
1476 self->incr_count = newAV ();
1477 self->incr_pos = 0;
1478 self->incr_need = 1;
1479
1480 av_push (self->incr_count, newSViv (1));
1481 }
1482
1483 do
1484 {
1485 if (!incr_parse (self, cborstr))
1486 {
1487 if (self->incr_need > self->max_size && self->max_size)
1488 croak ("attempted decode of CBOR text of %lu bytes size, but max_size is set to %lu",
1489 (unsigned long)self->incr_need, (unsigned long)self->max_size);
1490
1491 break;
1492 }
1493
1494 SV *sv;
1495 char *offset;
1496
1497 PUTBACK; sv = decode_cbor (cborstr, self, &offset); SPAGAIN;
1498 XPUSHs (sv);
1499
1500 sv_chop (cborstr, offset);
1501
1502 av_clear (self->incr_count);
1503 av_push (self->incr_count, newSViv (1));
1504
1505 self->incr_pos = 0;
1506 self->incr_need = self->incr_pos + 1;
1507 }
1508 while (ix);
1509}
1510
1511void incr_reset (CBOR *self)
1512 CODE:
1513{
1514 SvREFCNT_dec (self->incr_count);
1515 self->incr_count = 0;
1516}
1517
1287void DESTROY (CBOR *self) 1518void DESTROY (CBOR *self)
1288 PPCODE: 1519 PPCODE:
1289 cbor_free (self); 1520 cbor_free (self);
1290 1521
1291PROTOTYPES: ENABLE 1522PROTOTYPES: ENABLE

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines