ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/CBOR-XS/XS.pm
(Generate patch)

Comparing CBOR-XS/XS.pm (file contents):
Revision 1.40 by root, Sun Jan 5 14:24:54 2014 UTC vs.
Revision 1.86 by root, Wed Aug 3 12:28:32 2022 UTC

38with the added ability of supporting serialisation of Perl objects. (JSON 38with the added ability of supporting serialisation of Perl objects. (JSON
39often compresses better than CBOR though, so if you plan to compress the 39often compresses better than CBOR though, so if you plan to compress the
40data later and speed is less important you might want to compare both 40data later and speed is less important you might want to compare both
41formats first). 41formats first).
42 42
43The primary goal of this module is to be I<correct> and the secondary goal
44is to be I<fast>. To reach the latter goal it was written in C.
45
43To give you a general idea about speed, with texts in the megabyte range, 46To give you a general idea about speed, with texts in the megabyte range,
44C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or 47C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or
45L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the 48L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the
46data, the worse L<Storable> performs in comparison. 49data, the worse L<Storable> performs in comparison.
47 50
52In addition to the core CBOR data format, this module implements a 55In addition to the core CBOR data format, this module implements a
53number of extensions, to support cyclic and shared data structures 56number of extensions, to support cyclic and shared data structures
54(see C<allow_sharing> and C<allow_cycles>), string deduplication (see 57(see C<allow_sharing> and C<allow_cycles>), string deduplication (see
55C<pack_strings>) and scalar references (always enabled). 58C<pack_strings>) and scalar references (always enabled).
56 59
57The primary goal of this module is to be I<correct> and the secondary goal
58is to be I<fast>. To reach the latter goal it was written in C.
59
60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and 60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and
61vice versa. 61vice versa.
62 62
63=cut 63=cut
64 64
65package CBOR::XS; 65package CBOR::XS;
66 66
67use common::sense; 67use common::sense;
68 68
69our $VERSION = 1.25; 69our $VERSION = 1.86;
70our @ISA = qw(Exporter); 70our @ISA = qw(Exporter);
71 71
72our @EXPORT = qw(encode_cbor decode_cbor); 72our @EXPORT = qw(encode_cbor decode_cbor);
73 73
74use Exporter; 74use Exporter;
112 112
113The mutators for flags all return the CBOR object again and thus calls can 113The mutators for flags all return the CBOR object again and thus calls can
114be chained: 114be chained:
115 115
116 my $cbor = CBOR::XS->new->encode ({a => [1,2]}); 116 my $cbor = CBOR::XS->new->encode ({a => [1,2]});
117
118=item $cbor = new_safe CBOR::XS
119
120Create a new, safe/secure CBOR::XS object. This is similar to C<new>,
121but configures the coder object to be safe to use with untrusted
122data. Currently, this is equivalent to:
123
124 my $cbor = CBOR::XS
125 ->new
126 ->validate_utf8
127 ->forbid_objects
128 ->filter (\&CBOR::XS::safe_filter)
129 ->max_size (1e8);
130
131But is more future proof (it is better to crash because of a change than
132to be exploited in other ways).
133
134=cut
135
136sub new_safe {
137 CBOR::XS
138 ->new
139 ->validate_utf8
140 ->forbid_objects
141 ->filter (\&CBOR::XS::safe_filter)
142 ->max_size (1e8)
143}
117 144
118=item $cbor = $cbor->max_depth ([$maximum_nesting_depth]) 145=item $cbor = $cbor->max_depth ([$maximum_nesting_depth])
119 146
120=item $max_depth = $cbor->get_max_depth 147=item $max_depth = $cbor->get_max_depth
121 148
137 164
138Note that nesting is implemented by recursion in C. The default value has 165Note that nesting is implemented by recursion in C. The default value has
139been chosen to be as large as typical operating systems allow without 166been chosen to be as large as typical operating systems allow without
140crashing. 167crashing.
141 168
142See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 169See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
143 170
144=item $cbor = $cbor->max_size ([$maximum_string_size]) 171=item $cbor = $cbor->max_size ([$maximum_string_size])
145 172
146=item $max_size = $cbor->get_max_size 173=item $max_size = $cbor->get_max_size
147 174
152effect on C<encode> (yet). 179effect on C<encode> (yet).
153 180
154If no argument is given, the limit check will be deactivated (same as when 181If no argument is given, the limit check will be deactivated (same as when
155C<0> is specified). 182C<0> is specified).
156 183
157See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 184See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
158 185
159=item $cbor = $cbor->allow_unknown ([$enable]) 186=item $cbor = $cbor->allow_unknown ([$enable])
160 187
161=item $enabled = $cbor->get_allow_unknown 188=item $enabled = $cbor->get_allow_unknown
162 189
180reference to the earlier value. 207reference to the earlier value.
181 208
182This means that such values will only be encoded once, and will not result 209This means that such values will only be encoded once, and will not result
183in a deep cloning of the value on decode, in decoders supporting the value 210in a deep cloning of the value on decode, in decoders supporting the value
184sharing extension. This also makes it possible to encode cyclic data 211sharing extension. This also makes it possible to encode cyclic data
185structures (which need C<allow_cycles> to ne enabled to be decoded by this 212structures (which need C<allow_cycles> to be enabled to be decoded by this
186module). 213module).
187 214
188It is recommended to leave it off unless you know your 215It is recommended to leave it off unless you know your
189communication partner supports the value sharing extensions to CBOR 216communication partner supports the value sharing extensions to CBOR
190(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the 217(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the
191resulting data structure might be unusable. 218resulting data structure might be unusable.
192 219
193Detecting shared values incurs a runtime overhead when values are encoded 220Detecting shared values incurs a runtime overhead when values are encoded
194that have a reference counter large than one, and might unnecessarily 221that have a reference counter large than one, and might unnecessarily
195increase the encoded size, as potentially shared values are encode as 222increase the encoded size, as potentially shared values are encoded as
196shareable whether or not they are actually shared. 223shareable whether or not they are actually shared.
197 224
198At the moment, only targets of references can be shared (e.g. scalars, 225At the moment, only targets of references can be shared (e.g. scalars,
199arrays or hashes pointed to by a reference). Weirder constructs, such as 226arrays or hashes pointed to by a reference). Weirder constructs, such as
200an array with multiple "copies" of the I<same> string, which are hard but 227an array with multiple "copies" of the I<same> string, which are hard but
218isn't prepared for this will not leak memory. 245isn't prepared for this will not leak memory.
219 246
220If C<$enable> is false (the default), then C<decode> will throw an error 247If C<$enable> is false (the default), then C<decode> will throw an error
221when it encounters a self-referential/cyclic data structure. 248when it encounters a self-referential/cyclic data structure.
222 249
250FUTURE DIRECTION: the motivation behind this option is to avoid I<real>
251cycles - future versions of this module might chose to decode cyclic data
252structures using weak references when this option is off, instead of
253throwing an error.
254
223This option does not affect C<encode> in any way - shared values and 255This option does not affect C<encode> in any way - shared values and
224references will always be decoded properly if present. 256references will always be encoded properly if present.
257
258=item $cbor = $cbor->forbid_objects ([$enable])
259
260=item $enabled = $cbor->get_forbid_objects
261
262Disables the use of the object serialiser protocol.
263
264If C<$enable> is true (or missing), then C<encode> will will throw an
265exception when it encounters perl objects that would be encoded using the
266perl-object tag (26). When C<decode> encounters such tags, it will fall
267back to the general filter/tagged logic as if this were an unknown tag (by
268default resulting in a C<CBOR::XC::Tagged> object).
269
270If C<$enable> is false (the default), then C<encode> will use the
271L<Types::Serialiser> object serialisation protocol to serialise objects
272into perl-object tags, and C<decode> will do the same to decode such tags.
273
274See L<SECURITY CONSIDERATIONS>, below, for more info on why forbidding this
275protocol can be useful.
225 276
226=item $cbor = $cbor->pack_strings ([$enable]) 277=item $cbor = $cbor->pack_strings ([$enable])
227 278
228=item $enabled = $cbor->get_pack_strings 279=item $enabled = $cbor->get_pack_strings
229 280
242the standard CBOR way. 293the standard CBOR way.
243 294
244This option does not affect C<decode> in any way - string references will 295This option does not affect C<decode> in any way - string references will
245always be decoded properly if present. 296always be decoded properly if present.
246 297
298=item $cbor = $cbor->text_keys ([$enable])
299
300=item $enabled = $cbor->get_text_keys
301
302If C<$enabled> is true (or missing), then C<encode> will encode all
303perl hash keys as CBOR text strings/UTF-8 string, upgrading them as needed.
304
305If C<$enable> is false (the default), then C<encode> will encode hash keys
306normally - upgraded perl strings (strings internally encoded as UTF-8) as
307CBOR text strings, and downgraded perl strings as CBOR byte strings.
308
309This option does not affect C<decode> in any way.
310
311This option is useful for interoperability with CBOR decoders that don't
312treat byte strings as a form of text. It is especially useful as Perl
313gives very little control over hash keys.
314
315Enabling this option can be slow, as all downgraded hash keys that are
316encoded need to be scanned and converted to UTF-8.
317
318=item $cbor = $cbor->text_strings ([$enable])
319
320=item $enabled = $cbor->get_text_strings
321
322This option works similar to C<text_keys>, above, but works on all strings
323(including hash keys), so C<text_keys> has no further effect after
324enabling C<text_strings>.
325
326If C<$enabled> is true (or missing), then C<encode> will encode all perl
327strings as CBOR text strings/UTF-8 strings, upgrading them as needed.
328
329If C<$enable> is false (the default), then C<encode> will encode strings
330normally (but see C<text_keys>) - upgraded perl strings (strings
331internally encoded as UTF-8) as CBOR text strings, and downgraded perl
332strings as CBOR byte strings.
333
334This option does not affect C<decode> in any way.
335
336This option has similar advantages and disadvantages as C<text_keys>. In
337addition, this option effectively removes the ability to automatically
338encode byte strings, which might break some C<FREEZE> and C<TO_CBOR>
339methods that rely on this.
340
341A workaround is to use explicit type casts, which are unaffected by this option.
342
247=item $cbor = $cbor->validate_utf8 ([$enable]) 343=item $cbor = $cbor->validate_utf8 ([$enable])
248 344
249=item $enabled = $cbor->get_validate_utf8 345=item $enabled = $cbor->get_validate_utf8
250 346
251If C<$enable> is true (or missing), then C<decode> will validate that 347If C<$enable> is true (or missing), then C<decode> will validate that
256The concept of "valid UTF-8" used is perl's concept, which is a superset 352The concept of "valid UTF-8" used is perl's concept, which is a superset
257of the official UTF-8. 353of the official UTF-8.
258 354
259If C<$enable> is false (the default), then C<decode> will blindly accept 355If C<$enable> is false (the default), then C<decode> will blindly accept
260UTF-8 data, marking them as valid UTF-8 in the resulting data structure 356UTF-8 data, marking them as valid UTF-8 in the resulting data structure
261regardless of whether thats true or not. 357regardless of whether that's true or not.
262 358
263Perl isn't too happy about corrupted UTF-8 in strings, but should 359Perl isn't too happy about corrupted UTF-8 in strings, but should
264generally not crash or do similarly evil things. Extensions might be not 360generally not crash or do similarly evil things. Extensions might be not
265so forgiving, so it's recommended to turn on this setting if you receive 361so forgiving, so it's recommended to turn on this setting if you receive
266untrusted CBOR. 362untrusted CBOR.
288replace the tagged value in the decoded data structure, or no values, 384replace the tagged value in the decoded data structure, or no values,
289which will result in default handling, which currently means the decoder 385which will result in default handling, which currently means the decoder
290creates a C<CBOR::XS::Tagged> object to hold the tag and the value. 386creates a C<CBOR::XS::Tagged> object to hold the tag and the value.
291 387
292When the filter is cleared (the default state), the default filter 388When the filter is cleared (the default state), the default filter
293function, C<CBOR::XS::default_filter>, is used. This function simply looks 389function, C<CBOR::XS::default_filter>, is used. This function simply
294up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists it must be 390looks up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists
295a code reference that is called with tag and value, and is responsible for 391it must be a code reference that is called with tag and value, and is
296decoding the value. If no entry exists, it returns no values. 392responsible for decoding the value. If no entry exists, it returns no
393values. C<CBOR::XS> provides a number of default filter functions already,
394the the C<%CBOR::XS::FILTER> hash can be freely extended with more.
395
396C<CBOR::XS> additionally provides an alternative filter function that is
397supposed to be safe to use with untrusted data (which the default filter
398might not), called C<CBOR::XS::safe_filter>, which works the same as
399the C<default_filter> but uses the C<%CBOR::XS::SAFE_FILTER> variable
400instead. It is prepopulated with the tag decoding functions that are
401deemed safe (basically the same as C<%CBOR::XS::FILTER> without all
402the bignum tags), and can be extended by user code as wlel, although,
403obviously, one should be very careful about adding decoding functions
404here, since the expectation is that they are safe to use on untrusted
405data, after all.
297 406
298Example: decode all tags not handled internally into C<CBOR::XS::Tagged> 407Example: decode all tags not handled internally into C<CBOR::XS::Tagged>
299objects, with no other special handling (useful when working with 408objects, with no other special handling (useful when working with
300potentially "unsafe" CBOR data). 409potentially "unsafe" CBOR data).
301 410
308 my ($tag, $value); 417 my ($tag, $value);
309 418
310 "tag 1347375694 value $value" 419 "tag 1347375694 value $value"
311 }; 420 };
312 421
422Example: provide your own filter function that looks up tags in your own
423hash:
424
425 my %my_filter = (
426 998347484 => sub {
427 my ($tag, $value);
428
429 "tag 998347484 value $value"
430 };
431 );
432
433 my $coder = CBOR::XS->new->filter (sub {
434 &{ $my_filter{$_[0]} or return }
435 });
436
437
438Example: use the safe filter function (see L<SECURITY CONSIDERATIONS> for
439more considerations on security).
440
441 CBOR::XS->new->filter (\&CBOR::XS::safe_filter)->decode ($cbor_data);
442
313=item $cbor_data = $cbor->encode ($perl_scalar) 443=item $cbor_data = $cbor->encode ($perl_scalar)
314 444
315Converts the given Perl data structure (a scalar value) to its CBOR 445Converts the given Perl data structure (a scalar value) to its CBOR
316representation. 446representation.
317 447
326when there is trailing garbage after the CBOR string, it will silently 456when there is trailing garbage after the CBOR string, it will silently
327stop parsing there and return the number of characters consumed so far. 457stop parsing there and return the number of characters consumed so far.
328 458
329This is useful if your CBOR texts are not delimited by an outer protocol 459This is useful if your CBOR texts are not delimited by an outer protocol
330and you need to know where the first CBOR string ends amd the next one 460and you need to know where the first CBOR string ends amd the next one
331starts. 461starts - CBOR strings are self-delimited, so it is possible to concatenate
462CBOR strings without any delimiters or size fields and recover their data.
332 463
333 CBOR::XS->new->decode_prefix ("......") 464 CBOR::XS->new->decode_prefix ("......")
334 => ("...", 3) 465 => ("...", 3)
335 466
336=back 467=back
342Perl data structure in memory at one time, it does allow you to parse a 473Perl data structure in memory at one time, it does allow you to parse a
343CBOR stream incrementally, using a similar to using "decode_prefix" to see 474CBOR stream incrementally, using a similar to using "decode_prefix" to see
344if a full CBOR object is available, but is much more efficient. 475if a full CBOR object is available, but is much more efficient.
345 476
346It basically works by parsing as much of a CBOR string as possible - if 477It basically works by parsing as much of a CBOR string as possible - if
347the CBOR data is not complete yet, the pasrer will remember where it was, 478the CBOR data is not complete yet, the parser will remember where it was,
348to be able to restart when more data has been accumulated. Once enough 479to be able to restart when more data has been accumulated. Once enough
349data is available to either decode a complete CBOR value or raise an 480data is available to either decode a complete CBOR value or raise an
350error, a real decode will be attempted. 481error, a real decode will be attempted.
351 482
352A typical use case would be a network protocol that consists of sending 483A typical use case would be a network protocol that consists of sending
391 522
392Resets the incremental decoder. This throws away any saved state, so that 523Resets the incremental decoder. This throws away any saved state, so that
393subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse 524subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse
394a new CBOR value from the beginning of the C<$buffer> again. 525a new CBOR value from the beginning of the C<$buffer> again.
395 526
396This method can be caled at any time, but it I<must> be called if you want 527This method can be called at any time, but it I<must> be called if you want
397to change your C<$buffer> or there was a decoding error and you want to 528to change your C<$buffer> or there was a decoding error and you want to
398reuse the C<$cbor> object for future incremental parsings. 529reuse the C<$cbor> object for future incremental parsings.
399 530
400=back 531=back
401 532
476 607
477=item hash references 608=item hash references
478 609
479Perl hash references become CBOR maps. As there is no inherent ordering in 610Perl hash references become CBOR maps. As there is no inherent ordering in
480hash keys (or CBOR maps), they will usually be encoded in a pseudo-random 611hash keys (or CBOR maps), they will usually be encoded in a pseudo-random
481order. This order can be different each time a hahs is encoded. 612order. This order can be different each time a hash is encoded.
482 613
483Currently, tied hashes will use the indefinite-length format, while normal 614Currently, tied hashes will use the indefinite-length format, while normal
484hashes will use the fixed-length format. 615hashes will use the fixed-length format.
485 616
486=item array references 617=item array references
539 my $x = 3.1; # some variable containing a number 670 my $x = 3.1; # some variable containing a number
540 "$x"; # stringified 671 "$x"; # stringified
541 $x .= ""; # another, more awkward way to stringify 672 $x .= ""; # another, more awkward way to stringify
542 print $x; # perl does it for you, too, quite often 673 print $x; # perl does it for you, too, quite often
543 674
544You can force whether a string ie encoded as byte or text string by using 675You can force whether a string is encoded as byte or text string by using
545C<utf8::upgrade> and C<utf8::downgrade>): 676C<utf8::upgrade> and C<utf8::downgrade> (if C<text_strings> is disabled).
546 677
547 utf8::upgrade $x; # encode $x as text string 678 utf8::upgrade $x; # encode $x as text string
548 utf8::downgrade $x; # encode $x as byte string 679 utf8::downgrade $x; # encode $x as byte string
549 680
681More options are available, see L<TYPE CASTS>, below, and the C<text_keys>
682and C<text_strings> options.
683
550Perl doesn't define what operations up- and downgrade strings, so if the 684Perl doesn't define what operations up- and downgrade strings, so if the
551difference between byte and text is important, you should up- or downgrade 685difference between byte and text is important, you should up- or downgrade
552your string as late as possible before encoding. 686your string as late as possible before encoding. You can also force the
687use of CBOR text strings by using C<text_keys> or C<text_strings>.
553 688
554You can force the type to be a CBOR number by numifying it: 689You can force the type to be a CBOR number by numifying it:
555 690
556 my $x = "3"; # some variable containing a string 691 my $x = "3"; # some variable containing a string
557 $x += 0; # numify it, ensuring it will be dumped as a number 692 $x += 0; # numify it, ensuring it will be dumped as a number
568represent numerical values are supported, but might suffer loss of 703represent numerical values are supported, but might suffer loss of
569precision. 704precision.
570 705
571=back 706=back
572 707
708=head2 TYPE CASTS
709
710B<EXPERIMENTAL>: As an experimental extension, C<CBOR::XS> allows you to
711force specific CBOR types to be used when encoding. That allows you to
712encode types not normally accessible (e.g. half floats) as well as force
713string types even when C<text_strings> is in effect.
714
715Type forcing is done by calling a special "cast" function which keeps a
716copy of the value and returns a new value that can be handed over to any
717CBOR encoder function.
718
719The following casts are currently available (all of which are unary
720operators, that is, have a prototype of C<$>):
721
722=over
723
724=item CBOR::XS::as_int $value
725
726Forces the value to be encoded as some form of (basic, not bignum) integer
727type.
728
729=item CBOR::XS::as_text $value
730
731Forces the value to be encoded as (UTF-8) text values.
732
733=item CBOR::XS::as_bytes $value
734
735Forces the value to be encoded as a (binary) string value.
736
737Example: encode a perl string as binary even though C<text_strings> is in
738effect.
739
740 CBOR::XS->new->text_strings->encode ([4, "text", CBOR::XS::bytes "bytevalue"]);
741
742=item CBOR::XS::as_bool $value
743
744Converts a Perl boolean (which can be any kind of scalar) into a CBOR
745boolean. Strictly the same, but shorter to write, than:
746
747 $value ? Types::Serialiser::true : Types::Serialiser::false
748
749=item CBOR::XS::as_float16 $value
750
751Forces half-float (IEEE 754 binary16) encoding of the given value.
752
753=item CBOR::XS::as_float32 $value
754
755Forces single-float (IEEE 754 binary32) encoding of the given value.
756
757=item CBOR::XS::as_float64 $value
758
759Forces double-float (IEEE 754 binary64) encoding of the given value.
760
761=item CBOR::XS::as_cbor $cbor_text
762
763Not a type cast per-se, this type cast forces the argument to be encoded
764as-is. This can be used to embed pre-encoded CBOR data.
765
766Note that no checking on the validity of the C<$cbor_text> is done - it's
767the callers responsibility to correctly encode values.
768
769=item CBOR::XS::as_map [key => value...]
770
771Treat the array reference as key value pairs and output a CBOR map. This
772allows you to generate CBOR maps with arbitrary key types (or, if you
773don't care about semantics, duplicate keys or pairs in a custom order),
774which is otherwise hard to do with Perl.
775
776The single argument must be an array reference with an even number of
777elements.
778
779Note that only the reference to the array is copied, the array itself is
780not. Modifications done to the array before calling an encoding function
781will be reflected in the encoded output.
782
783Example: encode a CBOR map with a string and an integer as keys.
784
785 encode_cbor CBOR::XS::as_map [string => "value", 5 => "value"]
786
787=back
788
789=cut
790
791sub CBOR::XS::as_cbor ($) { bless [$_[0], 0, undef], CBOR::XS::Tagged:: }
792sub CBOR::XS::as_int ($) { bless [$_[0], 1, undef], CBOR::XS::Tagged:: }
793sub CBOR::XS::as_bytes ($) { bless [$_[0], 2, undef], CBOR::XS::Tagged:: }
794sub CBOR::XS::as_text ($) { bless [$_[0], 3, undef], CBOR::XS::Tagged:: }
795sub CBOR::XS::as_float16 ($) { bless [$_[0], 4, undef], CBOR::XS::Tagged:: }
796sub CBOR::XS::as_float32 ($) { bless [$_[0], 5, undef], CBOR::XS::Tagged:: }
797sub CBOR::XS::as_float64 ($) { bless [$_[0], 6, undef], CBOR::XS::Tagged:: }
798
799sub CBOR::XS::as_bool ($) { $_[0] ? $Types::Serialiser::true : $Types::Serialiser::false }
800
801sub CBOR::XS::as_map ($) {
802 ARRAY:: eq ref $_[0]
803 and $#{ $_[0] } & 1
804 or do { require Carp; Carp::croak ("CBOR::XS::as_map only acepts array references with an even number of elements, caught") };
805
806 bless [$_[0], 7, undef], CBOR::XS::Tagged::
807}
808
573=head2 OBJECT SERIALISATION 809=head2 OBJECT SERIALISATION
574 810
575This module implements both a CBOR-specific and the generic 811This module implements both a CBOR-specific and the generic
576L<Types::Serialier> object serialisation protocol. The following 812L<Types::Serialier> object serialisation protocol. The following
577subsections explain both methods. 813subsections explain both methods.
658 "$self" # encode url string 894 "$self" # encode url string
659 } 895 }
660 896
661 sub URI::THAW { 897 sub URI::THAW {
662 my ($class, $serialiser, $uri) = @_; 898 my ($class, $serialiser, $uri) = @_;
663
664 $class->new ($uri) 899 $class->new ($uri)
665 } 900 }
666 901
667Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For 902Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For
668example, a C<FREEZE> method that returns "type", "id" and "variant" values 903example, a C<FREEZE> method that returns "type", "id" and "variant" values
799additional tags (such as base64url). 1034additional tags (such as base64url).
800 1035
801=head2 ENFORCED TAGS 1036=head2 ENFORCED TAGS
802 1037
803These tags are always handled when decoding, and their handling cannot be 1038These tags are always handled when decoding, and their handling cannot be
804overriden by the user. 1039overridden by the user.
805 1040
806=over 4 1041=over 4
807 1042
808=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>) 1043=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>)
809 1044
810These tags are automatically created (and decoded) for serialisable 1045These tags are automatically created (and decoded) for serialisable
811objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object 1046objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object
812serialisation protocol). See L<OBJECT SERIALISATION> for details. 1047serialisation protocol). See L<OBJECT SERIALISATION> for details.
813 1048
814=item 28, 29 (shareable, sharedref, L <http://cbor.schmorp.de/value-sharing>) 1049=item 28, 29 (shareable, sharedref, L<http://cbor.schmorp.de/value-sharing>)
815 1050
816These tags are automatically decoded when encountered (and they do not 1051These tags are automatically decoded when encountered (and they do not
817result in a cyclic data structure, see C<allow_cycles>), resulting in 1052result in a cyclic data structure, see C<allow_cycles>), resulting in
818shared values in the decoded object. They are only encoded, however, when 1053shared values in the decoded object. They are only encoded, however, when
819C<allow_sharing> is enabled. 1054C<allow_sharing> is enabled.
829will be shared, others will not. While non-reference shared values can be 1064will be shared, others will not. While non-reference shared values can be
830generated in Perl with some effort, they were considered too unimportant 1065generated in Perl with some effort, they were considered too unimportant
831to be supported in the encoder. The decoder, however, will decode these 1066to be supported in the encoder. The decoder, however, will decode these
832values as shared values. 1067values as shared values.
833 1068
834=item 256, 25 (stringref-namespace, stringref, L <http://cbor.schmorp.de/stringref>) 1069=item 256, 25 (stringref-namespace, stringref, L<http://cbor.schmorp.de/stringref>)
835 1070
836These tags are automatically decoded when encountered. They are only 1071These tags are automatically decoded when encountered. They are only
837encoded, however, when C<pack_strings> is enabled. 1072encoded, however, when C<pack_strings> is enabled.
838 1073
839=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>) 1074=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>)
840 1075
841This tag is automatically generated when a reference are encountered (with 1076This tag is automatically generated when a reference are encountered (with
842the exception of hash and array refernces). It is converted to a reference 1077the exception of hash and array references). It is converted to a reference
843when decoding. 1078when decoding.
844 1079
845=item 55799 (self-describe CBOR, RFC 7049) 1080=item 55799 (self-describe CBOR, RFC 7049)
846 1081
847This value is not generated on encoding (unless explicitly requested by 1082This value is not generated on encoding (unless explicitly requested by
850=back 1085=back
851 1086
852=head2 NON-ENFORCED TAGS 1087=head2 NON-ENFORCED TAGS
853 1088
854These tags have default filters provided when decoding. Their handling can 1089These tags have default filters provided when decoding. Their handling can
855be overriden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by 1090be overridden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by
856providing a custom C<filter> callback when decoding. 1091providing a custom C<filter> callback when decoding.
857 1092
858When they result in decoding into a specific Perl class, the module 1093When they result in decoding into a specific Perl class, the module
859usually provides a corresponding C<TO_CBOR> method as well. 1094usually provides a corresponding C<TO_CBOR> method as well.
860 1095
878 1113
879These tags are decoded into L<Math::BigInt> objects. The corresponding 1114These tags are decoded into L<Math::BigInt> objects. The corresponding
880C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR 1115C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR
881integers, and others into positive/negative CBOR bignums. 1116integers, and others into positive/negative CBOR bignums.
882 1117
883=item 4, 5 (decimal fraction/bigfloat) 1118=item 4, 5, 264, 265 (decimal fraction/bigfloat)
884 1119
885Both decimal fractions and bigfloats are decoded into L<Math::BigFloat> 1120Both decimal fractions and bigfloats are decoded into L<Math::BigFloat>
886objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always> 1121objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always>
887encodes into a decimal fraction. 1122encodes into a decimal fraction (either tag 4 or 264).
888 1123
889CBOR cannot represent bigfloats with I<very> large exponents - conversion 1124NaN and infinities are not encoded properly, as they cannot be represented
890of such big float objects is undefined. 1125in CBOR.
891 1126
892Also, NaN and infinities are not encoded properly. 1127See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
1128
1129=item 30 (rational numbers)
1130
1131These tags are decoded into L<Math::BigRat> objects. The corresponding
1132C<Math::BigRat::TO_CBOR> method encodes rational numbers with denominator
1133C<1> via their numerator only, i.e., they become normal integers or
1134C<bignums>.
1135
1136See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
893 1137
894=item 21, 22, 23 (expected later JSON conversion) 1138=item 21, 22, 23 (expected later JSON conversion)
895 1139
896CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these 1140CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these
897tags. 1141tags.
902C<URI::TO_CBOR> method again results in a CBOR URI value. 1146C<URI::TO_CBOR> method again results in a CBOR URI value.
903 1147
904=back 1148=back
905 1149
906=cut 1150=cut
907
908our %FILTER = (
909 # 0 # rfc4287 datetime, utf-8
910 # 1 # unix timestamp, any
911
912 2 => sub { # pos bigint
913 require Math::BigInt;
914 Math::BigInt->new ("0x" . unpack "H*", pop)
915 },
916
917 3 => sub { # neg bigint
918 require Math::BigInt;
919 -Math::BigInt->new ("0x" . unpack "H*", pop)
920 },
921
922 4 => sub { # decimal fraction, array
923 require Math::BigFloat;
924 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
925 },
926
927 5 => sub { # bigfloat, array
928 require Math::BigFloat;
929 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2)
930 },
931
932 21 => sub { pop }, # expected conversion to base64url encoding
933 22 => sub { pop }, # expected conversion to base64 encoding
934 23 => sub { pop }, # expected conversion to base16 encoding
935
936 # 24 # embedded cbor, byte string
937
938 32 => sub {
939 require URI;
940 URI->new (pop)
941 },
942
943 # 33 # base64url rfc4648, utf-8
944 # 34 # base64 rfc46484, utf-8
945 # 35 # regex pcre/ecma262, utf-8
946 # 36 # mime message rfc2045, utf-8
947);
948
949 1151
950=head1 CBOR and JSON 1152=head1 CBOR and JSON
951 1153
952CBOR is supposed to implement a superset of the JSON data model, and is, 1154CBOR is supposed to implement a superset of the JSON data model, and is,
953with some coercion, able to represent all JSON texts (something that other 1155with some coercion, able to represent all JSON texts (something that other
962CBOR intact. 1164CBOR intact.
963 1165
964 1166
965=head1 SECURITY CONSIDERATIONS 1167=head1 SECURITY CONSIDERATIONS
966 1168
967When you are using CBOR in a protocol, talking to untrusted potentially 1169Tl;dr... if you want to decode or encode CBOR from untrusted sources, you
968hostile creatures requires relatively few measures. 1170should start with a coder object created via C<new_safe> (which implements
1171the mitigations explained below):
969 1172
1173 my $coder = CBOR::XS->new_safe;
1174
1175 my $data = $coder->decode ($cbor_text);
1176 my $cbor = $coder->encode ($data);
1177
1178Longer version: When you are using CBOR in a protocol, talking to
1179untrusted potentially hostile creatures requires some thought:
1180
1181=over 4
1182
1183=item Security of the CBOR decoder itself
1184
970First of all, your CBOR decoder should be secure, that is, should not have 1185First and foremost, your CBOR decoder should be secure, that is, should
1186not have any buffer overflows or similar bugs that could potentially be
971any buffer overflows. Obviously, this module should ensure that and I am 1187exploited. Obviously, this module should ensure that and I am trying hard
972trying hard on making that true, but you never know. 1188on making that true, but you never know.
973 1189
1190=item CBOR::XS can invoke almost arbitrary callbacks during decoding
1191
1192CBOR::XS supports object serialisation - decoding CBOR can cause calls
1193to I<any> C<THAW> method in I<any> package that exists in your process
1194(that is, CBOR::XS will not try to load modules, but any existing C<THAW>
1195method or function can be called, so they all have to be secure).
1196
1197Less obviously, it will also invoke C<TO_CBOR> and C<FREEZE> methods -
1198even if all your C<THAW> methods are secure, encoding data structures from
1199untrusted sources can invoke those and trigger bugs in those.
1200
1201So, if you are not sure about the security of all the modules you
1202have loaded (you shouldn't), you should disable this part using
1203C<forbid_objects> or using C<new_safe>.
1204
1205=item CBOR can be extended with tags that call library code
1206
1207CBOR can be extended with tags, and C<CBOR::XS> has a registry of
1208conversion functions for many existing tags that can be extended via
1209third-party modules (see the C<filter> method).
1210
1211If you don't trust these, you should configure the "safe" filter function,
1212C<CBOR::XS::safe_filter> (C<new_safe> does this), which by default only
1213includes conversion functions that are considered "safe" by the author
1214(but again, they can be extended by third party modules).
1215
1216Depending on your level of paranoia, you can use the "safe" filter:
1217
1218 $cbor->filter (\&CBOR::XS::safe_filter);
1219
1220... your own filter...
1221
1222 $cbor->filter (sub { ... do your stuffs here ... });
1223
1224... or even no filter at all, disabling all tag decoding:
1225
1226 $cbor->filter (sub { });
1227
1228This is never a problem for encoding, as the tag mechanism only exists in
1229CBOR texts.
1230
1231=item Resource-starving attacks: object memory usage
1232
974Second, you need to avoid resource-starving attacks. That means you should 1233You need to avoid resource-starving attacks. That means you should limit
975limit the size of CBOR data you accept, or make sure then when your 1234the size of CBOR data you accept, or make sure then when your resources
976resources run out, that's just fine (e.g. by using a separate process that 1235run out, that's just fine (e.g. by using a separate process that can
977can crash safely). The size of a CBOR string in octets is usually a good 1236crash safely). The size of a CBOR string in octets is usually a good
978indication of the size of the resources required to decode it into a Perl 1237indication of the size of the resources required to decode it into a Perl
979structure. While CBOR::XS can check the size of the CBOR text, it might be 1238structure. While CBOR::XS can check the size of the CBOR text (using
980too late when you already have it in memory, so you might want to check 1239C<max_size> - done by C<new_safe>), it might be too late when you already
981the size before you accept the string. 1240have it in memory, so you might want to check the size before you accept
1241the string.
982 1242
1243As for encoding, it is possible to construct data structures that are
1244relatively small but result in large CBOR texts (for example by having an
1245array full of references to the same big data structure, which will all be
1246deep-cloned during encoding by default). This is rarely an actual issue
1247(and the worst case is still just running out of memory), but you can
1248reduce this risk by using C<allow_sharing>.
1249
1250=item Resource-starving attacks: stack overflows
1251
983Third, CBOR::XS recurses using the C stack when decoding objects and 1252CBOR::XS recurses using the C stack when decoding objects and arrays. The
984arrays. The C stack is a limited resource: for instance, on my amd64 1253C stack is a limited resource: for instance, on my amd64 machine with 8MB
985machine with 8MB of stack size I can decode around 180k nested arrays but 1254of stack size I can decode around 180k nested arrays but only 14k nested
986only 14k nested CBOR objects (due to perl itself recursing deeply on croak 1255CBOR objects (due to perl itself recursing deeply on croak to free the
987to free the temporary). If that is exceeded, the program crashes. To be 1256temporary). If that is exceeded, the program crashes. To be conservative,
988conservative, the default nesting limit is set to 512. If your process 1257the default nesting limit is set to 512. If your process has a smaller
989has a smaller stack, you should adjust this setting accordingly with the 1258stack, you should adjust this setting accordingly with the C<max_depth>
990C<max_depth> method. 1259method.
1260
1261=item Resource-starving attacks: CPU en-/decoding complexity
1262
1263CBOR::XS will use the L<Math::BigInt>, L<Math::BigFloat> and
1264L<Math::BigRat> libraries to represent encode/decode bignums. These can be
1265very slow (as in, centuries of CPU time) and can even crash your program
1266(and are generally not very trustworthy). See the next section on bignum
1267security for details.
1268
1269=item Data breaches: leaking information in error messages
1270
1271CBOR::XS might leak contents of your Perl data structures in its error
1272messages, so when you serialise sensitive information you might want to
1273make sure that exceptions thrown by CBOR::XS will not end up in front of
1274untrusted eyes.
1275
1276=item Something else...
991 1277
992Something else could bomb you, too, that I forgot to think of. In that 1278Something else could bomb you, too, that I forgot to think of. In that
993case, you get to keep the pieces. I am always open for hints, though... 1279case, you get to keep the pieces. I am always open for hints, though...
994 1280
995Also keep in mind that CBOR::XS might leak contents of your Perl data 1281=back
996structures in its error messages, so when you serialise sensitive 1282
997information you might want to make sure that exceptions thrown by CBOR::XS 1283
998will not end up in front of untrusted eyes. 1284=head1 BIGNUM SECURITY CONSIDERATIONS
1285
1286CBOR::XS provides a C<TO_CBOR> method for both L<Math::BigInt> and
1287L<Math::BigFloat> that tries to encode the number in the simplest possible
1288way, that is, either a CBOR integer, a CBOR bigint/decimal fraction (tag
12894) or an arbitrary-exponent decimal fraction (tag 264). Rational numbers
1290(L<Math::BigRat>, tag 30) can also contain bignums as members.
1291
1292CBOR::XS will also understand base-2 bigfloat or arbitrary-exponent
1293bigfloats (tags 5 and 265), but it will never generate these on its own.
1294
1295Using the built-in L<Math::BigInt::Calc> support, encoding and decoding
1296decimal fractions is generally fast. Decoding bigints can be slow for very
1297big numbers (tens of thousands of digits, something that could potentially
1298be caught by limiting the size of CBOR texts), and decoding bigfloats or
1299arbitrary-exponent bigfloats can be I<extremely> slow (minutes, decades)
1300for large exponents (roughly 40 bit and longer).
1301
1302Additionally, L<Math::BigInt> can take advantage of other bignum
1303libraries, such as L<Math::GMP>, which cannot handle big floats with large
1304exponents, and might simply abort or crash your program, due to their code
1305quality.
1306
1307This can be a concern if you want to parse untrusted CBOR. If it is, you
1308might want to disable decoding of tag 2 (bigint) and 3 (negative bigint)
1309types. You should also disable types 5 and 265, as these can be slow even
1310without bigints.
1311
1312Disabling bigints will also partially or fully disable types that rely on
1313them, e.g. rational numbers that use bignums.
1314
999 1315
1000=head1 CBOR IMPLEMENTATION NOTES 1316=head1 CBOR IMPLEMENTATION NOTES
1001 1317
1002This section contains some random implementation notes. They do not 1318This section contains some random implementation notes. They do not
1003describe guaranteed behaviour, but merely behaviour as-is implemented 1319describe guaranteed behaviour, but merely behaviour as-is implemented
1017 1333
1018 1334
1019=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT 1335=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT
1020 1336
1021On perls that were built without 64 bit integer support (these are rare 1337On perls that were built without 64 bit integer support (these are rare
1022nowadays, even on 32 bit architectures), support for any kind of 64 bit 1338nowadays, even on 32 bit architectures, as all major Perl distributions
1339are built with 64 bit integer support), support for any kind of 64 bit
1023integer in CBOR is very limited - most likely, these 64 bit values will 1340value in CBOR is very limited - most likely, these 64 bit values will
1024be truncated, corrupted, or otherwise not decoded correctly. This also 1341be truncated, corrupted, or otherwise not decoded correctly. This also
1025includes string, array and map sizes that are stored as 64 bit integers. 1342includes string, float, array and map sizes that are stored as 64 bit
1343integers.
1026 1344
1027 1345
1028=head1 THREADS 1346=head1 THREADS
1029 1347
1030This module is I<not> guaranteed to be thread safe and there are no 1348This module is I<not> guaranteed to be thread safe and there are no
1043 1361
1044Please refrain from using rt.cpan.org or any other bug reporting 1362Please refrain from using rt.cpan.org or any other bug reporting
1045service. I put the contact address into my modules for a reason. 1363service. I put the contact address into my modules for a reason.
1046 1364
1047=cut 1365=cut
1366
1367# clumsy and slow hv_store-in-hash helper function
1368sub _hv_store {
1369 $_[0]{$_[1]} = $_[2];
1370}
1048 1371
1049our %FILTER = ( 1372our %FILTER = (
1050 0 => sub { # rfc4287 datetime, utf-8 1373 0 => sub { # rfc4287 datetime, utf-8
1051 require Time::Piece; 1374 require Time::Piece;
1052 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine" 1375 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine"
1053 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything 1376 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything
1054 # else either. Whats incredibe over standard strptime totally escapes me. 1377 # else either. Whats incredibe over standard strptime totally escapes me.
1055 # doesn't do fractional times, either. sigh. 1378 # doesn't do fractional times, either. sigh.
1056 # In fact, it's all a lie, it uses whatever strptime it wants, and of course, 1379 # In fact, it's all a lie, it uses whatever strptime it wants, and of course,
1057 # they are all incomptible. The openbsd one simply ignores %z (but according to the 1380 # they are all incompatible. The openbsd one simply ignores %z (but according to the
1058 # docs, it would be much more incredibly flexible indeed. If it worked, that is.). 1381 # docs, it would be much more incredibly flexible indeed. If it worked, that is.).
1059 scalar eval { 1382 scalar eval {
1060 my $s = $_[1]; 1383 my $s = $_[1];
1061 1384
1062 $s =~ s/Z$/+00:00/; 1385 $s =~ s/Z$/+00:00/;
1088 4 => sub { # decimal fraction, array 1411 4 => sub { # decimal fraction, array
1089 require Math::BigFloat; 1412 require Math::BigFloat;
1090 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0]) 1413 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1091 }, 1414 },
1092 1415
1416 264 => sub { # decimal fraction with arbitrary exponent
1417 require Math::BigFloat;
1418 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1419 },
1420
1093 5 => sub { # bigfloat, array 1421 5 => sub { # bigfloat, array
1094 require Math::BigFloat; 1422 require Math::BigFloat;
1095 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2) 1423 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1424 },
1425
1426 265 => sub { # bigfloat with arbitrary exponent
1427 require Math::BigFloat;
1428 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1429 },
1430
1431 30 => sub { # rational number
1432 require Math::BigRat;
1433 Math::BigRat->new ("$_[1][0]/$_[1][1]") # separate parameters only work in recent versons
1096 }, 1434 },
1097 1435
1098 21 => sub { pop }, # expected conversion to base64url encoding 1436 21 => sub { pop }, # expected conversion to base64url encoding
1099 22 => sub { pop }, # expected conversion to base64 encoding 1437 22 => sub { pop }, # expected conversion to base64 encoding
1100 23 => sub { pop }, # expected conversion to base16 encoding 1438 23 => sub { pop }, # expected conversion to base16 encoding
1110 # 34 # base64 rfc46484, utf-8 1448 # 34 # base64 rfc46484, utf-8
1111 # 35 # regex pcre/ecma262, utf-8 1449 # 35 # regex pcre/ecma262, utf-8
1112 # 36 # mime message rfc2045, utf-8 1450 # 36 # mime message rfc2045, utf-8
1113); 1451);
1114 1452
1115sub CBOR::XS::default_filter { 1453sub default_filter {
1116 &{ $FILTER{$_[0]} or return } 1454 &{ $FILTER{$_[0]} or return }
1455}
1456
1457our %SAFE_FILTER = map { $_ => $FILTER{$_} } 0, 1, 21, 22, 23, 32;
1458
1459sub safe_filter {
1460 &{ $SAFE_FILTER{$_[0]} or return }
1117} 1461}
1118 1462
1119sub URI::TO_CBOR { 1463sub URI::TO_CBOR {
1120 my $uri = $_[0]->as_string; 1464 my $uri = $_[0]->as_string;
1121 utf8::upgrade $uri; 1465 utf8::upgrade $uri;
1122 tag 32, $uri 1466 tag 32, $uri
1123} 1467}
1124 1468
1125sub Math::BigInt::TO_CBOR { 1469sub Math::BigInt::TO_CBOR {
1126 if ($_[0] >= -2147483648 && $_[0] <= 2147483647) { 1470 if (-2147483648 <= $_[0] && $_[0] <= 2147483647) {
1127 $_[0]->numify 1471 $_[0]->numify
1128 } else { 1472 } else {
1129 my $hex = substr $_[0]->as_hex, 2; 1473 my $hex = substr $_[0]->as_hex, 2;
1130 $hex = "0$hex" if 1 & length $hex; # sigh 1474 $hex = "0$hex" if 1 & length $hex; # sigh
1131 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex 1475 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex
1132 } 1476 }
1133} 1477}
1134 1478
1135sub Math::BigFloat::TO_CBOR { 1479sub Math::BigFloat::TO_CBOR {
1136 my ($m, $e) = $_[0]->parts; 1480 my ($m, $e) = $_[0]->parts;
1481
1482 -9223372036854775808 <= $e && $e <= 18446744073709551615
1137 tag 4, [$e->numify, $m] 1483 ? tag 4, [$e->numify, $m]
1484 : tag 264, [$e, $m]
1485}
1486
1487sub Math::BigRat::TO_CBOR {
1488 my ($n, $d) = $_[0]->parts;
1489
1490 # older versions of BigRat need *1, as they not always return numbers
1491
1492 $d*1 == 1
1493 ? $n*1
1494 : tag 30, [$n*1, $d*1]
1138} 1495}
1139 1496
1140sub Time::Piece::TO_CBOR { 1497sub Time::Piece::TO_CBOR {
1141 tag 1, 0 + $_[0]->epoch 1498 tag 1, 0 + $_[0]->epoch
1142} 1499}

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines