ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/CBOR-XS/XS.pm
(Generate patch)

Comparing CBOR-XS/XS.pm (file contents):
Revision 1.46 by root, Mon Feb 8 04:11:11 2016 UTC vs.
Revision 1.88 by root, Thu Sep 7 23:52:24 2023 UTC

38with the added ability of supporting serialisation of Perl objects. (JSON 38with the added ability of supporting serialisation of Perl objects. (JSON
39often compresses better than CBOR though, so if you plan to compress the 39often compresses better than CBOR though, so if you plan to compress the
40data later and speed is less important you might want to compare both 40data later and speed is less important you might want to compare both
41formats first). 41formats first).
42 42
43The primary goal of this module is to be I<correct> and the secondary goal
44is to be I<fast>. To reach the latter goal it was written in C.
45
43To give you a general idea about speed, with texts in the megabyte range, 46To give you a general idea about speed, with texts in the megabyte range,
44C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or 47C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or
45L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the 48L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the
46data, the worse L<Storable> performs in comparison. 49data, the worse L<Storable> performs in comparison.
47 50
52In addition to the core CBOR data format, this module implements a 55In addition to the core CBOR data format, this module implements a
53number of extensions, to support cyclic and shared data structures 56number of extensions, to support cyclic and shared data structures
54(see C<allow_sharing> and C<allow_cycles>), string deduplication (see 57(see C<allow_sharing> and C<allow_cycles>), string deduplication (see
55C<pack_strings>) and scalar references (always enabled). 58C<pack_strings>) and scalar references (always enabled).
56 59
57The primary goal of this module is to be I<correct> and the secondary goal
58is to be I<fast>. To reach the latter goal it was written in C.
59
60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and 60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and
61vice versa. 61vice versa.
62 62
63=cut 63=cut
64 64
65package CBOR::XS; 65package CBOR::XS;
66 66
67use common::sense; 67use common::sense;
68 68
69our $VERSION = 1.4; 69our $VERSION = 1.86;
70our @ISA = qw(Exporter); 70our @ISA = qw(Exporter);
71 71
72our @EXPORT = qw(encode_cbor decode_cbor); 72our @EXPORT = qw(encode_cbor decode_cbor);
73 73
74use Exporter; 74use Exporter;
112 112
113The mutators for flags all return the CBOR object again and thus calls can 113The mutators for flags all return the CBOR object again and thus calls can
114be chained: 114be chained:
115 115
116 my $cbor = CBOR::XS->new->encode ({a => [1,2]}); 116 my $cbor = CBOR::XS->new->encode ({a => [1,2]});
117
118=item $cbor = new_safe CBOR::XS
119
120Create a new, safe/secure CBOR::XS object. This is similar to C<new>,
121but configures the coder object to be safe to use with untrusted
122data. Currently, this is equivalent to:
123
124 my $cbor = CBOR::XS
125 ->new
126 ->validate_utf8
127 ->forbid_objects
128 ->filter (\&CBOR::XS::safe_filter)
129 ->max_size (1e8);
130
131But is more future proof (it is better to crash because of a change than
132to be exploited in other ways).
133
134=cut
135
136sub new_safe {
137 CBOR::XS
138 ->new
139 ->validate_utf8
140 ->forbid_objects
141 ->filter (\&CBOR::XS::safe_filter)
142 ->max_size (1e8)
143}
117 144
118=item $cbor = $cbor->max_depth ([$maximum_nesting_depth]) 145=item $cbor = $cbor->max_depth ([$maximum_nesting_depth])
119 146
120=item $max_depth = $cbor->get_max_depth 147=item $max_depth = $cbor->get_max_depth
121 148
137 164
138Note that nesting is implemented by recursion in C. The default value has 165Note that nesting is implemented by recursion in C. The default value has
139been chosen to be as large as typical operating systems allow without 166been chosen to be as large as typical operating systems allow without
140crashing. 167crashing.
141 168
142See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 169See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
143 170
144=item $cbor = $cbor->max_size ([$maximum_string_size]) 171=item $cbor = $cbor->max_size ([$maximum_string_size])
145 172
146=item $max_size = $cbor->get_max_size 173=item $max_size = $cbor->get_max_size
147 174
152effect on C<encode> (yet). 179effect on C<encode> (yet).
153 180
154If no argument is given, the limit check will be deactivated (same as when 181If no argument is given, the limit check will be deactivated (same as when
155C<0> is specified). 182C<0> is specified).
156 183
157See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 184See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
158 185
159=item $cbor = $cbor->allow_unknown ([$enable]) 186=item $cbor = $cbor->allow_unknown ([$enable])
160 187
161=item $enabled = $cbor->get_allow_unknown 188=item $enabled = $cbor->get_allow_unknown
162 189
180reference to the earlier value. 207reference to the earlier value.
181 208
182This means that such values will only be encoded once, and will not result 209This means that such values will only be encoded once, and will not result
183in a deep cloning of the value on decode, in decoders supporting the value 210in a deep cloning of the value on decode, in decoders supporting the value
184sharing extension. This also makes it possible to encode cyclic data 211sharing extension. This also makes it possible to encode cyclic data
185structures (which need C<allow_cycles> to ne enabled to be decoded by this 212structures (which need C<allow_cycles> to be enabled to be decoded by this
186module). 213module).
187 214
188It is recommended to leave it off unless you know your 215It is recommended to leave it off unless you know your
189communication partner supports the value sharing extensions to CBOR 216communication partner supports the value sharing extensions to CBOR
190(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the 217(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the
191resulting data structure might be unusable. 218resulting data structure might be unusable.
192 219
193Detecting shared values incurs a runtime overhead when values are encoded 220Detecting shared values incurs a runtime overhead when values are encoded
194that have a reference counter large than one, and might unnecessarily 221that have a reference counter larger than one, and might unnecessarily
195increase the encoded size, as potentially shared values are encode as 222increase the encoded size, as potentially shared values are encoded as
196shareable whether or not they are actually shared. 223shareable whether or not they are actually shared.
197 224
198At the moment, only targets of references can be shared (e.g. scalars, 225At the moment, only targets of references can be shared (e.g. scalars,
199arrays or hashes pointed to by a reference). Weirder constructs, such as 226arrays or hashes pointed to by a reference). Weirder constructs, such as
200an array with multiple "copies" of the I<same> string, which are hard but 227an array with multiple "copies" of the I<same> string, which are hard but
226throwing an error. 253throwing an error.
227 254
228This option does not affect C<encode> in any way - shared values and 255This option does not affect C<encode> in any way - shared values and
229references will always be encoded properly if present. 256references will always be encoded properly if present.
230 257
258=item $cbor = $cbor->forbid_objects ([$enable])
259
260=item $enabled = $cbor->get_forbid_objects
261
262Disables the use of the object serialiser protocol.
263
264If C<$enable> is true (or missing), then C<encode> will will throw an
265exception when it encounters perl objects that would be encoded using the
266perl-object tag (26). When C<decode> encounters such tags, it will fall
267back to the general filter/tagged logic as if this were an unknown tag (by
268default resulting in a C<CBOR::XC::Tagged> object).
269
270If C<$enable> is false (the default), then C<encode> will use the
271L<Types::Serialiser> object serialisation protocol to serialise objects
272into perl-object tags, and C<decode> will do the same to decode such tags.
273
274See L<SECURITY CONSIDERATIONS>, below, for more info on why forbidding this
275protocol can be useful.
276
231=item $cbor = $cbor->pack_strings ([$enable]) 277=item $cbor = $cbor->pack_strings ([$enable])
232 278
233=item $enabled = $cbor->get_pack_strings 279=item $enabled = $cbor->get_pack_strings
234 280
235If C<$enable> is true (or missing), then C<encode> will try not to encode 281If C<$enable> is true (or missing), then C<encode> will try not to encode
247the standard CBOR way. 293the standard CBOR way.
248 294
249This option does not affect C<decode> in any way - string references will 295This option does not affect C<decode> in any way - string references will
250always be decoded properly if present. 296always be decoded properly if present.
251 297
298=item $cbor = $cbor->text_keys ([$enable])
299
300=item $enabled = $cbor->get_text_keys
301
302If C<$enabled> is true (or missing), then C<encode> will encode all
303perl hash keys as CBOR text strings/UTF-8 string, upgrading them as needed.
304
305If C<$enable> is false (the default), then C<encode> will encode hash keys
306normally - upgraded perl strings (strings internally encoded as UTF-8) as
307CBOR text strings, and downgraded perl strings as CBOR byte strings.
308
309This option does not affect C<decode> in any way.
310
311This option is useful for interoperability with CBOR decoders that don't
312treat byte strings as a form of text. It is especially useful as Perl
313gives very little control over hash keys.
314
315Enabling this option can be slow, as all downgraded hash keys that are
316encoded need to be scanned and converted to UTF-8.
317
318=item $cbor = $cbor->text_strings ([$enable])
319
320=item $enabled = $cbor->get_text_strings
321
322This option works similar to C<text_keys>, above, but works on all strings
323(including hash keys), so C<text_keys> has no further effect after
324enabling C<text_strings>.
325
326If C<$enabled> is true (or missing), then C<encode> will encode all perl
327strings as CBOR text strings/UTF-8 strings, upgrading them as needed.
328
329If C<$enable> is false (the default), then C<encode> will encode strings
330normally (but see C<text_keys>) - upgraded perl strings (strings
331internally encoded as UTF-8) as CBOR text strings, and downgraded perl
332strings as CBOR byte strings.
333
334This option does not affect C<decode> in any way.
335
336This option has similar advantages and disadvantages as C<text_keys>. In
337addition, this option effectively removes the ability to automatically
338encode byte strings, which might break some C<FREEZE> and C<TO_CBOR>
339methods that rely on this.
340
341A workaround is to use explicit type casts, which are unaffected by this option.
342
252=item $cbor = $cbor->validate_utf8 ([$enable]) 343=item $cbor = $cbor->validate_utf8 ([$enable])
253 344
254=item $enabled = $cbor->get_validate_utf8 345=item $enabled = $cbor->get_validate_utf8
255 346
256If C<$enable> is true (or missing), then C<decode> will validate that 347If C<$enable> is true (or missing), then C<decode> will validate that
261The concept of "valid UTF-8" used is perl's concept, which is a superset 352The concept of "valid UTF-8" used is perl's concept, which is a superset
262of the official UTF-8. 353of the official UTF-8.
263 354
264If C<$enable> is false (the default), then C<decode> will blindly accept 355If C<$enable> is false (the default), then C<decode> will blindly accept
265UTF-8 data, marking them as valid UTF-8 in the resulting data structure 356UTF-8 data, marking them as valid UTF-8 in the resulting data structure
266regardless of whether thats true or not. 357regardless of whether that's true or not.
267 358
268Perl isn't too happy about corrupted UTF-8 in strings, but should 359Perl isn't too happy about corrupted UTF-8 in strings, but should
269generally not crash or do similarly evil things. Extensions might be not 360generally not crash or do similarly evil things. Extensions might be not
270so forgiving, so it's recommended to turn on this setting if you receive 361so forgiving, so it's recommended to turn on this setting if you receive
271untrusted CBOR. 362untrusted CBOR.
293replace the tagged value in the decoded data structure, or no values, 384replace the tagged value in the decoded data structure, or no values,
294which will result in default handling, which currently means the decoder 385which will result in default handling, which currently means the decoder
295creates a C<CBOR::XS::Tagged> object to hold the tag and the value. 386creates a C<CBOR::XS::Tagged> object to hold the tag and the value.
296 387
297When the filter is cleared (the default state), the default filter 388When the filter is cleared (the default state), the default filter
298function, C<CBOR::XS::default_filter>, is used. This function simply looks 389function, C<CBOR::XS::default_filter>, is used. This function simply
299up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists it must be 390looks up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists
300a code reference that is called with tag and value, and is responsible for 391it must be a code reference that is called with tag and value, and is
301decoding the value. If no entry exists, it returns no values. 392responsible for decoding the value. If no entry exists, it returns no
393values. C<CBOR::XS> provides a number of default filter functions already,
394the the C<%CBOR::XS::FILTER> hash can be freely extended with more.
395
396C<CBOR::XS> additionally provides an alternative filter function that is
397supposed to be safe to use with untrusted data (which the default filter
398might not), called C<CBOR::XS::safe_filter>, which works the same as
399the C<default_filter> but uses the C<%CBOR::XS::SAFE_FILTER> variable
400instead. It is prepopulated with the tag decoding functions that are
401deemed safe (basically the same as C<%CBOR::XS::FILTER> without all
402the bignum tags), and can be extended by user code as wlel, although,
403obviously, one should be very careful about adding decoding functions
404here, since the expectation is that they are safe to use on untrusted
405data, after all.
302 406
303Example: decode all tags not handled internally into C<CBOR::XS::Tagged> 407Example: decode all tags not handled internally into C<CBOR::XS::Tagged>
304objects, with no other special handling (useful when working with 408objects, with no other special handling (useful when working with
305potentially "unsafe" CBOR data). 409potentially "unsafe" CBOR data).
306 410
313 my ($tag, $value); 417 my ($tag, $value);
314 418
315 "tag 1347375694 value $value" 419 "tag 1347375694 value $value"
316 }; 420 };
317 421
422Example: provide your own filter function that looks up tags in your own
423hash:
424
425 my %my_filter = (
426 998347484 => sub {
427 my ($tag, $value);
428
429 "tag 998347484 value $value"
430 };
431 );
432
433 my $coder = CBOR::XS->new->filter (sub {
434 &{ $my_filter{$_[0]} or return }
435 });
436
437
438Example: use the safe filter function (see L<SECURITY CONSIDERATIONS> for
439more considerations on security).
440
441 CBOR::XS->new->filter (\&CBOR::XS::safe_filter)->decode ($cbor_data);
442
318=item $cbor_data = $cbor->encode ($perl_scalar) 443=item $cbor_data = $cbor->encode ($perl_scalar)
319 444
320Converts the given Perl data structure (a scalar value) to its CBOR 445Converts the given Perl data structure (a scalar value) to its CBOR
321representation. 446representation.
322 447
331when there is trailing garbage after the CBOR string, it will silently 456when there is trailing garbage after the CBOR string, it will silently
332stop parsing there and return the number of characters consumed so far. 457stop parsing there and return the number of characters consumed so far.
333 458
334This is useful if your CBOR texts are not delimited by an outer protocol 459This is useful if your CBOR texts are not delimited by an outer protocol
335and you need to know where the first CBOR string ends amd the next one 460and you need to know where the first CBOR string ends amd the next one
336starts. 461starts - CBOR strings are self-delimited, so it is possible to concatenate
462CBOR strings without any delimiters or size fields and recover their data.
337 463
338 CBOR::XS->new->decode_prefix ("......") 464 CBOR::XS->new->decode_prefix ("......")
339 => ("...", 3) 465 => ("...", 3)
340 466
341=back 467=back
347Perl data structure in memory at one time, it does allow you to parse a 473Perl data structure in memory at one time, it does allow you to parse a
348CBOR stream incrementally, using a similar to using "decode_prefix" to see 474CBOR stream incrementally, using a similar to using "decode_prefix" to see
349if a full CBOR object is available, but is much more efficient. 475if a full CBOR object is available, but is much more efficient.
350 476
351It basically works by parsing as much of a CBOR string as possible - if 477It basically works by parsing as much of a CBOR string as possible - if
352the CBOR data is not complete yet, the pasrer will remember where it was, 478the CBOR data is not complete yet, the parser will remember where it was,
353to be able to restart when more data has been accumulated. Once enough 479to be able to restart when more data has been accumulated. Once enough
354data is available to either decode a complete CBOR value or raise an 480data is available to either decode a complete CBOR value or raise an
355error, a real decode will be attempted. 481error, a real decode will be attempted.
356 482
357A typical use case would be a network protocol that consists of sending 483A typical use case would be a network protocol that consists of sending
396 522
397Resets the incremental decoder. This throws away any saved state, so that 523Resets the incremental decoder. This throws away any saved state, so that
398subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse 524subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse
399a new CBOR value from the beginning of the C<$buffer> again. 525a new CBOR value from the beginning of the C<$buffer> again.
400 526
401This method can be caled at any time, but it I<must> be called if you want 527This method can be called at any time, but it I<must> be called if you want
402to change your C<$buffer> or there was a decoding error and you want to 528to change your C<$buffer> or there was a decoding error and you want to
403reuse the C<$cbor> object for future incremental parsings. 529reuse the C<$cbor> object for future incremental parsings.
404 530
405=back 531=back
406 532
481 607
482=item hash references 608=item hash references
483 609
484Perl hash references become CBOR maps. As there is no inherent ordering in 610Perl hash references become CBOR maps. As there is no inherent ordering in
485hash keys (or CBOR maps), they will usually be encoded in a pseudo-random 611hash keys (or CBOR maps), they will usually be encoded in a pseudo-random
486order. This order can be different each time a hahs is encoded. 612order. This order can be different each time a hash is encoded.
487 613
488Currently, tied hashes will use the indefinite-length format, while normal 614Currently, tied hashes will use the indefinite-length format, while normal
489hashes will use the fixed-length format. 615hashes will use the fixed-length format.
490 616
491=item array references 617=item array references
509create such objects. 635create such objects.
510 636
511=item Types::Serialiser::true, Types::Serialiser::false, Types::Serialiser::error 637=item Types::Serialiser::true, Types::Serialiser::false, Types::Serialiser::error
512 638
513These special values become CBOR true, CBOR false and CBOR undefined 639These special values become CBOR true, CBOR false and CBOR undefined
514values, respectively. You can also use C<\1>, C<\0> and C<\undef> directly 640values, respectively.
515if you want.
516 641
517=item other blessed objects 642=item other blessed objects
518 643
519Other blessed objects are serialised via C<TO_CBOR> or C<FREEZE>. See 644Other blessed objects are serialised via C<TO_CBOR> or C<FREEZE>. See
520L<TAG HANDLING AND EXTENSIONS> for specific classes handled by this 645L<TAG HANDLING AND EXTENSIONS> for specific classes handled by this
544 my $x = 3.1; # some variable containing a number 669 my $x = 3.1; # some variable containing a number
545 "$x"; # stringified 670 "$x"; # stringified
546 $x .= ""; # another, more awkward way to stringify 671 $x .= ""; # another, more awkward way to stringify
547 print $x; # perl does it for you, too, quite often 672 print $x; # perl does it for you, too, quite often
548 673
549You can force whether a string ie encoded as byte or text string by using 674You can force whether a string is encoded as byte or text string by using
550C<utf8::upgrade> and C<utf8::downgrade>): 675C<utf8::upgrade> and C<utf8::downgrade> (if C<text_strings> is disabled).
551 676
552 utf8::upgrade $x; # encode $x as text string 677 utf8::upgrade $x; # encode $x as text string
553 utf8::downgrade $x; # encode $x as byte string 678 utf8::downgrade $x; # encode $x as byte string
554 679
680More options are available, see L<TYPE CASTS>, below, and the C<text_keys>
681and C<text_strings> options.
682
555Perl doesn't define what operations up- and downgrade strings, so if the 683Perl doesn't define what operations up- and downgrade strings, so if the
556difference between byte and text is important, you should up- or downgrade 684difference between byte and text is important, you should up- or downgrade
557your string as late as possible before encoding. 685your string as late as possible before encoding. You can also force the
686use of CBOR text strings by using C<text_keys> or C<text_strings>.
558 687
559You can force the type to be a CBOR number by numifying it: 688You can force the type to be a CBOR number by numifying it:
560 689
561 my $x = "3"; # some variable containing a string 690 my $x = "3"; # some variable containing a string
562 $x += 0; # numify it, ensuring it will be dumped as a number 691 $x += 0; # numify it, ensuring it will be dumped as a number
573represent numerical values are supported, but might suffer loss of 702represent numerical values are supported, but might suffer loss of
574precision. 703precision.
575 704
576=back 705=back
577 706
707=head2 TYPE CASTS
708
709B<EXPERIMENTAL>: As an experimental extension, C<CBOR::XS> allows you to
710force specific CBOR types to be used when encoding. That allows you to
711encode types not normally accessible (e.g. half floats) as well as force
712string types even when C<text_strings> is in effect.
713
714Type forcing is done by calling a special "cast" function which keeps a
715copy of the value and returns a new value that can be handed over to any
716CBOR encoder function.
717
718The following casts are currently available (all of which are unary
719operators, that is, have a prototype of C<$>):
720
721=over
722
723=item CBOR::XS::as_int $value
724
725Forces the value to be encoded as some form of (basic, not bignum) integer
726type.
727
728=item CBOR::XS::as_text $value
729
730Forces the value to be encoded as (UTF-8) text values.
731
732=item CBOR::XS::as_bytes $value
733
734Forces the value to be encoded as a (binary) string value.
735
736Example: encode a perl string as binary even though C<text_strings> is in
737effect.
738
739 CBOR::XS->new->text_strings->encode ([4, "text", CBOR::XS::bytes "bytevalue"]);
740
741=item CBOR::XS::as_bool $value
742
743Converts a Perl boolean (which can be any kind of scalar) into a CBOR
744boolean. Strictly the same, but shorter to write, than:
745
746 $value ? Types::Serialiser::true : Types::Serialiser::false
747
748=item CBOR::XS::as_float16 $value
749
750Forces half-float (IEEE 754 binary16) encoding of the given value.
751
752=item CBOR::XS::as_float32 $value
753
754Forces single-float (IEEE 754 binary32) encoding of the given value.
755
756=item CBOR::XS::as_float64 $value
757
758Forces double-float (IEEE 754 binary64) encoding of the given value.
759
760=item CBOR::XS::as_cbor $cbor_text
761
762Not a type cast per-se, this type cast forces the argument to be encoded
763as-is. This can be used to embed pre-encoded CBOR data.
764
765Note that no checking on the validity of the C<$cbor_text> is done - it's
766the callers responsibility to correctly encode values.
767
768=item CBOR::XS::as_map [key => value...]
769
770Treat the array reference as key value pairs and output a CBOR map. This
771allows you to generate CBOR maps with arbitrary key types (or, if you
772don't care about semantics, duplicate keys or pairs in a custom order),
773which is otherwise hard to do with Perl.
774
775The single argument must be an array reference with an even number of
776elements.
777
778Note that only the reference to the array is copied, the array itself is
779not. Modifications done to the array before calling an encoding function
780will be reflected in the encoded output.
781
782Example: encode a CBOR map with a string and an integer as keys.
783
784 encode_cbor CBOR::XS::as_map [string => "value", 5 => "value"]
785
786=back
787
788=cut
789
790sub CBOR::XS::as_cbor ($) { bless [$_[0], 0, undef], CBOR::XS::Tagged:: }
791sub CBOR::XS::as_int ($) { bless [$_[0], 1, undef], CBOR::XS::Tagged:: }
792sub CBOR::XS::as_bytes ($) { bless [$_[0], 2, undef], CBOR::XS::Tagged:: }
793sub CBOR::XS::as_text ($) { bless [$_[0], 3, undef], CBOR::XS::Tagged:: }
794sub CBOR::XS::as_float16 ($) { bless [$_[0], 4, undef], CBOR::XS::Tagged:: }
795sub CBOR::XS::as_float32 ($) { bless [$_[0], 5, undef], CBOR::XS::Tagged:: }
796sub CBOR::XS::as_float64 ($) { bless [$_[0], 6, undef], CBOR::XS::Tagged:: }
797
798sub CBOR::XS::as_bool ($) { $_[0] ? $Types::Serialiser::true : $Types::Serialiser::false }
799
800sub CBOR::XS::as_map ($) {
801 ARRAY:: eq ref $_[0]
802 and $#{ $_[0] } & 1
803 or do { require Carp; Carp::croak ("CBOR::XS::as_map only acepts array references with an even number of elements, caught") };
804
805 bless [$_[0], 7, undef], CBOR::XS::Tagged::
806}
807
578=head2 OBJECT SERIALISATION 808=head2 OBJECT SERIALISATION
579 809
580This module implements both a CBOR-specific and the generic 810This module implements both a CBOR-specific and the generic
581L<Types::Serialier> object serialisation protocol. The following 811L<Types::Serialier> object serialisation protocol. The following
582subsections explain both methods. 812subsections explain both methods.
663 "$self" # encode url string 893 "$self" # encode url string
664 } 894 }
665 895
666 sub URI::THAW { 896 sub URI::THAW {
667 my ($class, $serialiser, $uri) = @_; 897 my ($class, $serialiser, $uri) = @_;
668
669 $class->new ($uri) 898 $class->new ($uri)
670 } 899 }
671 900
672Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For 901Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For
673example, a C<FREEZE> method that returns "type", "id" and "variant" values 902example, a C<FREEZE> method that returns "type", "id" and "variant" values
804additional tags (such as base64url). 1033additional tags (such as base64url).
805 1034
806=head2 ENFORCED TAGS 1035=head2 ENFORCED TAGS
807 1036
808These tags are always handled when decoding, and their handling cannot be 1037These tags are always handled when decoding, and their handling cannot be
809overriden by the user. 1038overridden by the user.
810 1039
811=over 4 1040=over 4
812 1041
813=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>) 1042=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>)
814 1043
842encoded, however, when C<pack_strings> is enabled. 1071encoded, however, when C<pack_strings> is enabled.
843 1072
844=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>) 1073=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>)
845 1074
846This tag is automatically generated when a reference are encountered (with 1075This tag is automatically generated when a reference are encountered (with
847the exception of hash and array refernces). It is converted to a reference 1076the exception of hash and array references). It is converted to a reference
848when decoding. 1077when decoding.
849 1078
850=item 55799 (self-describe CBOR, RFC 7049) 1079=item 55799 (self-describe CBOR, RFC 7049)
851 1080
852This value is not generated on encoding (unless explicitly requested by 1081This value is not generated on encoding (unless explicitly requested by
855=back 1084=back
856 1085
857=head2 NON-ENFORCED TAGS 1086=head2 NON-ENFORCED TAGS
858 1087
859These tags have default filters provided when decoding. Their handling can 1088These tags have default filters provided when decoding. Their handling can
860be overriden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by 1089be overridden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by
861providing a custom C<filter> callback when decoding. 1090providing a custom C<filter> callback when decoding.
862 1091
863When they result in decoding into a specific Perl class, the module 1092When they result in decoding into a specific Perl class, the module
864usually provides a corresponding C<TO_CBOR> method as well. 1093usually provides a corresponding C<TO_CBOR> method as well.
865 1094
883 1112
884These tags are decoded into L<Math::BigInt> objects. The corresponding 1113These tags are decoded into L<Math::BigInt> objects. The corresponding
885C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR 1114C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR
886integers, and others into positive/negative CBOR bignums. 1115integers, and others into positive/negative CBOR bignums.
887 1116
888=item 4, 5 (decimal fraction/bigfloat) 1117=item 4, 5, 264, 265 (decimal fraction/bigfloat)
889 1118
890Both decimal fractions and bigfloats are decoded into L<Math::BigFloat> 1119Both decimal fractions and bigfloats are decoded into L<Math::BigFloat>
891objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always> 1120objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always>
892encodes into a decimal fraction. 1121encodes into a decimal fraction (either tag 4 or 264).
893 1122
894CBOR cannot represent bigfloats with I<very> large exponents - conversion 1123NaN and infinities are not encoded properly, as they cannot be represented
895of such big float objects is undefined. 1124in CBOR.
896 1125
897Also, NaN and infinities are not encoded properly. 1126See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
1127
1128=item 30 (rational numbers)
1129
1130These tags are decoded into L<Math::BigRat> objects. The corresponding
1131C<Math::BigRat::TO_CBOR> method encodes rational numbers with denominator
1132C<1> via their numerator only, i.e., they become normal integers or
1133C<bignums>.
1134
1135See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
898 1136
899=item 21, 22, 23 (expected later JSON conversion) 1137=item 21, 22, 23 (expected later JSON conversion)
900 1138
901CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these 1139CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these
902tags. 1140tags.
907C<URI::TO_CBOR> method again results in a CBOR URI value. 1145C<URI::TO_CBOR> method again results in a CBOR URI value.
908 1146
909=back 1147=back
910 1148
911=cut 1149=cut
912
913our %FILTER = (
914 # 0 # rfc4287 datetime, utf-8
915 # 1 # unix timestamp, any
916
917 2 => sub { # pos bigint
918 require Math::BigInt;
919 Math::BigInt->new ("0x" . unpack "H*", pop)
920 },
921
922 3 => sub { # neg bigint
923 require Math::BigInt;
924 -Math::BigInt->new ("0x" . unpack "H*", pop)
925 },
926
927 4 => sub { # decimal fraction, array
928 require Math::BigFloat;
929 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
930 },
931
932 5 => sub { # bigfloat, array
933 require Math::BigFloat;
934 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2)
935 },
936
937 21 => sub { pop }, # expected conversion to base64url encoding
938 22 => sub { pop }, # expected conversion to base64 encoding
939 23 => sub { pop }, # expected conversion to base16 encoding
940
941 # 24 # embedded cbor, byte string
942
943 32 => sub {
944 require URI;
945 URI->new (pop)
946 },
947
948 # 33 # base64url rfc4648, utf-8
949 # 34 # base64 rfc46484, utf-8
950 # 35 # regex pcre/ecma262, utf-8
951 # 36 # mime message rfc2045, utf-8
952);
953
954 1150
955=head1 CBOR and JSON 1151=head1 CBOR and JSON
956 1152
957CBOR is supposed to implement a superset of the JSON data model, and is, 1153CBOR is supposed to implement a superset of the JSON data model, and is,
958with some coercion, able to represent all JSON texts (something that other 1154with some coercion, able to represent all JSON texts (something that other
967CBOR intact. 1163CBOR intact.
968 1164
969 1165
970=head1 SECURITY CONSIDERATIONS 1166=head1 SECURITY CONSIDERATIONS
971 1167
972When you are using CBOR in a protocol, talking to untrusted potentially 1168Tl;dr... if you want to decode or encode CBOR from untrusted sources, you
973hostile creatures requires relatively few measures. 1169should start with a coder object created via C<new_safe> (which implements
1170the mitigations explained below):
974 1171
1172 my $coder = CBOR::XS->new_safe;
1173
1174 my $data = $coder->decode ($cbor_text);
1175 my $cbor = $coder->encode ($data);
1176
1177Longer version: When you are using CBOR in a protocol, talking to
1178untrusted potentially hostile creatures requires some thought:
1179
1180=over 4
1181
1182=item Security of the CBOR decoder itself
1183
975First of all, your CBOR decoder should be secure, that is, should not have 1184First and foremost, your CBOR decoder should be secure, that is, should
1185not have any buffer overflows or similar bugs that could potentially be
976any buffer overflows. Obviously, this module should ensure that and I am 1186exploited. Obviously, this module should ensure that and I am trying hard
977trying hard on making that true, but you never know. 1187on making that true, but you never know.
978 1188
1189=item CBOR::XS can invoke almost arbitrary callbacks during decoding
1190
1191CBOR::XS supports object serialisation - decoding CBOR can cause calls
1192to I<any> C<THAW> method in I<any> package that exists in your process
1193(that is, CBOR::XS will not try to load modules, but any existing C<THAW>
1194method or function can be called, so they all have to be secure).
1195
1196Less obviously, it will also invoke C<TO_CBOR> and C<FREEZE> methods -
1197even if all your C<THAW> methods are secure, encoding data structures from
1198untrusted sources can invoke those and trigger bugs in those.
1199
1200So, if you are not sure about the security of all the modules you
1201have loaded (you shouldn't), you should disable this part using
1202C<forbid_objects> or using C<new_safe>.
1203
1204=item CBOR can be extended with tags that call library code
1205
1206CBOR can be extended with tags, and C<CBOR::XS> has a registry of
1207conversion functions for many existing tags that can be extended via
1208third-party modules (see the C<filter> method).
1209
1210If you don't trust these, you should configure the "safe" filter function,
1211C<CBOR::XS::safe_filter> (C<new_safe> does this), which by default only
1212includes conversion functions that are considered "safe" by the author
1213(but again, they can be extended by third party modules).
1214
1215Depending on your level of paranoia, you can use the "safe" filter:
1216
1217 $cbor->filter (\&CBOR::XS::safe_filter);
1218
1219... your own filter...
1220
1221 $cbor->filter (sub { ... do your stuffs here ... });
1222
1223... or even no filter at all, disabling all tag decoding:
1224
1225 $cbor->filter (sub { });
1226
1227This is never a problem for encoding, as the tag mechanism only exists in
1228CBOR texts.
1229
1230=item Resource-starving attacks: object memory usage
1231
979Second, you need to avoid resource-starving attacks. That means you should 1232You need to avoid resource-starving attacks. That means you should limit
980limit the size of CBOR data you accept, or make sure then when your 1233the size of CBOR data you accept, or make sure then when your resources
981resources run out, that's just fine (e.g. by using a separate process that 1234run out, that's just fine (e.g. by using a separate process that can
982can crash safely). The size of a CBOR string in octets is usually a good 1235crash safely). The size of a CBOR string in octets is usually a good
983indication of the size of the resources required to decode it into a Perl 1236indication of the size of the resources required to decode it into a Perl
984structure. While CBOR::XS can check the size of the CBOR text, it might be 1237structure. While CBOR::XS can check the size of the CBOR text (using
985too late when you already have it in memory, so you might want to check 1238C<max_size> - done by C<new_safe>), it might be too late when you already
986the size before you accept the string. 1239have it in memory, so you might want to check the size before you accept
1240the string.
987 1241
1242As for encoding, it is possible to construct data structures that are
1243relatively small but result in large CBOR texts (for example by having an
1244array full of references to the same big data structure, which will all be
1245deep-cloned during encoding by default). This is rarely an actual issue
1246(and the worst case is still just running out of memory), but you can
1247reduce this risk by using C<allow_sharing>.
1248
1249=item Resource-starving attacks: stack overflows
1250
988Third, CBOR::XS recurses using the C stack when decoding objects and 1251CBOR::XS recurses using the C stack when decoding objects and arrays. The
989arrays. The C stack is a limited resource: for instance, on my amd64 1252C stack is a limited resource: for instance, on my amd64 machine with 8MB
990machine with 8MB of stack size I can decode around 180k nested arrays but 1253of stack size I can decode around 180k nested arrays but only 14k nested
991only 14k nested CBOR objects (due to perl itself recursing deeply on croak 1254CBOR objects (due to perl itself recursing deeply on croak to free the
992to free the temporary). If that is exceeded, the program crashes. To be 1255temporary). If that is exceeded, the program crashes. To be conservative,
993conservative, the default nesting limit is set to 512. If your process 1256the default nesting limit is set to 512. If your process has a smaller
994has a smaller stack, you should adjust this setting accordingly with the 1257stack, you should adjust this setting accordingly with the C<max_depth>
995C<max_depth> method. 1258method.
1259
1260=item Resource-starving attacks: CPU en-/decoding complexity
1261
1262CBOR::XS will use the L<Math::BigInt>, L<Math::BigFloat> and
1263L<Math::BigRat> libraries to represent encode/decode bignums. These can be
1264very slow (as in, centuries of CPU time) and can even crash your program
1265(and are generally not very trustworthy). See the next section on bignum
1266security for details.
1267
1268=item Data breaches: leaking information in error messages
1269
1270CBOR::XS might leak contents of your Perl data structures in its error
1271messages, so when you serialise sensitive information you might want to
1272make sure that exceptions thrown by CBOR::XS will not end up in front of
1273untrusted eyes.
1274
1275=item Something else...
996 1276
997Something else could bomb you, too, that I forgot to think of. In that 1277Something else could bomb you, too, that I forgot to think of. In that
998case, you get to keep the pieces. I am always open for hints, though... 1278case, you get to keep the pieces. I am always open for hints, though...
999 1279
1000Also keep in mind that CBOR::XS might leak contents of your Perl data 1280=back
1001structures in its error messages, so when you serialise sensitive 1281
1002information you might want to make sure that exceptions thrown by CBOR::XS 1282
1003will not end up in front of untrusted eyes. 1283=head1 BIGNUM SECURITY CONSIDERATIONS
1284
1285CBOR::XS provides a C<TO_CBOR> method for both L<Math::BigInt> and
1286L<Math::BigFloat> that tries to encode the number in the simplest possible
1287way, that is, either a CBOR integer, a CBOR bigint/decimal fraction (tag
12884) or an arbitrary-exponent decimal fraction (tag 264). Rational numbers
1289(L<Math::BigRat>, tag 30) can also contain bignums as members.
1290
1291CBOR::XS will also understand base-2 bigfloat or arbitrary-exponent
1292bigfloats (tags 5 and 265), but it will never generate these on its own.
1293
1294Using the built-in L<Math::BigInt::Calc> support, encoding and decoding
1295decimal fractions is generally fast. Decoding bigints can be slow for very
1296big numbers (tens of thousands of digits, something that could potentially
1297be caught by limiting the size of CBOR texts), and decoding bigfloats or
1298arbitrary-exponent bigfloats can be I<extremely> slow (minutes, decades)
1299for large exponents (roughly 40 bit and longer).
1300
1301Additionally, L<Math::BigInt> can take advantage of other bignum
1302libraries, such as L<Math::GMP>, which cannot handle big floats with large
1303exponents, and might simply abort or crash your program, due to their code
1304quality.
1305
1306This can be a concern if you want to parse untrusted CBOR. If it is, you
1307might want to disable decoding of tag 2 (bigint) and 3 (negative bigint)
1308types. You should also disable types 5 and 265, as these can be slow even
1309without bigints.
1310
1311Disabling bigints will also partially or fully disable types that rely on
1312them, e.g. rational numbers that use bignums.
1313
1004 1314
1005=head1 CBOR IMPLEMENTATION NOTES 1315=head1 CBOR IMPLEMENTATION NOTES
1006 1316
1007This section contains some random implementation notes. They do not 1317This section contains some random implementation notes. They do not
1008describe guaranteed behaviour, but merely behaviour as-is implemented 1318describe guaranteed behaviour, but merely behaviour as-is implemented
1024=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT 1334=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT
1025 1335
1026On perls that were built without 64 bit integer support (these are rare 1336On perls that were built without 64 bit integer support (these are rare
1027nowadays, even on 32 bit architectures, as all major Perl distributions 1337nowadays, even on 32 bit architectures, as all major Perl distributions
1028are built with 64 bit integer support), support for any kind of 64 bit 1338are built with 64 bit integer support), support for any kind of 64 bit
1029integer in CBOR is very limited - most likely, these 64 bit values will 1339value in CBOR is very limited - most likely, these 64 bit values will
1030be truncated, corrupted, or otherwise not decoded correctly. This also 1340be truncated, corrupted, or otherwise not decoded correctly. This also
1031includes string, array and map sizes that are stored as 64 bit integers. 1341includes string, float, array and map sizes that are stored as 64 bit
1342integers.
1032 1343
1033 1344
1034=head1 THREADS 1345=head1 THREADS
1035 1346
1036This module is I<not> guaranteed to be thread safe and there are no 1347This module is I<not> guaranteed to be thread safe and there are no
1049 1360
1050Please refrain from using rt.cpan.org or any other bug reporting 1361Please refrain from using rt.cpan.org or any other bug reporting
1051service. I put the contact address into my modules for a reason. 1362service. I put the contact address into my modules for a reason.
1052 1363
1053=cut 1364=cut
1365
1366# clumsy and slow hv_store-in-hash helper function
1367sub _hv_store {
1368 $_[0]{$_[1]} = $_[2];
1369}
1054 1370
1055our %FILTER = ( 1371our %FILTER = (
1056 0 => sub { # rfc4287 datetime, utf-8 1372 0 => sub { # rfc4287 datetime, utf-8
1057 require Time::Piece; 1373 require Time::Piece;
1058 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine" 1374 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine"
1059 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything 1375 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything
1060 # else either. Whats incredibe over standard strptime totally escapes me. 1376 # else either. Whats incredibe over standard strptime totally escapes me.
1061 # doesn't do fractional times, either. sigh. 1377 # doesn't do fractional times, either. sigh.
1062 # In fact, it's all a lie, it uses whatever strptime it wants, and of course, 1378 # In fact, it's all a lie, it uses whatever strptime it wants, and of course,
1063 # they are all incomptible. The openbsd one simply ignores %z (but according to the 1379 # they are all incompatible. The openbsd one simply ignores %z (but according to the
1064 # docs, it would be much more incredibly flexible indeed. If it worked, that is.). 1380 # docs, it would be much more incredibly flexible indeed. If it worked, that is.).
1065 scalar eval { 1381 scalar eval {
1066 my $s = $_[1]; 1382 my $s = $_[1];
1067 1383
1068 $s =~ s/Z$/+00:00/; 1384 $s =~ s/Z$/+00:00/;
1094 4 => sub { # decimal fraction, array 1410 4 => sub { # decimal fraction, array
1095 require Math::BigFloat; 1411 require Math::BigFloat;
1096 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0]) 1412 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1097 }, 1413 },
1098 1414
1415 264 => sub { # decimal fraction with arbitrary exponent
1416 require Math::BigFloat;
1417 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1418 },
1419
1099 5 => sub { # bigfloat, array 1420 5 => sub { # bigfloat, array
1100 require Math::BigFloat; 1421 require Math::BigFloat;
1101 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2) 1422 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1423 },
1424
1425 265 => sub { # bigfloat with arbitrary exponent
1426 require Math::BigFloat;
1427 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1428 },
1429
1430 30 => sub { # rational number
1431 require Math::BigRat;
1432 Math::BigRat->new ("$_[1][0]/$_[1][1]") # separate parameters only work in recent versons
1102 }, 1433 },
1103 1434
1104 21 => sub { pop }, # expected conversion to base64url encoding 1435 21 => sub { pop }, # expected conversion to base64url encoding
1105 22 => sub { pop }, # expected conversion to base64 encoding 1436 22 => sub { pop }, # expected conversion to base64 encoding
1106 23 => sub { pop }, # expected conversion to base16 encoding 1437 23 => sub { pop }, # expected conversion to base16 encoding
1116 # 34 # base64 rfc46484, utf-8 1447 # 34 # base64 rfc46484, utf-8
1117 # 35 # regex pcre/ecma262, utf-8 1448 # 35 # regex pcre/ecma262, utf-8
1118 # 36 # mime message rfc2045, utf-8 1449 # 36 # mime message rfc2045, utf-8
1119); 1450);
1120 1451
1121sub CBOR::XS::default_filter { 1452sub default_filter {
1122 &{ $FILTER{$_[0]} or return } 1453 &{ $FILTER{$_[0]} or return }
1454}
1455
1456our %SAFE_FILTER = map { $_ => $FILTER{$_} } 0, 1, 21, 22, 23, 32;
1457
1458sub safe_filter {
1459 &{ $SAFE_FILTER{$_[0]} or return }
1123} 1460}
1124 1461
1125sub URI::TO_CBOR { 1462sub URI::TO_CBOR {
1126 my $uri = $_[0]->as_string; 1463 my $uri = $_[0]->as_string;
1127 utf8::upgrade $uri; 1464 utf8::upgrade $uri;
1128 tag 32, $uri 1465 tag 32, $uri
1129} 1466}
1130 1467
1131sub Math::BigInt::TO_CBOR { 1468sub Math::BigInt::TO_CBOR {
1132 if ($_[0] >= -2147483648 && $_[0] <= 2147483647) { 1469 if (-2147483648 <= $_[0] && $_[0] <= 2147483647) {
1133 $_[0]->numify 1470 $_[0]->numify
1134 } else { 1471 } else {
1135 my $hex = substr $_[0]->as_hex, 2; 1472 my $hex = substr $_[0]->as_hex, 2;
1136 $hex = "0$hex" if 1 & length $hex; # sigh 1473 $hex = "0$hex" if 1 & length $hex; # sigh
1137 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex 1474 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex
1138 } 1475 }
1139} 1476}
1140 1477
1141sub Math::BigFloat::TO_CBOR { 1478sub Math::BigFloat::TO_CBOR {
1142 my ($m, $e) = $_[0]->parts; 1479 my ($m, $e) = $_[0]->parts;
1480
1481 -9223372036854775808 <= $e && $e <= 18446744073709551615
1143 tag 4, [$e->numify, $m] 1482 ? tag 4, [$e->numify, $m]
1483 : tag 264, [$e, $m]
1484}
1485
1486sub Math::BigRat::TO_CBOR {
1487 my ($n, $d) = $_[0]->parts;
1488
1489 # older versions of BigRat need *1, as they not always return numbers
1490
1491 $d*1 == 1
1492 ? $n*1
1493 : tag 30, [$n*1, $d*1]
1144} 1494}
1145 1495
1146sub Time::Piece::TO_CBOR { 1496sub Time::Piece::TO_CBOR {
1147 tag 1, 0 + $_[0]->epoch 1497 tag 1, 0 + $_[0]->epoch
1148} 1498}

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines