ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/CBOR-XS/XS.pm
(Generate patch)

Comparing CBOR-XS/XS.pm (file contents):
Revision 1.39 by root, Tue Dec 10 15:31:40 2013 UTC vs.
Revision 1.77 by root, Fri Dec 4 02:57:14 2020 UTC

38with the added ability of supporting serialisation of Perl objects. (JSON 38with the added ability of supporting serialisation of Perl objects. (JSON
39often compresses better than CBOR though, so if you plan to compress the 39often compresses better than CBOR though, so if you plan to compress the
40data later and speed is less important you might want to compare both 40data later and speed is less important you might want to compare both
41formats first). 41formats first).
42 42
43The primary goal of this module is to be I<correct> and the secondary goal
44is to be I<fast>. To reach the latter goal it was written in C.
45
43To give you a general idea about speed, with texts in the megabyte range, 46To give you a general idea about speed, with texts in the megabyte range,
44C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or 47C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or
45L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the 48L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the
46data, the worse L<Storable> performs in comparison. 49data, the worse L<Storable> performs in comparison.
47 50
52In addition to the core CBOR data format, this module implements a 55In addition to the core CBOR data format, this module implements a
53number of extensions, to support cyclic and shared data structures 56number of extensions, to support cyclic and shared data structures
54(see C<allow_sharing> and C<allow_cycles>), string deduplication (see 57(see C<allow_sharing> and C<allow_cycles>), string deduplication (see
55C<pack_strings>) and scalar references (always enabled). 58C<pack_strings>) and scalar references (always enabled).
56 59
57The primary goal of this module is to be I<correct> and the secondary goal
58is to be I<fast>. To reach the latter goal it was written in C.
59
60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and 60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and
61vice versa. 61vice versa.
62 62
63=cut 63=cut
64 64
65package CBOR::XS; 65package CBOR::XS;
66 66
67use common::sense; 67use common::sense;
68 68
69our $VERSION = 1.12; 69our $VERSION = 1.82;
70our @ISA = qw(Exporter); 70our @ISA = qw(Exporter);
71 71
72our @EXPORT = qw(encode_cbor decode_cbor); 72our @EXPORT = qw(encode_cbor decode_cbor);
73 73
74use Exporter; 74use Exporter;
112 112
113The mutators for flags all return the CBOR object again and thus calls can 113The mutators for flags all return the CBOR object again and thus calls can
114be chained: 114be chained:
115 115
116 my $cbor = CBOR::XS->new->encode ({a => [1,2]}); 116 my $cbor = CBOR::XS->new->encode ({a => [1,2]});
117
118=item $cbor = new_safe CBOR::XS
119
120Create a new, safe/secure CBOR::XS object. This is similar to C<new>,
121but configures the coder object to be safe to use with untrusted
122data. Currently, this is equivalent to:
123
124 my $cbor = CBOR::XS
125 ->new
126 ->forbid_objects
127 ->filter (\&CBOR::XS::safe_filter)
128 ->max_size (1e8);
129
130But is more future proof (it is better to crash because of a change than
131to be exploited in other ways).
132
133=cut
134
135sub new_safe {
136 CBOR::XS
137 ->new
138 ->forbid_objects
139 ->filter (\&CBOR::XS::safe_filter)
140 ->max_size (1e8)
141}
117 142
118=item $cbor = $cbor->max_depth ([$maximum_nesting_depth]) 143=item $cbor = $cbor->max_depth ([$maximum_nesting_depth])
119 144
120=item $max_depth = $cbor->get_max_depth 145=item $max_depth = $cbor->get_max_depth
121 146
137 162
138Note that nesting is implemented by recursion in C. The default value has 163Note that nesting is implemented by recursion in C. The default value has
139been chosen to be as large as typical operating systems allow without 164been chosen to be as large as typical operating systems allow without
140crashing. 165crashing.
141 166
142See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 167See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
143 168
144=item $cbor = $cbor->max_size ([$maximum_string_size]) 169=item $cbor = $cbor->max_size ([$maximum_string_size])
145 170
146=item $max_size = $cbor->get_max_size 171=item $max_size = $cbor->get_max_size
147 172
152effect on C<encode> (yet). 177effect on C<encode> (yet).
153 178
154If no argument is given, the limit check will be deactivated (same as when 179If no argument is given, the limit check will be deactivated (same as when
155C<0> is specified). 180C<0> is specified).
156 181
157See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 182See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
158 183
159=item $cbor = $cbor->allow_unknown ([$enable]) 184=item $cbor = $cbor->allow_unknown ([$enable])
160 185
161=item $enabled = $cbor->get_allow_unknown 186=item $enabled = $cbor->get_allow_unknown
162 187
180reference to the earlier value. 205reference to the earlier value.
181 206
182This means that such values will only be encoded once, and will not result 207This means that such values will only be encoded once, and will not result
183in a deep cloning of the value on decode, in decoders supporting the value 208in a deep cloning of the value on decode, in decoders supporting the value
184sharing extension. This also makes it possible to encode cyclic data 209sharing extension. This also makes it possible to encode cyclic data
185structures (which need C<allow_cycles> to ne enabled to be decoded by this 210structures (which need C<allow_cycles> to be enabled to be decoded by this
186module). 211module).
187 212
188It is recommended to leave it off unless you know your 213It is recommended to leave it off unless you know your
189communication partner supports the value sharing extensions to CBOR 214communication partner supports the value sharing extensions to CBOR
190(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the 215(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the
191resulting data structure might be unusable. 216resulting data structure might be unusable.
192 217
193Detecting shared values incurs a runtime overhead when values are encoded 218Detecting shared values incurs a runtime overhead when values are encoded
194that have a reference counter large than one, and might unnecessarily 219that have a reference counter large than one, and might unnecessarily
195increase the encoded size, as potentially shared values are encode as 220increase the encoded size, as potentially shared values are encoded as
196shareable whether or not they are actually shared. 221shareable whether or not they are actually shared.
197 222
198At the moment, only targets of references can be shared (e.g. scalars, 223At the moment, only targets of references can be shared (e.g. scalars,
199arrays or hashes pointed to by a reference). Weirder constructs, such as 224arrays or hashes pointed to by a reference). Weirder constructs, such as
200an array with multiple "copies" of the I<same> string, which are hard but 225an array with multiple "copies" of the I<same> string, which are hard but
218isn't prepared for this will not leak memory. 243isn't prepared for this will not leak memory.
219 244
220If C<$enable> is false (the default), then C<decode> will throw an error 245If C<$enable> is false (the default), then C<decode> will throw an error
221when it encounters a self-referential/cyclic data structure. 246when it encounters a self-referential/cyclic data structure.
222 247
248FUTURE DIRECTION: the motivation behind this option is to avoid I<real>
249cycles - future versions of this module might chose to decode cyclic data
250structures using weak references when this option is off, instead of
251throwing an error.
252
223This option does not affect C<encode> in any way - shared values and 253This option does not affect C<encode> in any way - shared values and
224references will always be decoded properly if present. 254references will always be encoded properly if present.
255
256=item $cbor = $cbor->forbid_objects ([$enable])
257
258=item $enabled = $cbor->get_forbid_objects
259
260Disables the use of the object serialiser protocol.
261
262If C<$enable> is true (or missing), then C<encode> will will throw an
263exception when it encounters perl objects that would be encoded using the
264perl-object tag (26). When C<decode> encounters such tags, it will fall
265back to the general filter/tagged logic as if this were an unknown tag (by
266default resulting in a C<CBOR::XC::Tagged> object).
267
268If C<$enable> is false (the default), then C<encode> will use the
269L<Types::Serialiser> object serialisation protocol to serialise objects
270into perl-object tags, and C<decode> will do the same to decode such tags.
271
272See L<SECURITY CONSIDERATIONS>, below, for more info on why forbidding this
273protocol can be useful.
225 274
226=item $cbor = $cbor->pack_strings ([$enable]) 275=item $cbor = $cbor->pack_strings ([$enable])
227 276
228=item $enabled = $cbor->get_pack_strings 277=item $enabled = $cbor->get_pack_strings
229 278
242the standard CBOR way. 291the standard CBOR way.
243 292
244This option does not affect C<decode> in any way - string references will 293This option does not affect C<decode> in any way - string references will
245always be decoded properly if present. 294always be decoded properly if present.
246 295
296=item $cbor = $cbor->text_keys ([$enable])
297
298=item $enabled = $cbor->get_text_keys
299
300If C<$enabled> is true (or missing), then C<encode> will encode all
301perl hash keys as CBOR text strings/UTF-8 string, upgrading them as needed.
302
303If C<$enable> is false (the default), then C<encode> will encode hash keys
304normally - upgraded perl strings (strings internally encoded as UTF-8) as
305CBOR text strings, and downgraded perl strings as CBOR byte strings.
306
307This option does not affect C<decode> in any way.
308
309This option is useful for interoperability with CBOR decoders that don't
310treat byte strings as a form of text. It is especially useful as Perl
311gives very little control over hash keys.
312
313Enabling this option can be slow, as all downgraded hash keys that are
314encoded need to be scanned and converted to UTF-8.
315
316=item $cbor = $cbor->text_strings ([$enable])
317
318=item $enabled = $cbor->get_text_strings
319
320This option works similar to C<text_keys>, above, but works on all strings
321(including hash keys), so C<text_keys> has no further effect after
322enabling C<text_strings>.
323
324If C<$enabled> is true (or missing), then C<encode> will encode all perl
325strings as CBOR text strings/UTF-8 strings, upgrading them as needed.
326
327If C<$enable> is false (the default), then C<encode> will encode strings
328normally (but see C<text_keys>) - upgraded perl strings (strings
329internally encoded as UTF-8) as CBOR text strings, and downgraded perl
330strings as CBOR byte strings.
331
332This option does not affect C<decode> in any way.
333
334This option has similar advantages and disadvantages as C<text_keys>. In
335addition, this option effectively removes the ability to automatically
336encode byte strings, which might break some C<FREEZE> and C<TO_CBOR>
337methods that rely on this.
338
339A workaround is to use explicit type casts, which are unaffected by this option.
340
247=item $cbor = $cbor->validate_utf8 ([$enable]) 341=item $cbor = $cbor->validate_utf8 ([$enable])
248 342
249=item $enabled = $cbor->get_validate_utf8 343=item $enabled = $cbor->get_validate_utf8
250 344
251If C<$enable> is true (or missing), then C<decode> will validate that 345If C<$enable> is true (or missing), then C<decode> will validate that
256The concept of "valid UTF-8" used is perl's concept, which is a superset 350The concept of "valid UTF-8" used is perl's concept, which is a superset
257of the official UTF-8. 351of the official UTF-8.
258 352
259If C<$enable> is false (the default), then C<decode> will blindly accept 353If C<$enable> is false (the default), then C<decode> will blindly accept
260UTF-8 data, marking them as valid UTF-8 in the resulting data structure 354UTF-8 data, marking them as valid UTF-8 in the resulting data structure
261regardless of whether thats true or not. 355regardless of whether that's true or not.
262 356
263Perl isn't too happy about corrupted UTF-8 in strings, but should 357Perl isn't too happy about corrupted UTF-8 in strings, but should
264generally not crash or do similarly evil things. Extensions might be not 358generally not crash or do similarly evil things. Extensions might be not
265so forgiving, so it's recommended to turn on this setting if you receive 359so forgiving, so it's recommended to turn on this setting if you receive
266untrusted CBOR. 360untrusted CBOR.
288replace the tagged value in the decoded data structure, or no values, 382replace the tagged value in the decoded data structure, or no values,
289which will result in default handling, which currently means the decoder 383which will result in default handling, which currently means the decoder
290creates a C<CBOR::XS::Tagged> object to hold the tag and the value. 384creates a C<CBOR::XS::Tagged> object to hold the tag and the value.
291 385
292When the filter is cleared (the default state), the default filter 386When the filter is cleared (the default state), the default filter
293function, C<CBOR::XS::default_filter>, is used. This function simply looks 387function, C<CBOR::XS::default_filter>, is used. This function simply
294up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists it must be 388looks up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists
295a code reference that is called with tag and value, and is responsible for 389it must be a code reference that is called with tag and value, and is
296decoding the value. If no entry exists, it returns no values. 390responsible for decoding the value. If no entry exists, it returns no
391values. C<CBOR::XS> provides a number of default filter functions already,
392the the C<%CBOR::XS::FILTER> hash can be freely extended with more.
393
394C<CBOR::XS> additionally provides an alternative filter function that is
395supposed to be safe to use with untrusted data (which the default filter
396might not), called C<CBOR::XS::safe_filter>, which works the same as
397the C<default_filter> but uses the C<%CBOR::XS::SAFE_FILTER> variable
398instead. It is prepopulated with the tag decoding functions that are
399deemed safe (basically the same as C<%CBOR::XS::FILTER> without all
400the bignum tags), and can be extended by user code as wlel, although,
401obviously, one should be very careful about adding decoding functions
402here, since the expectation is that they are safe to use on untrusted
403data, after all.
297 404
298Example: decode all tags not handled internally into C<CBOR::XS::Tagged> 405Example: decode all tags not handled internally into C<CBOR::XS::Tagged>
299objects, with no other special handling (useful when working with 406objects, with no other special handling (useful when working with
300potentially "unsafe" CBOR data). 407potentially "unsafe" CBOR data).
301 408
308 my ($tag, $value); 415 my ($tag, $value);
309 416
310 "tag 1347375694 value $value" 417 "tag 1347375694 value $value"
311 }; 418 };
312 419
420Example: provide your own filter function that looks up tags in your own
421hash:
422
423 my %my_filter = (
424 998347484 => sub {
425 my ($tag, $value);
426
427 "tag 998347484 value $value"
428 };
429 );
430
431 my $coder = CBOR::XS->new->filter (sub {
432 &{ $my_filter{$_[0]} or return }
433 });
434
435
436Example: use the safe filter function (see L<SECURITY CONSIDERATIONS> for
437more considerations on security).
438
439 CBOR::XS->new->filter (\&CBOR::XS::safe_filter)->decode ($cbor_data);
440
313=item $cbor_data = $cbor->encode ($perl_scalar) 441=item $cbor_data = $cbor->encode ($perl_scalar)
314 442
315Converts the given Perl data structure (a scalar value) to its CBOR 443Converts the given Perl data structure (a scalar value) to its CBOR
316representation. 444representation.
317 445
326when there is trailing garbage after the CBOR string, it will silently 454when there is trailing garbage after the CBOR string, it will silently
327stop parsing there and return the number of characters consumed so far. 455stop parsing there and return the number of characters consumed so far.
328 456
329This is useful if your CBOR texts are not delimited by an outer protocol 457This is useful if your CBOR texts are not delimited by an outer protocol
330and you need to know where the first CBOR string ends amd the next one 458and you need to know where the first CBOR string ends amd the next one
331starts. 459starts - CBOR strings are self-delimited, so it is possible to concatenate
460CBOR strings without any delimiters or size fields and recover their data.
332 461
333 CBOR::XS->new->decode_prefix ("......") 462 CBOR::XS->new->decode_prefix ("......")
334 => ("...", 3) 463 => ("...", 3)
335 464
336=back 465=back
391 520
392Resets the incremental decoder. This throws away any saved state, so that 521Resets the incremental decoder. This throws away any saved state, so that
393subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse 522subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse
394a new CBOR value from the beginning of the C<$buffer> again. 523a new CBOR value from the beginning of the C<$buffer> again.
395 524
396This method can be caled at any time, but it I<must> be called if you want 525This method can be called at any time, but it I<must> be called if you want
397to change your C<$buffer> or there was a decoding error and you want to 526to change your C<$buffer> or there was a decoding error and you want to
398reuse the C<$cbor> object for future incremental parsings. 527reuse the C<$cbor> object for future incremental parsings.
399 528
400=back 529=back
401 530
476 605
477=item hash references 606=item hash references
478 607
479Perl hash references become CBOR maps. As there is no inherent ordering in 608Perl hash references become CBOR maps. As there is no inherent ordering in
480hash keys (or CBOR maps), they will usually be encoded in a pseudo-random 609hash keys (or CBOR maps), they will usually be encoded in a pseudo-random
481order. This order can be different each time a hahs is encoded. 610order. This order can be different each time a hash is encoded.
482 611
483Currently, tied hashes will use the indefinite-length format, while normal 612Currently, tied hashes will use the indefinite-length format, while normal
484hashes will use the fixed-length format. 613hashes will use the fixed-length format.
485 614
486=item array references 615=item array references
539 my $x = 3.1; # some variable containing a number 668 my $x = 3.1; # some variable containing a number
540 "$x"; # stringified 669 "$x"; # stringified
541 $x .= ""; # another, more awkward way to stringify 670 $x .= ""; # another, more awkward way to stringify
542 print $x; # perl does it for you, too, quite often 671 print $x; # perl does it for you, too, quite often
543 672
544You can force whether a string ie encoded as byte or text string by using 673You can force whether a string is encoded as byte or text string by using
545C<utf8::upgrade> and C<utf8::downgrade>): 674C<utf8::upgrade> and C<utf8::downgrade> (if C<text_strings> is disabled).
546 675
547 utf8::upgrade $x; # encode $x as text string 676 utf8::upgrade $x; # encode $x as text string
548 utf8::downgrade $x; # encode $x as byte string 677 utf8::downgrade $x; # encode $x as byte string
549 678
679More options are available, see L<TYPE CASTS>, below, and the C<text_keys>
680and C<text_strings> options.
681
550Perl doesn't define what operations up- and downgrade strings, so if the 682Perl doesn't define what operations up- and downgrade strings, so if the
551difference between byte and text is important, you should up- or downgrade 683difference between byte and text is important, you should up- or downgrade
552your string as late as possible before encoding. 684your string as late as possible before encoding. You can also force the
685use of CBOR text strings by using C<text_keys> or C<text_strings>.
553 686
554You can force the type to be a CBOR number by numifying it: 687You can force the type to be a CBOR number by numifying it:
555 688
556 my $x = "3"; # some variable containing a string 689 my $x = "3"; # some variable containing a string
557 $x += 0; # numify it, ensuring it will be dumped as a number 690 $x += 0; # numify it, ensuring it will be dumped as a number
568represent numerical values are supported, but might suffer loss of 701represent numerical values are supported, but might suffer loss of
569precision. 702precision.
570 703
571=back 704=back
572 705
706=head2 TYPE CASTS
707
708B<EXPERIMENTAL>: As an experimental extension, C<CBOR::XS> allows you to
709force specific cbor types to be used when encoding. That allows you to
710encode types not normally accessible (e.g. half floats) as well as force
711string types even when C<text_strings> is in effect.
712
713Type forcing is done by calling a special "cast" function which keeps a
714copy of the value and returns a new value that can be handed over to any
715CBOR encoder function.
716
717The following casts are currently available (all of which are unary operators):
718
719=over
720
721=item CBOR::XS::as_int $value
722
723Forces the value to be encoded as some form of (basic, not bignum) integer
724type.
725
726=item CBOR::XS::as_text $value
727
728Forces the value to be encoded as (UTF-8) text values.
729
730=item CBOR::XS::as_bytes $value
731
732Forces the value to be encoded as a (binary) string value.
733
734Example: encode a perl string as binary even though C<text_strings> is in
735effect.
736
737 CBOR::XS->new->text_strings->encode ([4, "text", CBOR::XS::bytes "bytevalue"]);
738
739=item CBOR::XS::as_bool $value
740
741Converts a Perl boolean (which can be any kind of scalar) into a CBOR
742boolean. Strictly the same, but shorter to write, than:
743
744 $value ? Types::Serialiser::true : Types::Serialiser::false
745
746=item CBOR::XS::as_float16 $value
747
748Forces half-float (IEEE 754 binary16) encoding of the given value.
749
750=item CBOR::XS::as_float32 $value
751
752Forces single-float (IEEE 754 binary32) encoding of the given value.
753
754=item CBOR::XS::as_float64 $value
755
756Forces double-float (IEEE 754 binary64) encoding of the given value.
757
758=item CBOR::XS::as_cbor $cbor_text
759
760Not a type cast per-se, this type cast forces the argument to eb encoded
761as-is. This can be used to embed pre-encoded CBOR data.
762
763Note that no checking on the validity of the C<$cbor_text> is done - it's
764the callers responsibility to correctly encode values.
765
766=item CBOR::XS::as_map [key => value...]
767
768Treat the array reference as key value pairs and output a CBOR map. This
769allows you to generate CBOR maps with arbitrary key types (or, if you
770don't care about semantics, duplicate keys or prairs in a custom order),
771which is otherwise hard to do with Perl.
772
773The single argument must be an array reference with an even number of
774elements.
775
776Example: encode a CBOR map with a string and an integer as keys.
777
778 encode_cbor CBOR::XS::as_map [string => "value", 5 => "value"]
779
780=back
781
782=cut
783
784sub CBOR::XS::as_cbor ($) { bless [$_[0], 0, undef], CBOR::XS::Tagged:: }
785sub CBOR::XS::as_int ($) { bless [$_[0], 1, undef], CBOR::XS::Tagged:: }
786sub CBOR::XS::as_bytes ($) { bless [$_[0], 2, undef], CBOR::XS::Tagged:: }
787sub CBOR::XS::as_text ($) { bless [$_[0], 3, undef], CBOR::XS::Tagged:: }
788sub CBOR::XS::as_float16 ($) { bless [$_[0], 4, undef], CBOR::XS::Tagged:: }
789sub CBOR::XS::as_float32 ($) { bless [$_[0], 5, undef], CBOR::XS::Tagged:: }
790sub CBOR::XS::as_float64 ($) { bless [$_[0], 6, undef], CBOR::XS::Tagged:: }
791
792sub CBOR::XS::as_bool ($) { $_[0] ? $Types::Serialiser::true : $Types::Serialiser::false }
793
794sub CBOR::XS::as_map ($) {
795 ARRAY:: eq ref $_[0]
796 and $#{ $_[0] } & 1
797 or do { require Carp; Carp::croak ("CBOR::XS::as_map only acepts array references with an even number of elements, found ") };
798
799 bless [$_[0], 7, undef], CBOR::XS::Tagged::
800}
801
573=head2 OBJECT SERIALISATION 802=head2 OBJECT SERIALISATION
574 803
575This module implements both a CBOR-specific and the generic 804This module implements both a CBOR-specific and the generic
576L<Types::Serialier> object serialisation protocol. The following 805L<Types::Serialier> object serialisation protocol. The following
577subsections explain both methods. 806subsections explain both methods.
658 "$self" # encode url string 887 "$self" # encode url string
659 } 888 }
660 889
661 sub URI::THAW { 890 sub URI::THAW {
662 my ($class, $serialiser, $uri) = @_; 891 my ($class, $serialiser, $uri) = @_;
663
664 $class->new ($uri) 892 $class->new ($uri)
665 } 893 }
666 894
667Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For 895Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For
668example, a C<FREEZE> method that returns "type", "id" and "variant" values 896example, a C<FREEZE> method that returns "type", "id" and "variant" values
799additional tags (such as base64url). 1027additional tags (such as base64url).
800 1028
801=head2 ENFORCED TAGS 1029=head2 ENFORCED TAGS
802 1030
803These tags are always handled when decoding, and their handling cannot be 1031These tags are always handled when decoding, and their handling cannot be
804overriden by the user. 1032overridden by the user.
805 1033
806=over 4 1034=over 4
807 1035
808=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>) 1036=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>)
809 1037
810These tags are automatically created (and decoded) for serialisable 1038These tags are automatically created (and decoded) for serialisable
811objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object 1039objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object
812serialisation protocol). See L<OBJECT SERIALISATION> for details. 1040serialisation protocol). See L<OBJECT SERIALISATION> for details.
813 1041
814=item 28, 29 (shareable, sharedref, L <http://cbor.schmorp.de/value-sharing>) 1042=item 28, 29 (shareable, sharedref, L<http://cbor.schmorp.de/value-sharing>)
815 1043
816These tags are automatically decoded when encountered (and they do not 1044These tags are automatically decoded when encountered (and they do not
817result in a cyclic data structure, see C<allow_cycles>), resulting in 1045result in a cyclic data structure, see C<allow_cycles>), resulting in
818shared values in the decoded object. They are only encoded, however, when 1046shared values in the decoded object. They are only encoded, however, when
819C<allow_sharing> is enabled. 1047C<allow_sharing> is enabled.
829will be shared, others will not. While non-reference shared values can be 1057will be shared, others will not. While non-reference shared values can be
830generated in Perl with some effort, they were considered too unimportant 1058generated in Perl with some effort, they were considered too unimportant
831to be supported in the encoder. The decoder, however, will decode these 1059to be supported in the encoder. The decoder, however, will decode these
832values as shared values. 1060values as shared values.
833 1061
834=item 256, 25 (stringref-namespace, stringref, L <http://cbor.schmorp.de/stringref>) 1062=item 256, 25 (stringref-namespace, stringref, L<http://cbor.schmorp.de/stringref>)
835 1063
836These tags are automatically decoded when encountered. They are only 1064These tags are automatically decoded when encountered. They are only
837encoded, however, when C<pack_strings> is enabled. 1065encoded, however, when C<pack_strings> is enabled.
838 1066
839=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>) 1067=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>)
840 1068
841This tag is automatically generated when a reference are encountered (with 1069This tag is automatically generated when a reference are encountered (with
842the exception of hash and array refernces). It is converted to a reference 1070the exception of hash and array references). It is converted to a reference
843when decoding. 1071when decoding.
844 1072
845=item 55799 (self-describe CBOR, RFC 7049) 1073=item 55799 (self-describe CBOR, RFC 7049)
846 1074
847This value is not generated on encoding (unless explicitly requested by 1075This value is not generated on encoding (unless explicitly requested by
850=back 1078=back
851 1079
852=head2 NON-ENFORCED TAGS 1080=head2 NON-ENFORCED TAGS
853 1081
854These tags have default filters provided when decoding. Their handling can 1082These tags have default filters provided when decoding. Their handling can
855be overriden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by 1083be overridden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by
856providing a custom C<filter> callback when decoding. 1084providing a custom C<filter> callback when decoding.
857 1085
858When they result in decoding into a specific Perl class, the module 1086When they result in decoding into a specific Perl class, the module
859usually provides a corresponding C<TO_CBOR> method as well. 1087usually provides a corresponding C<TO_CBOR> method as well.
860 1088
878 1106
879These tags are decoded into L<Math::BigInt> objects. The corresponding 1107These tags are decoded into L<Math::BigInt> objects. The corresponding
880C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR 1108C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR
881integers, and others into positive/negative CBOR bignums. 1109integers, and others into positive/negative CBOR bignums.
882 1110
883=item 4, 5 (decimal fraction/bigfloat) 1111=item 4, 5, 264, 265 (decimal fraction/bigfloat)
884 1112
885Both decimal fractions and bigfloats are decoded into L<Math::BigFloat> 1113Both decimal fractions and bigfloats are decoded into L<Math::BigFloat>
886objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always> 1114objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always>
887encodes into a decimal fraction. 1115encodes into a decimal fraction (either tag 4 or 264).
888 1116
889CBOR cannot represent bigfloats with I<very> large exponents - conversion 1117NaN and infinities are not encoded properly, as they cannot be represented
890of such big float objects is undefined. 1118in CBOR.
891 1119
892Also, NaN and infinities are not encoded properly. 1120See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
1121
1122=item 30 (rational numbers)
1123
1124These tags are decoded into L<Math::BigRat> objects. The corresponding
1125C<Math::BigRat::TO_CBOR> method encodes rational numbers with denominator
1126C<1> via their numerator only, i.e., they become normal integers or
1127C<bignums>.
1128
1129See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
893 1130
894=item 21, 22, 23 (expected later JSON conversion) 1131=item 21, 22, 23 (expected later JSON conversion)
895 1132
896CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these 1133CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these
897tags. 1134tags.
902C<URI::TO_CBOR> method again results in a CBOR URI value. 1139C<URI::TO_CBOR> method again results in a CBOR URI value.
903 1140
904=back 1141=back
905 1142
906=cut 1143=cut
907
908our %FILTER = (
909 # 0 # rfc4287 datetime, utf-8
910 # 1 # unix timestamp, any
911
912 2 => sub { # pos bigint
913 require Math::BigInt;
914 Math::BigInt->new ("0x" . unpack "H*", pop)
915 },
916
917 3 => sub { # neg bigint
918 require Math::BigInt;
919 -Math::BigInt->new ("0x" . unpack "H*", pop)
920 },
921
922 4 => sub { # decimal fraction, array
923 require Math::BigFloat;
924 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
925 },
926
927 5 => sub { # bigfloat, array
928 require Math::BigFloat;
929 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2)
930 },
931
932 21 => sub { pop }, # expected conversion to base64url encoding
933 22 => sub { pop }, # expected conversion to base64 encoding
934 23 => sub { pop }, # expected conversion to base16 encoding
935
936 # 24 # embedded cbor, byte string
937
938 32 => sub {
939 require URI;
940 URI->new (pop)
941 },
942
943 # 33 # base64url rfc4648, utf-8
944 # 34 # base64 rfc46484, utf-8
945 # 35 # regex pcre/ecma262, utf-8
946 # 36 # mime message rfc2045, utf-8
947);
948
949 1144
950=head1 CBOR and JSON 1145=head1 CBOR and JSON
951 1146
952CBOR is supposed to implement a superset of the JSON data model, and is, 1147CBOR is supposed to implement a superset of the JSON data model, and is,
953with some coercion, able to represent all JSON texts (something that other 1148with some coercion, able to represent all JSON texts (something that other
962CBOR intact. 1157CBOR intact.
963 1158
964 1159
965=head1 SECURITY CONSIDERATIONS 1160=head1 SECURITY CONSIDERATIONS
966 1161
967When you are using CBOR in a protocol, talking to untrusted potentially 1162Tl;dr... if you want to decode or encode CBOR from untrusted sources, you
968hostile creatures requires relatively few measures. 1163should start with a coder object created via C<new_safe> (which implements
1164the mitigations explained below):
969 1165
1166 my $coder = CBOR::XS->new_safe;
1167
1168 my $data = $coder->decode ($cbor_text);
1169 my $cbor = $coder->encode ($data);
1170
1171Longer version: When you are using CBOR in a protocol, talking to
1172untrusted potentially hostile creatures requires some thought:
1173
1174=over 4
1175
1176=item Security of the CBOR decoder itself
1177
970First of all, your CBOR decoder should be secure, that is, should not have 1178First and foremost, your CBOR decoder should be secure, that is, should
1179not have any buffer overflows or similar bugs that could potentially be
971any buffer overflows. Obviously, this module should ensure that and I am 1180exploited. Obviously, this module should ensure that and I am trying hard
972trying hard on making that true, but you never know. 1181on making that true, but you never know.
973 1182
1183=item CBOR::XS can invoke almost arbitrary callbacks during decoding
1184
1185CBOR::XS supports object serialisation - decoding CBOR can cause calls
1186to I<any> C<THAW> method in I<any> package that exists in your process
1187(that is, CBOR::XS will not try to load modules, but any existing C<THAW>
1188method or function can be called, so they all have to be secure).
1189
1190Less obviously, it will also invoke C<TO_CBOR> and C<FREEZE> methods -
1191even if all your C<THAW> methods are secure, encoding data structures from
1192untrusted sources can invoke those and trigger bugs in those.
1193
1194So, if you are not sure about the security of all the modules you
1195have loaded (you shouldn't), you should disable this part using
1196C<forbid_objects> or using C<new_safe>.
1197
1198=item CBOR can be extended with tags that call library code
1199
1200CBOR can be extended with tags, and C<CBOR::XS> has a registry of
1201conversion functions for many existing tags that can be extended via
1202third-party modules (see the C<filter> method).
1203
1204If you don't trust these, you should configure the "safe" filter function,
1205C<CBOR::XS::safe_filter> (C<new_safe> does this), which by default only
1206includes conversion functions that are considered "safe" by the author
1207(but again, they can be extended by third party modules).
1208
1209Depending on your level of paranoia, you can use the "safe" filter:
1210
1211 $cbor->filter (\&CBOR::XS::safe_filter);
1212
1213... your own filter...
1214
1215 $cbor->filter (sub { ... do your stuffs here ... });
1216
1217... or even no filter at all, disabling all tag decoding:
1218
1219 $cbor->filter (sub { });
1220
1221This is never a problem for encoding, as the tag mechanism only exists in
1222CBOR texts.
1223
1224=item Resource-starving attacks: object memory usage
1225
974Second, you need to avoid resource-starving attacks. That means you should 1226You need to avoid resource-starving attacks. That means you should limit
975limit the size of CBOR data you accept, or make sure then when your 1227the size of CBOR data you accept, or make sure then when your resources
976resources run out, that's just fine (e.g. by using a separate process that 1228run out, that's just fine (e.g. by using a separate process that can
977can crash safely). The size of a CBOR string in octets is usually a good 1229crash safely). The size of a CBOR string in octets is usually a good
978indication of the size of the resources required to decode it into a Perl 1230indication of the size of the resources required to decode it into a Perl
979structure. While CBOR::XS can check the size of the CBOR text, it might be 1231structure. While CBOR::XS can check the size of the CBOR text (using
980too late when you already have it in memory, so you might want to check 1232C<max_size> - done by C<new_safe>), it might be too late when you already
981the size before you accept the string. 1233have it in memory, so you might want to check the size before you accept
1234the string.
982 1235
1236As for encoding, it is possible to construct data structures that are
1237relatively small but result in large CBOR texts (for example by having an
1238array full of references to the same big data structure, which will all be
1239deep-cloned during encoding by default). This is rarely an actual issue
1240(and the worst case is still just running out of memory), but you can
1241reduce this risk by using C<allow_sharing>.
1242
1243=item Resource-starving attacks: stack overflows
1244
983Third, CBOR::XS recurses using the C stack when decoding objects and 1245CBOR::XS recurses using the C stack when decoding objects and arrays. The
984arrays. The C stack is a limited resource: for instance, on my amd64 1246C stack is a limited resource: for instance, on my amd64 machine with 8MB
985machine with 8MB of stack size I can decode around 180k nested arrays but 1247of stack size I can decode around 180k nested arrays but only 14k nested
986only 14k nested CBOR objects (due to perl itself recursing deeply on croak 1248CBOR objects (due to perl itself recursing deeply on croak to free the
987to free the temporary). If that is exceeded, the program crashes. To be 1249temporary). If that is exceeded, the program crashes. To be conservative,
988conservative, the default nesting limit is set to 512. If your process 1250the default nesting limit is set to 512. If your process has a smaller
989has a smaller stack, you should adjust this setting accordingly with the 1251stack, you should adjust this setting accordingly with the C<max_depth>
990C<max_depth> method. 1252method.
1253
1254=item Resource-starving attacks: CPU en-/decoding complexity
1255
1256CBOR::XS will use the L<Math::BigInt>, L<Math::BigFloat> and
1257L<Math::BigRat> libraries to represent encode/decode bignums. These can be
1258very slow (as in, centuries of CPU time) and can even crash your program
1259(and are generally not very trustworthy). See the next section on bignum
1260security for details.
1261
1262=item Data breaches: leaking information in error messages
1263
1264CBOR::XS might leak contents of your Perl data structures in its error
1265messages, so when you serialise sensitive information you might want to
1266make sure that exceptions thrown by CBOR::XS will not end up in front of
1267untrusted eyes.
1268
1269=item Something else...
991 1270
992Something else could bomb you, too, that I forgot to think of. In that 1271Something else could bomb you, too, that I forgot to think of. In that
993case, you get to keep the pieces. I am always open for hints, though... 1272case, you get to keep the pieces. I am always open for hints, though...
994 1273
995Also keep in mind that CBOR::XS might leak contents of your Perl data 1274=back
996structures in its error messages, so when you serialise sensitive 1275
997information you might want to make sure that exceptions thrown by CBOR::XS 1276
998will not end up in front of untrusted eyes. 1277=head1 BIGNUM SECURITY CONSIDERATIONS
1278
1279CBOR::XS provides a C<TO_CBOR> method for both L<Math::BigInt> and
1280L<Math::BigFloat> that tries to encode the number in the simplest possible
1281way, that is, either a CBOR integer, a CBOR bigint/decimal fraction (tag
12824) or an arbitrary-exponent decimal fraction (tag 264). Rational numbers
1283(L<Math::BigRat>, tag 30) can also contain bignums as members.
1284
1285CBOR::XS will also understand base-2 bigfloat or arbitrary-exponent
1286bigfloats (tags 5 and 265), but it will never generate these on its own.
1287
1288Using the built-in L<Math::BigInt::Calc> support, encoding and decoding
1289decimal fractions is generally fast. Decoding bigints can be slow for very
1290big numbers (tens of thousands of digits, something that could potentially
1291be caught by limiting the size of CBOR texts), and decoding bigfloats or
1292arbitrary-exponent bigfloats can be I<extremely> slow (minutes, decades)
1293for large exponents (roughly 40 bit and longer).
1294
1295Additionally, L<Math::BigInt> can take advantage of other bignum
1296libraries, such as L<Math::GMP>, which cannot handle big floats with large
1297exponents, and might simply abort or crash your program, due to their code
1298quality.
1299
1300This can be a concern if you want to parse untrusted CBOR. If it is, you
1301might want to disable decoding of tag 2 (bigint) and 3 (negative bigint)
1302types. You should also disable types 5 and 265, as these can be slow even
1303without bigints.
1304
1305Disabling bigints will also partially or fully disable types that rely on
1306them, e.g. rational numbers that use bignums.
1307
999 1308
1000=head1 CBOR IMPLEMENTATION NOTES 1309=head1 CBOR IMPLEMENTATION NOTES
1001 1310
1002This section contains some random implementation notes. They do not 1311This section contains some random implementation notes. They do not
1003describe guaranteed behaviour, but merely behaviour as-is implemented 1312describe guaranteed behaviour, but merely behaviour as-is implemented
1017 1326
1018 1327
1019=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT 1328=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT
1020 1329
1021On perls that were built without 64 bit integer support (these are rare 1330On perls that were built without 64 bit integer support (these are rare
1022nowadays, even on 32 bit architectures), support for any kind of 64 bit 1331nowadays, even on 32 bit architectures, as all major Perl distributions
1332are built with 64 bit integer support), support for any kind of 64 bit
1023integer in CBOR is very limited - most likely, these 64 bit values will 1333value in CBOR is very limited - most likely, these 64 bit values will
1024be truncated, corrupted, or otherwise not decoded correctly. This also 1334be truncated, corrupted, or otherwise not decoded correctly. This also
1025includes string, array and map sizes that are stored as 64 bit integers. 1335includes string, float, array and map sizes that are stored as 64 bit
1336integers.
1026 1337
1027 1338
1028=head1 THREADS 1339=head1 THREADS
1029 1340
1030This module is I<not> guaranteed to be thread safe and there are no 1341This module is I<not> guaranteed to be thread safe and there are no
1043 1354
1044Please refrain from using rt.cpan.org or any other bug reporting 1355Please refrain from using rt.cpan.org or any other bug reporting
1045service. I put the contact address into my modules for a reason. 1356service. I put the contact address into my modules for a reason.
1046 1357
1047=cut 1358=cut
1359
1360# clumsy and slow hv_store-in-hash helper function
1361sub _hv_store {
1362 $_[0]{$_[1]} = $_[2];
1363}
1048 1364
1049our %FILTER = ( 1365our %FILTER = (
1050 0 => sub { # rfc4287 datetime, utf-8 1366 0 => sub { # rfc4287 datetime, utf-8
1051 require Time::Piece; 1367 require Time::Piece;
1052 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine" 1368 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine"
1053 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything 1369 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything
1054 # else either. Whats incredibe over standard strptime totally escapes me. 1370 # else either. Whats incredibe over standard strptime totally escapes me.
1055 # doesn't do fractional times, either. sigh. 1371 # doesn't do fractional times, either. sigh.
1056 # In fact, it's all a lie, it uses whatever strptime it wants, and of course, 1372 # In fact, it's all a lie, it uses whatever strptime it wants, and of course,
1057 # they are all incomptible. The openbsd one simply ignores %z (but according to the 1373 # they are all incompatible. The openbsd one simply ignores %z (but according to the
1058 # docs, it would be much more incredibly flexible indeed. If it worked, that is.). 1374 # docs, it would be much more incredibly flexible indeed. If it worked, that is.).
1059 scalar eval { 1375 scalar eval {
1060 my $s = $_[1]; 1376 my $s = $_[1];
1061 1377
1062 $s =~ s/Z$/+00:00/; 1378 $s =~ s/Z$/+00:00/;
1088 4 => sub { # decimal fraction, array 1404 4 => sub { # decimal fraction, array
1089 require Math::BigFloat; 1405 require Math::BigFloat;
1090 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0]) 1406 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1091 }, 1407 },
1092 1408
1409 264 => sub { # decimal fraction with arbitrary exponent
1410 require Math::BigFloat;
1411 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1412 },
1413
1093 5 => sub { # bigfloat, array 1414 5 => sub { # bigfloat, array
1094 require Math::BigFloat; 1415 require Math::BigFloat;
1095 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2) 1416 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1417 },
1418
1419 265 => sub { # bigfloat with arbitrary exponent
1420 require Math::BigFloat;
1421 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1422 },
1423
1424 30 => sub { # rational number
1425 require Math::BigRat;
1426 Math::BigRat->new ("$_[1][0]/$_[1][1]") # separate parameters only work in recent versons
1096 }, 1427 },
1097 1428
1098 21 => sub { pop }, # expected conversion to base64url encoding 1429 21 => sub { pop }, # expected conversion to base64url encoding
1099 22 => sub { pop }, # expected conversion to base64 encoding 1430 22 => sub { pop }, # expected conversion to base64 encoding
1100 23 => sub { pop }, # expected conversion to base16 encoding 1431 23 => sub { pop }, # expected conversion to base16 encoding
1110 # 34 # base64 rfc46484, utf-8 1441 # 34 # base64 rfc46484, utf-8
1111 # 35 # regex pcre/ecma262, utf-8 1442 # 35 # regex pcre/ecma262, utf-8
1112 # 36 # mime message rfc2045, utf-8 1443 # 36 # mime message rfc2045, utf-8
1113); 1444);
1114 1445
1115sub CBOR::XS::default_filter { 1446sub default_filter {
1116 &{ $FILTER{$_[0]} or return } 1447 &{ $FILTER{$_[0]} or return }
1448}
1449
1450our %SAFE_FILTER = map { $_ => $FILTER{$_} } 0, 1, 21, 22, 23, 32;
1451
1452sub safe_filter {
1453 &{ $SAFE_FILTER{$_[0]} or return }
1117} 1454}
1118 1455
1119sub URI::TO_CBOR { 1456sub URI::TO_CBOR {
1120 my $uri = $_[0]->as_string; 1457 my $uri = $_[0]->as_string;
1121 utf8::upgrade $uri; 1458 utf8::upgrade $uri;
1122 tag 32, $uri 1459 tag 32, $uri
1123} 1460}
1124 1461
1125sub Math::BigInt::TO_CBOR { 1462sub Math::BigInt::TO_CBOR {
1126 if ($_[0] >= -2147483648 && $_[0] <= 2147483647) { 1463 if (-2147483648 <= $_[0] && $_[0] <= 2147483647) {
1127 $_[0]->numify 1464 $_[0]->numify
1128 } else { 1465 } else {
1129 my $hex = substr $_[0]->as_hex, 2; 1466 my $hex = substr $_[0]->as_hex, 2;
1130 $hex = "0$hex" if 1 & length $hex; # sigh 1467 $hex = "0$hex" if 1 & length $hex; # sigh
1131 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex 1468 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex
1132 } 1469 }
1133} 1470}
1134 1471
1135sub Math::BigFloat::TO_CBOR { 1472sub Math::BigFloat::TO_CBOR {
1136 my ($m, $e) = $_[0]->parts; 1473 my ($m, $e) = $_[0]->parts;
1474
1475 -9223372036854775808 <= $e && $e <= 18446744073709551615
1137 tag 4, [$e->numify, $m] 1476 ? tag 4, [$e->numify, $m]
1477 : tag 264, [$e, $m]
1138} 1478}
1139 1479
1480sub Math::BigRat::TO_CBOR {
1481 my ($n, $d) = $_[0]->parts;
1482
1483 # older versions of BigRat need *1, as they not always return numbers
1484
1485 $d*1 == 1
1486 ? $n*1
1487 : tag 30, [$n*1, $d*1]
1488}
1489
1140sub Time::Piece::TO_CBOR { 1490sub Time::Piece::TO_CBOR {
1141 tag 1, $_[0]->epoch 1491 tag 1, 0 + $_[0]->epoch
1142} 1492}
1143 1493
1144XSLoader::load "CBOR::XS", $VERSION; 1494XSLoader::load "CBOR::XS", $VERSION;
1145 1495
1146=head1 SEE ALSO 1496=head1 SEE ALSO

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines