ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/CBOR-XS/XS.pm
(Generate patch)

Comparing CBOR-XS/XS.pm (file contents):
Revision 1.41 by root, Mon Jan 6 04:15:31 2014 UTC vs.
Revision 1.84 by root, Sat Oct 23 03:00:31 2021 UTC

38with the added ability of supporting serialisation of Perl objects. (JSON 38with the added ability of supporting serialisation of Perl objects. (JSON
39often compresses better than CBOR though, so if you plan to compress the 39often compresses better than CBOR though, so if you plan to compress the
40data later and speed is less important you might want to compare both 40data later and speed is less important you might want to compare both
41formats first). 41formats first).
42 42
43The primary goal of this module is to be I<correct> and the secondary goal
44is to be I<fast>. To reach the latter goal it was written in C.
45
43To give you a general idea about speed, with texts in the megabyte range, 46To give you a general idea about speed, with texts in the megabyte range,
44C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or 47C<CBOR::XS> usually encodes roughly twice as fast as L<Storable> or
45L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the 48L<JSON::XS> and decodes about 15%-30% faster than those. The shorter the
46data, the worse L<Storable> performs in comparison. 49data, the worse L<Storable> performs in comparison.
47 50
52In addition to the core CBOR data format, this module implements a 55In addition to the core CBOR data format, this module implements a
53number of extensions, to support cyclic and shared data structures 56number of extensions, to support cyclic and shared data structures
54(see C<allow_sharing> and C<allow_cycles>), string deduplication (see 57(see C<allow_sharing> and C<allow_cycles>), string deduplication (see
55C<pack_strings>) and scalar references (always enabled). 58C<pack_strings>) and scalar references (always enabled).
56 59
57The primary goal of this module is to be I<correct> and the secondary goal
58is to be I<fast>. To reach the latter goal it was written in C.
59
60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and 60See MAPPING, below, on how CBOR::XS maps perl values to CBOR values and
61vice versa. 61vice versa.
62 62
63=cut 63=cut
64 64
65package CBOR::XS; 65package CBOR::XS;
66 66
67use common::sense; 67use common::sense;
68 68
69our $VERSION = 1.25; 69our $VERSION = 1.85;
70our @ISA = qw(Exporter); 70our @ISA = qw(Exporter);
71 71
72our @EXPORT = qw(encode_cbor decode_cbor); 72our @EXPORT = qw(encode_cbor decode_cbor);
73 73
74use Exporter; 74use Exporter;
112 112
113The mutators for flags all return the CBOR object again and thus calls can 113The mutators for flags all return the CBOR object again and thus calls can
114be chained: 114be chained:
115 115
116 my $cbor = CBOR::XS->new->encode ({a => [1,2]}); 116 my $cbor = CBOR::XS->new->encode ({a => [1,2]});
117
118=item $cbor = new_safe CBOR::XS
119
120Create a new, safe/secure CBOR::XS object. This is similar to C<new>,
121but configures the coder object to be safe to use with untrusted
122data. Currently, this is equivalent to:
123
124 my $cbor = CBOR::XS
125 ->new
126 ->validate_utf8
127 ->forbid_objects
128 ->filter (\&CBOR::XS::safe_filter)
129 ->max_size (1e8);
130
131But is more future proof (it is better to crash because of a change than
132to be exploited in other ways).
133
134=cut
135
136sub new_safe {
137 CBOR::XS
138 ->new
139 ->validate_utf8
140 ->forbid_objects
141 ->filter (\&CBOR::XS::safe_filter)
142 ->max_size (1e8)
143}
117 144
118=item $cbor = $cbor->max_depth ([$maximum_nesting_depth]) 145=item $cbor = $cbor->max_depth ([$maximum_nesting_depth])
119 146
120=item $max_depth = $cbor->get_max_depth 147=item $max_depth = $cbor->get_max_depth
121 148
137 164
138Note that nesting is implemented by recursion in C. The default value has 165Note that nesting is implemented by recursion in C. The default value has
139been chosen to be as large as typical operating systems allow without 166been chosen to be as large as typical operating systems allow without
140crashing. 167crashing.
141 168
142See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 169See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
143 170
144=item $cbor = $cbor->max_size ([$maximum_string_size]) 171=item $cbor = $cbor->max_size ([$maximum_string_size])
145 172
146=item $max_size = $cbor->get_max_size 173=item $max_size = $cbor->get_max_size
147 174
152effect on C<encode> (yet). 179effect on C<encode> (yet).
153 180
154If no argument is given, the limit check will be deactivated (same as when 181If no argument is given, the limit check will be deactivated (same as when
155C<0> is specified). 182C<0> is specified).
156 183
157See SECURITY CONSIDERATIONS, below, for more info on why this is useful. 184See L<SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
158 185
159=item $cbor = $cbor->allow_unknown ([$enable]) 186=item $cbor = $cbor->allow_unknown ([$enable])
160 187
161=item $enabled = $cbor->get_allow_unknown 188=item $enabled = $cbor->get_allow_unknown
162 189
180reference to the earlier value. 207reference to the earlier value.
181 208
182This means that such values will only be encoded once, and will not result 209This means that such values will only be encoded once, and will not result
183in a deep cloning of the value on decode, in decoders supporting the value 210in a deep cloning of the value on decode, in decoders supporting the value
184sharing extension. This also makes it possible to encode cyclic data 211sharing extension. This also makes it possible to encode cyclic data
185structures (which need C<allow_cycles> to ne enabled to be decoded by this 212structures (which need C<allow_cycles> to be enabled to be decoded by this
186module). 213module).
187 214
188It is recommended to leave it off unless you know your 215It is recommended to leave it off unless you know your
189communication partner supports the value sharing extensions to CBOR 216communication partner supports the value sharing extensions to CBOR
190(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the 217(L<http://cbor.schmorp.de/value-sharing>), as without decoder support, the
191resulting data structure might be unusable. 218resulting data structure might be unusable.
192 219
193Detecting shared values incurs a runtime overhead when values are encoded 220Detecting shared values incurs a runtime overhead when values are encoded
194that have a reference counter large than one, and might unnecessarily 221that have a reference counter large than one, and might unnecessarily
195increase the encoded size, as potentially shared values are encode as 222increase the encoded size, as potentially shared values are encoded as
196shareable whether or not they are actually shared. 223shareable whether or not they are actually shared.
197 224
198At the moment, only targets of references can be shared (e.g. scalars, 225At the moment, only targets of references can be shared (e.g. scalars,
199arrays or hashes pointed to by a reference). Weirder constructs, such as 226arrays or hashes pointed to by a reference). Weirder constructs, such as
200an array with multiple "copies" of the I<same> string, which are hard but 227an array with multiple "copies" of the I<same> string, which are hard but
224cycles - future versions of this module might chose to decode cyclic data 251cycles - future versions of this module might chose to decode cyclic data
225structures using weak references when this option is off, instead of 252structures using weak references when this option is off, instead of
226throwing an error. 253throwing an error.
227 254
228This option does not affect C<encode> in any way - shared values and 255This option does not affect C<encode> in any way - shared values and
229references will always be decoded properly if present. 256references will always be encoded properly if present.
257
258=item $cbor = $cbor->forbid_objects ([$enable])
259
260=item $enabled = $cbor->get_forbid_objects
261
262Disables the use of the object serialiser protocol.
263
264If C<$enable> is true (or missing), then C<encode> will will throw an
265exception when it encounters perl objects that would be encoded using the
266perl-object tag (26). When C<decode> encounters such tags, it will fall
267back to the general filter/tagged logic as if this were an unknown tag (by
268default resulting in a C<CBOR::XC::Tagged> object).
269
270If C<$enable> is false (the default), then C<encode> will use the
271L<Types::Serialiser> object serialisation protocol to serialise objects
272into perl-object tags, and C<decode> will do the same to decode such tags.
273
274See L<SECURITY CONSIDERATIONS>, below, for more info on why forbidding this
275protocol can be useful.
230 276
231=item $cbor = $cbor->pack_strings ([$enable]) 277=item $cbor = $cbor->pack_strings ([$enable])
232 278
233=item $enabled = $cbor->get_pack_strings 279=item $enabled = $cbor->get_pack_strings
234 280
247the standard CBOR way. 293the standard CBOR way.
248 294
249This option does not affect C<decode> in any way - string references will 295This option does not affect C<decode> in any way - string references will
250always be decoded properly if present. 296always be decoded properly if present.
251 297
298=item $cbor = $cbor->text_keys ([$enable])
299
300=item $enabled = $cbor->get_text_keys
301
302If C<$enabled> is true (or missing), then C<encode> will encode all
303perl hash keys as CBOR text strings/UTF-8 string, upgrading them as needed.
304
305If C<$enable> is false (the default), then C<encode> will encode hash keys
306normally - upgraded perl strings (strings internally encoded as UTF-8) as
307CBOR text strings, and downgraded perl strings as CBOR byte strings.
308
309This option does not affect C<decode> in any way.
310
311This option is useful for interoperability with CBOR decoders that don't
312treat byte strings as a form of text. It is especially useful as Perl
313gives very little control over hash keys.
314
315Enabling this option can be slow, as all downgraded hash keys that are
316encoded need to be scanned and converted to UTF-8.
317
318=item $cbor = $cbor->text_strings ([$enable])
319
320=item $enabled = $cbor->get_text_strings
321
322This option works similar to C<text_keys>, above, but works on all strings
323(including hash keys), so C<text_keys> has no further effect after
324enabling C<text_strings>.
325
326If C<$enabled> is true (or missing), then C<encode> will encode all perl
327strings as CBOR text strings/UTF-8 strings, upgrading them as needed.
328
329If C<$enable> is false (the default), then C<encode> will encode strings
330normally (but see C<text_keys>) - upgraded perl strings (strings
331internally encoded as UTF-8) as CBOR text strings, and downgraded perl
332strings as CBOR byte strings.
333
334This option does not affect C<decode> in any way.
335
336This option has similar advantages and disadvantages as C<text_keys>. In
337addition, this option effectively removes the ability to automatically
338encode byte strings, which might break some C<FREEZE> and C<TO_CBOR>
339methods that rely on this.
340
341A workaround is to use explicit type casts, which are unaffected by this option.
342
252=item $cbor = $cbor->validate_utf8 ([$enable]) 343=item $cbor = $cbor->validate_utf8 ([$enable])
253 344
254=item $enabled = $cbor->get_validate_utf8 345=item $enabled = $cbor->get_validate_utf8
255 346
256If C<$enable> is true (or missing), then C<decode> will validate that 347If C<$enable> is true (or missing), then C<decode> will validate that
261The concept of "valid UTF-8" used is perl's concept, which is a superset 352The concept of "valid UTF-8" used is perl's concept, which is a superset
262of the official UTF-8. 353of the official UTF-8.
263 354
264If C<$enable> is false (the default), then C<decode> will blindly accept 355If C<$enable> is false (the default), then C<decode> will blindly accept
265UTF-8 data, marking them as valid UTF-8 in the resulting data structure 356UTF-8 data, marking them as valid UTF-8 in the resulting data structure
266regardless of whether thats true or not. 357regardless of whether that's true or not.
267 358
268Perl isn't too happy about corrupted UTF-8 in strings, but should 359Perl isn't too happy about corrupted UTF-8 in strings, but should
269generally not crash or do similarly evil things. Extensions might be not 360generally not crash or do similarly evil things. Extensions might be not
270so forgiving, so it's recommended to turn on this setting if you receive 361so forgiving, so it's recommended to turn on this setting if you receive
271untrusted CBOR. 362untrusted CBOR.
293replace the tagged value in the decoded data structure, or no values, 384replace the tagged value in the decoded data structure, or no values,
294which will result in default handling, which currently means the decoder 385which will result in default handling, which currently means the decoder
295creates a C<CBOR::XS::Tagged> object to hold the tag and the value. 386creates a C<CBOR::XS::Tagged> object to hold the tag and the value.
296 387
297When the filter is cleared (the default state), the default filter 388When the filter is cleared (the default state), the default filter
298function, C<CBOR::XS::default_filter>, is used. This function simply looks 389function, C<CBOR::XS::default_filter>, is used. This function simply
299up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists it must be 390looks up the tag in the C<%CBOR::XS::FILTER> hash. If an entry exists
300a code reference that is called with tag and value, and is responsible for 391it must be a code reference that is called with tag and value, and is
301decoding the value. If no entry exists, it returns no values. 392responsible for decoding the value. If no entry exists, it returns no
393values. C<CBOR::XS> provides a number of default filter functions already,
394the the C<%CBOR::XS::FILTER> hash can be freely extended with more.
395
396C<CBOR::XS> additionally provides an alternative filter function that is
397supposed to be safe to use with untrusted data (which the default filter
398might not), called C<CBOR::XS::safe_filter>, which works the same as
399the C<default_filter> but uses the C<%CBOR::XS::SAFE_FILTER> variable
400instead. It is prepopulated with the tag decoding functions that are
401deemed safe (basically the same as C<%CBOR::XS::FILTER> without all
402the bignum tags), and can be extended by user code as wlel, although,
403obviously, one should be very careful about adding decoding functions
404here, since the expectation is that they are safe to use on untrusted
405data, after all.
302 406
303Example: decode all tags not handled internally into C<CBOR::XS::Tagged> 407Example: decode all tags not handled internally into C<CBOR::XS::Tagged>
304objects, with no other special handling (useful when working with 408objects, with no other special handling (useful when working with
305potentially "unsafe" CBOR data). 409potentially "unsafe" CBOR data).
306 410
313 my ($tag, $value); 417 my ($tag, $value);
314 418
315 "tag 1347375694 value $value" 419 "tag 1347375694 value $value"
316 }; 420 };
317 421
422Example: provide your own filter function that looks up tags in your own
423hash:
424
425 my %my_filter = (
426 998347484 => sub {
427 my ($tag, $value);
428
429 "tag 998347484 value $value"
430 };
431 );
432
433 my $coder = CBOR::XS->new->filter (sub {
434 &{ $my_filter{$_[0]} or return }
435 });
436
437
438Example: use the safe filter function (see L<SECURITY CONSIDERATIONS> for
439more considerations on security).
440
441 CBOR::XS->new->filter (\&CBOR::XS::safe_filter)->decode ($cbor_data);
442
318=item $cbor_data = $cbor->encode ($perl_scalar) 443=item $cbor_data = $cbor->encode ($perl_scalar)
319 444
320Converts the given Perl data structure (a scalar value) to its CBOR 445Converts the given Perl data structure (a scalar value) to its CBOR
321representation. 446representation.
322 447
331when there is trailing garbage after the CBOR string, it will silently 456when there is trailing garbage after the CBOR string, it will silently
332stop parsing there and return the number of characters consumed so far. 457stop parsing there and return the number of characters consumed so far.
333 458
334This is useful if your CBOR texts are not delimited by an outer protocol 459This is useful if your CBOR texts are not delimited by an outer protocol
335and you need to know where the first CBOR string ends amd the next one 460and you need to know where the first CBOR string ends amd the next one
336starts. 461starts - CBOR strings are self-delimited, so it is possible to concatenate
462CBOR strings without any delimiters or size fields and recover their data.
337 463
338 CBOR::XS->new->decode_prefix ("......") 464 CBOR::XS->new->decode_prefix ("......")
339 => ("...", 3) 465 => ("...", 3)
340 466
341=back 467=back
396 522
397Resets the incremental decoder. This throws away any saved state, so that 523Resets the incremental decoder. This throws away any saved state, so that
398subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse 524subsequent calls to C<incr_parse> or C<incr_parse_multiple> start to parse
399a new CBOR value from the beginning of the C<$buffer> again. 525a new CBOR value from the beginning of the C<$buffer> again.
400 526
401This method can be caled at any time, but it I<must> be called if you want 527This method can be called at any time, but it I<must> be called if you want
402to change your C<$buffer> or there was a decoding error and you want to 528to change your C<$buffer> or there was a decoding error and you want to
403reuse the C<$cbor> object for future incremental parsings. 529reuse the C<$cbor> object for future incremental parsings.
404 530
405=back 531=back
406 532
481 607
482=item hash references 608=item hash references
483 609
484Perl hash references become CBOR maps. As there is no inherent ordering in 610Perl hash references become CBOR maps. As there is no inherent ordering in
485hash keys (or CBOR maps), they will usually be encoded in a pseudo-random 611hash keys (or CBOR maps), they will usually be encoded in a pseudo-random
486order. This order can be different each time a hahs is encoded. 612order. This order can be different each time a hash is encoded.
487 613
488Currently, tied hashes will use the indefinite-length format, while normal 614Currently, tied hashes will use the indefinite-length format, while normal
489hashes will use the fixed-length format. 615hashes will use the fixed-length format.
490 616
491=item array references 617=item array references
544 my $x = 3.1; # some variable containing a number 670 my $x = 3.1; # some variable containing a number
545 "$x"; # stringified 671 "$x"; # stringified
546 $x .= ""; # another, more awkward way to stringify 672 $x .= ""; # another, more awkward way to stringify
547 print $x; # perl does it for you, too, quite often 673 print $x; # perl does it for you, too, quite often
548 674
549You can force whether a string ie encoded as byte or text string by using 675You can force whether a string is encoded as byte or text string by using
550C<utf8::upgrade> and C<utf8::downgrade>): 676C<utf8::upgrade> and C<utf8::downgrade> (if C<text_strings> is disabled).
551 677
552 utf8::upgrade $x; # encode $x as text string 678 utf8::upgrade $x; # encode $x as text string
553 utf8::downgrade $x; # encode $x as byte string 679 utf8::downgrade $x; # encode $x as byte string
554 680
681More options are available, see L<TYPE CASTS>, below, and the C<text_keys>
682and C<text_strings> options.
683
555Perl doesn't define what operations up- and downgrade strings, so if the 684Perl doesn't define what operations up- and downgrade strings, so if the
556difference between byte and text is important, you should up- or downgrade 685difference between byte and text is important, you should up- or downgrade
557your string as late as possible before encoding. 686your string as late as possible before encoding. You can also force the
687use of CBOR text strings by using C<text_keys> or C<text_strings>.
558 688
559You can force the type to be a CBOR number by numifying it: 689You can force the type to be a CBOR number by numifying it:
560 690
561 my $x = "3"; # some variable containing a string 691 my $x = "3"; # some variable containing a string
562 $x += 0; # numify it, ensuring it will be dumped as a number 692 $x += 0; # numify it, ensuring it will be dumped as a number
573represent numerical values are supported, but might suffer loss of 703represent numerical values are supported, but might suffer loss of
574precision. 704precision.
575 705
576=back 706=back
577 707
708=head2 TYPE CASTS
709
710B<EXPERIMENTAL>: As an experimental extension, C<CBOR::XS> allows you to
711force specific CBOR types to be used when encoding. That allows you to
712encode types not normally accessible (e.g. half floats) as well as force
713string types even when C<text_strings> is in effect.
714
715Type forcing is done by calling a special "cast" function which keeps a
716copy of the value and returns a new value that can be handed over to any
717CBOR encoder function.
718
719The following casts are currently available (all of which are unary
720operators, that is, have a prototype of C<$>):
721
722=over
723
724=item CBOR::XS::as_int $value
725
726Forces the value to be encoded as some form of (basic, not bignum) integer
727type.
728
729=item CBOR::XS::as_text $value
730
731Forces the value to be encoded as (UTF-8) text values.
732
733=item CBOR::XS::as_bytes $value
734
735Forces the value to be encoded as a (binary) string value.
736
737Example: encode a perl string as binary even though C<text_strings> is in
738effect.
739
740 CBOR::XS->new->text_strings->encode ([4, "text", CBOR::XS::bytes "bytevalue"]);
741
742=item CBOR::XS::as_bool $value
743
744Converts a Perl boolean (which can be any kind of scalar) into a CBOR
745boolean. Strictly the same, but shorter to write, than:
746
747 $value ? Types::Serialiser::true : Types::Serialiser::false
748
749=item CBOR::XS::as_float16 $value
750
751Forces half-float (IEEE 754 binary16) encoding of the given value.
752
753=item CBOR::XS::as_float32 $value
754
755Forces single-float (IEEE 754 binary32) encoding of the given value.
756
757=item CBOR::XS::as_float64 $value
758
759Forces double-float (IEEE 754 binary64) encoding of the given value.
760
761=item CBOR::XS::as_cbor $cbor_text
762
763Not a type cast per-se, this type cast forces the argument to be encoded
764as-is. This can be used to embed pre-encoded CBOR data.
765
766Note that no checking on the validity of the C<$cbor_text> is done - it's
767the callers responsibility to correctly encode values.
768
769=item CBOR::XS::as_map [key => value...]
770
771Treat the array reference as key value pairs and output a CBOR map. This
772allows you to generate CBOR maps with arbitrary key types (or, if you
773don't care about semantics, duplicate keys or pairs in a custom order),
774which is otherwise hard to do with Perl.
775
776The single argument must be an array reference with an even number of
777elements.
778
779Note that only the reference to the array is copied, the array itself is
780not. Modifications done to the array before calling an encoding function
781will be reflected in the encoded output.
782
783Example: encode a CBOR map with a string and an integer as keys.
784
785 encode_cbor CBOR::XS::as_map [string => "value", 5 => "value"]
786
787=back
788
789=cut
790
791sub CBOR::XS::as_cbor ($) { bless [$_[0], 0, undef], CBOR::XS::Tagged:: }
792sub CBOR::XS::as_int ($) { bless [$_[0], 1, undef], CBOR::XS::Tagged:: }
793sub CBOR::XS::as_bytes ($) { bless [$_[0], 2, undef], CBOR::XS::Tagged:: }
794sub CBOR::XS::as_text ($) { bless [$_[0], 3, undef], CBOR::XS::Tagged:: }
795sub CBOR::XS::as_float16 ($) { bless [$_[0], 4, undef], CBOR::XS::Tagged:: }
796sub CBOR::XS::as_float32 ($) { bless [$_[0], 5, undef], CBOR::XS::Tagged:: }
797sub CBOR::XS::as_float64 ($) { bless [$_[0], 6, undef], CBOR::XS::Tagged:: }
798
799sub CBOR::XS::as_bool ($) { $_[0] ? $Types::Serialiser::true : $Types::Serialiser::false }
800
801sub CBOR::XS::as_map ($) {
802 ARRAY:: eq ref $_[0]
803 and $#{ $_[0] } & 1
804 or do { require Carp; Carp::croak ("CBOR::XS::as_map only acepts array references with an even number of elements, caught") };
805
806 bless [$_[0], 7, undef], CBOR::XS::Tagged::
807}
808
578=head2 OBJECT SERIALISATION 809=head2 OBJECT SERIALISATION
579 810
580This module implements both a CBOR-specific and the generic 811This module implements both a CBOR-specific and the generic
581L<Types::Serialier> object serialisation protocol. The following 812L<Types::Serialier> object serialisation protocol. The following
582subsections explain both methods. 813subsections explain both methods.
663 "$self" # encode url string 894 "$self" # encode url string
664 } 895 }
665 896
666 sub URI::THAW { 897 sub URI::THAW {
667 my ($class, $serialiser, $uri) = @_; 898 my ($class, $serialiser, $uri) = @_;
668
669 $class->new ($uri) 899 $class->new ($uri)
670 } 900 }
671 901
672Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For 902Unlike C<TO_CBOR>, multiple values can be returned by C<FREEZE>. For
673example, a C<FREEZE> method that returns "type", "id" and "variant" values 903example, a C<FREEZE> method that returns "type", "id" and "variant" values
804additional tags (such as base64url). 1034additional tags (such as base64url).
805 1035
806=head2 ENFORCED TAGS 1036=head2 ENFORCED TAGS
807 1037
808These tags are always handled when decoding, and their handling cannot be 1038These tags are always handled when decoding, and their handling cannot be
809overriden by the user. 1039overridden by the user.
810 1040
811=over 4 1041=over 4
812 1042
813=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>) 1043=item 26 (perl-object, L<http://cbor.schmorp.de/perl-object>)
814 1044
815These tags are automatically created (and decoded) for serialisable 1045These tags are automatically created (and decoded) for serialisable
816objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object 1046objects using the C<FREEZE/THAW> methods (the L<Types::Serialier> object
817serialisation protocol). See L<OBJECT SERIALISATION> for details. 1047serialisation protocol). See L<OBJECT SERIALISATION> for details.
818 1048
819=item 28, 29 (shareable, sharedref, L <http://cbor.schmorp.de/value-sharing>) 1049=item 28, 29 (shareable, sharedref, L<http://cbor.schmorp.de/value-sharing>)
820 1050
821These tags are automatically decoded when encountered (and they do not 1051These tags are automatically decoded when encountered (and they do not
822result in a cyclic data structure, see C<allow_cycles>), resulting in 1052result in a cyclic data structure, see C<allow_cycles>), resulting in
823shared values in the decoded object. They are only encoded, however, when 1053shared values in the decoded object. They are only encoded, however, when
824C<allow_sharing> is enabled. 1054C<allow_sharing> is enabled.
834will be shared, others will not. While non-reference shared values can be 1064will be shared, others will not. While non-reference shared values can be
835generated in Perl with some effort, they were considered too unimportant 1065generated in Perl with some effort, they were considered too unimportant
836to be supported in the encoder. The decoder, however, will decode these 1066to be supported in the encoder. The decoder, however, will decode these
837values as shared values. 1067values as shared values.
838 1068
839=item 256, 25 (stringref-namespace, stringref, L <http://cbor.schmorp.de/stringref>) 1069=item 256, 25 (stringref-namespace, stringref, L<http://cbor.schmorp.de/stringref>)
840 1070
841These tags are automatically decoded when encountered. They are only 1071These tags are automatically decoded when encountered. They are only
842encoded, however, when C<pack_strings> is enabled. 1072encoded, however, when C<pack_strings> is enabled.
843 1073
844=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>) 1074=item 22098 (indirection, L<http://cbor.schmorp.de/indirection>)
845 1075
846This tag is automatically generated when a reference are encountered (with 1076This tag is automatically generated when a reference are encountered (with
847the exception of hash and array refernces). It is converted to a reference 1077the exception of hash and array references). It is converted to a reference
848when decoding. 1078when decoding.
849 1079
850=item 55799 (self-describe CBOR, RFC 7049) 1080=item 55799 (self-describe CBOR, RFC 7049)
851 1081
852This value is not generated on encoding (unless explicitly requested by 1082This value is not generated on encoding (unless explicitly requested by
855=back 1085=back
856 1086
857=head2 NON-ENFORCED TAGS 1087=head2 NON-ENFORCED TAGS
858 1088
859These tags have default filters provided when decoding. Their handling can 1089These tags have default filters provided when decoding. Their handling can
860be overriden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by 1090be overridden by changing the C<%CBOR::XS::FILTER> entry for the tag, or by
861providing a custom C<filter> callback when decoding. 1091providing a custom C<filter> callback when decoding.
862 1092
863When they result in decoding into a specific Perl class, the module 1093When they result in decoding into a specific Perl class, the module
864usually provides a corresponding C<TO_CBOR> method as well. 1094usually provides a corresponding C<TO_CBOR> method as well.
865 1095
883 1113
884These tags are decoded into L<Math::BigInt> objects. The corresponding 1114These tags are decoded into L<Math::BigInt> objects. The corresponding
885C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR 1115C<Math::BigInt::TO_CBOR> method encodes "small" bigints into normal CBOR
886integers, and others into positive/negative CBOR bignums. 1116integers, and others into positive/negative CBOR bignums.
887 1117
888=item 4, 5 (decimal fraction/bigfloat) 1118=item 4, 5, 264, 265 (decimal fraction/bigfloat)
889 1119
890Both decimal fractions and bigfloats are decoded into L<Math::BigFloat> 1120Both decimal fractions and bigfloats are decoded into L<Math::BigFloat>
891objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always> 1121objects. The corresponding C<Math::BigFloat::TO_CBOR> method I<always>
892encodes into a decimal fraction. 1122encodes into a decimal fraction (either tag 4 or 264).
893 1123
894CBOR cannot represent bigfloats with I<very> large exponents - conversion 1124NaN and infinities are not encoded properly, as they cannot be represented
895of such big float objects is undefined. 1125in CBOR.
896 1126
897Also, NaN and infinities are not encoded properly. 1127See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
1128
1129=item 30 (rational numbers)
1130
1131These tags are decoded into L<Math::BigRat> objects. The corresponding
1132C<Math::BigRat::TO_CBOR> method encodes rational numbers with denominator
1133C<1> via their numerator only, i.e., they become normal integers or
1134C<bignums>.
1135
1136See L<BIGNUM SECURITY CONSIDERATIONS> for more info.
898 1137
899=item 21, 22, 23 (expected later JSON conversion) 1138=item 21, 22, 23 (expected later JSON conversion)
900 1139
901CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these 1140CBOR::XS is not a CBOR-to-JSON converter, and will simply ignore these
902tags. 1141tags.
907C<URI::TO_CBOR> method again results in a CBOR URI value. 1146C<URI::TO_CBOR> method again results in a CBOR URI value.
908 1147
909=back 1148=back
910 1149
911=cut 1150=cut
912
913our %FILTER = (
914 # 0 # rfc4287 datetime, utf-8
915 # 1 # unix timestamp, any
916
917 2 => sub { # pos bigint
918 require Math::BigInt;
919 Math::BigInt->new ("0x" . unpack "H*", pop)
920 },
921
922 3 => sub { # neg bigint
923 require Math::BigInt;
924 -Math::BigInt->new ("0x" . unpack "H*", pop)
925 },
926
927 4 => sub { # decimal fraction, array
928 require Math::BigFloat;
929 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
930 },
931
932 5 => sub { # bigfloat, array
933 require Math::BigFloat;
934 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2)
935 },
936
937 21 => sub { pop }, # expected conversion to base64url encoding
938 22 => sub { pop }, # expected conversion to base64 encoding
939 23 => sub { pop }, # expected conversion to base16 encoding
940
941 # 24 # embedded cbor, byte string
942
943 32 => sub {
944 require URI;
945 URI->new (pop)
946 },
947
948 # 33 # base64url rfc4648, utf-8
949 # 34 # base64 rfc46484, utf-8
950 # 35 # regex pcre/ecma262, utf-8
951 # 36 # mime message rfc2045, utf-8
952);
953
954 1151
955=head1 CBOR and JSON 1152=head1 CBOR and JSON
956 1153
957CBOR is supposed to implement a superset of the JSON data model, and is, 1154CBOR is supposed to implement a superset of the JSON data model, and is,
958with some coercion, able to represent all JSON texts (something that other 1155with some coercion, able to represent all JSON texts (something that other
967CBOR intact. 1164CBOR intact.
968 1165
969 1166
970=head1 SECURITY CONSIDERATIONS 1167=head1 SECURITY CONSIDERATIONS
971 1168
972When you are using CBOR in a protocol, talking to untrusted potentially 1169Tl;dr... if you want to decode or encode CBOR from untrusted sources, you
973hostile creatures requires relatively few measures. 1170should start with a coder object created via C<new_safe> (which implements
1171the mitigations explained below):
974 1172
1173 my $coder = CBOR::XS->new_safe;
1174
1175 my $data = $coder->decode ($cbor_text);
1176 my $cbor = $coder->encode ($data);
1177
1178Longer version: When you are using CBOR in a protocol, talking to
1179untrusted potentially hostile creatures requires some thought:
1180
1181=over 4
1182
1183=item Security of the CBOR decoder itself
1184
975First of all, your CBOR decoder should be secure, that is, should not have 1185First and foremost, your CBOR decoder should be secure, that is, should
1186not have any buffer overflows or similar bugs that could potentially be
976any buffer overflows. Obviously, this module should ensure that and I am 1187exploited. Obviously, this module should ensure that and I am trying hard
977trying hard on making that true, but you never know. 1188on making that true, but you never know.
978 1189
1190=item CBOR::XS can invoke almost arbitrary callbacks during decoding
1191
1192CBOR::XS supports object serialisation - decoding CBOR can cause calls
1193to I<any> C<THAW> method in I<any> package that exists in your process
1194(that is, CBOR::XS will not try to load modules, but any existing C<THAW>
1195method or function can be called, so they all have to be secure).
1196
1197Less obviously, it will also invoke C<TO_CBOR> and C<FREEZE> methods -
1198even if all your C<THAW> methods are secure, encoding data structures from
1199untrusted sources can invoke those and trigger bugs in those.
1200
1201So, if you are not sure about the security of all the modules you
1202have loaded (you shouldn't), you should disable this part using
1203C<forbid_objects> or using C<new_safe>.
1204
1205=item CBOR can be extended with tags that call library code
1206
1207CBOR can be extended with tags, and C<CBOR::XS> has a registry of
1208conversion functions for many existing tags that can be extended via
1209third-party modules (see the C<filter> method).
1210
1211If you don't trust these, you should configure the "safe" filter function,
1212C<CBOR::XS::safe_filter> (C<new_safe> does this), which by default only
1213includes conversion functions that are considered "safe" by the author
1214(but again, they can be extended by third party modules).
1215
1216Depending on your level of paranoia, you can use the "safe" filter:
1217
1218 $cbor->filter (\&CBOR::XS::safe_filter);
1219
1220... your own filter...
1221
1222 $cbor->filter (sub { ... do your stuffs here ... });
1223
1224... or even no filter at all, disabling all tag decoding:
1225
1226 $cbor->filter (sub { });
1227
1228This is never a problem for encoding, as the tag mechanism only exists in
1229CBOR texts.
1230
1231=item Resource-starving attacks: object memory usage
1232
979Second, you need to avoid resource-starving attacks. That means you should 1233You need to avoid resource-starving attacks. That means you should limit
980limit the size of CBOR data you accept, or make sure then when your 1234the size of CBOR data you accept, or make sure then when your resources
981resources run out, that's just fine (e.g. by using a separate process that 1235run out, that's just fine (e.g. by using a separate process that can
982can crash safely). The size of a CBOR string in octets is usually a good 1236crash safely). The size of a CBOR string in octets is usually a good
983indication of the size of the resources required to decode it into a Perl 1237indication of the size of the resources required to decode it into a Perl
984structure. While CBOR::XS can check the size of the CBOR text, it might be 1238structure. While CBOR::XS can check the size of the CBOR text (using
985too late when you already have it in memory, so you might want to check 1239C<max_size> - done by C<new_safe>), it might be too late when you already
986the size before you accept the string. 1240have it in memory, so you might want to check the size before you accept
1241the string.
987 1242
1243As for encoding, it is possible to construct data structures that are
1244relatively small but result in large CBOR texts (for example by having an
1245array full of references to the same big data structure, which will all be
1246deep-cloned during encoding by default). This is rarely an actual issue
1247(and the worst case is still just running out of memory), but you can
1248reduce this risk by using C<allow_sharing>.
1249
1250=item Resource-starving attacks: stack overflows
1251
988Third, CBOR::XS recurses using the C stack when decoding objects and 1252CBOR::XS recurses using the C stack when decoding objects and arrays. The
989arrays. The C stack is a limited resource: for instance, on my amd64 1253C stack is a limited resource: for instance, on my amd64 machine with 8MB
990machine with 8MB of stack size I can decode around 180k nested arrays but 1254of stack size I can decode around 180k nested arrays but only 14k nested
991only 14k nested CBOR objects (due to perl itself recursing deeply on croak 1255CBOR objects (due to perl itself recursing deeply on croak to free the
992to free the temporary). If that is exceeded, the program crashes. To be 1256temporary). If that is exceeded, the program crashes. To be conservative,
993conservative, the default nesting limit is set to 512. If your process 1257the default nesting limit is set to 512. If your process has a smaller
994has a smaller stack, you should adjust this setting accordingly with the 1258stack, you should adjust this setting accordingly with the C<max_depth>
995C<max_depth> method. 1259method.
1260
1261=item Resource-starving attacks: CPU en-/decoding complexity
1262
1263CBOR::XS will use the L<Math::BigInt>, L<Math::BigFloat> and
1264L<Math::BigRat> libraries to represent encode/decode bignums. These can be
1265very slow (as in, centuries of CPU time) and can even crash your program
1266(and are generally not very trustworthy). See the next section on bignum
1267security for details.
1268
1269=item Data breaches: leaking information in error messages
1270
1271CBOR::XS might leak contents of your Perl data structures in its error
1272messages, so when you serialise sensitive information you might want to
1273make sure that exceptions thrown by CBOR::XS will not end up in front of
1274untrusted eyes.
1275
1276=item Something else...
996 1277
997Something else could bomb you, too, that I forgot to think of. In that 1278Something else could bomb you, too, that I forgot to think of. In that
998case, you get to keep the pieces. I am always open for hints, though... 1279case, you get to keep the pieces. I am always open for hints, though...
999 1280
1000Also keep in mind that CBOR::XS might leak contents of your Perl data 1281=back
1001structures in its error messages, so when you serialise sensitive 1282
1002information you might want to make sure that exceptions thrown by CBOR::XS 1283
1003will not end up in front of untrusted eyes. 1284=head1 BIGNUM SECURITY CONSIDERATIONS
1285
1286CBOR::XS provides a C<TO_CBOR> method for both L<Math::BigInt> and
1287L<Math::BigFloat> that tries to encode the number in the simplest possible
1288way, that is, either a CBOR integer, a CBOR bigint/decimal fraction (tag
12894) or an arbitrary-exponent decimal fraction (tag 264). Rational numbers
1290(L<Math::BigRat>, tag 30) can also contain bignums as members.
1291
1292CBOR::XS will also understand base-2 bigfloat or arbitrary-exponent
1293bigfloats (tags 5 and 265), but it will never generate these on its own.
1294
1295Using the built-in L<Math::BigInt::Calc> support, encoding and decoding
1296decimal fractions is generally fast. Decoding bigints can be slow for very
1297big numbers (tens of thousands of digits, something that could potentially
1298be caught by limiting the size of CBOR texts), and decoding bigfloats or
1299arbitrary-exponent bigfloats can be I<extremely> slow (minutes, decades)
1300for large exponents (roughly 40 bit and longer).
1301
1302Additionally, L<Math::BigInt> can take advantage of other bignum
1303libraries, such as L<Math::GMP>, which cannot handle big floats with large
1304exponents, and might simply abort or crash your program, due to their code
1305quality.
1306
1307This can be a concern if you want to parse untrusted CBOR. If it is, you
1308might want to disable decoding of tag 2 (bigint) and 3 (negative bigint)
1309types. You should also disable types 5 and 265, as these can be slow even
1310without bigints.
1311
1312Disabling bigints will also partially or fully disable types that rely on
1313them, e.g. rational numbers that use bignums.
1314
1004 1315
1005=head1 CBOR IMPLEMENTATION NOTES 1316=head1 CBOR IMPLEMENTATION NOTES
1006 1317
1007This section contains some random implementation notes. They do not 1318This section contains some random implementation notes. They do not
1008describe guaranteed behaviour, but merely behaviour as-is implemented 1319describe guaranteed behaviour, but merely behaviour as-is implemented
1022 1333
1023 1334
1024=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT 1335=head1 LIMITATIONS ON PERLS WITHOUT 64-BIT INTEGER SUPPORT
1025 1336
1026On perls that were built without 64 bit integer support (these are rare 1337On perls that were built without 64 bit integer support (these are rare
1027nowadays, even on 32 bit architectures), support for any kind of 64 bit 1338nowadays, even on 32 bit architectures, as all major Perl distributions
1339are built with 64 bit integer support), support for any kind of 64 bit
1028integer in CBOR is very limited - most likely, these 64 bit values will 1340value in CBOR is very limited - most likely, these 64 bit values will
1029be truncated, corrupted, or otherwise not decoded correctly. This also 1341be truncated, corrupted, or otherwise not decoded correctly. This also
1030includes string, array and map sizes that are stored as 64 bit integers. 1342includes string, float, array and map sizes that are stored as 64 bit
1343integers.
1031 1344
1032 1345
1033=head1 THREADS 1346=head1 THREADS
1034 1347
1035This module is I<not> guaranteed to be thread safe and there are no 1348This module is I<not> guaranteed to be thread safe and there are no
1048 1361
1049Please refrain from using rt.cpan.org or any other bug reporting 1362Please refrain from using rt.cpan.org or any other bug reporting
1050service. I put the contact address into my modules for a reason. 1363service. I put the contact address into my modules for a reason.
1051 1364
1052=cut 1365=cut
1366
1367# clumsy and slow hv_store-in-hash helper function
1368sub _hv_store {
1369 $_[0]{$_[1]} = $_[2];
1370}
1053 1371
1054our %FILTER = ( 1372our %FILTER = (
1055 0 => sub { # rfc4287 datetime, utf-8 1373 0 => sub { # rfc4287 datetime, utf-8
1056 require Time::Piece; 1374 require Time::Piece;
1057 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine" 1375 # Time::Piece::Strptime uses the "incredibly flexible date parsing routine"
1058 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything 1376 # from FreeBSD, which can't parse ISO 8601, RFC3339, RFC4287 or much of anything
1059 # else either. Whats incredibe over standard strptime totally escapes me. 1377 # else either. Whats incredibe over standard strptime totally escapes me.
1060 # doesn't do fractional times, either. sigh. 1378 # doesn't do fractional times, either. sigh.
1061 # In fact, it's all a lie, it uses whatever strptime it wants, and of course, 1379 # In fact, it's all a lie, it uses whatever strptime it wants, and of course,
1062 # they are all incomptible. The openbsd one simply ignores %z (but according to the 1380 # they are all incompatible. The openbsd one simply ignores %z (but according to the
1063 # docs, it would be much more incredibly flexible indeed. If it worked, that is.). 1381 # docs, it would be much more incredibly flexible indeed. If it worked, that is.).
1064 scalar eval { 1382 scalar eval {
1065 my $s = $_[1]; 1383 my $s = $_[1];
1066 1384
1067 $s =~ s/Z$/+00:00/; 1385 $s =~ s/Z$/+00:00/;
1093 4 => sub { # decimal fraction, array 1411 4 => sub { # decimal fraction, array
1094 require Math::BigFloat; 1412 require Math::BigFloat;
1095 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0]) 1413 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1096 }, 1414 },
1097 1415
1416 264 => sub { # decimal fraction with arbitrary exponent
1417 require Math::BigFloat;
1418 Math::BigFloat->new ($_[1][1] . "E" . $_[1][0])
1419 },
1420
1098 5 => sub { # bigfloat, array 1421 5 => sub { # bigfloat, array
1099 require Math::BigFloat; 1422 require Math::BigFloat;
1100 scalar Math::BigFloat->new ($_[1][1])->blsft ($_[1][0], 2) 1423 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1424 },
1425
1426 265 => sub { # bigfloat with arbitrary exponent
1427 require Math::BigFloat;
1428 scalar Math::BigFloat->new ($_[1][1]) * Math::BigFloat->new (2)->bpow ($_[1][0])
1429 },
1430
1431 30 => sub { # rational number
1432 require Math::BigRat;
1433 Math::BigRat->new ("$_[1][0]/$_[1][1]") # separate parameters only work in recent versons
1101 }, 1434 },
1102 1435
1103 21 => sub { pop }, # expected conversion to base64url encoding 1436 21 => sub { pop }, # expected conversion to base64url encoding
1104 22 => sub { pop }, # expected conversion to base64 encoding 1437 22 => sub { pop }, # expected conversion to base64 encoding
1105 23 => sub { pop }, # expected conversion to base16 encoding 1438 23 => sub { pop }, # expected conversion to base16 encoding
1115 # 34 # base64 rfc46484, utf-8 1448 # 34 # base64 rfc46484, utf-8
1116 # 35 # regex pcre/ecma262, utf-8 1449 # 35 # regex pcre/ecma262, utf-8
1117 # 36 # mime message rfc2045, utf-8 1450 # 36 # mime message rfc2045, utf-8
1118); 1451);
1119 1452
1120sub CBOR::XS::default_filter { 1453sub default_filter {
1121 &{ $FILTER{$_[0]} or return } 1454 &{ $FILTER{$_[0]} or return }
1455}
1456
1457our %SAFE_FILTER = map { $_ => $FILTER{$_} } 0, 1, 21, 22, 23, 32;
1458
1459sub safe_filter {
1460 &{ $SAFE_FILTER{$_[0]} or return }
1122} 1461}
1123 1462
1124sub URI::TO_CBOR { 1463sub URI::TO_CBOR {
1125 my $uri = $_[0]->as_string; 1464 my $uri = $_[0]->as_string;
1126 utf8::upgrade $uri; 1465 utf8::upgrade $uri;
1127 tag 32, $uri 1466 tag 32, $uri
1128} 1467}
1129 1468
1130sub Math::BigInt::TO_CBOR { 1469sub Math::BigInt::TO_CBOR {
1131 if ($_[0] >= -2147483648 && $_[0] <= 2147483647) { 1470 if (-2147483648 <= $_[0] && $_[0] <= 2147483647) {
1132 $_[0]->numify 1471 $_[0]->numify
1133 } else { 1472 } else {
1134 my $hex = substr $_[0]->as_hex, 2; 1473 my $hex = substr $_[0]->as_hex, 2;
1135 $hex = "0$hex" if 1 & length $hex; # sigh 1474 $hex = "0$hex" if 1 & length $hex; # sigh
1136 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex 1475 tag $_[0] >= 0 ? 2 : 3, pack "H*", $hex
1137 } 1476 }
1138} 1477}
1139 1478
1140sub Math::BigFloat::TO_CBOR { 1479sub Math::BigFloat::TO_CBOR {
1141 my ($m, $e) = $_[0]->parts; 1480 my ($m, $e) = $_[0]->parts;
1481
1482 -9223372036854775808 <= $e && $e <= 18446744073709551615
1142 tag 4, [$e->numify, $m] 1483 ? tag 4, [$e->numify, $m]
1484 : tag 264, [$e, $m]
1485}
1486
1487sub Math::BigRat::TO_CBOR {
1488 my ($n, $d) = $_[0]->parts;
1489
1490 # older versions of BigRat need *1, as they not always return numbers
1491
1492 $d*1 == 1
1493 ? $n*1
1494 : tag 30, [$n*1, $d*1]
1143} 1495}
1144 1496
1145sub Time::Piece::TO_CBOR { 1497sub Time::Piece::TO_CBOR {
1146 tag 1, 0 + $_[0]->epoch 1498 tag 1, 0 + $_[0]->epoch
1147} 1499}

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines